diff --git a/README.md b/README.md index 5fc88902..ed69524d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,28 @@ This monorepo contains the following components: [Bugout](https://bugout.dev). For more information on how that data is processed, check how the API inserts events from those sources into a stream. +### Installation and setup + +#### Run server with Docker Compose + +If you want to deploy Moonstream in isolation against live services, then docker compose is your choice! + +- Run script `backend/configs/docker_generate_env.bash` which prepare for you: + - `backend/configs/docker.moonstreamapi.env` with environment variables +- Run script `db/configs/docker_generate_env.bash` which prepare for you: + - `db/configs/alembic.moonstreamdb.ini` with postgresql uri + +```bash +./backend/configs/docker_generate_env.bash +./db/configs/docker_generate_env.bash +``` + +- Run local setup + +```bash +docker-compose up --build +``` + ## Contributing If you would like to contribute to Moonstream, please reach out to @zomglings on the [Moonstream Discord](https://discord.gg/pYE65FuNSz). diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..da3de864 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,220 @@ +# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### Pycharm ### +.idea + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python + +### VSCode ### +.vscode + +# Git +.git +.github + +# Envionments +.moonstream +.moonstreamapi +.venv +.env + +# Environment variables +test.env +dev.env +prod.env + +configs/test.env +configs/dev.env +configs/prod.env +configs/moonstreamapi.env +configs/docker.env +configs/docker.dev.env +configs/docker.test.env +configs/docker.prod.env + +.secrets/ + +Dockerfile +docker-compose.yml diff --git a/backend/.gitignore b/backend/.gitignore index 01ef8896..fad97d64 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -160,9 +160,21 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode -# Custom +# Envionments +.moonstream/ +.moonstreamapi/ +.venv/ +.env/ + +# Environment variables dev.env +test.env prod.env -.moonstream -.venv -.secrets +moonstreamapi.env +docker.env +docker.dev.env +docker.test.dev +docker.prod.env +docker.moonstreamapi.env + +.secrets/ diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..fa529ace --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.8-slim-buster + +# Update packages and +# prepare alembic for docker compose setup +RUN apt-get update && \ + apt-get install -y libpq-dev gcc curl && \ + rm -rf /var/lib/apt/lists/* && \ + pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 install --no-cache-dir psycopg2-binary alembic + +WORKDIR /usr/src/moonstreamapi + +COPY . /usr/src/moonstreamapi + +# Install Moonstream API package +RUN pip3 install --no-cache-dir -e . + +EXPOSE 7481 + +ENTRYPOINT ["./dev.sh"] \ No newline at end of file diff --git a/backend/README.md b/backend/README.md index 960f3b2c..a2e6e4d5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1 +1,43 @@ # moonstream backend + +### Installation and setup + +To set up Moonstream API for development, do the following: + +- Clone the git repository +- Install postgresql (https://www.postgresql.org/download/linux/ubuntu/) + +#### Run server with Docker + +To be able to run Moonstream API with your existing local or development services as database, you need to build your own setup. **Be aware! The files with environment variables `docker.dev.env` lives inside your docker container!** + +- Copy `configs/sample.env` to `configs/docker.dev.env`, or use your local configs from `configs/dev.env` to `configs/docker.dev.env` +- Edit in `docker.dev.env` file `MOONSTREAM_DB_URI` and other variables if required +- Clean environment file from `export ` prefix and quotation marks to be able to use it with Docker + +```bash +sed --in-place 's|^export * ||' configs/docker.dev.env +sed --in-place 's|"||g' configs/docker.dev.env +``` + +Build container on your machine + +```bash +docker build -t moonstreamapi-dev . +``` + +Run `moonstreamapi-dev` container, with following command we specified `--network="host"` setting which allows to Docker container use localhost interface of your machine (https://docs.docker.com/network/host/) + +```bash +docker run --name moonstreamapi-dev \ + --network="host" \ + --env-file="configs/docker.dev.env" \ + -p 7481:7481/tcp \ + -ti -d moonstreamapi-dev +``` + +Attach to container to see logs + +```bash +docker container attach moonstreamapi-dev +``` diff --git a/backend/configs/docker_generate_env.bash b/backend/configs/docker_generate_env.bash new file mode 100755 index 00000000..f3db3b53 --- /dev/null +++ b/backend/configs/docker_generate_env.bash @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Prepare Moonstream API application for docker-compose use + +# Print help message +function usage { + echo "Usage: $0 [-h] -d DATABASE_NAME" + echo + echo "CLI to generate environment variables" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -d Database name for postgres in docker-compose setup" +} + +FLAG_DATABASE_NAME="moonstream_dev" + +while getopts 'd:' flag; do + case "${flag}" in + d) FLAG_DATABASE_NAME="${OPTARG}" ;; + h) usage + exit 1 ;; + *) usage + exit 1 ;; + esac +done + +set -e + +SCRIPT_DIR="$(realpath $(dirname $0))" +DOCKER_MOONSTREAM_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME" +DOCKER_MOONSTREAM_ENV_FILE="docker.moonstreamapi.env" +BUGOUT_BROOD_URL="http://brood:7474" +BUGOUT_SPIRE_URL="http://spire:7475" + +# Generate environment variables + +cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" + +# Clean file with variables from export prefix and quotation marks +sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" + +sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^BUGOUT_BROOD_URL=.*|BUGOUT_BROOD_URL=$BUGOUT_BROOD_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^BUGOUT_SPIRE_URL=.*|BUGOUT_SPIRE_URL=$BUGOUT_SPIRE_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" diff --git a/backend/configs/docker_wait.sh b/backend/configs/docker_wait.sh new file mode 100755 index 00000000..86fd6d52 --- /dev/null +++ b/backend/configs/docker_wait.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -e + +HOST="$1" +shift + +until curl --request GET --url "http://$HOST/ping"; do + >&2 echo "$HOST is unavailable, sleeping" + sleep 1 +done + +>&2 echo "$HOST is up" diff --git a/backend/configs/sample.env b/backend/configs/sample.env new file mode 100644 index 00000000..e4d7418a --- /dev/null +++ b/backend/configs/sample.env @@ -0,0 +1,23 @@ +# Required environment variables +export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" +export BUGOUT_BROOD_URL="https://auth.bugout.dev" +export BUGOUT_SPIRE_URL="https://spire.bugout.dev" +export MOONSTREAM_APPLICATION_ID="" +export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export MOONSTREAM_POOL_SIZE=0 + +# Blockchain, txpool, whalewatch data depends variables +export MOONSTREAM_DATA_JOURNAL_ID="" +export HUMBUG_TXPOOL_CLIENT_ID="" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT=8545 + +# Set following parameters if AWS node instance and S3 smartcontracts configured +export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" +export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" + +# Set the following variables in the most reasonable manner for your development environment +export HUMBUG_REPORTER_BACKEND_TOKEN="" diff --git a/backend/dev.sh b/backend/dev.sh index 90b30c34..3080c352 100755 --- a/backend/dev.sh +++ b/backend/dev.sh @@ -1,9 +1,19 @@ #!/usr/bin/env sh -# Expects access to Python environment with the requirements for this project installed. +# Sets up Moonstream API server +# Expects access to Python environment with the requirements +# for this project installed. set -e -MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}" -MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}" +MOONSTREAMAPI_HOST="${MOONSTREAMAPI_HOST:-127.0.0.1}" +MOONSTREAMAPI_PORT="${MOONSTREAMAPI_PORT:-7481}" +MOONSTREAMAPI_APP_DIR="${MOONSTREAMAPI_APP_DIR:-$PWD}" +MOONSTREAMAPI_ASGI_APP="${MOONSTREAMAPI_ASGI_APP:-moonstreamapi.api:app}" +MOONSTREAMAPI_UVICORN_WORKERS="${MOONSTREAMAPI_UVICORN_WORKERS:-2}" -uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload +uvicorn --reload \ + --port "$MOONSTREAMAPI_PORT" \ + --host "$MOONSTREAMAPI_HOST" \ + --app-dir "$MOONSTREAMAPI_APP_DIR" \ + --workers "$MOONSTREAMAPI_UVICORN_WORKERS" \ + "$MOONSTREAMAPI_ASGI_APP" \ No newline at end of file diff --git a/backend/moonstreamapi/providers/bugout.py b/backend/moonstreamapi/providers/bugout.py index da00a721..371cdec7 100644 --- a/backend/moonstreamapi/providers/bugout.py +++ b/backend/moonstreamapi/providers/bugout.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session from .. import data from ..stream_queries import StreamQuery -from ..settings import ETHTXPOOL_HUMBUG_CLIENT_ID +from ..settings import HUMBUG_TXPOOL_CLIENT_ID logger = logging.getLogger(__name__) logger.setLevel(logging.WARN) @@ -362,5 +362,5 @@ ethereum_txpool_provider = EthereumTXPoolProvider( description=ethereum_txpool_description, default_time_interval_seconds=5, estimated_events_per_time_interval=50, - tags=[f"client:{ETHTXPOOL_HUMBUG_CLIENT_ID}"], + tags=[f"client:{HUMBUG_TXPOOL_CLIENT_ID}"], ) diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index c491ddc7..c2bfc750 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -39,14 +39,14 @@ DOCS_TARGET_PATH = "docs" DEFAULT_STREAM_TIMEINTERVAL = 5 * 60 -ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get( - "ETHTXPOOL_HUMBUG_CLIENT_ID", "client:ethereum-txpool-crawler-0" +HUMBUG_TXPOOL_CLIENT_ID = os.environ.get( + "HUMBUG_TXPOOL_CLIENT_ID", "client:ethereum-txpool-crawler-0" ) # S3 Bucket -ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") +ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET") if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: - raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") + raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET is not set") MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get( "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", "" diff --git a/backend/moonstreamapi/version.py b/backend/moonstreamapi/version.py index 50195ae0..a90de673 100644 --- a/backend/moonstreamapi/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.1.0" +MOONSTREAMAPI_VERSION = "0.1.1" diff --git a/backend/moonstreamapi/web3_provider.py b/backend/moonstreamapi/web3_provider.py index 719de079..946eb380 100644 --- a/backend/moonstreamapi/web3_provider.py +++ b/backend/moonstreamapi/web3_provider.py @@ -32,7 +32,7 @@ def fetch_web3_provider_ip(): return record_value -if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric(): +if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.startswith("http"): web3_provider_ip = fetch_web3_provider_ip() if web3_provider_ip is None: raise ValueError("Unable to extract web3 provider IP") diff --git a/backend/sample.env b/backend/sample.env deleted file mode 100644 index 8b3d4fa4..00000000 --- a/backend/sample.env +++ /dev/null @@ -1,15 +0,0 @@ -export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" -export MOONSTREAM_APPLICATION_ID="" -export MOONSTREAM_DATA_JOURNAL_ID="" -export MOONSTREAM_DB_URI="postgresql://:@:/" -export MOONSTREAM_POOL_SIZE=0 -export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" -export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="" -export AWS_S3_SMARTCONTRACT_BUCKET="" -export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" -export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" -export BUGOUT_BROOD_URL="https://auth.bugout.dev" -export BUGOUT_SPIRE_URL="https://spire.bugout.dev" -export HUMBUG_REPORTER_BACKEND_TOKEN="" -export ETHTXPOOL_HUMBUG_CLIENT_ID="" diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index 5ec96ed6..0ee24441 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -29,9 +29,9 @@ BASE_API_URL = "https://api.etherscan.io/api?module=contract&action=getsourcecod ETHERSCAN_SMARTCONTRACTS_LABEL_NAME = "etherscan_smartcontract" -bucket = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") +bucket = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET") if bucket is None: - raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET must be set") + raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET must be set") @dataclass diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index f415c35c..dc17504b 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -8,7 +8,7 @@ import logging import time from datetime import datetime, timedelta from enum import Enum -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, Union from uuid import UUID import boto3 # type: ignore @@ -411,6 +411,46 @@ def get_unique_address( ) +def get_blocks_state( + db_session: Session, blockchain_type: AvailableBlockchainType +) -> Dict[str, int]: + + """ + Generate meta information about + """ + + blocks_state = { + "latest_stored_block": 0, + "latest_labelled_block": 0, + "earliest_labelled_block": 0, + } + + label_model = get_label_model(blockchain_type) + + transactions_model = get_transaction_model(blockchain_type) + + max_transactions_number = db_session.query( + func.max(transactions_model.block_number).label("block_number") + ).scalar() + + result = ( + db_session.query( + func.min(label_model.block_number).label("earliest_labelled_block"), + func.max(label_model.block_number).label("latest_labelled_block"), + max_transactions_number, + ).filter(label_model.label == CRAWLER_LABEL) + ).one_or_none() + + if result: + earliest_labelled_block, latest_labelled_block, latest_stored_block = result + blocks_state = { + "latest_stored_block": latest_stored_block, + "latest_labelled_block": latest_labelled_block, + "earliest_labelled_block": earliest_labelled_block, + } + return blocks_state + + def generate_list_of_names( type: str, subscription_filters: Dict[str, Any], read_abi: bool, abi_json: Any ): @@ -577,7 +617,7 @@ def stats_generate_handler(args: argparse.Namespace): # Meen it's are different blockchain type continue - s3_data_object = {} + s3_data_object: Dict[str, Any] = {} extention_data = [] @@ -677,6 +717,10 @@ def stats_generate_handler(args: argparse.Namespace): } ) + current_blocks_state = get_blocks_state( + db_session=db_session, blockchain_type=blockchain_type + ) + for timescale in [timescale.value for timescale in TimeScale]: start_date = ( @@ -685,6 +729,8 @@ def stats_generate_handler(args: argparse.Namespace): print(f"Timescale: {timescale}") + s3_data_object["blocks_state"] = current_blocks_state + s3_data_object["web3_metric"] = extention_data functions_calls_data = generate_data( diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index f6ff4d9b..a58f54ce 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -8,7 +8,7 @@ export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN="" -export AWS_S3_SMARTCONTRACT_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" diff --git a/db/.dockerignore b/db/.dockerignore new file mode 100644 index 00000000..984ce5b4 --- /dev/null +++ b/db/.dockerignore @@ -0,0 +1,225 @@ +# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### Pycharm ### +.idea + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python + +### VSCode ### +.vscode + +# Git +.git +.github + +# Envionments +.moonstream +.moonstreamdb +.venv +.env + +# Environment variables +test.env +dev.env +prod.env + +configs/test.env +configs/dev.env +configs/prod.env +configs/moonstreamdb.env +configs/docker.env +configs/docker.dev.env +configs/docker.test.env +configs/docker.prod.env + +.secrets/ + +# Alembic migrations +configs/alembic.test.ini +configs/alembic.dev.ini +configs/alembic.prod.ini + +Dockerfile +docker-compose.yml diff --git a/db/.gitignore b/db/.gitignore index dfdac9a2..512e870f 100644 --- a/db/.gitignore +++ b/db/.gitignore @@ -160,14 +160,29 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode -# Custom -dev.env -prod.env -dev.env.ps1 -prod.env.ps1 -alembic.dev.ini -alembic.prod.ini +# Envionments +.moonstream/ +.moonstreamdb/ .db/ .venv/ +.env/ + +# Environment variables +dev.env +test.env +prod.env +moonstreamdb.env +docker.env +docker.dev.env +docker.test.dev +docker.prod.env +docker.moonstreamdb.env + .secrets/ -.moonstreamdb + +# Alembic migrations +alembic.test.ini +alembic.dev.ini +alembic.prod.ini +alembic.moonstreamdb.ini +alembic.docker.ini diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 00000000..df664b86 --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.8-slim-buster + +# Update packages and +# prepare alembic for docker compose setup +RUN apt-get update && \ + apt-get install -y libpq-dev gcc && \ + rm -rf /var/lib/apt/lists/* && \ + pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 install --no-cache-dir psycopg2-binary alembic + +WORKDIR /usr/src/moonstreamdb + +COPY . /usr/src/moonstreamdb + +# Install Moonstream DB package +RUN pip3 install --no-cache-dir -e . + +ENTRYPOINT ["./migrate.sh"] \ No newline at end of file diff --git a/db/alembic.sample.ini b/db/configs/alembic.sample.ini similarity index 100% rename from db/alembic.sample.ini rename to db/configs/alembic.sample.ini diff --git a/db/configs/docker_generate_env.bash b/db/configs/docker_generate_env.bash new file mode 100755 index 00000000..0ef9c16f --- /dev/null +++ b/db/configs/docker_generate_env.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Prepare Moonstream DB application for docker-compose use + +# Print help message +function usage { + echo "Usage: $0 [-h] -d DATABASE_NAME" + echo + echo "CLI to generate environment variables" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -d Database name for postgres in docker-compose setup" +} + +FLAG_DATABASE_NAME="moonstream_dev" + +while getopts 'd:' flag; do + case "${flag}" in + d) FLAG_DATABASE_NAME="${OPTARG}" ;; + h) usage + exit 1 ;; + *) usage + exit 1 ;; + esac +done + +set -e + +SCRIPT_DIR="$(realpath $(dirname $0))" +DOCKER_MOONSTREAMDB_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME" +DOCKER_MOONSTREAMDB_ENV_FILE="docker.moonstreamdb.env" +DOCKER_MOONSTREAMDB_ALEMBIC_FILE="alembic.moonstreamdb.ini" + +# Generate environment variables + +cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +# Clean file with variables from export prefix and quotation marks +sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" +sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +# Generate alembic config + +cp "$SCRIPT_DIR/alembic.sample.ini" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE" + +sed -i "s|^sqlalchemy.url =.*|sqlalchemy.url = $DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE" diff --git a/db/sample.env b/db/configs/sample.env similarity index 67% rename from db/sample.env rename to db/configs/sample.env index 4e3db953..c5409401 100644 --- a/db/sample.env +++ b/db/configs/sample.env @@ -1,2 +1,3 @@ +# Required environment variables to work with database CLI export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 diff --git a/db/migrate.sh b/db/migrate.sh new file mode 100755 index 00000000..5f8e573a --- /dev/null +++ b/db/migrate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Sets up Brood server for docker compose: +# 1. Running alembic migrations to head (using config file specified +# by the ALEMBIC_CONFIG environment variable) +# 2. Running dev.sh (from the directory from which this script was called) + +set -e + +if [ -z "$ALEMBIC_CONFIG" ] +then + echo "Please explicitly set the ALEMBIC_CONFIG environment variable to point to an alembic configuration file" + exit 1 +fi + +ALEMBIC_CONFIG="$ALEMBIC_CONFIG" sh alembic.sh upgrade head diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index e8c442cb..66c24df8 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.0" +MOONSTREAMDB_VERSION = "0.2.1" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..614e353e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ +# Compose version +version: "3" + +services: + # Moonstream API application + moonstreamapi: + build: + context: ./backend/ + dockerfile: ./Dockerfile + image: moonstreamapi:latest + ports: + - "127.0.0.1:7481:7481" + # Specify environment file for compose setup + env_file: ./backend/configs/docker.moonstreamapi.env + environment: + MOONSTREAMAPI_HOST: 0.0.0.0 + MOONSTREAMAPI_PORT: 7481 + MOONSTREAMAPI_UVICORN_WORKERS: 1 + healthcheck: + test: ["CMD", "curl", "-f", "http://moonstreamapi:7481/ping"] + interval: 5s + timeout: 1s + retries: 2 + start_period: 2s + depends_on: + db: + condition: service_healthy + + # Moonstream DB application + moonstreamdb: + build: + context: ./db/ + dockerfile: ./Dockerfile + image: moonstreamdb:latest + # Specify environment file for compose setup + env_file: ./db/configs/docker.moonstreamdb.env + environment: + ALEMBIC_CONFIG: ./configs/alembic.moonstreamdb.ini + depends_on: + db: + condition: service_healthy + + # DB postgres application + db: + image: postgres:13 + ports: + - "127.0.0.1:5432:5432" + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: moonstream_dev + healthcheck: + test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT 1;"] + interval: 5s + timeout: 1s + retries: 3 + start_period: 2s diff --git a/frontend/pages/status/index.js b/frontend/pages/status/index.js index c0c41882..8f9765a8 100644 --- a/frontend/pages/status/index.js +++ b/frontend/pages/status/index.js @@ -1,11 +1,14 @@ -import React from "react"; +import React, { useContext } from "react"; import { useStatus } from "../../src/core/hooks"; import { Heading, Text, Flex, Spacer, chakra, Spinner } from "@chakra-ui/react"; import { getLayout, getLayoutProps } from "../../src/layouts/InfoPageLayout"; +import UserContext from "../../src/core/providers/UserProvider/context"; const Status = () => { + const user = useContext(UserContext); const healthyStatusText = "Available"; const downStatusText = "Unavailable"; + const unauthorizedText = "Please login"; const healthyStatusColor = "green.900"; const downStatusColor = "red.600"; @@ -14,14 +17,43 @@ const Status = () => { }; const { - apiServerStatusCache, - ethereumClusterServerStatusCache, - gethStatusCache, + serverListStatusCache, crawlersStatusCache, dbServerStatusCache, latestBlockDBStatusCache, } = useStatus(); + const moonstreamapiStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "moonstreamapi" + )[0]; + const moonstreamCrawlersStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "moonstream_crawlers" + )[0]; + const nodeEthereumAStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_ethereum_a" + )[0]; + const nodeEthereumAGeth = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_ethereum_a_geth" + )[0]; + const nodeEthereumBStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_ethereum_b" + )[0]; + const nodeEthereumBGeth = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_ethereum_b_geth" + )[0]; + const nodePolygonAStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_polygon_a" + )[0]; + const nodePolygonAGeth = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_polygon_a_geth" + )[0]; + const nodePolygonBStatus = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_polygon_b" + )[0]; + const nodePolygonBGeth = serverListStatusCache?.data?.filter( + (i) => i.status.name === "node_polygon_b_geth" + )[0]; + const StatusRow = (props) => { console.log(props.cache.data); return ( @@ -46,50 +78,44 @@ const Status = () => { {`Status page`} - + - {apiServerStatusCache?.data?.status == "ok" + {moonstreamapiStatus?.status.body.status == "ok" ? healthyStatusText : downStatusText} +
- + + - {ethereumClusterServerStatusCache?.data + {moonstreamCrawlersStatus?.status.body.status == "ok" ? healthyStatusText : downStatusText} - - - {gethStatusCache?.data?.current_block - ? gethStatusCache.data.current_block - : 0} - - - {crawlersStatusCache?.data?.ethereum_txpool_timestamp - ? shortTimestamp( - crawlersStatusCache?.data?.ethereum_txpool_timestamp - ) - : downStatusText} + {!user + ? crawlersStatusCache?.data?.ethereum_txpool_timestamp + ? shortTimestamp( + crawlersStatusCache?.data?.ethereum_txpool_timestamp + ) + : downStatusText + : unauthorizedText} { cache={crawlersStatusCache} > - {crawlersStatusCache?.data?.ethereum_trending_timestamp - ? shortTimestamp( - crawlersStatusCache?.data?.ethereum_trending_timestamp - ) - : downStatusText} + {!user + ? crawlersStatusCache?.data?.ethereum_trending_timestamp + ? shortTimestamp( + crawlersStatusCache?.data?.ethereum_trending_timestamp + ) + : downStatusText + : unauthorizedText}
+ + + + {nodeEthereumAStatus?.status.body.status == "ok" + ? healthyStatusText + : downStatusText} + + + + + {nodeEthereumAGeth?.status.body.current_block + ? nodeEthereumAGeth.status.body.current_block + : 0} + + +
+ + + {nodeEthereumBStatus?.status.body.status == "ok" + ? healthyStatusText + : downStatusText} + + + + + {nodeEthereumBGeth?.status.body.current_block + ? nodeEthereumBGeth.status.body.current_block + : 0} + + +
+ + + {nodePolygonAStatus?.status.body.status == "ok" + ? healthyStatusText + : downStatusText} + + + + + {nodePolygonAGeth?.status.body.current_block + ? nodePolygonAGeth.status.body.current_block + : 0} + + +
+ + + {nodePolygonBStatus?.status.body.status == "ok" + ? healthyStatusText + : downStatusText} + + + + + {nodePolygonBGeth?.status.body.current_block + ? nodePolygonBGeth.status.body.current_block + : 0} + + + +
+ { - const getAPIServerStatus = async () => { - const response = await StatusService.apiServerStatus(); - return response.data; - }; - const getEthereumClusterServerStatus = async () => { - const response = await StatusService.ethereumClusterServerStatus(); - return response.data; - }; - const getGethStatus = async () => { - const response = await StatusService.gethStatus(); + const getServerListStatus = async () => { + const response = await StatusService.serverListStatus(); return response.data; }; const getCrawlersStatus = async () => { @@ -28,22 +20,14 @@ const useStatus = () => { return response.data; }; - const apiServerStatusCache = useQuery("apiServer", getAPIServerStatus, { - ...queryCacheProps, - retry: 0, - }); - const ethereumClusterServerStatusCache = useQuery( - "ethereumClusterServer", - getEthereumClusterServerStatus, + const serverListStatusCache = useQuery( + "serverListStatus", + getServerListStatus, { ...queryCacheProps, retry: 0, } ); - const gethStatusCache = useQuery("geth", getGethStatus, { - ...queryCacheProps, - retry: 0, - }); const crawlersStatusCache = useQuery("crawlers", getCrawlersStatus, { ...queryCacheProps, retry: 0, @@ -62,9 +46,7 @@ const useStatus = () => { ); return { - apiServerStatusCache, - ethereumClusterServerStatusCache, - gethStatusCache, + serverListStatusCache, crawlersStatusCache, dbServerStatusCache, latestBlockDBStatusCache, diff --git a/frontend/src/core/services/status.service.js b/frontend/src/core/services/status.service.js index 8666ccd7..9dff55ca 100644 --- a/frontend/src/core/services/status.service.js +++ b/frontend/src/core/services/status.service.js @@ -1,28 +1,13 @@ import { http } from "../utils"; +const BUGOUT_STATUS_URL = process.env.NEXT_PUBLIC_BUGOUT_STATUS_URL; const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL; const DB_URL = process.env.NEXT_PUBLIC_MOONSTREAM_DB_URL; -const ETHEREUM_CLUSTER_URL = - process.env.NEXT_PUBLIC_MOONSTREAM_ETHEREUM_CLUSTER_URL; -export const apiServerStatus = () => { +export const serverListStatus = () => { return http({ method: "GET", - url: `${API_URL}/ping`, - }); -}; - -export const ethereumClusterServerStatus = () => { - return http({ - method: "GET", - url: `${ETHEREUM_CLUSTER_URL}/ping`, - }); -}; - -export const gethStatus = () => { - return http({ - method: "GET", - url: `${ETHEREUM_CLUSTER_URL}/status`, + url: `${BUGOUT_STATUS_URL}`, }); };