diff --git a/README.md b/README.md index 5fc88902..ed69524d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,28 @@ This monorepo contains the following components: [Bugout](https://bugout.dev). For more information on how that data is processed, check how the API inserts events from those sources into a stream. +### Installation and setup + +#### Run server with Docker Compose + +If you want to deploy Moonstream in isolation against live services, then docker compose is your choice! + +- Run script `backend/configs/docker_generate_env.bash` which prepare for you: + - `backend/configs/docker.moonstreamapi.env` with environment variables +- Run script `db/configs/docker_generate_env.bash` which prepare for you: + - `db/configs/alembic.moonstreamdb.ini` with postgresql uri + +```bash +./backend/configs/docker_generate_env.bash +./db/configs/docker_generate_env.bash +``` + +- Run local setup + +```bash +docker-compose up --build +``` + ## Contributing If you would like to contribute to Moonstream, please reach out to @zomglings on the [Moonstream Discord](https://discord.gg/pYE65FuNSz). diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..da3de864 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,220 @@ +# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### Pycharm ### +.idea + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python + +### VSCode ### +.vscode + +# Git +.git +.github + +# Envionments +.moonstream +.moonstreamapi +.venv +.env + +# Environment variables +test.env +dev.env +prod.env + +configs/test.env +configs/dev.env +configs/prod.env +configs/moonstreamapi.env +configs/docker.env +configs/docker.dev.env +configs/docker.test.env +configs/docker.prod.env + +.secrets/ + +Dockerfile +docker-compose.yml diff --git a/backend/.gitignore b/backend/.gitignore index 01ef8896..fad97d64 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -160,9 +160,21 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode -# Custom +# Envionments +.moonstream/ +.moonstreamapi/ +.venv/ +.env/ + +# Environment variables dev.env +test.env prod.env -.moonstream -.venv -.secrets +moonstreamapi.env +docker.env +docker.dev.env +docker.test.dev +docker.prod.env +docker.moonstreamapi.env + +.secrets/ diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..fa529ace --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.8-slim-buster + +# Update packages and +# prepare alembic for docker compose setup +RUN apt-get update && \ + apt-get install -y libpq-dev gcc curl && \ + rm -rf /var/lib/apt/lists/* && \ + pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 install --no-cache-dir psycopg2-binary alembic + +WORKDIR /usr/src/moonstreamapi + +COPY . /usr/src/moonstreamapi + +# Install Moonstream API package +RUN pip3 install --no-cache-dir -e . + +EXPOSE 7481 + +ENTRYPOINT ["./dev.sh"] \ No newline at end of file diff --git a/backend/README.md b/backend/README.md index 960f3b2c..a2e6e4d5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1 +1,43 @@ # moonstream backend + +### Installation and setup + +To set up Moonstream API for development, do the following: + +- Clone the git repository +- Install postgresql (https://www.postgresql.org/download/linux/ubuntu/) + +#### Run server with Docker + +To be able to run Moonstream API with your existing local or development services as database, you need to build your own setup. **Be aware! The files with environment variables `docker.dev.env` lives inside your docker container!** + +- Copy `configs/sample.env` to `configs/docker.dev.env`, or use your local configs from `configs/dev.env` to `configs/docker.dev.env` +- Edit in `docker.dev.env` file `MOONSTREAM_DB_URI` and other variables if required +- Clean environment file from `export ` prefix and quotation marks to be able to use it with Docker + +```bash +sed --in-place 's|^export * ||' configs/docker.dev.env +sed --in-place 's|"||g' configs/docker.dev.env +``` + +Build container on your machine + +```bash +docker build -t moonstreamapi-dev . +``` + +Run `moonstreamapi-dev` container, with following command we specified `--network="host"` setting which allows to Docker container use localhost interface of your machine (https://docs.docker.com/network/host/) + +```bash +docker run --name moonstreamapi-dev \ + --network="host" \ + --env-file="configs/docker.dev.env" \ + -p 7481:7481/tcp \ + -ti -d moonstreamapi-dev +``` + +Attach to container to see logs + +```bash +docker container attach moonstreamapi-dev +``` diff --git a/backend/configs/docker_generate_env.bash b/backend/configs/docker_generate_env.bash new file mode 100755 index 00000000..f3db3b53 --- /dev/null +++ b/backend/configs/docker_generate_env.bash @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Prepare Moonstream API application for docker-compose use + +# Print help message +function usage { + echo "Usage: $0 [-h] -d DATABASE_NAME" + echo + echo "CLI to generate environment variables" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -d Database name for postgres in docker-compose setup" +} + +FLAG_DATABASE_NAME="moonstream_dev" + +while getopts 'd:' flag; do + case "${flag}" in + d) FLAG_DATABASE_NAME="${OPTARG}" ;; + h) usage + exit 1 ;; + *) usage + exit 1 ;; + esac +done + +set -e + +SCRIPT_DIR="$(realpath $(dirname $0))" +DOCKER_MOONSTREAM_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME" +DOCKER_MOONSTREAM_ENV_FILE="docker.moonstreamapi.env" +BUGOUT_BROOD_URL="http://brood:7474" +BUGOUT_SPIRE_URL="http://spire:7475" + +# Generate environment variables + +cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" + +# Clean file with variables from export prefix and quotation marks +sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" + +sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^BUGOUT_BROOD_URL=.*|BUGOUT_BROOD_URL=$BUGOUT_BROOD_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^BUGOUT_SPIRE_URL=.*|BUGOUT_SPIRE_URL=$BUGOUT_SPIRE_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" diff --git a/backend/configs/docker_wait.sh b/backend/configs/docker_wait.sh new file mode 100755 index 00000000..86fd6d52 --- /dev/null +++ b/backend/configs/docker_wait.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -e + +HOST="$1" +shift + +until curl --request GET --url "http://$HOST/ping"; do + >&2 echo "$HOST is unavailable, sleeping" + sleep 1 +done + +>&2 echo "$HOST is up" diff --git a/backend/configs/sample.env b/backend/configs/sample.env new file mode 100644 index 00000000..e4d7418a --- /dev/null +++ b/backend/configs/sample.env @@ -0,0 +1,23 @@ +# Required environment variables +export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" +export BUGOUT_BROOD_URL="https://auth.bugout.dev" +export BUGOUT_SPIRE_URL="https://spire.bugout.dev" +export MOONSTREAM_APPLICATION_ID="" +export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export MOONSTREAM_POOL_SIZE=0 + +# Blockchain, txpool, whalewatch data depends variables +export MOONSTREAM_DATA_JOURNAL_ID="" +export HUMBUG_TXPOOL_CLIENT_ID="" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT=8545 + +# Set following parameters if AWS node instance and S3 smartcontracts configured +export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" +export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" + +# Set the following variables in the most reasonable manner for your development environment +export HUMBUG_REPORTER_BACKEND_TOKEN="" diff --git a/backend/dev.sh b/backend/dev.sh index 90b30c34..3080c352 100755 --- a/backend/dev.sh +++ b/backend/dev.sh @@ -1,9 +1,19 @@ #!/usr/bin/env sh -# Expects access to Python environment with the requirements for this project installed. +# Sets up Moonstream API server +# Expects access to Python environment with the requirements +# for this project installed. set -e -MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}" -MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}" +MOONSTREAMAPI_HOST="${MOONSTREAMAPI_HOST:-127.0.0.1}" +MOONSTREAMAPI_PORT="${MOONSTREAMAPI_PORT:-7481}" +MOONSTREAMAPI_APP_DIR="${MOONSTREAMAPI_APP_DIR:-$PWD}" +MOONSTREAMAPI_ASGI_APP="${MOONSTREAMAPI_ASGI_APP:-moonstreamapi.api:app}" +MOONSTREAMAPI_UVICORN_WORKERS="${MOONSTREAMAPI_UVICORN_WORKERS:-2}" -uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload +uvicorn --reload \ + --port "$MOONSTREAMAPI_PORT" \ + --host "$MOONSTREAMAPI_HOST" \ + --app-dir "$MOONSTREAMAPI_APP_DIR" \ + --workers "$MOONSTREAMAPI_UVICORN_WORKERS" \ + "$MOONSTREAMAPI_ASGI_APP" \ No newline at end of file diff --git a/backend/moonstreamapi/providers/bugout.py b/backend/moonstreamapi/providers/bugout.py index df73711c..ca01a6e5 100644 --- a/backend/moonstreamapi/providers/bugout.py +++ b/backend/moonstreamapi/providers/bugout.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session from .. import data from ..stream_queries import StreamQuery -from ..settings import ETHTXPOOL_HUMBUG_CLIENT_ID +from ..settings import HUMBUG_TXPOOL_CLIENT_ID logger = logging.getLogger(__name__) logger.setLevel(logging.WARN) @@ -354,5 +354,5 @@ ethereum_txpool_provider = EthereumTXPoolProvider( description=ethereum_txpool_description, default_time_interval_seconds=5, estimated_events_per_time_interval=50, - tags=[f"client:{ETHTXPOOL_HUMBUG_CLIENT_ID}"], + tags=[f"client:{HUMBUG_TXPOOL_CLIENT_ID}"], ) diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index c491ddc7..c2bfc750 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -39,14 +39,14 @@ DOCS_TARGET_PATH = "docs" DEFAULT_STREAM_TIMEINTERVAL = 5 * 60 -ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get( - "ETHTXPOOL_HUMBUG_CLIENT_ID", "client:ethereum-txpool-crawler-0" +HUMBUG_TXPOOL_CLIENT_ID = os.environ.get( + "HUMBUG_TXPOOL_CLIENT_ID", "client:ethereum-txpool-crawler-0" ) # S3 Bucket -ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") +ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET") if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: - raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") + raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET is not set") MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get( "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", "" diff --git a/backend/moonstreamapi/version.py b/backend/moonstreamapi/version.py index 50195ae0..a90de673 100644 --- a/backend/moonstreamapi/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.1.0" +MOONSTREAMAPI_VERSION = "0.1.1" diff --git a/backend/moonstreamapi/web3_provider.py b/backend/moonstreamapi/web3_provider.py index 719de079..946eb380 100644 --- a/backend/moonstreamapi/web3_provider.py +++ b/backend/moonstreamapi/web3_provider.py @@ -32,7 +32,7 @@ def fetch_web3_provider_ip(): return record_value -if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric(): +if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.startswith("http"): web3_provider_ip = fetch_web3_provider_ip() if web3_provider_ip is None: raise ValueError("Unable to extract web3 provider IP") diff --git a/backend/sample.env b/backend/sample.env deleted file mode 100644 index 8b3d4fa4..00000000 --- a/backend/sample.env +++ /dev/null @@ -1,15 +0,0 @@ -export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" -export MOONSTREAM_APPLICATION_ID="" -export MOONSTREAM_DATA_JOURNAL_ID="" -export MOONSTREAM_DB_URI="postgresql://:@:/" -export MOONSTREAM_POOL_SIZE=0 -export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" -export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="" -export AWS_S3_SMARTCONTRACT_BUCKET="" -export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" -export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" -export BUGOUT_BROOD_URL="https://auth.bugout.dev" -export BUGOUT_SPIRE_URL="https://spire.bugout.dev" -export HUMBUG_REPORTER_BACKEND_TOKEN="" -export ETHTXPOOL_HUMBUG_CLIENT_ID="" diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index 5ec96ed6..0ee24441 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -29,9 +29,9 @@ BASE_API_URL = "https://api.etherscan.io/api?module=contract&action=getsourcecod ETHERSCAN_SMARTCONTRACTS_LABEL_NAME = "etherscan_smartcontract" -bucket = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") +bucket = os.environ.get("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET") if bucket is None: - raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET must be set") + raise ValueError("MOONSTREAM_S3_SMARTCONTRACTS_BUCKET must be set") @dataclass diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index f6ff4d9b..a58f54ce 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -8,7 +8,7 @@ export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN="" -export AWS_S3_SMARTCONTRACT_BUCKET="" +export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" diff --git a/db/.dockerignore b/db/.dockerignore new file mode 100644 index 00000000..984ce5b4 --- /dev/null +++ b/db/.dockerignore @@ -0,0 +1,225 @@ +# Created by https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,visualstudiocode,linux,python + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### Pycharm ### +.idea + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.toptal.com/developers/gitignore/api/vim,visualstudiocode,linux,python + +### VSCode ### +.vscode + +# Git +.git +.github + +# Envionments +.moonstream +.moonstreamdb +.venv +.env + +# Environment variables +test.env +dev.env +prod.env + +configs/test.env +configs/dev.env +configs/prod.env +configs/moonstreamdb.env +configs/docker.env +configs/docker.dev.env +configs/docker.test.env +configs/docker.prod.env + +.secrets/ + +# Alembic migrations +configs/alembic.test.ini +configs/alembic.dev.ini +configs/alembic.prod.ini + +Dockerfile +docker-compose.yml diff --git a/db/.gitignore b/db/.gitignore index dfdac9a2..512e870f 100644 --- a/db/.gitignore +++ b/db/.gitignore @@ -160,14 +160,29 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode -# Custom -dev.env -prod.env -dev.env.ps1 -prod.env.ps1 -alembic.dev.ini -alembic.prod.ini +# Envionments +.moonstream/ +.moonstreamdb/ .db/ .venv/ +.env/ + +# Environment variables +dev.env +test.env +prod.env +moonstreamdb.env +docker.env +docker.dev.env +docker.test.dev +docker.prod.env +docker.moonstreamdb.env + .secrets/ -.moonstreamdb + +# Alembic migrations +alembic.test.ini +alembic.dev.ini +alembic.prod.ini +alembic.moonstreamdb.ini +alembic.docker.ini diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 00000000..df664b86 --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.8-slim-buster + +# Update packages and +# prepare alembic for docker compose setup +RUN apt-get update && \ + apt-get install -y libpq-dev gcc && \ + rm -rf /var/lib/apt/lists/* && \ + pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 install --no-cache-dir psycopg2-binary alembic + +WORKDIR /usr/src/moonstreamdb + +COPY . /usr/src/moonstreamdb + +# Install Moonstream DB package +RUN pip3 install --no-cache-dir -e . + +ENTRYPOINT ["./migrate.sh"] \ No newline at end of file diff --git a/db/alembic.sample.ini b/db/configs/alembic.sample.ini similarity index 100% rename from db/alembic.sample.ini rename to db/configs/alembic.sample.ini diff --git a/db/configs/docker_generate_env.bash b/db/configs/docker_generate_env.bash new file mode 100755 index 00000000..0ef9c16f --- /dev/null +++ b/db/configs/docker_generate_env.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Prepare Moonstream DB application for docker-compose use + +# Print help message +function usage { + echo "Usage: $0 [-h] -d DATABASE_NAME" + echo + echo "CLI to generate environment variables" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -d Database name for postgres in docker-compose setup" +} + +FLAG_DATABASE_NAME="moonstream_dev" + +while getopts 'd:' flag; do + case "${flag}" in + d) FLAG_DATABASE_NAME="${OPTARG}" ;; + h) usage + exit 1 ;; + *) usage + exit 1 ;; + esac +done + +set -e + +SCRIPT_DIR="$(realpath $(dirname $0))" +DOCKER_MOONSTREAMDB_DB_URI="postgresql://postgres:postgres@db/$FLAG_DATABASE_NAME" +DOCKER_MOONSTREAMDB_ENV_FILE="docker.moonstreamdb.env" +DOCKER_MOONSTREAMDB_ALEMBIC_FILE="alembic.moonstreamdb.ini" + +# Generate environment variables + +cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +# Clean file with variables from export prefix and quotation marks +sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" +sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" + +# Generate alembic config + +cp "$SCRIPT_DIR/alembic.sample.ini" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE" + +sed -i "s|^sqlalchemy.url =.*|sqlalchemy.url = $DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE" diff --git a/db/sample.env b/db/configs/sample.env similarity index 67% rename from db/sample.env rename to db/configs/sample.env index 4e3db953..c5409401 100644 --- a/db/sample.env +++ b/db/configs/sample.env @@ -1,2 +1,3 @@ +# Required environment variables to work with database CLI export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 diff --git a/db/migrate.sh b/db/migrate.sh new file mode 100755 index 00000000..5f8e573a --- /dev/null +++ b/db/migrate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Sets up Brood server for docker compose: +# 1. Running alembic migrations to head (using config file specified +# by the ALEMBIC_CONFIG environment variable) +# 2. Running dev.sh (from the directory from which this script was called) + +set -e + +if [ -z "$ALEMBIC_CONFIG" ] +then + echo "Please explicitly set the ALEMBIC_CONFIG environment variable to point to an alembic configuration file" + exit 1 +fi + +ALEMBIC_CONFIG="$ALEMBIC_CONFIG" sh alembic.sh upgrade head diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index e8c442cb..66c24df8 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.0" +MOONSTREAMDB_VERSION = "0.2.1" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..614e353e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ +# Compose version +version: "3" + +services: + # Moonstream API application + moonstreamapi: + build: + context: ./backend/ + dockerfile: ./Dockerfile + image: moonstreamapi:latest + ports: + - "127.0.0.1:7481:7481" + # Specify environment file for compose setup + env_file: ./backend/configs/docker.moonstreamapi.env + environment: + MOONSTREAMAPI_HOST: 0.0.0.0 + MOONSTREAMAPI_PORT: 7481 + MOONSTREAMAPI_UVICORN_WORKERS: 1 + healthcheck: + test: ["CMD", "curl", "-f", "http://moonstreamapi:7481/ping"] + interval: 5s + timeout: 1s + retries: 2 + start_period: 2s + depends_on: + db: + condition: service_healthy + + # Moonstream DB application + moonstreamdb: + build: + context: ./db/ + dockerfile: ./Dockerfile + image: moonstreamdb:latest + # Specify environment file for compose setup + env_file: ./db/configs/docker.moonstreamdb.env + environment: + ALEMBIC_CONFIG: ./configs/alembic.moonstreamdb.ini + depends_on: + db: + condition: service_healthy + + # DB postgres application + db: + image: postgres:13 + ports: + - "127.0.0.1:5432:5432" + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: moonstream_dev + healthcheck: + test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT 1;"] + interval: 5s + timeout: 1s + retries: 3 + start_period: 2s