diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index db7b099d..c0ad2e06 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -3,20 +3,25 @@ # Deployment script - intended to run on Moonstream servers # Main -APP_DIR="${APP_DIR:-/home/ubuntu/app}" +APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" -PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/app-env}" +PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}" PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/app-secrets}" +SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SERVICE_FILE="${SCRIPT_DIR}/moonstream.service" set -eu +echo +echo +echo "Updating pip and setuptools" +"${PIP}" install -U pip setuptools + echo echo echo "Updating Python dependencies" diff --git a/backend/deploy/moonstream.service b/backend/deploy/moonstream.service index f92c48e4..f7b565aa 100644 --- a/backend/deploy/moonstream.service +++ b/backend/deploy/moonstream.service @@ -5,9 +5,9 @@ After=network.target [Service] User=ubuntu Group=www-data -WorkingDirectory=/home/ubuntu/app -EnvironmentFile=/home/ubuntu/secrets/app.env -ExecStart=/home/ubuntu/server-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app +WorkingDirectory=/home/ubuntu/moonstream +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app SyslogIdentifier=moonstream [Install] diff --git a/backend/requirements.txt b/backend/requirements.txt index 649acbd9..783e88cb 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,6 +8,7 @@ certifi==2021.5.30 charset-normalizer==2.0.3 click==8.0.1 fastapi==0.66.0 +-e git+https://git@github.com/bugout-dev/moonstream.git@60e90219a8e24077a1ab046463775f837df5f03e#egg=moonstreamdb&subdirectory=db h11==0.12.0 idna==3.2 jmespath==0.10.0 diff --git a/backend/setup.py b/backend/setup.py index 5c8d9371..f0b66219 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -29,5 +29,4 @@ setup( "Topic :: Software Development :: Libraries", ], url="https://github.com/bugout-dev/moonstream", - entry_points={"console_scripts": ["moonstream=moonstream.cli:main"]}, ) diff --git a/crawlers/.gitignore b/crawlers/.gitignore index 49a73033..6ecbec4f 100644 --- a/crawlers/.gitignore +++ b/crawlers/.gitignore @@ -162,4 +162,6 @@ cython_debug/ # Custom dev.env prod.env +prod.env.ps1 +dev.env.ps1 .venv diff --git a/crawlers/moonstreamcrawlers/version.py b/crawlers/moonstreamcrawlers/version.py new file mode 100644 index 00000000..960ebf6d --- /dev/null +++ b/crawlers/moonstreamcrawlers/version.py @@ -0,0 +1,5 @@ +""" +Moonstream crawlers version. +""" + +MOONSTREAMCRAWLERS_VERSION = "0.0.1" diff --git a/crawlers/requirements.txt b/crawlers/requirements.txt index 3bd0b16c..976fc0cc 100644 Binary files a/crawlers/requirements.txt and b/crawlers/requirements.txt differ diff --git a/crawlers/sample.env b/crawlers/sample.env index 7d907c50..f78af6d4 100644 --- a/crawlers/sample.env +++ b/crawlers/sample.env @@ -1,2 +1,3 @@ export MOONSTREAM_IPC_PATH=null -export MOONSTREAM_CRAWL_WORKERS_RAW=4 +export MOONSTREAM_CRAWL_WORKERS=4 +export MOONSTREAM_DB_URI="" diff --git a/crawlers/setup.py b/crawlers/setup.py index 7be14a58..4d4b93a4 100644 --- a/crawlers/setup.py +++ b/crawlers/setup.py @@ -1,12 +1,14 @@ from setuptools import find_packages, setup +from moonstreamcrawlers.version import MOONSTREAMCRAWLERS_VERSION + long_description = "" with open("README.md") as ifp: long_description = ifp.read() setup( name="moonstreamcrawlers", - version="0.0.1", + version=MOONSTREAMCRAWLERS_VERSION, author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0", @@ -28,7 +30,7 @@ setup( ], python_requires=">=3.6", packages=find_packages(), - package_data={"bugout": ["py.typed"]}, + package_data={"moonstreamcrawlers": ["py.typed"]}, zip_safe=False, install_requires=["web3"], extras_require={"dev": ["black", "mypy"]}, diff --git a/db/alembic.sh b/db/alembic.sh new file mode 100755 index 00000000..9ca8b9b8 --- /dev/null +++ b/db/alembic.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +PYTHONPATH=".:$PYTHONPATH" alembic "$@" diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index ad00e567..3b1e9e6c 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -1,5 +1,5 @@ """ -Exploration database connection. +Moonstream database connection. """ from contextlib import contextmanager import os @@ -7,20 +7,20 @@ import os from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session -EXPLORATION_DB_URI = os.environ.get("EXPLORATION_DB_URI") -if EXPLORATION_DB_URI is None: - raise ValueError("EXPLORATION_DB_URI environment variable must be set") -EXPLORATION_POOL_SIZE_RAW = os.environ.get("EXPLORATION_POOL_SIZE", 0) +MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") +if MOONSTREAM_DB_URI is None: + raise ValueError("MOONSTREAM_DB_URI environment variable must be set") +MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0) try: - if EXPLORATION_POOL_SIZE_RAW is not None: - EXPLORATION_POOL_SIZE = int(EXPLORATION_POOL_SIZE_RAW) + if MOONSTREAM_POOL_SIZE_RAW is not None: + MOONSTREAM_POOL_SIZE = int(MOONSTREAM_POOL_SIZE_RAW) except: raise Exception( - f"Could not parse EXPLORATION_POOL_SIZE as int: {EXPLORATION_POOL_SIZE_RAW}" + f"Could not parse MOONSTREAM_POOL_SIZE as int: {MOONSTREAM_POOL_SIZE_RAW}" ) # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool -engine = create_engine(EXPLORATION_DB_URI, pool_size=EXPLORATION_POOL_SIZE) +engine = create_engine(MOONSTREAM_DB_URI, pool_size=MOONSTREAM_POOL_SIZE) SessionLocal = sessionmaker(bind=engine) diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py new file mode 100644 index 00000000..5fcc4759 --- /dev/null +++ b/db/moonstreamdb/version.py @@ -0,0 +1,5 @@ +""" +Moonstream database version. +""" + +MOONSTREAMDB_VERSION = "0.0.1" diff --git a/db/sample.env b/db/sample.env index e5402a4a..852bb72d 100644 --- a/db/sample.env +++ b/db/sample.env @@ -1 +1 @@ -export EXPLORATION_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI="postgresql://:@:/" diff --git a/db/setup.py b/db/setup.py index c800c42a..abab0340 100644 --- a/db/setup.py +++ b/db/setup.py @@ -1,12 +1,14 @@ from setuptools import find_packages, setup +from moonstreamdb.version import MOONSTREAMDB_VERSION + long_description = "" with open("README.md") as ifp: long_description = ifp.read() setup( name="moonstreamdb", - version="0.0.1", + version=MOONSTREAMDB_VERSION, author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0", @@ -28,7 +30,7 @@ setup( ], python_requires=">=3.6", packages=find_packages(), - package_data={"bugout": ["py.typed"]}, + package_data={"moonstreamdb": ["py.typed"]}, zip_safe=False, install_requires=["alembic", "psycopg2-binary", "sqlalchemy"], extras_require={"dev": ["black", "mypy"]},