Merge branch 'main' into smart-contract-crawlers

backend/deploy/deploy.bash

@@ -3,20 +3,25 @@
 # Deployment script - intended to run on Moonstream servers
 
 # Main
-APP_DIR="${APP_DIR:-/home/ubuntu/app}"
+APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
-PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/app-env}"
+PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
 PYTHON="${PYTHON_ENV_DIR}/bin/python"
 PIP="${PYTHON_ENV_DIR}/bin/pip"
 SCRIPT_DIR="$(realpath $(dirname $0))"
 PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/app-secrets}"
+SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
 SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
 
 set -eu
 
+echo
+echo
+echo "Updating pip and setuptools"
+"${PIP}" install -U pip setuptools
+
 echo
 echo
 echo "Updating Python dependencies"

backend/deploy/moonstream.service

@@ -5,9 +5,9 @@ After=network.target
 [Service]
 User=ubuntu
 Group=www-data
-WorkingDirectory=/home/ubuntu/app
-EnvironmentFile=/home/ubuntu/secrets/app.env
-ExecStart=/home/ubuntu/server-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
+WorkingDirectory=/home/ubuntu/moonstream
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
 SyslogIdentifier=moonstream
 
 [Install]

backend/requirements.txt

@@ -8,6 +8,7 @@ certifi==2021.5.30
 charset-normalizer==2.0.3
 click==8.0.1
 fastapi==0.66.0
+-e git+https://git@github.com/bugout-dev/moonstream.git@60e90219a8e24077a1ab046463775f837df5f03e#egg=moonstreamdb&subdirectory=db
 h11==0.12.0
 idna==3.2
 jmespath==0.10.0

backend/setup.py

@@ -29,5 +29,4 @@ setup(
         "Topic :: Software Development :: Libraries",
     ],
     url="https://github.com/bugout-dev/moonstream",
-    entry_points={"console_scripts": ["moonstream=moonstream.cli:main"]},
 )

crawlers/.gitignore

@@ -162,4 +162,6 @@ cython_debug/
 # Custom
 dev.env
 prod.env
+prod.env.ps1
+dev.env.ps1
 .venv

crawlers/moonstreamcrawlers/version.py

@@ -0,0 +1,5 @@
+"""
+Moonstream crawlers version.
+"""
+
+MOONSTREAMCRAWLERS_VERSION = "0.0.1"

crawlers/sample.env

@@ -1,2 +1,3 @@
 export MOONSTREAM_IPC_PATH=null
-export MOONSTREAM_CRAWL_WORKERS_RAW=4
+export MOONSTREAM_CRAWL_WORKERS=4
+export MOONSTREAM_DB_URI="<moonstream_database_uri>"

crawlers/setup.py

@@ -1,12 +1,14 @@
 from setuptools import find_packages, setup
 
+from moonstreamcrawlers.version import MOONSTREAMCRAWLERS_VERSION
+
 long_description = ""
 with open("README.md") as ifp:
     long_description = ifp.read()
 
 setup(
     name="moonstreamcrawlers",
-    version="0.0.1",
+    version=MOONSTREAMCRAWLERS_VERSION,
     author="Bugout.dev",
     author_email="engineers@bugout.dev",
     license="Apache License 2.0",
@@ -28,7 +30,7 @@ setup(
     ],
     python_requires=">=3.6",
     packages=find_packages(),
-    package_data={"bugout": ["py.typed"]},
+    package_data={"moonstreamcrawlers": ["py.typed"]},
     zip_safe=False,
     install_requires=["web3"],
     extras_require={"dev": ["black", "mypy"]},

db/alembic.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+PYTHONPATH=".:$PYTHONPATH" alembic "$@"

db/moonstreamdb/db.py

@@ -1,5 +1,5 @@
 """
-Exploration database connection.
+Moonstream database connection.
 """
 from contextlib import contextmanager
 import os
@@ -7,20 +7,20 @@ import os
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
 
-EXPLORATION_DB_URI = os.environ.get("EXPLORATION_DB_URI")
-if EXPLORATION_DB_URI is None:
-    raise ValueError("EXPLORATION_DB_URI environment variable must be set")
-EXPLORATION_POOL_SIZE_RAW = os.environ.get("EXPLORATION_POOL_SIZE", 0)
+MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI")
+if MOONSTREAM_DB_URI is None:
+    raise ValueError("MOONSTREAM_DB_URI environment variable must be set")
+MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0)
 try:
-    if EXPLORATION_POOL_SIZE_RAW is not None:
-        EXPLORATION_POOL_SIZE = int(EXPLORATION_POOL_SIZE_RAW)
+    if MOONSTREAM_POOL_SIZE_RAW is not None:
+        MOONSTREAM_POOL_SIZE = int(MOONSTREAM_POOL_SIZE_RAW)
 except:
     raise Exception(
-        f"Could not parse EXPLORATION_POOL_SIZE as int: {EXPLORATION_POOL_SIZE_RAW}"
+        f"Could not parse MOONSTREAM_POOL_SIZE as int: {MOONSTREAM_POOL_SIZE_RAW}"
     )
 
 # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool
-engine = create_engine(EXPLORATION_DB_URI, pool_size=EXPLORATION_POOL_SIZE)
+engine = create_engine(MOONSTREAM_DB_URI, pool_size=MOONSTREAM_POOL_SIZE)
 SessionLocal = sessionmaker(bind=engine)
 
 

db/moonstreamdb/version.py

@@ -0,0 +1,5 @@
+"""
+Moonstream database version.
+"""
+
+MOONSTREAMDB_VERSION = "0.0.1"

db/sample.env

@@ -1 +1 @@
-export EXPLORATION_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
+export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"

db/setup.py

@@ -1,12 +1,14 @@
 from setuptools import find_packages, setup
 
+from moonstreamdb.version import MOONSTREAMDB_VERSION
+
 long_description = ""
 with open("README.md") as ifp:
     long_description = ifp.read()
 
 setup(
     name="moonstreamdb",
-    version="0.0.1",
+    version=MOONSTREAMDB_VERSION,
     author="Bugout.dev",
     author_email="engineers@bugout.dev",
     license="Apache License 2.0",
@@ -28,7 +30,7 @@ setup(
     ],
     python_requires=">=3.6",
     packages=find_packages(),
-    package_data={"bugout": ["py.typed"]},
+    package_data={"moonstreamdb": ["py.typed"]},
     zip_safe=False,
     install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
     extras_require={"dev": ["black", "mypy"]},