diff --git a/.github/workflows/lint.backend.yml b/.github/workflows/lint.backend.yml index 8aae8ed9..5e77057c 100644 --- a/.github/workflows/lint.backend.yml +++ b/.github/workflows/lint.backend.yml @@ -16,12 +16,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./backend - run: pip install -r requirements.txt + run: pip install -e .[dev] - name: Mypy type check working-directory: ./backend - run: mypy moonstream/ + run: mypy moonstreamapi/ - name: Black syntax check working-directory: ./backend - run: black --check moonstream/ + run: black --check moonstreamapi/ diff --git a/.github/workflows/lint.crawlers.yml b/.github/workflows/lint.crawlers.yml index a1d563d8..c8d202c6 100644 --- a/.github/workflows/lint.crawlers.yml +++ b/.github/workflows/lint.crawlers.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./crawlers/mooncrawl run: pip install -e .[dev] diff --git a/.github/workflows/lint.db.yml b/.github/workflows/lint.db.yml index 62937a51..99c26d72 100644 --- a/.github/workflows/lint.db.yml +++ b/.github/workflows/lint.db.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./db run: pip install -e .[dev] diff --git a/.github/workflows/release.mooncrawl.yml b/.github/workflows/release.mooncrawl.yml new file mode 100644 index 00000000..dde24973 --- /dev/null +++ b/.github/workflows/release.mooncrawl.yml @@ -0,0 +1,30 @@ +name: Release mooncrawl package + +on: + push: + tags: + - 'mooncrawl/v*' + +defaults: + run: + working-directory: crawlers/mooncrawl + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/release.moonstreamapi.yml b/.github/workflows/release.moonstreamapi.yml new file mode 100644 index 00000000..754ecbcb --- /dev/null +++ b/.github/workflows/release.moonstreamapi.yml @@ -0,0 +1,30 @@ +name: Release moonstreamapi package + +on: + push: + tags: + - 'moonstreamapi/v*' + +defaults: + run: + working-directory: backend + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/release.moonstreamdb.yml b/.github/workflows/release.moonstreamdb.yml new file mode 100644 index 00000000..3a1bc05c --- /dev/null +++ b/.github/workflows/release.moonstreamdb.yml @@ -0,0 +1,30 @@ +name: Release moonstreamdb package + +on: + push: + tags: + - 'moonstreamdb/v*' + +defaults: + run: + working-directory: db + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index bb113a64..00a8ac56 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Deployment script - intended to run on Moonstream servers +# Deployment script - intended to run on Moonstream API server # Colors C_RESET='\033[0m' @@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}" PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" -PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" -SERVICE_FILE="${SCRIPT_DIR}/moonstream.service" + +# Parameters scripts +PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# API server service file +SERVICE_FILE="${SCRIPT_DIR}/moonstreamapi.service" set -eu echo echo -echo -e "${PREFIX_INFO} Updating pip and setuptools" -"${PIP}" install -U pip setuptools +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt" +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_BACKEND_DIR}/" echo echo @@ -49,13 +54,18 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$ echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo -echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Updating nodes connection parameters" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Moonstream API service definition with ${SERVICE_FILE}" chmod 644 "${SERVICE_FILE}" -cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service +cp "${SERVICE_FILE}" /etc/systemd/system/moonstreamapi.service systemctl daemon-reload -systemctl restart moonstream.service -systemctl status moonstream.service +systemctl restart moonstreamapi.service +systemctl status moonstreamapi.service diff --git a/backend/deploy/moonstream.service b/backend/deploy/moonstreamapi.service similarity index 70% rename from backend/deploy/moonstream.service rename to backend/deploy/moonstreamapi.service index 7e94ff8b..c03eecf2 100644 --- a/backend/deploy/moonstream.service +++ b/backend/deploy/moonstreamapi.service @@ -1,5 +1,5 @@ [Unit] -Description=moonstream-service +Description=moonstreamapi-service After=network.target [Service] @@ -7,8 +7,8 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/backend EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app -SyslogIdentifier=moonstream +ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstreamapi.api:app +SyslogIdentifier=moonstreamapi [Install] WantedBy=multi-user.target diff --git a/backend/deploy/nodes-connections.bash b/backend/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/backend/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/backend/deploy/parameters.bash b/backend/deploy/parameters.bash index cb6ea2e9..dc401951 100755 --- a/backend/deploy/parameters.bash +++ b/backend/deploy/parameters.bash @@ -1,6 +1,9 @@ #!/usr/bin/env bash # -# Collect secrets from AWS SSM Parameter Store and output as environment variable exports. +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' # Colors C_RESET='\033[0m' @@ -26,6 +29,7 @@ and output as environment variable exports" echo " -o Output file name environment variables export to" } +# TODO(kompotkot): Flag for export prefix product_flag="" output_flag="" verbose_flag="false" @@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then exit 1 fi +verbose "${PREFIX_INFO} Script version: v${VERSION}" + verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" ENV_PARAMETERS=$(aws ssm describe-parameters \ --parameter-filters Key=tag:Product,Values=${product_flag} \ diff --git a/backend/dev.sh b/backend/dev.sh index bf4043c6..90b30c34 100755 --- a/backend/dev.sh +++ b/backend/dev.sh @@ -6,4 +6,4 @@ set -e MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}" MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}" -uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --reload +uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload diff --git a/backend/moonstream/__init__.py b/backend/moonstream/__init__.py deleted file mode 100644 index 5df6f535..00000000 --- a/backend/moonstream/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .reporter import reporter -from .version import MOONSTREAM_VERSION - -# Reporting -reporter.tags.append(f"version:{MOONSTREAM_VERSION}") -reporter.system_report(publish=True) -reporter.setup_excepthook(publish=True) diff --git a/backend/moonstreamapi/__init__.py b/backend/moonstreamapi/__init__.py new file mode 100644 index 00000000..e8612006 --- /dev/null +++ b/backend/moonstreamapi/__init__.py @@ -0,0 +1,11 @@ +try: + from .reporter import reporter + from .version import MOONSTREAMAPI_VERSION + + # Reporting + reporter.tags.append(f"version:{MOONSTREAMAPI_VERSION}") + reporter.system_report(publish=True) + reporter.setup_excepthook(publish=True) +except: + # Pass it to be able import MOONSTREAMAPI_VERSION in setup.py with pip + pass diff --git a/backend/moonstream/abi_decoder.py b/backend/moonstreamapi/abi_decoder.py similarity index 100% rename from backend/moonstream/abi_decoder.py rename to backend/moonstreamapi/abi_decoder.py diff --git a/backend/moonstream/actions.py b/backend/moonstreamapi/actions.py similarity index 100% rename from backend/moonstream/actions.py rename to backend/moonstreamapi/actions.py diff --git a/backend/moonstream/admin/__init__.py b/backend/moonstreamapi/admin/__init__.py similarity index 100% rename from backend/moonstream/admin/__init__.py rename to backend/moonstreamapi/admin/__init__.py diff --git a/backend/moonstream/admin/cli.py b/backend/moonstreamapi/admin/cli.py similarity index 100% rename from backend/moonstream/admin/cli.py rename to backend/moonstreamapi/admin/cli.py diff --git a/backend/moonstream/admin/migrations/__init__.py b/backend/moonstreamapi/admin/migrations/__init__.py similarity index 100% rename from backend/moonstream/admin/migrations/__init__.py rename to backend/moonstreamapi/admin/migrations/__init__.py diff --git a/backend/moonstream/admin/migrations/checksum_address.py b/backend/moonstreamapi/admin/migrations/checksum_address.py similarity index 100% rename from backend/moonstream/admin/migrations/checksum_address.py rename to backend/moonstreamapi/admin/migrations/checksum_address.py diff --git a/backend/moonstream/admin/subscription_types.py b/backend/moonstreamapi/admin/subscription_types.py similarity index 100% rename from backend/moonstream/admin/subscription_types.py rename to backend/moonstreamapi/admin/subscription_types.py diff --git a/backend/moonstream/api.py b/backend/moonstreamapi/api.py similarity index 96% rename from backend/moonstream/api.py rename to backend/moonstreamapi/api.py index 15bf4f4a..3629d922 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstreamapi/api.py @@ -19,7 +19,7 @@ from .routes.users import router as users_router from .routes.whales import router as whales_router from .middleware import BroodAuthMiddleware, MoonstreamHTTPException from .settings import DOCS_TARGET_PATH, ORIGINS -from .version import MOONSTREAM_VERSION +from .version import MOONSTREAMAPI_VERSION logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ tags_metadata = [ app = FastAPI( title=f"Moonstream API", description="Moonstream API endpoints.", - version=MOONSTREAM_VERSION, + version=MOONSTREAMAPI_VERSION, openapi_tags=tags_metadata, openapi_url="/openapi.json", docs_url=None, @@ -90,7 +90,7 @@ async def version_handler() -> data.VersionResponse: """ Get server version. """ - return data.VersionResponse(version=MOONSTREAM_VERSION) + return data.VersionResponse(version=MOONSTREAMAPI_VERSION) @app.get("/now", tags=["time"]) diff --git a/backend/moonstream/data.py b/backend/moonstreamapi/data.py similarity index 100% rename from backend/moonstream/data.py rename to backend/moonstreamapi/data.py diff --git a/backend/moonstream/middleware.py b/backend/moonstreamapi/middleware.py similarity index 100% rename from backend/moonstream/middleware.py rename to backend/moonstreamapi/middleware.py diff --git a/backend/moonstream/providers/__init__.py b/backend/moonstreamapi/providers/__init__.py similarity index 100% rename from backend/moonstream/providers/__init__.py rename to backend/moonstreamapi/providers/__init__.py diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstreamapi/providers/bugout.py similarity index 100% rename from backend/moonstream/providers/bugout.py rename to backend/moonstreamapi/providers/bugout.py diff --git a/backend/moonstream/providers/ethereum_blockchain.py b/backend/moonstreamapi/providers/ethereum_blockchain.py similarity index 100% rename from backend/moonstream/providers/ethereum_blockchain.py rename to backend/moonstreamapi/providers/ethereum_blockchain.py diff --git a/backend/moonstream/py.typed b/backend/moonstreamapi/py.typed similarity index 100% rename from backend/moonstream/py.typed rename to backend/moonstreamapi/py.typed diff --git a/backend/moonstream/reporter.py b/backend/moonstreamapi/reporter.py similarity index 93% rename from backend/moonstream/reporter.py rename to backend/moonstreamapi/reporter.py index 1ba0997b..0a67cc3e 100644 --- a/backend/moonstream/reporter.py +++ b/backend/moonstreamapi/reporter.py @@ -9,7 +9,7 @@ session_id = str(uuid.uuid4()) client_id = "moonstream-backend" reporter = HumbugReporter( - name="moonstream", + name="moonstreamapi", consent=HumbugConsent(True), client_id=client_id, session_id=session_id, diff --git a/backend/moonstream/routes/__init__.py b/backend/moonstreamapi/routes/__init__.py similarity index 100% rename from backend/moonstream/routes/__init__.py rename to backend/moonstreamapi/routes/__init__.py diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstreamapi/routes/address_info.py similarity index 100% rename from backend/moonstream/routes/address_info.py rename to backend/moonstreamapi/routes/address_info.py diff --git a/backend/moonstream/routes/nft.py b/backend/moonstreamapi/routes/nft.py similarity index 100% rename from backend/moonstream/routes/nft.py rename to backend/moonstreamapi/routes/nft.py diff --git a/backend/moonstream/routes/streams.py b/backend/moonstreamapi/routes/streams.py similarity index 100% rename from backend/moonstream/routes/streams.py rename to backend/moonstreamapi/routes/streams.py diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstreamapi/routes/subscriptions.py similarity index 100% rename from backend/moonstream/routes/subscriptions.py rename to backend/moonstreamapi/routes/subscriptions.py diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstreamapi/routes/txinfo.py similarity index 100% rename from backend/moonstream/routes/txinfo.py rename to backend/moonstreamapi/routes/txinfo.py diff --git a/backend/moonstream/routes/users.py b/backend/moonstreamapi/routes/users.py similarity index 100% rename from backend/moonstream/routes/users.py rename to backend/moonstreamapi/routes/users.py diff --git a/backend/moonstream/routes/whales.py b/backend/moonstreamapi/routes/whales.py similarity index 100% rename from backend/moonstream/routes/whales.py rename to backend/moonstreamapi/routes/whales.py diff --git a/backend/moonstream/settings.py b/backend/moonstreamapi/settings.py similarity index 100% rename from backend/moonstream/settings.py rename to backend/moonstreamapi/settings.py diff --git a/backend/moonstream/stream_boundaries.py b/backend/moonstreamapi/stream_boundaries.py similarity index 100% rename from backend/moonstream/stream_boundaries.py rename to backend/moonstreamapi/stream_boundaries.py diff --git a/backend/moonstream/stream_queries.py b/backend/moonstreamapi/stream_queries.py similarity index 100% rename from backend/moonstream/stream_queries.py rename to backend/moonstreamapi/stream_queries.py diff --git a/backend/moonstream/test_stream_boundaries.py b/backend/moonstreamapi/test_stream_boundaries.py similarity index 100% rename from backend/moonstream/test_stream_boundaries.py rename to backend/moonstreamapi/test_stream_boundaries.py diff --git a/backend/moonstream/test_stream_queries.py b/backend/moonstreamapi/test_stream_queries.py similarity index 100% rename from backend/moonstream/test_stream_queries.py rename to backend/moonstreamapi/test_stream_queries.py diff --git a/backend/moonstream/version.py b/backend/moonstreamapi/version.py similarity index 58% rename from backend/moonstream/version.py rename to backend/moonstreamapi/version.py index 20be0ba5..50195ae0 100644 --- a/backend/moonstream/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAM_VERSION = "0.0.2" +MOONSTREAMAPI_VERSION = "0.1.0" diff --git a/backend/moonstream/web3_provider.py b/backend/moonstreamapi/web3_provider.py similarity index 100% rename from backend/moonstream/web3_provider.py rename to backend/moonstreamapi/web3_provider.py diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index e4c0b435..00000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,35 +0,0 @@ -appdirs==1.4.4 -asgiref==3.4.1 -black==21.7b0 -boto3==1.18.1 -botocore==1.21.1 -bugout==0.1.18 -certifi==2021.5.30 -charset-normalizer==2.0.3 -click==8.0.1 -fastapi==0.66.0 -h11==0.12.0 -idna==3.2 -jmespath==0.10.0 -humbug==0.2.7 --e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db -mypy==0.910 -mypy-extensions==0.4.3 -pathspec==0.9.0 -pydantic==1.8.2 -pyevmasm==0.2.3 -python-dateutil==2.8.2 -python-multipart==0.0.5 -regex==2021.7.6 -requests==2.26.0 -s3transfer==0.5.0 -six==1.16.0 -starlette==0.14.2 -toml==0.10.2 -tomli==1.0.4 -types-python-dateutil==0.1.6 -typing-extensions==3.10.0.0 -types-requests==2.25.6 -urllib3==1.26.6 -uvicorn==0.14.0 -web3==5.24.0 \ No newline at end of file diff --git a/backend/setup.py b/backend/setup.py index c86bdfef..5730087a 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -1,30 +1,34 @@ from setuptools import find_packages, setup -from moonstream.version import MOONSTREAM_VERSION +from moonstreamapi.version import MOONSTREAMAPI_VERSION long_description = "" with open("README.md") as ifp: long_description = ifp.read() setup( - name="moonstream", - version=MOONSTREAM_VERSION, + name="moonstreamapi", + version=MOONSTREAMAPI_VERSION, packages=find_packages(), install_requires=[ + "appdirs", "boto3", - "bugout >= 0.1.18", + "bugout", "fastapi", - "humbug>=0.2.7", + "moonstreamdb", + "humbug", + "pydantic", + "pyevmasm", "python-dateutil", + "python-multipart", "uvicorn", - "types-python-dateutil", - "types-requests", + "web3", ], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"], "distribute": ["setuptools", "twine", "wheel"], }, - package_data={"moonstream": ["py.typed"]}, + package_data={"moonstreamapi": ["py.typed"]}, zip_safe=False, description="The Bugout blockchain inspector API.", long_description=long_description, @@ -38,5 +42,5 @@ setup( "Topic :: Software Development :: Libraries", ], url="https://github.com/bugout-dev/moonstream", - entry_points={"console_scripts": ["mnstr=moonstream.admin.cli:main"]}, + entry_points={"console_scripts": ["mnstr=moonstreamapi.admin.cli:main"]}, ) diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index f38a61a4..60e50f43 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -24,10 +24,14 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" + +# Parameters scripts PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts" -CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash" -CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# Crawlers server service file +CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" @@ -38,9 +42,6 @@ ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" # Polygon service file POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service" -CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" - - set -eu @@ -62,9 +63,13 @@ cd "${EXEC_DIR}" echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install --upgrade pip -"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt" +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools + +echo +echo +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/" echo echo @@ -75,12 +80,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo echo -e "${PREFIX_INFO} Updating nodes connection parameters" -curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" echo echo @@ -123,4 +128,3 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV systemctl daemon-reload systemctl restart "${CRAWLERS_SERVICE_FILE}" systemctl status "${CRAWLERS_SERVICE_FILE}" - diff --git a/crawlers/deploy/nodes-connections.bash b/crawlers/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/crawlers/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/crawlers/deploy/parameters.bash b/crawlers/deploy/parameters.bash new file mode 100755 index 00000000..dc401951 --- /dev/null +++ b/crawlers/deploy/parameters.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT" + echo + echo "CLI to collect secrets from AWS SSM Parameter Store +and output as environment variable exports" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -p Product tag (moonstream, spire, brood, drones)" + echo " -o Output file name environment variables export to" +} + +# TODO(kompotkot): Flag for export prefix +product_flag="" +output_flag="" +verbose_flag="false" + +while getopts 'p:o:v' flag; do + case "${flag}" in + p) product_flag="${OPTARG}" ;; + o) output_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# Product flag should be specified +# TODO(kompotkot): Extend script to work with few product at once +if [ -z "${product_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify product tag" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=${product_flag} \ + | jq -r .Parameters[].Name) +if [ -z "${ENV_PARAMETERS}" ]; then + verbose "${PREFIX_CRIT} There no parameters for provided product tag" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving parameters values" +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names ${ENV_PARAMETERS} \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length) +verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do + param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name) + param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value) + if [ -z "${output_flag}" ]; then + echo "${param_key}=${param_value}" + else + echo "${param_key}=${param_value}" >> "${output_flag}" + fi +done diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt deleted file mode 100644 index 79549178..00000000 --- a/crawlers/mooncrawl/requirements.txt +++ /dev/null @@ -1,60 +0,0 @@ -aiohttp==3.7.4.post0 -async-timeout==3.0.1 -attrs==21.2.0 -base58==2.1.0 -bitarray==1.2.2 -black==21.8b0 -boto3==1.18.40 -botocore==1.21.40 -bugout==0.1.17 -certifi==2021.5.30 -chardet==4.0.0 -charset-normalizer==2.0.4 -click==8.0.1 -cytoolz==0.11.0 -eth-abi==2.1.1 -eth-account==0.5.5 -eth-hash==0.3.2 -eth-keyfile==0.5.1 -eth-keys==0.3.3 -eth-rlp==0.2.1 -eth-typing==2.2.2 -eth-utils==1.10.0 -hexbytes==0.2.2 -humbug==0.2.7 -idna==3.2 -ipfshttpclient==0.8.0a2 -jmespath==0.10.0 -jsonschema==3.2.0 -isort==5.10.0 -lru-dict==1.1.7 -moonstreamdb==0.2.0 -multiaddr==0.0.9 -multidict==5.1.0 -mypy==0.910 -mypy-extensions==0.4.3 -netaddr==0.8.0 -parsimonious==0.8.1 -pathspec==0.9.0 -platformdirs==2.3.0 -protobuf==3.17.3 -pycryptodome==3.10.1 -pyrsistent==0.18.0 -python-dateutil==2.8.2 -regex==2021.8.28 -requests==2.26.0 -rlp==2.0.1 -s3transfer==0.5.0 -six==1.16.0 -toml==0.10.2 -tomli==1.2.1 -toolz==0.11.1 -tqdm==4.62.2 -types-python-dateutil==2.8.0 -types-requests==2.25.6 -typing-extensions==3.10.0.2 -urllib3==1.26.6 -varint==1.0.2 -web3==5.23.1 -websockets==9.1 -yarl==1.6.3 diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index d6e0f305..8f755014 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,8 +34,9 @@ setup( zip_safe=False, install_requires=[ "boto3", - "bugout >= 0.1.17", - "moonstreamdb == 0.2.0", + "bugout", + "chardet", + "moonstreamdb", "humbug", "python-dateutil", "requests", @@ -43,7 +44,8 @@ setup( "web3", ], extras_require={ - "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"] + "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"], + "distribute": ["setuptools", "twine", "wheel"], }, entry_points={ "console_scripts": [ diff --git a/db/setup.py b/db/setup.py index 601c4c14..8f8e70a5 100644 --- a/db/setup.py +++ b/db/setup.py @@ -34,7 +34,7 @@ setup( zip_safe=False, install_requires=["alembic", "psycopg2-binary", "sqlalchemy"], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy"], "distribute": ["setuptools", "twine", "wheel"], }, entry_points={