diff --git a/.github/workflows/lint.backend.yml b/.github/workflows/lint.backend.yml index 8aae8ed9..5e77057c 100644 --- a/.github/workflows/lint.backend.yml +++ b/.github/workflows/lint.backend.yml @@ -16,12 +16,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./backend - run: pip install -r requirements.txt + run: pip install -e .[dev] - name: Mypy type check working-directory: ./backend - run: mypy moonstream/ + run: mypy moonstreamapi/ - name: Black syntax check working-directory: ./backend - run: black --check moonstream/ + run: black --check moonstreamapi/ diff --git a/.github/workflows/lint.crawlers.yml b/.github/workflows/lint.crawlers.yml index a1d563d8..c8d202c6 100644 --- a/.github/workflows/lint.crawlers.yml +++ b/.github/workflows/lint.crawlers.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./crawlers/mooncrawl run: pip install -e .[dev] diff --git a/.github/workflows/lint.db.yml b/.github/workflows/lint.db.yml index 62937a51..99c26d72 100644 --- a/.github/workflows/lint.db.yml +++ b/.github/workflows/lint.db.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./db run: pip install -e .[dev] diff --git a/.github/workflows/release.mooncrawl.yml b/.github/workflows/release.mooncrawl.yml new file mode 100644 index 00000000..dde24973 --- /dev/null +++ b/.github/workflows/release.mooncrawl.yml @@ -0,0 +1,30 @@ +name: Release mooncrawl package + +on: + push: + tags: + - 'mooncrawl/v*' + +defaults: + run: + working-directory: crawlers/mooncrawl + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/release.moonstreamapi.yml b/.github/workflows/release.moonstreamapi.yml new file mode 100644 index 00000000..754ecbcb --- /dev/null +++ b/.github/workflows/release.moonstreamapi.yml @@ -0,0 +1,30 @@ +name: Release moonstreamapi package + +on: + push: + tags: + - 'moonstreamapi/v*' + +defaults: + run: + working-directory: backend + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/release.moonstreamdb.yml b/.github/workflows/release.moonstreamdb.yml new file mode 100644 index 00000000..3a1bc05c --- /dev/null +++ b/.github/workflows/release.moonstreamdb.yml @@ -0,0 +1,30 @@ +name: Release moonstreamdb package + +on: + push: + tags: + - 'moonstreamdb/v*' + +defaults: + run: + working-directory: db + +jobs: + publish: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[distribute] + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index bb113a64..00a8ac56 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Deployment script - intended to run on Moonstream servers +# Deployment script - intended to run on Moonstream API server # Colors C_RESET='\033[0m' @@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}" PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" -PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" -SERVICE_FILE="${SCRIPT_DIR}/moonstream.service" + +# Parameters scripts +PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# API server service file +SERVICE_FILE="${SCRIPT_DIR}/moonstreamapi.service" set -eu echo echo -echo -e "${PREFIX_INFO} Updating pip and setuptools" -"${PIP}" install -U pip setuptools +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt" +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_BACKEND_DIR}/" echo echo @@ -49,13 +54,18 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$ echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo -echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Updating nodes connection parameters" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Moonstream API service definition with ${SERVICE_FILE}" chmod 644 "${SERVICE_FILE}" -cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service +cp "${SERVICE_FILE}" /etc/systemd/system/moonstreamapi.service systemctl daemon-reload -systemctl restart moonstream.service -systemctl status moonstream.service +systemctl restart moonstreamapi.service +systemctl status moonstreamapi.service diff --git a/backend/deploy/moonstream.service b/backend/deploy/moonstreamapi.service similarity index 70% rename from backend/deploy/moonstream.service rename to backend/deploy/moonstreamapi.service index 7e94ff8b..c03eecf2 100644 --- a/backend/deploy/moonstream.service +++ b/backend/deploy/moonstreamapi.service @@ -1,5 +1,5 @@ [Unit] -Description=moonstream-service +Description=moonstreamapi-service After=network.target [Service] @@ -7,8 +7,8 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/backend EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app -SyslogIdentifier=moonstream +ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstreamapi.api:app +SyslogIdentifier=moonstreamapi [Install] WantedBy=multi-user.target diff --git a/backend/deploy/nodes-connections.bash b/backend/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/backend/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/backend/deploy/parameters.bash b/backend/deploy/parameters.bash index cb6ea2e9..dc401951 100755 --- a/backend/deploy/parameters.bash +++ b/backend/deploy/parameters.bash @@ -1,6 +1,9 @@ #!/usr/bin/env bash # -# Collect secrets from AWS SSM Parameter Store and output as environment variable exports. +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' # Colors C_RESET='\033[0m' @@ -26,6 +29,7 @@ and output as environment variable exports" echo " -o Output file name environment variables export to" } +# TODO(kompotkot): Flag for export prefix product_flag="" output_flag="" verbose_flag="false" @@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then exit 1 fi +verbose "${PREFIX_INFO} Script version: v${VERSION}" + verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" ENV_PARAMETERS=$(aws ssm describe-parameters \ --parameter-filters Key=tag:Product,Values=${product_flag} \ diff --git a/backend/dev.sh b/backend/dev.sh index bf4043c6..90b30c34 100755 --- a/backend/dev.sh +++ b/backend/dev.sh @@ -6,4 +6,4 @@ set -e MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}" MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}" -uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --reload +uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload diff --git a/backend/moonstream/__init__.py b/backend/moonstream/__init__.py deleted file mode 100644 index 5df6f535..00000000 --- a/backend/moonstream/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .reporter import reporter -from .version import MOONSTREAM_VERSION - -# Reporting -reporter.tags.append(f"version:{MOONSTREAM_VERSION}") -reporter.system_report(publish=True) -reporter.setup_excepthook(publish=True) diff --git a/backend/moonstreamapi/__init__.py b/backend/moonstreamapi/__init__.py new file mode 100644 index 00000000..e8612006 --- /dev/null +++ b/backend/moonstreamapi/__init__.py @@ -0,0 +1,11 @@ +try: + from .reporter import reporter + from .version import MOONSTREAMAPI_VERSION + + # Reporting + reporter.tags.append(f"version:{MOONSTREAMAPI_VERSION}") + reporter.system_report(publish=True) + reporter.setup_excepthook(publish=True) +except: + # Pass it to be able import MOONSTREAMAPI_VERSION in setup.py with pip + pass diff --git a/backend/moonstream/abi_decoder.py b/backend/moonstreamapi/abi_decoder.py similarity index 100% rename from backend/moonstream/abi_decoder.py rename to backend/moonstreamapi/abi_decoder.py diff --git a/backend/moonstream/actions.py b/backend/moonstreamapi/actions.py similarity index 100% rename from backend/moonstream/actions.py rename to backend/moonstreamapi/actions.py diff --git a/backend/moonstream/admin/__init__.py b/backend/moonstreamapi/admin/__init__.py similarity index 100% rename from backend/moonstream/admin/__init__.py rename to backend/moonstreamapi/admin/__init__.py diff --git a/backend/moonstream/admin/cli.py b/backend/moonstreamapi/admin/cli.py similarity index 100% rename from backend/moonstream/admin/cli.py rename to backend/moonstreamapi/admin/cli.py diff --git a/backend/moonstream/admin/migrations/1_add_abi_backet_and_s3_path.json b/backend/moonstreamapi/admin/migrations/1_add_abi_backet_and_s3_path.json similarity index 100% rename from backend/moonstream/admin/migrations/1_add_abi_backet_and_s3_path.json rename to backend/moonstreamapi/admin/migrations/1_add_abi_backet_and_s3_path.json diff --git a/backend/moonstream/admin/migrations/__init__.py b/backend/moonstreamapi/admin/migrations/__init__.py similarity index 100% rename from backend/moonstream/admin/migrations/__init__.py rename to backend/moonstreamapi/admin/migrations/__init__.py diff --git a/backend/moonstream/admin/migrations/checksum_address.py b/backend/moonstreamapi/admin/migrations/checksum_address.py similarity index 100% rename from backend/moonstream/admin/migrations/checksum_address.py rename to backend/moonstreamapi/admin/migrations/checksum_address.py diff --git a/backend/moonstream/admin/subscription_types.py b/backend/moonstreamapi/admin/subscription_types.py similarity index 100% rename from backend/moonstream/admin/subscription_types.py rename to backend/moonstreamapi/admin/subscription_types.py diff --git a/backend/moonstream/admin/subscriptions.py b/backend/moonstreamapi/admin/subscriptions.py similarity index 100% rename from backend/moonstream/admin/subscriptions.py rename to backend/moonstreamapi/admin/subscriptions.py diff --git a/backend/moonstream/api.py b/backend/moonstreamapi/api.py similarity index 96% rename from backend/moonstream/api.py rename to backend/moonstreamapi/api.py index 7acc1af4..5e281725 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstreamapi/api.py @@ -20,7 +20,7 @@ from .routes.whales import router as whales_router from .routes.dashboards import router as dashboards_router from .middleware import BroodAuthMiddleware, MoonstreamHTTPException from .settings import DOCS_TARGET_PATH, ORIGINS -from .version import MOONSTREAM_VERSION +from .version import MOONSTREAMAPI_VERSION logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -45,7 +45,7 @@ tags_metadata = [ app = FastAPI( title=f"Moonstream API", description="Moonstream API endpoints.", - version=MOONSTREAM_VERSION, + version=MOONSTREAMAPI_VERSION, openapi_tags=tags_metadata, openapi_url="/openapi.json", docs_url=None, @@ -91,7 +91,7 @@ async def version_handler() -> data.VersionResponse: """ Get server version. """ - return data.VersionResponse(version=MOONSTREAM_VERSION) + return data.VersionResponse(version=MOONSTREAMAPI_VERSION) @app.get("/now", tags=["time"]) diff --git a/backend/moonstream/data.py b/backend/moonstreamapi/data.py similarity index 100% rename from backend/moonstream/data.py rename to backend/moonstreamapi/data.py diff --git a/backend/moonstream/middleware.py b/backend/moonstreamapi/middleware.py similarity index 100% rename from backend/moonstream/middleware.py rename to backend/moonstreamapi/middleware.py diff --git a/backend/moonstream/providers/__init__.py b/backend/moonstreamapi/providers/__init__.py similarity index 100% rename from backend/moonstream/providers/__init__.py rename to backend/moonstreamapi/providers/__init__.py diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstreamapi/providers/bugout.py similarity index 100% rename from backend/moonstream/providers/bugout.py rename to backend/moonstreamapi/providers/bugout.py diff --git a/backend/moonstream/providers/ethereum_blockchain.py b/backend/moonstreamapi/providers/ethereum_blockchain.py similarity index 100% rename from backend/moonstream/providers/ethereum_blockchain.py rename to backend/moonstreamapi/providers/ethereum_blockchain.py diff --git a/backend/moonstream/py.typed b/backend/moonstreamapi/py.typed similarity index 100% rename from backend/moonstream/py.typed rename to backend/moonstreamapi/py.typed diff --git a/backend/moonstream/reporter.py b/backend/moonstreamapi/reporter.py similarity index 93% rename from backend/moonstream/reporter.py rename to backend/moonstreamapi/reporter.py index 1ba0997b..0a67cc3e 100644 --- a/backend/moonstream/reporter.py +++ b/backend/moonstreamapi/reporter.py @@ -9,7 +9,7 @@ session_id = str(uuid.uuid4()) client_id = "moonstream-backend" reporter = HumbugReporter( - name="moonstream", + name="moonstreamapi", consent=HumbugConsent(True), client_id=client_id, session_id=session_id, diff --git a/backend/moonstream/routes/__init__.py b/backend/moonstreamapi/routes/__init__.py similarity index 100% rename from backend/moonstream/routes/__init__.py rename to backend/moonstreamapi/routes/__init__.py diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstreamapi/routes/address_info.py similarity index 100% rename from backend/moonstream/routes/address_info.py rename to backend/moonstreamapi/routes/address_info.py diff --git a/backend/moonstream/routes/dashboards.py b/backend/moonstreamapi/routes/dashboards.py similarity index 100% rename from backend/moonstream/routes/dashboards.py rename to backend/moonstreamapi/routes/dashboards.py diff --git a/backend/moonstream/routes/nft.py b/backend/moonstreamapi/routes/nft.py similarity index 100% rename from backend/moonstream/routes/nft.py rename to backend/moonstreamapi/routes/nft.py diff --git a/backend/moonstream/routes/streams.py b/backend/moonstreamapi/routes/streams.py similarity index 100% rename from backend/moonstream/routes/streams.py rename to backend/moonstreamapi/routes/streams.py diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstreamapi/routes/subscriptions.py similarity index 100% rename from backend/moonstream/routes/subscriptions.py rename to backend/moonstreamapi/routes/subscriptions.py diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstreamapi/routes/txinfo.py similarity index 100% rename from backend/moonstream/routes/txinfo.py rename to backend/moonstreamapi/routes/txinfo.py diff --git a/backend/moonstream/routes/users.py b/backend/moonstreamapi/routes/users.py similarity index 100% rename from backend/moonstream/routes/users.py rename to backend/moonstreamapi/routes/users.py diff --git a/backend/moonstream/routes/whales.py b/backend/moonstreamapi/routes/whales.py similarity index 100% rename from backend/moonstream/routes/whales.py rename to backend/moonstreamapi/routes/whales.py diff --git a/backend/moonstream/settings.py b/backend/moonstreamapi/settings.py similarity index 100% rename from backend/moonstream/settings.py rename to backend/moonstreamapi/settings.py diff --git a/backend/moonstream/stream_boundaries.py b/backend/moonstreamapi/stream_boundaries.py similarity index 100% rename from backend/moonstream/stream_boundaries.py rename to backend/moonstreamapi/stream_boundaries.py diff --git a/backend/moonstream/stream_queries.py b/backend/moonstreamapi/stream_queries.py similarity index 100% rename from backend/moonstream/stream_queries.py rename to backend/moonstreamapi/stream_queries.py diff --git a/backend/moonstream/test_actions_abi_validations.py b/backend/moonstreamapi/test_actions_abi_validations.py similarity index 100% rename from backend/moonstream/test_actions_abi_validations.py rename to backend/moonstreamapi/test_actions_abi_validations.py diff --git a/backend/moonstream/test_stream_boundaries.py b/backend/moonstreamapi/test_stream_boundaries.py similarity index 100% rename from backend/moonstream/test_stream_boundaries.py rename to backend/moonstreamapi/test_stream_boundaries.py diff --git a/backend/moonstream/test_stream_queries.py b/backend/moonstreamapi/test_stream_queries.py similarity index 100% rename from backend/moonstream/test_stream_queries.py rename to backend/moonstreamapi/test_stream_queries.py diff --git a/backend/moonstream/version.py b/backend/moonstreamapi/version.py similarity index 58% rename from backend/moonstream/version.py rename to backend/moonstreamapi/version.py index 20be0ba5..50195ae0 100644 --- a/backend/moonstream/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAM_VERSION = "0.0.2" +MOONSTREAMAPI_VERSION = "0.1.0" diff --git a/backend/moonstream/web3_provider.py b/backend/moonstreamapi/web3_provider.py similarity index 100% rename from backend/moonstream/web3_provider.py rename to backend/moonstreamapi/web3_provider.py diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index e4c0b435..00000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,35 +0,0 @@ -appdirs==1.4.4 -asgiref==3.4.1 -black==21.7b0 -boto3==1.18.1 -botocore==1.21.1 -bugout==0.1.18 -certifi==2021.5.30 -charset-normalizer==2.0.3 -click==8.0.1 -fastapi==0.66.0 -h11==0.12.0 -idna==3.2 -jmespath==0.10.0 -humbug==0.2.7 --e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db -mypy==0.910 -mypy-extensions==0.4.3 -pathspec==0.9.0 -pydantic==1.8.2 -pyevmasm==0.2.3 -python-dateutil==2.8.2 -python-multipart==0.0.5 -regex==2021.7.6 -requests==2.26.0 -s3transfer==0.5.0 -six==1.16.0 -starlette==0.14.2 -toml==0.10.2 -tomli==1.0.4 -types-python-dateutil==0.1.6 -typing-extensions==3.10.0.0 -types-requests==2.25.6 -urllib3==1.26.6 -uvicorn==0.14.0 -web3==5.24.0 \ No newline at end of file diff --git a/backend/setup.py b/backend/setup.py index c86bdfef..5730087a 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -1,30 +1,34 @@ from setuptools import find_packages, setup -from moonstream.version import MOONSTREAM_VERSION +from moonstreamapi.version import MOONSTREAMAPI_VERSION long_description = "" with open("README.md") as ifp: long_description = ifp.read() setup( - name="moonstream", - version=MOONSTREAM_VERSION, + name="moonstreamapi", + version=MOONSTREAMAPI_VERSION, packages=find_packages(), install_requires=[ + "appdirs", "boto3", - "bugout >= 0.1.18", + "bugout", "fastapi", - "humbug>=0.2.7", + "moonstreamdb", + "humbug", + "pydantic", + "pyevmasm", "python-dateutil", + "python-multipart", "uvicorn", - "types-python-dateutil", - "types-requests", + "web3", ], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"], "distribute": ["setuptools", "twine", "wheel"], }, - package_data={"moonstream": ["py.typed"]}, + package_data={"moonstreamapi": ["py.typed"]}, zip_safe=False, description="The Bugout blockchain inspector API.", long_description=long_description, @@ -38,5 +42,5 @@ setup( "Topic :: Software Development :: Libraries", ], url="https://github.com/bugout-dev/moonstream", - entry_points={"console_scripts": ["mnstr=moonstream.admin.cli:main"]}, + entry_points={"console_scripts": ["mnstr=moonstreamapi.admin.cli:main"]}, ) diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index d0467a87..f8c5dc50 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -24,15 +24,28 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" + +# Parameters scripts PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts" -CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash" -CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# Crawlers server service file +CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" + +# Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" -ETHEREUM_TRENDING_SERVICE="ethereum-trending.service" -ETHEREUM_TRENDING_TIMER="ethereum-trending.service" -ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service" -ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" +ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service" +ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer" +ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" +ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service" +ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer" + +# Polygon service file +POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service" +POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" +POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" + set -eu @@ -54,9 +67,13 @@ cd "${EXEC_DIR}" echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install --upgrade pip -"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt" +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools + +echo +echo +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/" echo echo @@ -67,12 +84,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo echo -e "${PREFIX_INFO} Updating nodes connection parameters" -curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" echo echo @@ -84,27 +101,52 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}" echo echo -echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}" -chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" -cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}" -cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER_FILE}" systemctl daemon-reload -systemctl restart "${ETHEREUM_TRENDING_TIMER}" +systemctl restart "${ETHEREUM_TRENDING_TIMER_FILE}" echo echo -echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}" -chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" -cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE_FILE}" systemctl daemon-reload -systemctl restart "${ETHEREUM_TXPOOL_SERVICE}" +systemctl restart "${ETHEREUM_TXPOOL_SERVICE_FILE}" echo echo -echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}" -chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}" systemctl daemon-reload -systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}" -systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}" +systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}" +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${POLYGON_SYNCHRONIZE_SERVICE}" +systemctl daemon-reload +systemctl restart "${POLYGON_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERVICE_FILE}" +systemctl daemon-reload +systemctl restart "${CRAWLERS_SERVICE_FILE}" +systemctl status "${CRAWLERS_SERVICE_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Polygon missing service and timer with: ${POLYGON_MISSING_SERVICE_FILE}, ${POLYGON_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}" +systemctl daemon-reload +systemctl restart "${POLYGON_MISSING_TIMER_FILE}" diff --git a/crawlers/deploy/ethereum-missing.service b/crawlers/deploy/ethereum-missing.service new file mode 100644 index 00000000..c52eb23b --- /dev/null +++ b/crawlers/deploy/ethereum-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Ethereum database +After=network.target + +[Service] +Type=oneshot +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n \ No newline at end of file diff --git a/crawlers/deploy/ethereum-missing.timer b/crawlers/deploy/ethereum-missing.timer new file mode 100644 index 00000000..87fc6c16 --- /dev/null +++ b/crawlers/deploy/ethereum-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Ethereum database + +[Timer] +OnBootSec=10s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/ethereum-synchronize.service b/crawlers/deploy/ethereum-synchronize.service index 20b7aa0e..825e3f35 100644 --- a/crawlers/deploy/ethereum-synchronize.service +++ b/crawlers/deploy/ethereum-synchronize.service @@ -7,7 +7,7 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler blocks synchronize -c 6 -j 1 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 1 SyslogIdentifier=ethereum-synchronize [Install] diff --git a/crawlers/deploy/ethereum-trending.service b/crawlers/deploy/ethereum-trending.service index 579cc63d..1bb59d79 100644 --- a/crawlers/deploy/ethereum-trending.service +++ b/crawlers/deploy/ethereum-trending.service @@ -8,4 +8,4 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler trending \ No newline at end of file +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler trending \ No newline at end of file diff --git a/crawlers/deploy/nodes-connections.bash b/crawlers/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/crawlers/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/crawlers/deploy/parameters.bash b/crawlers/deploy/parameters.bash new file mode 100755 index 00000000..dc401951 --- /dev/null +++ b/crawlers/deploy/parameters.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT" + echo + echo "CLI to collect secrets from AWS SSM Parameter Store +and output as environment variable exports" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -p Product tag (moonstream, spire, brood, drones)" + echo " -o Output file name environment variables export to" +} + +# TODO(kompotkot): Flag for export prefix +product_flag="" +output_flag="" +verbose_flag="false" + +while getopts 'p:o:v' flag; do + case "${flag}" in + p) product_flag="${OPTARG}" ;; + o) output_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# Product flag should be specified +# TODO(kompotkot): Extend script to work with few product at once +if [ -z "${product_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify product tag" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=${product_flag} \ + | jq -r .Parameters[].Name) +if [ -z "${ENV_PARAMETERS}" ]; then + verbose "${PREFIX_CRIT} There no parameters for provided product tag" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving parameters values" +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names ${ENV_PARAMETERS} \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length) +verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do + param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name) + param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value) + if [ -z "${output_flag}" ]; then + echo "${param_key}=${param_value}" + else + echo "${param_key}=${param_value}" >> "${output_flag}" + fi +done diff --git a/crawlers/deploy/polygon-missing.service b/crawlers/deploy/polygon-missing.service new file mode 100644 index 00000000..2b90adcb --- /dev/null +++ b/crawlers/deploy/polygon-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Polygon database +After=network.target + +[Service] +Type=oneshot +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n \ No newline at end of file diff --git a/crawlers/deploy/polygon-missing.timer b/crawlers/deploy/polygon-missing.timer new file mode 100644 index 00000000..9fd8a52e --- /dev/null +++ b/crawlers/deploy/polygon-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Polygon database + +[Timer] +OnBootSec=10s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/polygon-synchronize.service b/crawlers/deploy/polygon-synchronize.service new file mode 100644 index 00000000..3f002132 --- /dev/null +++ b/crawlers/deploy/polygon-synchronize.service @@ -0,0 +1,14 @@ +[Unit] +Description=Polygon block with transactions synchronizer +After=network.target + +[Service] +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon -c 6 -j 1 +SyslogIdentifier=polygon-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/mooncrawl/.isort.cfg b/crawlers/mooncrawl/.isort.cfg new file mode 100644 index 00000000..81d54de1 --- /dev/null +++ b/crawlers/mooncrawl/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile = black +multi_line_output = 3 \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/__init__.py b/crawlers/mooncrawl/mooncrawl/__init__.py index 9548e297..a93e3aa7 100644 --- a/crawlers/mooncrawl/mooncrawl/__init__.py +++ b/crawlers/mooncrawl/mooncrawl/__init__.py @@ -1,7 +1,11 @@ -from .reporter import reporter -from .version import MOONCRAWL_VERSION +try: + from .reporter import reporter + from .version import MOONCRAWL_VERSION -# Reporting -reporter.tags.append(f"version:{MOONCRAWL_VERSION}") -reporter.system_report(publish=True) -reporter.setup_excepthook(publish=True) + # Reporting + reporter.tags.append(f"version:{MOONCRAWL_VERSION}") + reporter.system_report(publish=True) + reporter.setup_excepthook(publish=True) +except: + # Pass it to be able import MOONCRAWL_VERSION in setup.py with pip + pass diff --git a/crawlers/mooncrawl/mooncrawl/ethereum.py b/crawlers/mooncrawl/mooncrawl/blockchain.py similarity index 67% rename from crawlers/mooncrawl/mooncrawl/ethereum.py rename to crawlers/mooncrawl/mooncrawl/blockchain.py index c02a9eb5..0aef1ab4 100644 --- a/crawlers/mooncrawl/mooncrawl/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -1,64 +1,118 @@ -from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait -from dataclasses import dataclass -from datetime import datetime import logging -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union -from psycopg2.errors import UniqueViolation # type: ignore -from sqlalchemy import desc, Column -from sqlalchemy import func -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session, Query -from tqdm import tqdm -from web3 import Web3, IPCProvider, HTTPProvider -from web3.types import BlockData - -from .settings import MOONSTREAM_IPC_PATH, MOONSTREAM_CRAWL_WORKERS from moonstreamdb.db import yield_db_session, yield_db_session_ctx from moonstreamdb.models import ( EthereumBlock, EthereumTransaction, + PolygonBlock, + PolygonTransaction, +) +from psycopg2.errors import UniqueViolation # type: ignore +from sqlalchemy import Column, desc, func +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session +from tqdm import tqdm +from web3 import HTTPProvider, IPCProvider, Web3 +from web3.middleware import geth_poa_middleware +from web3.types import BlockData + +from .data import AvailableBlockchainType, DateRange +from .settings import ( + MOONSTREAM_CRAWL_WORKERS, + MOONSTREAM_ETHEREUM_IPC_PATH, + MOONSTREAM_POLYGON_IPC_PATH, ) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class EthereumBlockCrawlError(Exception): +class BlockCrawlError(Exception): """ - Raised when there is a problem crawling Ethereum blocks. + Raised when there is a problem crawling blocks. """ -@dataclass -class DateRange: - start_time: datetime - end_time: datetime - include_start: bool - include_end: bool - - -def connect(web3_uri: Optional[str] = MOONSTREAM_IPC_PATH): +def connect(blockchain_type: AvailableBlockchainType, web3_uri: Optional[str] = None): web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider() - if web3_uri is not None: - if web3_uri.startswith("http://") or web3_uri.startswith("https://"): - web3_provider = Web3.HTTPProvider(web3_uri) + + if web3_uri is None: + if blockchain_type == AvailableBlockchainType.ETHEREUM: + web3_uri = MOONSTREAM_ETHEREUM_IPC_PATH + elif blockchain_type == AvailableBlockchainType.POLYGON: + web3_uri = MOONSTREAM_POLYGON_IPC_PATH else: - web3_provider = Web3.IPCProvider(web3_uri) + raise Exception("Wrong blockchain type provided for web3 URI") + + if web3_uri.startswith("http://") or web3_uri.startswith("https://"): + web3_provider = Web3.HTTPProvider(web3_uri) + else: + web3_provider = Web3.IPCProvider(web3_uri) web3_client = Web3(web3_provider) + + # Inject --dev middleware if it is not Ethereum mainnet + # Docs: https://web3py.readthedocs.io/en/stable/middleware.html#geth-style-proof-of-authority + if blockchain_type != AvailableBlockchainType.ETHEREUM: + web3_client.middleware_onion.inject(geth_poa_middleware, layer=0) + return web3_client -def add_block(db_session, block: Any) -> None: +def get_block_model( + blockchain_type: AvailableBlockchainType, +) -> Type[Union[EthereumBlock, PolygonBlock]]: + """ + Depends on provided blockchain type: Ethereum or Polygon, + set proper blocks model: EthereumBlock or PolygonBlock. + """ + block_model: Type[Union[EthereumBlock, PolygonBlock]] + if blockchain_type == AvailableBlockchainType.ETHEREUM: + block_model = EthereumBlock + elif blockchain_type == AvailableBlockchainType.POLYGON: + block_model = PolygonBlock + else: + raise Exception("Unsupported blockchain type provided") + + return block_model + + +def get_transaction_model( + blockchain_type: AvailableBlockchainType, +) -> Type[Union[EthereumTransaction, PolygonTransaction]]: + """ + Depends on provided blockchain type: Ethereum or Polygon, + set proper block transactions model: EthereumTransaction or PolygonTransaction. + """ + transaction_model: Type[Union[EthereumTransaction, PolygonTransaction]] + if blockchain_type == AvailableBlockchainType.ETHEREUM: + transaction_model = EthereumTransaction + elif blockchain_type == AvailableBlockchainType.POLYGON: + transaction_model = PolygonTransaction + else: + raise Exception("Unsupported blockchain type provided") + + return transaction_model + + +def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None: """ Add block if doesn't presented in database. block: web3.types.BlockData """ - block_obj = EthereumBlock( + block_model = get_block_model(blockchain_type) + + # BlockData.extraData doesn't exist at Polygon mainnet + extra_data = None + if block.get("extraData", None) is not None: + extra_data = block.get("extraData").hex() + + block_obj = block_model( block_number=block.number, difficulty=block.difficulty, - extra_data=block.extraData.hex(), + extra_data=extra_data, gas_limit=block.gasLimit, gas_used=block.gasUsed, base_fee_per_gas=block.get("baseFeePerGas", None), @@ -78,14 +132,17 @@ def add_block(db_session, block: Any) -> None: db_session.add(block_obj) -def add_block_transactions(db_session, block: Any) -> None: +def add_block_transactions( + db_session, block: Any, blockchain_type: AvailableBlockchainType +) -> None: """ Add block transactions. block: web3.types.BlockData """ + transaction_model = get_transaction_model(blockchain_type) for tx in block.transactions: - tx_obj = EthereumTransaction( + tx_obj = transaction_model( hash=tx.hash.hex(), block_number=block.number, from_address=tx["from"], @@ -103,22 +160,25 @@ def add_block_transactions(db_session, block: Any) -> None: db_session.add(tx_obj) -def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]: +def get_latest_blocks( + blockchain_type: AvailableBlockchainType, confirmations: int = 0 +) -> Tuple[Optional[int], int]: """ - Retrieve the latest block from the connected node (connection is created by the connect() method). + Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method). If confirmations > 0, and the latest block on the node has block number N, this returns the block with block_number (N - confirmations) """ - web3_client = connect() + web3_client = connect(blockchain_type) latest_block_number: int = web3_client.eth.block_number if confirmations > 0: latest_block_number -= confirmations + block_model = get_block_model(blockchain_type) with yield_db_session_ctx() as db_session: latest_stored_block_row = ( - db_session.query(EthereumBlock.block_number) - .order_by(EthereumBlock.block_number.desc()) + db_session.query(block_model.block_number) + .order_by(block_model.block_number.desc()) .first() ) latest_stored_block_number = ( @@ -128,11 +188,15 @@ def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]: return latest_stored_block_number, latest_block_number -def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> None: +def crawl_blocks( + blockchain_type: AvailableBlockchainType, + blocks_numbers: List[int], + with_transactions: bool = False, +) -> None: """ Open database and geth sessions and fetch block data from blockchain. """ - web3_client = connect() + web3_client = connect(blockchain_type) with yield_db_session_ctx() as db_session: pbar = tqdm(total=len(blocks_numbers)) for block_number in blocks_numbers: @@ -143,10 +207,10 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> block: BlockData = web3_client.eth.get_block( block_number, full_transactions=with_transactions ) - add_block(db_session, block) + add_block(db_session, block, blockchain_type) if with_transactions: - add_block_transactions(db_session, block) + add_block_transactions(db_session, block, blockchain_type) db_session.commit() except IntegrityError as err: @@ -157,7 +221,7 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> except Exception as err: db_session.rollback() message = f"Error adding block (number={block_number}) to database:\n{repr(err)}" - raise EthereumBlockCrawlError(message) + raise BlockCrawlError(message) except: db_session.rollback() logger.error( @@ -168,7 +232,11 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> pbar.close() -def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> List[int]: +def check_missing_blocks( + blockchain_type: AvailableBlockchainType, + blocks_numbers: List[int], + notransactions=False, +) -> List[int]: """ Query block from postgres. If block does not presented in database, add to missing blocks numbers list. @@ -178,33 +246,35 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis bottom_block = min(blocks_numbers[-1], blocks_numbers[0]) top_block = max(blocks_numbers[-1], blocks_numbers[0]) + block_model = get_block_model(blockchain_type) + transaction_model = get_transaction_model(blockchain_type) with yield_db_session_ctx() as db_session: if notransactions: blocks_exist_raw_query = ( - db_session.query(EthereumBlock.block_number) - .filter(EthereumBlock.block_number >= bottom_block) - .filter(EthereumBlock.block_number <= top_block) + db_session.query(block_model.block_number) + .filter(block_model.block_number >= bottom_block) + .filter(block_model.block_number <= top_block) ) blocks_exist = [[block[0]] for block in blocks_exist_raw_query.all()] else: corrupted_blocks = [] blocks_exist_raw_query = ( db_session.query( - EthereumBlock.block_number, func.count(EthereumTransaction.hash) + block_model.block_number, func.count(transaction_model.hash) ) .join( - EthereumTransaction, - EthereumTransaction.block_number == EthereumBlock.block_number, + transaction_model, + transaction_model.block_number == block_model.block_number, ) - .filter(EthereumBlock.block_number >= bottom_block) - .filter(EthereumBlock.block_number <= top_block) - .group_by(EthereumBlock.block_number) + .filter(block_model.block_number >= bottom_block) + .filter(block_model.block_number <= top_block) + .group_by(block_model.block_number) ) blocks_exist = [ [block[0], block[1]] for block in blocks_exist_raw_query.all() ] - web3_client = connect() + web3_client = connect(blockchain_type) blocks_exist_len = len(blocks_exist) pbar = tqdm(total=blocks_exist_len) @@ -218,8 +288,8 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis corrupted_blocks.append(block_in_db[0]) # Delete existing corrupted block and add to missing list del_block = ( - db_session.query(EthereumBlock) - .filter(EthereumBlock.block_number == block_in_db[0]) + db_session.query(block_model) + .filter(block_model.block_number == block_in_db[0]) .one() ) db_session.delete(del_block) @@ -242,6 +312,7 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis def crawl_blocks_executor( + blockchain_type: AvailableBlockchainType, block_numbers_list: List[int], with_transactions: bool = False, num_processes: int = MOONSTREAM_CRAWL_WORKERS, @@ -272,13 +343,15 @@ def crawl_blocks_executor( results: List[Future] = [] if num_processes == 1: logger.warning("Executing block crawler in lazy mod") - return crawl_blocks(block_numbers_list, with_transactions) + return crawl_blocks(blockchain_type, block_numbers_list, with_transactions) else: with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor: for worker in worker_indices: block_chunk = worker_job_lists[worker] logger.info(f"Spawned process for {len(block_chunk)} blocks") - result = executor.submit(crawl_blocks, block_chunk, with_transactions) + result = executor.submit( + crawl_blocks, blockchain_type, block_chunk, with_transactions + ) result.add_done_callback(record_error) results.append(result) @@ -286,7 +359,7 @@ def crawl_blocks_executor( if len(errors) > 0: error_messages = "\n".join([f"- {error}" for error in errors]) message = f"Error processing blocks in list:\n{error_messages}" - raise EthereumBlockCrawlError(message) + raise BlockCrawlError(message) def trending( diff --git a/crawlers/mooncrawl/mooncrawl/contract/cli.py b/crawlers/mooncrawl/mooncrawl/contract/cli.py index 0fd1ead4..5e315071 100644 --- a/crawlers/mooncrawl/mooncrawl/contract/cli.py +++ b/crawlers/mooncrawl/mooncrawl/contract/cli.py @@ -8,7 +8,8 @@ from moonstreamdb.db import yield_db_session_ctx from sqlalchemy.orm.session import Session from web3 import Web3 -from ..ethereum import connect +from ..blockchain import connect +from ..data import AvailableBlockchainType from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore logging.basicConfig(level=logging.INFO) @@ -116,7 +117,7 @@ def run_crawler_desc( def handle_parser(args: argparse.Namespace): with yield_db_session_ctx() as session: - w3 = connect() + w3 = connect(AvailableBlockchainType.ETHEREUM) if args.order == "asc": run_crawler_asc( w3=w3, diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/crawler.py similarity index 72% rename from crawlers/mooncrawl/mooncrawl/ethcrawler.py rename to crawlers/mooncrawl/mooncrawl/crawler.py index 373612a2..b2071553 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/crawler.py @@ -2,24 +2,25 @@ Moonstream crawlers CLI. """ import argparse -from datetime import datetime, timedelta, timezone -from enum import Enum import json import logging import os import sys import time +from datetime import datetime, timedelta, timezone +from enum import Enum from typing import Iterator, List import dateutil.parser -from .ethereum import ( - crawl_blocks_executor, - check_missing_blocks, - get_latest_blocks, +from .blockchain import ( DateRange, + check_missing_blocks, + crawl_blocks_executor, + get_latest_blocks, trending, ) +from .data import AvailableBlockchainType from .publish import publish_json from .settings import MOONSTREAM_CRAWL_WORKERS from .version import MOONCRAWL_VERSION @@ -86,13 +87,13 @@ def yield_blocks_numbers_lists( current_block -= block_step -def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: +def crawler_blocks_sync_handler(args: argparse.Namespace) -> None: """ - Synchronize latest Ethereum blocks with database. + Synchronize latest Blockchain blocks with database. """ while True: latest_stored_block_number, latest_block_number = get_latest_blocks( - args.confirmations + AvailableBlockchainType(args.blockchain), args.confirmations ) if latest_stored_block_number is None: latest_stored_block_number = 0 @@ -132,6 +133,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: ) # TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize. crawl_blocks_executor( + blockchain_type=AvailableBlockchainType(args.blockchain), block_numbers_list=blocks_numbers_list, with_transactions=True, num_processes=args.jobs, @@ -141,7 +143,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: ) -def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: +def crawler_blocks_add_handler(args: argparse.Namespace) -> None: """ Add blocks to moonstream database. """ @@ -150,7 +152,9 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks): logger.info(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}") crawl_blocks_executor( - block_numbers_list=blocks_numbers_list, with_transactions=True + blockchain_type=AvailableBlockchainType(args.blockchain), + block_numbers_list=blocks_numbers_list, + with_transactions=True, ) logger.info( @@ -158,19 +162,32 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: ) -def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: +def crawler_blocks_missing_handler(args: argparse.Namespace) -> None: """ Check missing blocks and missing transactions in each block. + If block range doesn't provided, get latest block from blockchain minus 50, + and check last 2000 blocks behind. """ startTime = time.time() missing_blocks_numbers_total = [] - for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks): + + block_range = args.blocks + if block_range is None: + confirmations = 50 + shift = 2000 + _, latest_block_number = get_latest_blocks( + AvailableBlockchainType(args.blockchain), confirmations + ) + block_range = f"{latest_block_number-shift}-{latest_block_number}" + + for blocks_numbers_list in yield_blocks_numbers_lists(block_range): logger.info( f"Checking missing blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]} " f"with comparing transactions: {not args.notransactions}" ) missing_blocks_numbers = check_missing_blocks( + blockchain_type=AvailableBlockchainType(args.blockchain), blocks_numbers=blocks_numbers_list, notransactions=args.notransactions, ) @@ -185,7 +202,8 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: if (len(missing_blocks_numbers_total)) > 0: time.sleep(5) crawl_blocks_executor( - missing_blocks_numbers_total, + blockchain_type=AvailableBlockchainType(args.blockchain), + block_numbers_list=missing_blocks_numbers_total, with_transactions=True, num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS, ) @@ -195,7 +213,7 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: ) -def ethcrawler_trending_handler(args: argparse.Namespace) -> None: +def crawler_trending_handler(args: argparse.Namespace) -> None: date_range = DateRange( start_time=args.start, end_time=args.end, @@ -228,15 +246,15 @@ def main() -> None: time_now = datetime.now(timezone.utc) - # Ethereum blocks parser - parser_ethcrawler_blocks = subcommands.add_parser( - "blocks", description="Ethereum blocks commands" + # Blockchain blocks parser + parser_crawler_blocks = subcommands.add_parser( + "blocks", description="Blockchain blocks commands" ) - parser_ethcrawler_blocks.set_defaults( - func=lambda _: parser_ethcrawler_blocks.print_help() + parser_crawler_blocks.set_defaults( + func=lambda _: parser_crawler_blocks.print_help() ) - subcommands_ethcrawler_blocks = parser_ethcrawler_blocks.add_subparsers( - description="Ethereum blocks commands" + subcommands_crawler_blocks = parser_crawler_blocks.add_subparsers( + description="Blockchain blocks commands" ) valid_processing_orders = { @@ -251,29 +269,29 @@ def main() -> None: f"Invalid processing order ({raw_order}). Valid choices: {valid_processing_orders.keys()}" ) - parser_ethcrawler_blocks_sync = subcommands_ethcrawler_blocks.add_parser( - "synchronize", description="Synchronize to latest ethereum block commands" + parser_crawler_blocks_sync = subcommands_crawler_blocks.add_parser( + "synchronize", description="Synchronize to latest blockchain block commands" ) - parser_ethcrawler_blocks_sync.add_argument( + parser_crawler_blocks_sync.add_argument( "-s", "--start", type=int, - help="(Optional) Block to start synchronization from. Default: None - current Ethereum block minus confirmations ", + help="(Optional) Block to start synchronization from. Default: None - current Blockchain block minus confirmations ", ) - parser_ethcrawler_blocks_sync.add_argument( + parser_crawler_blocks_sync.add_argument( "-c", "--confirmations", type=int, default=0, help="Number of confirmations we require before storing a block in the database. (Default: 0)", ) - parser_ethcrawler_blocks_sync.add_argument( + parser_crawler_blocks_sync.add_argument( "--order", type=processing_order, default=ProcessingOrder.ASCENDING, help="Order in which to process blocks (choices: desc, asc; default: asc)", ) - parser_ethcrawler_blocks_sync.add_argument( + parser_crawler_blocks_sync.add_argument( "-j", "--jobs", type=int, @@ -283,72 +301,85 @@ def main() -> None: " If you set to 1, the main process handles synchronization without spawning subprocesses." ), ) - parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler) - - parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser( - "add", description="Add ethereum blocks commands" + parser_crawler_blocks_sync.add_argument( + "--blockchain", + required=True, + help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}", ) - parser_ethcrawler_blocks_add.add_argument( + parser_crawler_blocks_sync.set_defaults(func=crawler_blocks_sync_handler) + + parser_crawler_blocks_add = subcommands_crawler_blocks.add_parser( + "add", description="Add blockchain blocks commands" + ) + parser_crawler_blocks_add.add_argument( "-b", "--blocks", required=True, help="List of blocks range in format {bottom_block}-{top_block}", ) - parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler) - - parser_ethcrawler_blocks_missing = subcommands_ethcrawler_blocks.add_parser( - "missing", description="Add missing ethereum blocks with transactions commands" + parser_crawler_blocks_add.add_argument( + "--blockchain", + required=True, + help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}", ) - parser_ethcrawler_blocks_missing.add_argument( + parser_crawler_blocks_add.set_defaults(func=crawler_blocks_add_handler) + + parser_crawler_blocks_missing = subcommands_crawler_blocks.add_parser( + "missing", + description="Add missing Blockchain blocks with transactions commands", + ) + parser_crawler_blocks_missing.add_argument( "-b", "--blocks", - required=True, help="List of blocks range in format {bottom_block}-{top_block}", ) - parser_ethcrawler_blocks_missing.add_argument( + parser_crawler_blocks_missing.add_argument( "-n", "--notransactions", action="store_true", help="Skip crawling block transactions", ) - parser_ethcrawler_blocks_missing.add_argument( + parser_crawler_blocks_missing.add_argument( "-l", "--lazy", action="store_true", help="Lazy block adding one by one", ) - parser_ethcrawler_blocks_missing.set_defaults( - func=ethcrawler_blocks_missing_handler + parser_crawler_blocks_missing.add_argument( + "--blockchain", + required=True, + help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}", ) + parser_crawler_blocks_missing.set_defaults(func=crawler_blocks_missing_handler) - parser_ethcrawler_trending = subcommands.add_parser( - "trending", description="Trending addresses on the Ethereum blockchain" + parser_crawler_trending = subcommands.add_parser( + "trending", description="Trending addresses on the Blockchain blockchain" ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "-s", "--start", type=dateutil.parser.parse, default=(time_now - timedelta(hours=1, minutes=0)).isoformat(), help=f"Start time for window to calculate trending addresses in (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})", ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "--include-start", action="store_true", help="Set this flag if range should include start time", ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "-e", "--end", type=dateutil.parser.parse, default=time_now.isoformat(), help=f"End time for window to calculate trending addresses in (default: {time_now.isoformat()})", ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "--include-end", action="store_true", help="Set this flag if range should include end time", ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "--humbug", default=None, help=( @@ -357,14 +388,14 @@ def main() -> None: "MOONSTREAM_HUMBUG_TOKEN environment variable)" ), ) - parser_ethcrawler_trending.add_argument( + parser_crawler_trending.add_argument( "-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout, help="Optional file to write output to. By default, prints to stdout.", ) - parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler) + parser_crawler_trending.set_defaults(func=crawler_trending_handler) args = parser.parse_args() args.func(args) diff --git a/crawlers/mooncrawl/mooncrawl/data.py b/crawlers/mooncrawl/mooncrawl/data.py new file mode 100644 index 00000000..04e7235a --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/data.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from datetime import datetime +from enum import Enum + + +class AvailableBlockchainType(Enum): + ETHEREUM = "ethereum" + POLYGON = "polygon" + + +@dataclass +class DateRange: + start_time: datetime + end_time: datetime + include_start: bool + include_end: bool diff --git a/crawlers/mooncrawl/mooncrawl/esd.py b/crawlers/mooncrawl/mooncrawl/esd.py index 1a2fff6c..ace2f6f2 100644 --- a/crawlers/mooncrawl/mooncrawl/esd.py +++ b/crawlers/mooncrawl/mooncrawl/esd.py @@ -3,10 +3,10 @@ import sys import time from typing import Optional, Union +import requests from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature from sqlalchemy.orm import Session -import requests CRAWL_URLS = { "functions": "https://www.4byte.directory/api/v1/signatures/", diff --git a/crawlers/mooncrawl/mooncrawl/etherscan.py b/crawlers/mooncrawl/mooncrawl/etherscan.py index e3c46f94..5ec96ed6 100644 --- a/crawlers/mooncrawl/mooncrawl/etherscan.py +++ b/crawlers/mooncrawl/mooncrawl/etherscan.py @@ -1,23 +1,23 @@ import argparse import codecs import csv -from dataclasses import dataclass -from datetime import datetime import json import logging import os import sys import time -from typing import Any, List, Optional, Dict +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, List, Optional import boto3 # type: ignore +import requests from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumLabel -import requests from sqlalchemy.orm import Session -from .version import MOONCRAWL_VERSION from .settings import MOONSTREAM_ETHERSCAN_TOKEN +from .version import MOONCRAWL_VERSION logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/identity.py b/crawlers/mooncrawl/mooncrawl/identity.py index 8a8fb08e..f0a8807e 100644 --- a/crawlers/mooncrawl/mooncrawl/identity.py +++ b/crawlers/mooncrawl/mooncrawl/identity.py @@ -4,10 +4,9 @@ import os import time import requests -from sqlalchemy import text - from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumLabel +from sqlalchemy import text logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index caf5bfbe..0f06474f 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -2,12 +2,12 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse -from datetime import datetime, timezone import json import logging import sys import time -from typing import Any, cast, Dict, Optional +from datetime import datetime, timezone +from typing import Any, Dict, Optional, cast from bugout.app import Bugout from bugout.journal import SearchOrder @@ -16,26 +16,25 @@ from moonstreamdb.models import EthereumBlock from sqlalchemy.orm.session import Session from web3 import Web3 -from ..ethereum import connect +from ..blockchain import connect +from ..data import AvailableBlockchainType +from ..publish import publish_json +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_ETHEREUM_IPC_PATH, + NFT_HUMBUG_TOKEN, +) +from ..version import MOONCRAWL_VERSION from .ethereum import ( - summary as ethereum_summary, - add_labels, - MINT_LABEL, - TRANSFER_LABEL, SUMMARY_KEY_ARGS, + SUMMARY_KEY_END_BLOCK, SUMMARY_KEY_ID, SUMMARY_KEY_NUM_BLOCKS, SUMMARY_KEY_START_BLOCK, - SUMMARY_KEY_END_BLOCK, + add_labels, ) -from ..publish import publish_json -from ..settings import ( - MOONSTREAM_IPC_PATH, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - NFT_HUMBUG_TOKEN, - MOONSTREAM_DATA_JOURNAL_ID, -) -from ..version import MOONCRAWL_VERSION +from .ethereum import summary as ethereum_summary logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -46,17 +45,17 @@ BLOCKS_PER_SUMMARY = 40 def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: """ Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up - the MOONSTREAM_IPC_PATH environment variable. + the MOONSTREAM_ETHEREUM_IPC_PATH environment variable. """ - web3_connection_string = MOONSTREAM_IPC_PATH + web3_connection_string = MOONSTREAM_ETHEREUM_IPC_PATH args_web3 = vars(args).get("web3") if args_web3 is not None: web3_connection_string = cast(str, args_web3) if web3_connection_string is None: raise ValueError( - "Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable" + "Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_IPC_PATH environment variable" ) - return connect(web3_connection_string) + return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string) def get_latest_block_from_node(web3_client: Web3): @@ -295,7 +294,7 @@ def main() -> None: "--web3", type=str, default=None, - help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.", + help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_ETHEREUM_IPC_PATH environment variable.", ) parser_ethereum_label.set_defaults(func=ethereum_label_handler) diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index 5a92872f..f3520fcd 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -1,23 +1,18 @@ -from dataclasses import dataclass, asdict -from datetime import datetime, timedelta, timezone import json import logging -from hexbytes.main import HexBytes -from typing import Any, cast, Dict, List, Optional, Set, Tuple +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Set, Tuple, cast from eth_typing.encoding import HexStr -from moonstreamdb.models import ( - EthereumBlock, - EthereumLabel, - EthereumTransaction, -) +from hexbytes.main import HexBytes +from moonstreamdb.models import EthereumBlock, EthereumLabel, EthereumTransaction from sqlalchemy import and_, func, text -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session, Query +from sqlalchemy.orm import Query, Session from tqdm import tqdm from web3 import Web3 -from web3.types import FilterParams, LogReceipt from web3._utils.events import get_event_data +from web3.types import FilterParams, LogReceipt from ..reporter import reporter diff --git a/crawlers/mooncrawl/mooncrawl/publish.py b/crawlers/mooncrawl/mooncrawl/publish.py index 16a4ed2b..5e15607c 100644 --- a/crawlers/mooncrawl/mooncrawl/publish.py +++ b/crawlers/mooncrawl/mooncrawl/publish.py @@ -1,6 +1,6 @@ -from datetime import datetime import json import os +from datetime import datetime from typing import Any, Dict, List, Optional import requests diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index fe85b631..b2684104 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -11,9 +11,18 @@ MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get( MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 ) -MOONSTREAM_IPC_PATH = ( +MOONSTREAM_ETHEREUM_IPC_PATH = ( f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" ) +MOONSTREAM_NODE_POLYGON_IPC_ADDR = os.environ.get( + "MOONSTREAM_NODE_POLYGON_IPC_ADDR", "127.0.0.1" +) +MOONSTREAM_NODE_POLYGON_IPC_PORT = os.environ.get( + "MOONSTREAM_NODE_POLYGON_IPC_PORT", 8545 +) +MOONSTREAM_POLYGON_IPC_PATH = ( + f"http://{MOONSTREAM_NODE_POLYGON_IPC_ADDR}:{MOONSTREAM_NODE_POLYGON_IPC_PORT}" +) MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/mooncrawl/test_cli.py b/crawlers/mooncrawl/mooncrawl/test_cli.py index 6fa331a3..757ae163 100644 --- a/crawlers/mooncrawl/mooncrawl/test_cli.py +++ b/crawlers/mooncrawl/mooncrawl/test_cli.py @@ -1,13 +1,13 @@ import unittest -from . import ethcrawler +from . import crawler class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_10_6_step_4(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( + for block_numbers_list in crawler.yield_blocks_numbers_lists( "10-6", block_step=4 ) ] @@ -16,7 +16,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_10_6_step_3(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( + for block_numbers_list in crawler.yield_blocks_numbers_lists( "10-6", block_step=3 ) ] @@ -25,8 +25,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_10_6_descending_step_3(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "10-6", ethcrawler.ProcessingOrder.DESCENDING, 3 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "10-6", crawler.ProcessingOrder.DESCENDING, 3 ) ] self.assertListEqual(partition, [[10, 9, 8], [7, 6]]) @@ -34,8 +34,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_10_6_descending_step_10(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "10-6", ethcrawler.ProcessingOrder.DESCENDING, 10 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "10-6", crawler.ProcessingOrder.DESCENDING, 10 ) ] self.assertListEqual(partition, [[10, 9, 8, 7, 6]]) @@ -43,7 +43,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_6_10_step_4(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( + for block_numbers_list in crawler.yield_blocks_numbers_lists( "6-10", block_step=4 ) ] @@ -52,7 +52,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_6_10_step_3(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( + for block_numbers_list in crawler.yield_blocks_numbers_lists( "6-10", block_step=3 ) ] @@ -61,8 +61,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_6_10_descending_step_3(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "6-10", ethcrawler.ProcessingOrder.DESCENDING, 3 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "6-10", crawler.ProcessingOrder.DESCENDING, 3 ) ] self.assertListEqual(partition, [[10, 9, 8], [7, 6]]) @@ -70,8 +70,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_descending_6_10_descending_step_10(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "6-10", ethcrawler.ProcessingOrder.DESCENDING, 10 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "6-10", crawler.ProcessingOrder.DESCENDING, 10 ) ] self.assertListEqual(partition, [[10, 9, 8, 7, 6]]) @@ -79,8 +79,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_ascending_10_6_ascending_step_3(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "10-6", ethcrawler.ProcessingOrder.ASCENDING, 3 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "10-6", crawler.ProcessingOrder.ASCENDING, 3 ) ] self.assertListEqual(partition, [[6, 7, 8], [9, 10]]) @@ -88,8 +88,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_ascending_10_6_ascending_step_10(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "10-6", ethcrawler.ProcessingOrder.ASCENDING, 10 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "10-6", crawler.ProcessingOrder.ASCENDING, 10 ) ] self.assertListEqual(partition, [[6, 7, 8, 9, 10]]) @@ -97,8 +97,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_ascending_6_10_ascending_step_4(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "6-10", ethcrawler.ProcessingOrder.ASCENDING, 4 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "6-10", crawler.ProcessingOrder.ASCENDING, 4 ) ] self.assertListEqual(partition, [[6, 7, 8, 9], [10]]) @@ -106,8 +106,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase): def test_yield_ascending_6_10_ascending_step_10(self): partition = [ block_numbers_list - for block_numbers_list in ethcrawler.yield_blocks_numbers_lists( - "6-10", ethcrawler.ProcessingOrder.ASCENDING, 10 + for block_numbers_list in crawler.yield_blocks_numbers_lists( + "6-10", crawler.ProcessingOrder.ASCENDING, 10 ) ] self.assertListEqual(partition, [[6, 7, 8, 9, 10]]) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index b9be69f5..fd80ca83 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.0.5" +MOONCRAWL_VERSION = "0.1.0" diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt deleted file mode 100644 index 82fb4aa6..00000000 --- a/crawlers/mooncrawl/requirements.txt +++ /dev/null @@ -1,59 +0,0 @@ -aiohttp==3.7.4.post0 -async-timeout==3.0.1 -attrs==21.2.0 -base58==2.1.0 -bitarray==1.2.2 -black==21.8b0 -boto3==1.18.40 -botocore==1.21.40 -bugout==0.1.17 -certifi==2021.5.30 -chardet==4.0.0 -charset-normalizer==2.0.4 -click==8.0.1 -cytoolz==0.11.0 --e git+https://git@github.com/bugout-dev/moonstream.git@67fe019f1086c435dd3b58f1ade2778acc2167c7#egg=moonstreamdb&subdirectory=db -eth-abi==2.1.1 -eth-account==0.5.5 -eth-hash==0.3.2 -eth-keyfile==0.5.1 -eth-keys==0.3.3 -eth-rlp==0.2.1 -eth-typing==2.2.2 -eth-utils==1.10.0 -hexbytes==0.2.2 -humbug==0.2.7 -idna==3.2 -ipfshttpclient==0.8.0a2 -jmespath==0.10.0 -jsonschema==3.2.0 -lru-dict==1.1.7 -multiaddr==0.0.9 -multidict==5.1.0 -mypy==0.910 -mypy-extensions==0.4.3 -netaddr==0.8.0 -parsimonious==0.8.1 -pathspec==0.9.0 -platformdirs==2.3.0 -protobuf==3.17.3 -pycryptodome==3.10.1 -pyrsistent==0.18.0 -python-dateutil==2.8.2 -regex==2021.8.28 -requests==2.26.0 -rlp==2.0.1 -s3transfer==0.5.0 -six==1.16.0 -toml==0.10.2 -tomli==1.2.1 -toolz==0.11.1 -tqdm==4.62.2 -types-python-dateutil==2.8.0 -types-requests==2.25.6 -typing-extensions==3.10.0.2 -urllib3==1.26.6 -varint==1.0.2 -web3==5.23.1 -websockets==9.1 -yarl==1.6.3 diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 01cdfb64..5a2ce208 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -1,6 +1,8 @@ # Path to IPC socket to use for web3 connections export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" +export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN="" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 66be8b27..8f755014 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -1,5 +1,6 @@ from setuptools import find_packages, setup +from mooncrawl.version import MOONCRAWL_VERSION long_description = "" with open("README.md") as ifp: @@ -7,7 +8,7 @@ with open("README.md") as ifp: setup( name="mooncrawl", - version="0.0.5", + version=MOONCRAWL_VERSION, author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0", @@ -33,8 +34,9 @@ setup( zip_safe=False, install_requires=[ "boto3", - "bugout >= 0.1.17", - "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@0a771ddfbca1254be331149ccf2d162aa09b7bc0#egg=moonstreamdb&subdirectory=db", + "bugout", + "chardet", + "moonstreamdb", "humbug", "python-dateutil", "requests", @@ -42,11 +44,12 @@ setup( "web3", ], extras_require={ - "dev": ["black", "mypy", "types-requests", "types-python-dateutil"] + "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"], + "distribute": ["setuptools", "twine", "wheel"], }, entry_points={ "console_scripts": [ - "ethcrawler=mooncrawl.ethcrawler:main", + "crawler=mooncrawl.crawler:main", "esd=mooncrawl.esd:main", "identity=mooncrawl.identity:main", "etherscan=mooncrawl.etherscan:main", diff --git a/db/setup.py b/db/setup.py index 601c4c14..8f8e70a5 100644 --- a/db/setup.py +++ b/db/setup.py @@ -34,7 +34,7 @@ setup( zip_safe=False, install_requires=["alembic", "psycopg2-binary", "sqlalchemy"], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy"], "distribute": ["setuptools", "twine", "wheel"], }, entry_points={ diff --git a/frontend/pages/index.js b/frontend/pages/index.js index c528ea2f..225cfe21 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -19,6 +19,7 @@ import { Text, GridItem, SimpleGrid, + Button, Image as ChakraImage, } from "@chakra-ui/react"; import dynamic from "next/dynamic"; @@ -776,18 +777,18 @@ const Homepage = () => { Join our Discord - toggleModal("register")} size="lg" variant="solid" colorScheme="orange" > Sign up - +