mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into stats_data_worker

pull/391/head
kompotkot 2021-11-13 14:49:15 +00:00
rodzic 46ab3d72b0 8a4c8518d3
commit 77dd49405d
79 zmienionych plików z 885 dodań i 360 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

8
.github/workflows/lint.backend.yml vendored
Wyświetl plik

@ -16,12 +16,14 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./backend
run: pip install -r requirements.txt
run: pip install -e .[dev]
- name: Mypy type check
working-directory: ./backend
run: mypy moonstream/
run: mypy moonstreamapi/
- name: Black syntax check
working-directory: ./backend
run: black --check moonstream/
run: black --check moonstreamapi/

2
.github/workflows/lint.crawlers.yml vendored
Wyświetl plik

@ -16,6 +16,8 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./crawlers/mooncrawl
run: pip install -e .[dev]

2
.github/workflows/lint.db.yml vendored
Wyświetl plik

@ -16,6 +16,8 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./db
run: pip install -e .[dev]

30
.github/workflows/release.mooncrawl.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release mooncrawl package
on:
push:
tags:
- 'mooncrawl/v*'
defaults:
run:
working-directory: crawlers/mooncrawl
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

30
.github/workflows/release.moonstreamapi.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release moonstreamapi package
on:
push:
tags:
- 'moonstreamapi/v*'
defaults:
run:
working-directory: backend
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

30
.github/workflows/release.moonstreamdb.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release moonstreamdb package
on:
push:
tags:
- 'moonstreamdb/v*'
defaults:
run:
working-directory: db
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

36
backend/deploy/deploy.bash
Wyświetl plik

@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Deployment script - intended to run on Moonstream servers
# Deployment script - intended to run on Moonstream API server
# Colors
C_RESET='\033[0m'
@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
PYTHON="${PYTHON_ENV_DIR}/bin/python"
PIP="${PYTHON_ENV_DIR}/bin/pip"
SCRIPT_DIR="$(realpath $(dirname $0))"
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
# Parameters scripts
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# API server service file
SERVICE_FILE="${SCRIPT_DIR}/moonstreamapi.service"
set -eu
echo
echo
echo -e "${PREFIX_INFO} Updating pip and setuptools"
"${PIP}" install -U pip setuptools
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
"${PIP}" install --upgrade pip setuptools
echo
echo
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
echo -e "${PREFIX_INFO} Installing Python dependencies"
"${PIP}" install -e "${APP_BACKEND_DIR}/"
echo
echo
@ -49,13 +54,18 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$
echo
echo
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}"
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Moonstream API service definition with ${SERVICE_FILE}"
chmod 644 "${SERVICE_FILE}"
cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service
cp "${SERVICE_FILE}" /etc/systemd/system/moonstreamapi.service
systemctl daemon-reload
systemctl restart moonstream.service
systemctl status moonstream.service
systemctl restart moonstreamapi.service
systemctl status moonstreamapi.service

6
backend/deploy/moonstream.service → backend/deploy/moonstreamapi.service
Wyświetl plik

@ -1,5 +1,5 @@
[Unit]
Description=moonstream-service
Description=moonstreamapi-service
After=network.target
[Service]
@ -7,8 +7,8 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/backend
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
SyslogIdentifier=moonstream
ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstreamapi.api:app
SyslogIdentifier=moonstreamapi
[Install]
WantedBy=multi-user.target

89
backend/deploy/nodes-connections.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Update nodes connection address environment variables
# from AWS Route53 internal hosted zone
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
echo
echo "CLI to update nodes connection address environment
variables from AWS Route53 internal hosted zone"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -f File path where environment variables update at"
}
file_flag=""
verbose_flag="false"
while getopts 'f:v' flag; do
case "${flag}" in
f) file_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# File flag should be specified
if [ -z "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify file path"
usage
exit 1
fi
if [ ! -f "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Provided file does not exist"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Source environment variables"
. ${file_flag}
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving Polygon node address"
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
exit 1
fi
# TODO(kompotkot): Modify regexp to work with export prefix
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

8
backend/deploy/parameters.bash
Wyświetl plik

@ -1,6 +1,9 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
# Collect secrets from AWS SSM Parameter Store and
# opt out as environment variable exports.
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
@ -26,6 +29,7 @@ and output as environment variable exports"
echo " -o Output file name environment variables export to"
}
# TODO(kompotkot): Flag for export prefix
product_flag=""
output_flag=""
verbose_flag="false"
@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=${product_flag} \

2
backend/dev.sh
Wyświetl plik

@ -6,4 +6,4 @@ set -e
MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --reload
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload

7
backend/moonstream/__init__.py
Wyświetl plik

@ -1,7 +0,0 @@
from .reporter import reporter
from .version import MOONSTREAM_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)

11
backend/moonstreamapi/__init__.py 100644
Wyświetl plik

@ -0,0 +1,11 @@
try:
from .reporter import reporter
from .version import MOONSTREAMAPI_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAMAPI_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
except:
# Pass it to be able import MOONSTREAMAPI_VERSION in setup.py with pip
pass

0
backend/moonstream/abi_decoder.py → backend/moonstreamapi/abi_decoder.py
Wyświetl plik

0
backend/moonstream/actions.py → backend/moonstreamapi/actions.py
Wyświetl plik

0
backend/moonstream/admin/__init__.py → backend/moonstreamapi/admin/__init__.py
Wyświetl plik

0
backend/moonstream/admin/cli.py → backend/moonstreamapi/admin/cli.py
Wyświetl plik

0
backend/moonstream/admin/migrations/1_add_abi_backet_and_s3_path.json → backend/moonstreamapi/admin/migrations/1_add_abi_backet_and_s3_path.json
Wyświetl plik

0
backend/moonstream/admin/migrations/__init__.py → backend/moonstreamapi/admin/migrations/__init__.py
Wyświetl plik

0
backend/moonstream/admin/migrations/checksum_address.py → backend/moonstreamapi/admin/migrations/checksum_address.py
Wyświetl plik

0
backend/moonstream/admin/subscription_types.py → backend/moonstreamapi/admin/subscription_types.py
Wyświetl plik

0
backend/moonstream/admin/subscriptions.py → backend/moonstreamapi/admin/subscriptions.py
Wyświetl plik

6
backend/moonstream/api.py → backend/moonstreamapi/api.py
Wyświetl plik

@ -20,7 +20,7 @@ from .routes.whales import router as whales_router
from .routes.dashboards import router as dashboards_router
from .middleware import BroodAuthMiddleware, MoonstreamHTTPException
from .settings import DOCS_TARGET_PATH, ORIGINS
from .version import MOONSTREAM_VERSION
from .version import MOONSTREAMAPI_VERSION
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -45,7 +45,7 @@ tags_metadata = [
app = FastAPI(
title=f"Moonstream API",
description="Moonstream API endpoints.",
version=MOONSTREAM_VERSION,
version=MOONSTREAMAPI_VERSION,
openapi_tags=tags_metadata,
openapi_url="/openapi.json",
docs_url=None,
@ -91,7 +91,7 @@ async def version_handler() -> data.VersionResponse:
"""
Get server version.
"""
return data.VersionResponse(version=MOONSTREAM_VERSION)
return data.VersionResponse(version=MOONSTREAMAPI_VERSION)
@app.get("/now", tags=["time"])

0
backend/moonstream/data.py → backend/moonstreamapi/data.py
Wyświetl plik

0
backend/moonstream/middleware.py → backend/moonstreamapi/middleware.py
Wyświetl plik

0
backend/moonstream/providers/__init__.py → backend/moonstreamapi/providers/__init__.py
Wyświetl plik

0
backend/moonstream/providers/bugout.py → backend/moonstreamapi/providers/bugout.py
Wyświetl plik

0
backend/moonstream/providers/ethereum_blockchain.py → backend/moonstreamapi/providers/ethereum_blockchain.py
Wyświetl plik

0
backend/moonstream/py.typed → backend/moonstreamapi/py.typed
Wyświetl plik

2
backend/moonstream/reporter.py → backend/moonstreamapi/reporter.py
Wyświetl plik

@ -9,7 +9,7 @@ session_id = str(uuid.uuid4())
client_id = "moonstream-backend"
reporter = HumbugReporter(
name="moonstream",
name="moonstreamapi",
consent=HumbugConsent(True),
client_id=client_id,
session_id=session_id,

0
backend/moonstream/routes/__init__.py → backend/moonstreamapi/routes/__init__.py
Wyświetl plik

0
backend/moonstream/routes/address_info.py → backend/moonstreamapi/routes/address_info.py
Wyświetl plik

0
backend/moonstream/routes/dashboards.py → backend/moonstreamapi/routes/dashboards.py
Wyświetl plik

0
backend/moonstream/routes/nft.py → backend/moonstreamapi/routes/nft.py
Wyświetl plik

0
backend/moonstream/routes/streams.py → backend/moonstreamapi/routes/streams.py
Wyświetl plik

0
backend/moonstream/routes/subscriptions.py → backend/moonstreamapi/routes/subscriptions.py
Wyświetl plik

0
backend/moonstream/routes/txinfo.py → backend/moonstreamapi/routes/txinfo.py
Wyświetl plik

0
backend/moonstream/routes/users.py → backend/moonstreamapi/routes/users.py
Wyświetl plik

0
backend/moonstream/routes/whales.py → backend/moonstreamapi/routes/whales.py
Wyświetl plik

0
backend/moonstream/settings.py → backend/moonstreamapi/settings.py
Wyświetl plik

0
backend/moonstream/stream_boundaries.py → backend/moonstreamapi/stream_boundaries.py
Wyświetl plik

0
backend/moonstream/stream_queries.py → backend/moonstreamapi/stream_queries.py
Wyświetl plik

0
backend/moonstream/test_actions_abi_validations.py → backend/moonstreamapi/test_actions_abi_validations.py
Wyświetl plik

0
backend/moonstream/test_stream_boundaries.py → backend/moonstreamapi/test_stream_boundaries.py
Wyświetl plik

0
backend/moonstream/test_stream_queries.py → backend/moonstreamapi/test_stream_queries.py
Wyświetl plik

2
backend/moonstream/version.py → backend/moonstreamapi/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAM_VERSION = "0.0.2"
MOONSTREAMAPI_VERSION = "0.1.0"

0
backend/moonstream/web3_provider.py → backend/moonstreamapi/web3_provider.py
Wyświetl plik

35
backend/requirements.txt
Wyświetl plik

@ -1,35 +0,0 @@
appdirs==1.4.4
asgiref==3.4.1
black==21.7b0
boto3==1.18.1
botocore==1.21.1
bugout==0.1.18
certifi==2021.5.30
charset-normalizer==2.0.3
click==8.0.1
fastapi==0.66.0
h11==0.12.0
idna==3.2
jmespath==0.10.0
humbug==0.2.7
-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3
pathspec==0.9.0
pydantic==1.8.2
pyevmasm==0.2.3
python-dateutil==2.8.2
python-multipart==0.0.5
regex==2021.7.6
requests==2.26.0
s3transfer==0.5.0
six==1.16.0
starlette==0.14.2
toml==0.10.2
tomli==1.0.4
types-python-dateutil==0.1.6
typing-extensions==3.10.0.0
types-requests==2.25.6
urllib3==1.26.6
uvicorn==0.14.0
web3==5.24.0

24
backend/setup.py
Wyświetl plik

@ -1,30 +1,34 @@
from setuptools import find_packages, setup
from moonstream.version import MOONSTREAM_VERSION
from moonstreamapi.version import MOONSTREAMAPI_VERSION
long_description = ""
with open("README.md") as ifp:
long_description = ifp.read()
setup(
name="moonstream",
version=MOONSTREAM_VERSION,
name="moonstreamapi",
version=MOONSTREAMAPI_VERSION,
packages=find_packages(),
install_requires=[
"appdirs",
"boto3",
"bugout >= 0.1.18",
"bugout",
"fastapi",
"humbug>=0.2.7",
"moonstreamdb",
"humbug",
"pydantic",
"pyevmasm",
"python-dateutil",
"python-multipart",
"uvicorn",
"types-python-dateutil",
"types-requests",
"web3",
],
extras_require={
"dev": ["black", "mypy"],
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
"distribute": ["setuptools", "twine", "wheel"],
},
package_data={"moonstream": ["py.typed"]},
package_data={"moonstreamapi": ["py.typed"]},
zip_safe=False,
description="The Bugout blockchain inspector API.",
long_description=long_description,
@ -38,5 +42,5 @@ setup(
"Topic :: Software Development :: Libraries",
],
url="https://github.com/bugout-dev/moonstream",
entry_points={"console_scripts": ["mnstr=moonstream.admin.cli:main"]},
entry_points={"console_scripts": ["mnstr=moonstreamapi.admin.cli:main"]},
)

94
crawlers/deploy/deploy.bash
Wyświetl plik

@ -24,15 +24,28 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
SCRIPT_DIR="$(realpath $(dirname $0))"
# Parameters scripts
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts"
CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash"
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# Crawlers server service file
CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
# Ethereum service files
ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
# Polygon service file
POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
set -eu
@ -54,9 +67,13 @@ cd "${EXEC_DIR}"
echo
echo
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install --upgrade pip
"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
"${PIP}" install --upgrade pip setuptools
echo
echo
echo -e "${PREFIX_INFO} Installing Python dependencies"
"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/"
echo
echo
@ -67,12 +84,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex
echo
echo
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
@ -84,27 +101,52 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TRENDING_TIMER}"
systemctl restart "${ETHEREUM_TRENDING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
systemctl restart "${ETHEREUM_TXPOOL_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${POLYGON_SYNCHRONIZE_SERVICE}"
systemctl daemon-reload
systemctl restart "${POLYGON_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${CRAWLERS_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${CRAWLERS_SERVICE_FILE}"
systemctl status "${CRAWLERS_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon missing service and timer with: ${POLYGON_MISSING_SERVICE_FILE}, ${POLYGON_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${POLYGON_MISSING_TIMER_FILE}"

11
crawlers/deploy/ethereum-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Ethereum database
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n

9
crawlers/deploy/ethereum-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Ethereum database
[Timer]
OnBootSec=10s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

2
crawlers/deploy/ethereum-synchronize.service
Wyświetl plik

@ -7,7 +7,7 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler blocks synchronize -c 6 -j 1
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 1
SyslogIdentifier=ethereum-synchronize
[Install]

2
crawlers/deploy/ethereum-trending.service
Wyświetl plik

@ -8,4 +8,4 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler trending
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler trending

89
crawlers/deploy/nodes-connections.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Update nodes connection address environment variables
# from AWS Route53 internal hosted zone
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
echo
echo "CLI to update nodes connection address environment
variables from AWS Route53 internal hosted zone"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -f File path where environment variables update at"
}
file_flag=""
verbose_flag="false"
while getopts 'f:v' flag; do
case "${flag}" in
f) file_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# File flag should be specified
if [ -z "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify file path"
usage
exit 1
fi
if [ ! -f "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Provided file does not exist"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Source environment variables"
. ${file_flag}
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving Polygon node address"
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
exit 1
fi
# TODO(kompotkot): Modify regexp to work with export prefix
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

89
crawlers/deploy/parameters.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and
# opt out as environment variable exports.
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"
echo
echo "CLI to collect secrets from AWS SSM Parameter Store
and output as environment variable exports"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -p Product tag (moonstream, spire, brood, drones)"
echo " -o Output file name environment variables export to"
}
# TODO(kompotkot): Flag for export prefix
product_flag=""
output_flag=""
verbose_flag="false"
while getopts 'p:o:v' flag; do
case "${flag}" in
p) product_flag="${OPTARG}" ;;
o) output_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# Product flag should be specified
# TODO(kompotkot): Extend script to work with few product at once
if [ -z "${product_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify product tag"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=${product_flag} \
| jq -r .Parameters[].Name)
if [ -z "${ENV_PARAMETERS}" ]; then
verbose "${PREFIX_CRIT} There no parameters for provided product tag"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving parameters values"
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
--names ${ENV_PARAMETERS} \
--query "Parameters[*].{Name:Name,Value:Value}")
ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
if [ -z "${output_flag}" ]; then
echo "${param_key}=${param_value}"
else
echo "${param_key}=${param_value}" >> "${output_flag}"
fi
done

11
crawlers/deploy/polygon-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Polygon database
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n

9
crawlers/deploy/polygon-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Polygon database
[Timer]
OnBootSec=10s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

14
crawlers/deploy/polygon-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,14 @@
[Unit]
Description=Polygon block with transactions synchronizer
After=network.target
[Service]
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon -c 6 -j 1
SyslogIdentifier=polygon-synchronize
[Install]
WantedBy=multi-user.target

3
crawlers/mooncrawl/.isort.cfg 100644
Wyświetl plik

@ -0,0 +1,3 @@
[settings]
profile = black
multi_line_output = 3

16
crawlers/mooncrawl/mooncrawl/__init__.py
Wyświetl plik

@ -1,7 +1,11 @@
from .reporter import reporter
from .version import MOONCRAWL_VERSION
try:
from .reporter import reporter
from .version import MOONCRAWL_VERSION
# Reporting
reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
# Reporting
reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
except:
# Pass it to be able import MOONCRAWL_VERSION in setup.py with pip
pass

193
crawlers/mooncrawl/mooncrawl/ethereum.py → crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -1,64 +1,118 @@
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from dataclasses import dataclass
from datetime import datetime
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import desc, Column
from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, Query
from tqdm import tqdm
from web3 import Web3, IPCProvider, HTTPProvider
from web3.types import BlockData
from .settings import MOONSTREAM_IPC_PATH, MOONSTREAM_CRAWL_WORKERS
from moonstreamdb.db import yield_db_session, yield_db_session_ctx
from moonstreamdb.models import (
EthereumBlock,
EthereumTransaction,
PolygonBlock,
PolygonTransaction,
)
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import Column, desc, func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import HTTPProvider, IPCProvider, Web3
from web3.middleware import geth_poa_middleware
from web3.types import BlockData
from .data import AvailableBlockchainType, DateRange
from .settings import (
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_IPC_PATH,
MOONSTREAM_POLYGON_IPC_PATH,
)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class EthereumBlockCrawlError(Exception):
class BlockCrawlError(Exception):
"""
Raised when there is a problem crawling Ethereum blocks.
Raised when there is a problem crawling blocks.
"""
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool
def connect(web3_uri: Optional[str] = MOONSTREAM_IPC_PATH):
def connect(blockchain_type: AvailableBlockchainType, web3_uri: Optional[str] = None):
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
if web3_uri is not None:
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
if web3_uri is None:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
web3_uri = MOONSTREAM_ETHEREUM_IPC_PATH
elif blockchain_type == AvailableBlockchainType.POLYGON:
web3_uri = MOONSTREAM_POLYGON_IPC_PATH
else:
web3_provider = Web3.IPCProvider(web3_uri)
raise Exception("Wrong blockchain type provided for web3 URI")
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
else:
web3_provider = Web3.IPCProvider(web3_uri)
web3_client = Web3(web3_provider)
# Inject --dev middleware if it is not Ethereum mainnet
# Docs: https://web3py.readthedocs.io/en/stable/middleware.html#geth-style-proof-of-authority
if blockchain_type != AvailableBlockchainType.ETHEREUM:
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
return web3_client
def add_block(db_session, block: Any) -> None:
def get_block_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumBlock, PolygonBlock]]:
"""
Depends on provided blockchain type: Ethereum or Polygon,
set proper blocks model: EthereumBlock or PolygonBlock.
"""
block_model: Type[Union[EthereumBlock, PolygonBlock]]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
block_model = EthereumBlock
elif blockchain_type == AvailableBlockchainType.POLYGON:
block_model = PolygonBlock
else:
raise Exception("Unsupported blockchain type provided")
return block_model
def get_transaction_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumTransaction, PolygonTransaction]]:
"""
Depends on provided blockchain type: Ethereum or Polygon,
set proper block transactions model: EthereumTransaction or PolygonTransaction.
"""
transaction_model: Type[Union[EthereumTransaction, PolygonTransaction]]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
transaction_model = EthereumTransaction
elif blockchain_type == AvailableBlockchainType.POLYGON:
transaction_model = PolygonTransaction
else:
raise Exception("Unsupported blockchain type provided")
return transaction_model
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
"""
Add block if doesn't presented in database.
block: web3.types.BlockData
"""
block_obj = EthereumBlock(
block_model = get_block_model(blockchain_type)
# BlockData.extraData doesn't exist at Polygon mainnet
extra_data = None
if block.get("extraData", None) is not None:
extra_data = block.get("extraData").hex()
block_obj = block_model(
block_number=block.number,
difficulty=block.difficulty,
extra_data=block.extraData.hex(),
extra_data=extra_data,
gas_limit=block.gasLimit,
gas_used=block.gasUsed,
base_fee_per_gas=block.get("baseFeePerGas", None),
@ -78,14 +132,17 @@ def add_block(db_session, block: Any) -> None:
db_session.add(block_obj)
def add_block_transactions(db_session, block: Any) -> None:
def add_block_transactions(
db_session, block: Any, blockchain_type: AvailableBlockchainType
) -> None:
"""
Add block transactions.
block: web3.types.BlockData
"""
transaction_model = get_transaction_model(blockchain_type)
for tx in block.transactions:
tx_obj = EthereumTransaction(
tx_obj = transaction_model(
hash=tx.hash.hex(),
block_number=block.number,
from_address=tx["from"],
@ -103,22 +160,25 @@ def add_block_transactions(db_session, block: Any) -> None:
db_session.add(tx_obj)
def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
def get_latest_blocks(
blockchain_type: AvailableBlockchainType, confirmations: int = 0
) -> Tuple[Optional[int], int]:
"""
Retrieve the latest block from the connected node (connection is created by the connect() method).
Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method).
If confirmations > 0, and the latest block on the node has block number N, this returns the block
with block_number (N - confirmations)
"""
web3_client = connect()
web3_client = connect(blockchain_type)
latest_block_number: int = web3_client.eth.block_number
if confirmations > 0:
latest_block_number -= confirmations
block_model = get_block_model(blockchain_type)
with yield_db_session_ctx() as db_session:
latest_stored_block_row = (
db_session.query(EthereumBlock.block_number)
.order_by(EthereumBlock.block_number.desc())
db_session.query(block_model.block_number)
.order_by(block_model.block_number.desc())
.first()
)
latest_stored_block_number = (
@ -128,11 +188,15 @@ def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
return latest_stored_block_number, latest_block_number
def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> None:
def crawl_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
with_transactions: bool = False,
) -> None:
"""
Open database and geth sessions and fetch block data from blockchain.
"""
web3_client = connect()
web3_client = connect(blockchain_type)
with yield_db_session_ctx() as db_session:
pbar = tqdm(total=len(blocks_numbers))
for block_number in blocks_numbers:
@ -143,10 +207,10 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
block: BlockData = web3_client.eth.get_block(
block_number, full_transactions=with_transactions
)
add_block(db_session, block)
add_block(db_session, block, blockchain_type)
if with_transactions:
add_block_transactions(db_session, block)
add_block_transactions(db_session, block, blockchain_type)
db_session.commit()
except IntegrityError as err:
@ -157,7 +221,7 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
except Exception as err:
db_session.rollback()
message = f"Error adding block (number={block_number}) to database:\n{repr(err)}"
raise EthereumBlockCrawlError(message)
raise BlockCrawlError(message)
except:
db_session.rollback()
logger.error(
@ -168,7 +232,11 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
pbar.close()
def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> List[int]:
def check_missing_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
notransactions=False,
) -> List[int]:
"""
Query block from postgres. If block does not presented in database,
add to missing blocks numbers list.
@ -178,33 +246,35 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
bottom_block = min(blocks_numbers[-1], blocks_numbers[0])
top_block = max(blocks_numbers[-1], blocks_numbers[0])
block_model = get_block_model(blockchain_type)
transaction_model = get_transaction_model(blockchain_type)
with yield_db_session_ctx() as db_session:
if notransactions:
blocks_exist_raw_query = (
db_session.query(EthereumBlock.block_number)
.filter(EthereumBlock.block_number >= bottom_block)
.filter(EthereumBlock.block_number <= top_block)
db_session.query(block_model.block_number)
.filter(block_model.block_number >= bottom_block)
.filter(block_model.block_number <= top_block)
)
blocks_exist = [[block[0]] for block in blocks_exist_raw_query.all()]
else:
corrupted_blocks = []
blocks_exist_raw_query = (
db_session.query(
EthereumBlock.block_number, func.count(EthereumTransaction.hash)
block_model.block_number, func.count(transaction_model.hash)
)
.join(
EthereumTransaction,
EthereumTransaction.block_number == EthereumBlock.block_number,
transaction_model,
transaction_model.block_number == block_model.block_number,
)
.filter(EthereumBlock.block_number >= bottom_block)
.filter(EthereumBlock.block_number <= top_block)
.group_by(EthereumBlock.block_number)
.filter(block_model.block_number >= bottom_block)
.filter(block_model.block_number <= top_block)
.group_by(block_model.block_number)
)
blocks_exist = [
[block[0], block[1]] for block in blocks_exist_raw_query.all()
]
web3_client = connect()
web3_client = connect(blockchain_type)
blocks_exist_len = len(blocks_exist)
pbar = tqdm(total=blocks_exist_len)
@ -218,8 +288,8 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
corrupted_blocks.append(block_in_db[0])
# Delete existing corrupted block and add to missing list
del_block = (
db_session.query(EthereumBlock)
.filter(EthereumBlock.block_number == block_in_db[0])
db_session.query(block_model)
.filter(block_model.block_number == block_in_db[0])
.one()
)
db_session.delete(del_block)
@ -242,6 +312,7 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
def crawl_blocks_executor(
blockchain_type: AvailableBlockchainType,
block_numbers_list: List[int],
with_transactions: bool = False,
num_processes: int = MOONSTREAM_CRAWL_WORKERS,
@ -272,13 +343,15 @@ def crawl_blocks_executor(
results: List[Future] = []
if num_processes == 1:
logger.warning("Executing block crawler in lazy mod")
return crawl_blocks(block_numbers_list, with_transactions)
return crawl_blocks(blockchain_type, block_numbers_list, with_transactions)
else:
with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor:
for worker in worker_indices:
block_chunk = worker_job_lists[worker]
logger.info(f"Spawned process for {len(block_chunk)} blocks")
result = executor.submit(crawl_blocks, block_chunk, with_transactions)
result = executor.submit(
crawl_blocks, blockchain_type, block_chunk, with_transactions
)
result.add_done_callback(record_error)
results.append(result)
@ -286,7 +359,7 @@ def crawl_blocks_executor(
if len(errors) > 0:
error_messages = "\n".join([f"- {error}" for error in errors])
message = f"Error processing blocks in list:\n{error_messages}"
raise EthereumBlockCrawlError(message)
raise BlockCrawlError(message)
def trending(

5
crawlers/mooncrawl/mooncrawl/contract/cli.py
Wyświetl plik

@ -8,7 +8,8 @@ from moonstreamdb.db import yield_db_session_ctx
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from ..blockchain import connect
from ..data import AvailableBlockchainType
from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
logging.basicConfig(level=logging.INFO)
@ -116,7 +117,7 @@ def run_crawler_desc(
def handle_parser(args: argparse.Namespace):
with yield_db_session_ctx() as session:
w3 = connect()
w3 = connect(AvailableBlockchainType.ETHEREUM)
if args.order == "asc":
run_crawler_asc(
w3=w3,

137
crawlers/mooncrawl/mooncrawl/ethcrawler.py → crawlers/mooncrawl/mooncrawl/crawler.py
Wyświetl plik

@ -2,24 +2,25 @@
Moonstream crawlers CLI.
"""
import argparse
from datetime import datetime, timedelta, timezone
from enum import Enum
import json
import logging
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Iterator, List
import dateutil.parser
from .ethereum import (
crawl_blocks_executor,
check_missing_blocks,
get_latest_blocks,
from .blockchain import (
DateRange,
check_missing_blocks,
crawl_blocks_executor,
get_latest_blocks,
trending,
)
from .data import AvailableBlockchainType
from .publish import publish_json
from .settings import MOONSTREAM_CRAWL_WORKERS
from .version import MOONCRAWL_VERSION
@ -86,13 +87,13 @@ def yield_blocks_numbers_lists(
current_block -= block_step
def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
"""
Synchronize latest Ethereum blocks with database.
Synchronize latest Blockchain blocks with database.
"""
while True:
latest_stored_block_number, latest_block_number = get_latest_blocks(
args.confirmations
AvailableBlockchainType(args.blockchain), args.confirmations
)
if latest_stored_block_number is None:
latest_stored_block_number = 0
@ -132,6 +133,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
# TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize.
crawl_blocks_executor(
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=blocks_numbers_list,
with_transactions=True,
num_processes=args.jobs,
@ -141,7 +143,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
"""
Add blocks to moonstream database.
"""
@ -150,7 +152,9 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
logger.info(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}")
crawl_blocks_executor(
block_numbers_list=blocks_numbers_list, with_transactions=True
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=blocks_numbers_list,
with_transactions=True,
)
logger.info(
@ -158,19 +162,32 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
"""
Check missing blocks and missing transactions in each block.
If block range doesn't provided, get latest block from blockchain minus 50,
and check last 2000 blocks behind.
"""
startTime = time.time()
missing_blocks_numbers_total = []
for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
block_range = args.blocks
if block_range is None:
confirmations = 50
shift = 2000
_, latest_block_number = get_latest_blocks(
AvailableBlockchainType(args.blockchain), confirmations
)
block_range = f"{latest_block_number-shift}-{latest_block_number}"
for blocks_numbers_list in yield_blocks_numbers_lists(block_range):
logger.info(
f"Checking missing blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]} "
f"with comparing transactions: {not args.notransactions}"
)
missing_blocks_numbers = check_missing_blocks(
blockchain_type=AvailableBlockchainType(args.blockchain),
blocks_numbers=blocks_numbers_list,
notransactions=args.notransactions,
)
@ -185,7 +202,8 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
if (len(missing_blocks_numbers_total)) > 0:
time.sleep(5)
crawl_blocks_executor(
missing_blocks_numbers_total,
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=missing_blocks_numbers_total,
with_transactions=True,
num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS,
)
@ -195,7 +213,7 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
def crawler_trending_handler(args: argparse.Namespace) -> None:
date_range = DateRange(
start_time=args.start,
end_time=args.end,
@ -228,15 +246,15 @@ def main() -> None:
time_now = datetime.now(timezone.utc)
# Ethereum blocks parser
parser_ethcrawler_blocks = subcommands.add_parser(
"blocks", description="Ethereum blocks commands"
# Blockchain blocks parser
parser_crawler_blocks = subcommands.add_parser(
"blocks", description="Blockchain blocks commands"
)
parser_ethcrawler_blocks.set_defaults(
func=lambda _: parser_ethcrawler_blocks.print_help()
parser_crawler_blocks.set_defaults(
func=lambda _: parser_crawler_blocks.print_help()
)
subcommands_ethcrawler_blocks = parser_ethcrawler_blocks.add_subparsers(
description="Ethereum blocks commands"
subcommands_crawler_blocks = parser_crawler_blocks.add_subparsers(
description="Blockchain blocks commands"
)
valid_processing_orders = {
@ -251,29 +269,29 @@ def main() -> None:
f"Invalid processing order ({raw_order}). Valid choices: {valid_processing_orders.keys()}"
)
parser_ethcrawler_blocks_sync = subcommands_ethcrawler_blocks.add_parser(
"synchronize", description="Synchronize to latest ethereum block commands"
parser_crawler_blocks_sync = subcommands_crawler_blocks.add_parser(
"synchronize", description="Synchronize to latest blockchain block commands"
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-s",
"--start",
type=int,
help="(Optional) Block to start synchronization from. Default: None - current Ethereum block minus confirmations ",
help="(Optional) Block to start synchronization from. Default: None - current Blockchain block minus confirmations ",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-c",
"--confirmations",
type=int,
default=0,
help="Number of confirmations we require before storing a block in the database. (Default: 0)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"--order",
type=processing_order,
default=ProcessingOrder.ASCENDING,
help="Order in which to process blocks (choices: desc, asc; default: asc)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-j",
"--jobs",
type=int,
@ -283,72 +301,85 @@ def main() -> None:
" If you set to 1, the main process handles synchronization without spawning subprocesses."
),
)
parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler)
parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser(
"add", description="Add ethereum blocks commands"
parser_crawler_blocks_sync.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_ethcrawler_blocks_add.add_argument(
parser_crawler_blocks_sync.set_defaults(func=crawler_blocks_sync_handler)
parser_crawler_blocks_add = subcommands_crawler_blocks.add_parser(
"add", description="Add blockchain blocks commands"
)
parser_crawler_blocks_add.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler)
parser_ethcrawler_blocks_missing = subcommands_ethcrawler_blocks.add_parser(
"missing", description="Add missing ethereum blocks with transactions commands"
parser_crawler_blocks_add.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_add.set_defaults(func=crawler_blocks_add_handler)
parser_crawler_blocks_missing = subcommands_crawler_blocks.add_parser(
"missing",
description="Add missing Blockchain blocks with transactions commands",
)
parser_crawler_blocks_missing.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-l",
"--lazy",
action="store_true",
help="Lazy block adding one by one",
)
parser_ethcrawler_blocks_missing.set_defaults(
func=ethcrawler_blocks_missing_handler
parser_crawler_blocks_missing.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_crawler_blocks_missing.set_defaults(func=crawler_blocks_missing_handler)
parser_ethcrawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Ethereum blockchain"
parser_crawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Blockchain blockchain"
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-s",
"--start",
type=dateutil.parser.parse,
default=(time_now - timedelta(hours=1, minutes=0)).isoformat(),
help=f"Start time for window to calculate trending addresses in (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-start",
action="store_true",
help="Set this flag if range should include start time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-e",
"--end",
type=dateutil.parser.parse,
default=time_now.isoformat(),
help=f"End time for window to calculate trending addresses in (default: {time_now.isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-end",
action="store_true",
help="Set this flag if range should include end time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--humbug",
default=None,
help=(
@ -357,14 +388,14 @@ def main() -> None:
"MOONSTREAM_HUMBUG_TOKEN environment variable)"
),
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-o",
"--outfile",
type=argparse.FileType("w"),
default=sys.stdout,
help="Optional file to write output to. By default, prints to stdout.",
)
parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler)
parser_crawler_trending.set_defaults(func=crawler_trending_handler)
args = parser.parse_args()
args.func(args)

16
crawlers/mooncrawl/mooncrawl/data.py 100644
Wyświetl plik

@ -0,0 +1,16 @@
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
POLYGON = "polygon"
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool

2
crawlers/mooncrawl/mooncrawl/esd.py
Wyświetl plik

@ -3,10 +3,10 @@ import sys
import time
from typing import Optional, Union
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature
from sqlalchemy.orm import Session
import requests
CRAWL_URLS = {
"functions": "https://www.4byte.directory/api/v1/signatures/",

10
crawlers/mooncrawl/mooncrawl/etherscan.py
Wyświetl plik

@ -1,23 +1,23 @@
import argparse
import codecs
import csv
from dataclasses import dataclass
from datetime import datetime
import json
import logging
import os
import sys
import time
from typing import Any, List, Optional, Dict
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional
import boto3 # type: ignore
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
import requests
from sqlalchemy.orm import Session
from .version import MOONCRAWL_VERSION
from .settings import MOONSTREAM_ETHERSCAN_TOKEN
from .version import MOONCRAWL_VERSION
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

3
crawlers/mooncrawl/mooncrawl/identity.py
Wyświetl plik

@ -4,10 +4,9 @@ import os
import time
import requests
from sqlalchemy import text
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
from sqlalchemy import text
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

41
crawlers/mooncrawl/mooncrawl/nft/cli.py
Wyświetl plik

@ -2,12 +2,12 @@
A command line tool to crawl information about NFTs from various sources.
"""
import argparse
from datetime import datetime, timezone
import json
import logging
import sys
import time
from typing import Any, cast, Dict, Optional
from datetime import datetime, timezone
from typing import Any, Dict, Optional, cast
from bugout.app import Bugout
from bugout.journal import SearchOrder
@ -16,26 +16,25 @@ from moonstreamdb.models import EthereumBlock
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from ..blockchain import connect
from ..data import AvailableBlockchainType
from ..publish import publish_json
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ETHEREUM_IPC_PATH,
NFT_HUMBUG_TOKEN,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import (
summary as ethereum_summary,
add_labels,
MINT_LABEL,
TRANSFER_LABEL,
SUMMARY_KEY_ARGS,
SUMMARY_KEY_END_BLOCK,
SUMMARY_KEY_ID,
SUMMARY_KEY_NUM_BLOCKS,
SUMMARY_KEY_START_BLOCK,
SUMMARY_KEY_END_BLOCK,
add_labels,
)
from ..publish import publish_json
from ..settings import (
MOONSTREAM_IPC_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
NFT_HUMBUG_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import summary as ethereum_summary
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -46,17 +45,17 @@ BLOCKS_PER_SUMMARY = 40
def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
"""
Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up
the MOONSTREAM_IPC_PATH environment variable.
the MOONSTREAM_ETHEREUM_IPC_PATH environment variable.
"""
web3_connection_string = MOONSTREAM_IPC_PATH
web3_connection_string = MOONSTREAM_ETHEREUM_IPC_PATH
args_web3 = vars(args).get("web3")
if args_web3 is not None:
web3_connection_string = cast(str, args_web3)
if web3_connection_string is None:
raise ValueError(
"Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable"
"Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_IPC_PATH environment variable"
)
return connect(web3_connection_string)
return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string)
def get_latest_block_from_node(web3_client: Web3):
@ -295,7 +294,7 @@ def main() -> None:
"--web3",
type=str,
default=None,
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.",
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_ETHEREUM_IPC_PATH environment variable.",
)
parser_ethereum_label.set_defaults(func=ethereum_label_handler)

19
crawlers/mooncrawl/mooncrawl/nft/ethereum.py
Wyświetl plik

@ -1,23 +1,18 @@
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta, timezone
import json
import logging
from hexbytes.main import HexBytes
from typing import Any, cast, Dict, List, Optional, Set, Tuple
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set, Tuple, cast
from eth_typing.encoding import HexStr
from moonstreamdb.models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
)
from hexbytes.main import HexBytes
from moonstreamdb.models import EthereumBlock, EthereumLabel, EthereumTransaction
from sqlalchemy import and_, func, text
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Session, Query
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import Web3
from web3.types import FilterParams, LogReceipt
from web3._utils.events import get_event_data
from web3.types import FilterParams, LogReceipt
from ..reporter import reporter

2
crawlers/mooncrawl/mooncrawl/publish.py
Wyświetl plik

@ -1,6 +1,6 @@
from datetime import datetime
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
import requests

11
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -11,9 +11,18 @@ MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545
)
MOONSTREAM_IPC_PATH = (
MOONSTREAM_ETHEREUM_IPC_PATH = (
f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}"
)
MOONSTREAM_NODE_POLYGON_IPC_ADDR = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_ADDR", "127.0.0.1"
)
MOONSTREAM_NODE_POLYGON_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_PORT", 8545
)
MOONSTREAM_POLYGON_IPC_PATH = (
f"http://{MOONSTREAM_NODE_POLYGON_IPC_ADDR}:{MOONSTREAM_NODE_POLYGON_IPC_PORT}"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")

42
crawlers/mooncrawl/mooncrawl/test_cli.py
Wyświetl plik

@ -1,13 +1,13 @@
import unittest
from . import ethcrawler
from . import crawler
class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=4
)
]
@ -16,7 +16,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=3
)
]
@ -25,8 +25,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -34,8 +34,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -43,7 +43,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=4
)
]
@ -52,7 +52,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=3
)
]
@ -61,8 +61,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -70,8 +70,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -79,8 +79,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 3
)
]
self.assertListEqual(partition, [[6, 7, 8], [9, 10]])
@ -88,8 +88,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])
@ -97,8 +97,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 4
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 4
)
]
self.assertListEqual(partition, [[6, 7, 8, 9], [10]])
@ -106,8 +106,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.0.5"
MOONCRAWL_VERSION = "0.1.0"

59
crawlers/mooncrawl/requirements.txt
Wyświetl plik

@ -1,59 +0,0 @@
aiohttp==3.7.4.post0
async-timeout==3.0.1
attrs==21.2.0
base58==2.1.0
bitarray==1.2.2
black==21.8b0
boto3==1.18.40
botocore==1.21.40
bugout==0.1.17
certifi==2021.5.30
chardet==4.0.0
charset-normalizer==2.0.4
click==8.0.1
cytoolz==0.11.0
-e git+https://git@github.com/bugout-dev/moonstream.git@67fe019f1086c435dd3b58f1ade2778acc2167c7#egg=moonstreamdb&subdirectory=db
eth-abi==2.1.1
eth-account==0.5.5
eth-hash==0.3.2
eth-keyfile==0.5.1
eth-keys==0.3.3
eth-rlp==0.2.1
eth-typing==2.2.2
eth-utils==1.10.0
hexbytes==0.2.2
humbug==0.2.7
idna==3.2
ipfshttpclient==0.8.0a2
jmespath==0.10.0
jsonschema==3.2.0
lru-dict==1.1.7
multiaddr==0.0.9
multidict==5.1.0
mypy==0.910
mypy-extensions==0.4.3
netaddr==0.8.0
parsimonious==0.8.1
pathspec==0.9.0
platformdirs==2.3.0
protobuf==3.17.3
pycryptodome==3.10.1
pyrsistent==0.18.0
python-dateutil==2.8.2
regex==2021.8.28
requests==2.26.0
rlp==2.0.1
s3transfer==0.5.0
six==1.16.0
toml==0.10.2
tomli==1.2.1
toolz==0.11.1
tqdm==4.62.2
types-python-dateutil==2.8.0
types-requests==2.25.6
typing-extensions==3.10.0.2
urllib3==1.26.6
varint==1.0.2
web3==5.23.1
websockets==9.1
yarl==1.6.3

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -1,6 +1,8 @@
# Path to IPC socket to use for web3 connections
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545"
export MOONSTREAM_CRAWL_WORKERS=4
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"

13
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -1,5 +1,6 @@
from setuptools import find_packages, setup
from mooncrawl.version import MOONCRAWL_VERSION
long_description = ""
with open("README.md") as ifp:
@ -7,7 +8,7 @@ with open("README.md") as ifp:
setup(
name="mooncrawl",
version="0.0.5",
version=MOONCRAWL_VERSION,
author="Bugout.dev",
author_email="engineers@bugout.dev",
license="Apache License 2.0",
@ -33,8 +34,9 @@ setup(
zip_safe=False,
install_requires=[
"boto3",
"bugout >= 0.1.17",
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@0a771ddfbca1254be331149ccf2d162aa09b7bc0#egg=moonstreamdb&subdirectory=db",
"bugout",
"chardet",
"moonstreamdb",
"humbug",
"python-dateutil",
"requests",
@ -42,11 +44,12 @@ setup(
"web3",
],
extras_require={
"dev": ["black", "mypy", "types-requests", "types-python-dateutil"]
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
"distribute": ["setuptools", "twine", "wheel"],
},
entry_points={
"console_scripts": [
"ethcrawler=mooncrawl.ethcrawler:main",
"crawler=mooncrawl.crawler:main",
"esd=mooncrawl.esd:main",
"identity=mooncrawl.identity:main",
"etherscan=mooncrawl.etherscan:main",

2
db/setup.py
Wyświetl plik

@ -34,7 +34,7 @@ setup(
zip_safe=False,
install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
extras_require={
"dev": ["black", "mypy"],
"dev": ["black", "isort", "mypy"],
"distribute": ["setuptools", "twine", "wheel"],
},
entry_points={

7
frontend/pages/index.js
Wyświetl plik

@ -19,6 +19,7 @@ import {
Text,
GridItem,
SimpleGrid,
Button,
Image as ChakraImage,
} from "@chakra-ui/react";
import dynamic from "next/dynamic";
@ -776,18 +777,18 @@ const Homepage = () => {
Join our Discord
</RouteButton>
</Flex>
<RouteButton
<Button
placeSelf="center"
isExternal
w={["100%", "100%", "fit-content", null]}
maxW={["250px", null, "fit-content"]}
href={`https://github.com/bugout-dev/moonstream`}
onClick={() => toggleModal("register")}
size="lg"
variant="solid"
colorScheme="orange"
>
Sign up
</RouteButton>
</Button>
</Stack>
</GridItem>
</Grid>