mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into fix-modal-windows

pull/420/head
Tim Pechersky 2021-11-15 16:16:58 +00:00
rodzic 83b1790dad 3918124679
commit f33a0704cf
112 zmienionych plików z 2302 dodań i 528 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

8
.github/workflows/lint.backend.yml vendored
Wyświetl plik

@ -16,12 +16,14 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./backend
run: pip install -r requirements.txt
run: pip install -e .[dev]
- name: Mypy type check
working-directory: ./backend
run: mypy moonstream/
run: mypy moonstreamapi/
- name: Black syntax check
working-directory: ./backend
run: black --check moonstream/
run: black --check moonstreamapi/

2
.github/workflows/lint.crawlers.yml vendored
Wyświetl plik

@ -16,6 +16,8 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./crawlers/mooncrawl
run: pip install -e .[dev]

2
.github/workflows/lint.db.yml vendored
Wyświetl plik

@ -16,6 +16,8 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Upgrade pip and setuptools
run: pip install --upgrade pip setuptools
- name: Install test requirements
working-directory: ./db
run: pip install -e .[dev]

30
.github/workflows/release.mooncrawl.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release mooncrawl package
on:
push:
tags:
- 'mooncrawl/v*'
defaults:
run:
working-directory: crawlers/mooncrawl
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

30
.github/workflows/release.moonstreamapi.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release moonstreamapi package
on:
push:
tags:
- 'moonstreamapi/v*'
defaults:
run:
working-directory: backend
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

30
.github/workflows/release.moonstreamdb.yml vendored 100644
Wyświetl plik

@ -0,0 +1,30 @@
name: Release moonstreamdb package
on:
push:
tags:
- 'moonstreamdb/v*'
defaults:
run:
working-directory: db
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[distribute]
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

36
backend/deploy/deploy.bash
Wyświetl plik

@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Deployment script - intended to run on Moonstream servers
# Deployment script - intended to run on Moonstream API server
# Colors
C_RESET='\033[0m'
@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
PYTHON="${PYTHON_ENV_DIR}/bin/python"
PIP="${PYTHON_ENV_DIR}/bin/pip"
SCRIPT_DIR="$(realpath $(dirname $0))"
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
# Parameters scripts
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# API server service file
SERVICE_FILE="${SCRIPT_DIR}/moonstreamapi.service"
set -eu
echo
echo
echo -e "${PREFIX_INFO} Updating pip and setuptools"
"${PIP}" install -U pip setuptools
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
"${PIP}" install --upgrade pip setuptools
echo
echo
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
echo -e "${PREFIX_INFO} Installing Python dependencies"
"${PIP}" install -e "${APP_BACKEND_DIR}/"
echo
echo
@ -49,13 +54,18 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$
echo
echo
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}"
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Moonstream API service definition with ${SERVICE_FILE}"
chmod 644 "${SERVICE_FILE}"
cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service
cp "${SERVICE_FILE}" /etc/systemd/system/moonstreamapi.service
systemctl daemon-reload
systemctl restart moonstream.service
systemctl status moonstream.service
systemctl restart moonstreamapi.service
systemctl status moonstreamapi.service

6
backend/deploy/moonstream.service → backend/deploy/moonstreamapi.service
Wyświetl plik

@ -1,5 +1,5 @@
[Unit]
Description=moonstream-service
Description=moonstreamapi-service
After=network.target
[Service]
@ -7,8 +7,8 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/backend
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
SyslogIdentifier=moonstream
ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstreamapi.api:app
SyslogIdentifier=moonstreamapi
[Install]
WantedBy=multi-user.target

89
backend/deploy/nodes-connections.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Update nodes connection address environment variables
# from AWS Route53 internal hosted zone
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
echo
echo "CLI to update nodes connection address environment
variables from AWS Route53 internal hosted zone"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -f File path where environment variables update at"
}
file_flag=""
verbose_flag="false"
while getopts 'f:v' flag; do
case "${flag}" in
f) file_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# File flag should be specified
if [ -z "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify file path"
usage
exit 1
fi
if [ ! -f "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Provided file does not exist"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Source environment variables"
. ${file_flag}
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving Polygon node address"
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
exit 1
fi
# TODO(kompotkot): Modify regexp to work with export prefix
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

8
backend/deploy/parameters.bash
Wyświetl plik

@ -1,6 +1,9 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
# Collect secrets from AWS SSM Parameter Store and
# opt out as environment variable exports.
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
@ -26,6 +29,7 @@ and output as environment variable exports"
echo " -o Output file name environment variables export to"
}
# TODO(kompotkot): Flag for export prefix
product_flag=""
output_flag=""
verbose_flag="false"
@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=${product_flag} \

2
backend/dev.sh
Wyświetl plik

@ -6,4 +6,4 @@ set -e
MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --reload
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload

7
backend/moonstream/__init__.py
Wyświetl plik

@ -1,7 +0,0 @@
from .reporter import reporter
from .version import MOONSTREAM_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)

11
backend/moonstreamapi/__init__.py 100644
Wyświetl plik

@ -0,0 +1,11 @@
try:
from .reporter import reporter
from .version import MOONSTREAMAPI_VERSION
# Reporting
reporter.tags.append(f"version:{MOONSTREAMAPI_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
except:
# Pass it to be able import MOONSTREAMAPI_VERSION in setup.py with pip
pass

0
backend/moonstream/abi_decoder.py → backend/moonstreamapi/abi_decoder.py
Wyświetl plik

26
backend/moonstream/actions.py → backend/moonstreamapi/actions.py
Wyświetl plik

@ -30,14 +30,20 @@ from .settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
AWS_S3_SMARTCONTRACTS_ABI_BUCKET,
AWS_S3_SMARTCONTRACTS_ABI_PREFIX,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
)
from web3 import Web3
logger = logging.getLogger(__name__)
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
}
class StatusAPIException(Exception):
"""
Raised during checking Moonstream API statuses.
@ -379,15 +385,13 @@ def dashboards_abi_validation(
return True
def validate_abi_string(abi: str) -> None:
def validate_abi_json(abi: Any) -> None:
"""
Transform string to json and run validation
"""
try:
validate_abi(json.loads(abi))
except json.JSONDecodeError:
raise MoonstreamHTTPException(status_code=400, detail="Malformed abi body.")
validate_abi(abi)
except ValueError as e:
raise MoonstreamHTTPException(status_code=400, detail=e)
except:
@ -408,10 +412,10 @@ def upload_abi_to_s3(
s3_client = boto3.client("s3")
bucket = AWS_S3_SMARTCONTRACTS_ABI_BUCKET
bucket = MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET
result_bytes = abi.encode("utf-8")
result_key = f"{AWS_S3_SMARTCONTRACTS_ABI_PREFIX}/{resource.resource_data['address']}/{resource.id}/abi.json"
result_key = f"{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{blockchain_by_subscription_id[resource.resource_data['subscription_type_id']]}/abi/{resource.resource_data['address']}/{resource.id}/abi.json"
s3_client.put_object(
Body=result_bytes,
@ -423,9 +427,7 @@ def upload_abi_to_s3(
update["abi"] = True
update["bucket"] = AWS_S3_SMARTCONTRACTS_ABI_BUCKET
update[
"s3_path"
] = f"{AWS_S3_SMARTCONTRACTS_ABI_PREFIX}/{resource.resource_data['address']}/{resource.id}/abi.json"
update["bucket"] = MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET
update["s3_path"] = result_key
return update

0
backend/moonstream/admin/__init__.py → backend/moonstreamapi/admin/__init__.py
Wyświetl plik

0
backend/moonstream/admin/cli.py → backend/moonstreamapi/admin/cli.py
Wyświetl plik

0
backend/moonstream/admin/migrations/1_add_abi_backet_and_s3_path.json → backend/moonstreamapi/admin/migrations/1_add_abi_backet_and_s3_path.json
Wyświetl plik

0
backend/moonstream/admin/migrations/__init__.py → backend/moonstreamapi/admin/migrations/__init__.py
Wyświetl plik

0
backend/moonstream/admin/migrations/checksum_address.py → backend/moonstreamapi/admin/migrations/checksum_address.py
Wyświetl plik

10
backend/moonstream/admin/subscription_types.py → backend/moonstreamapi/admin/subscription_types.py
Wyświetl plik

@ -27,6 +27,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"polygon_blockchain": SubscriptionTypeResourceData(
id="polygon_blockchain",
name="Polygon transactions",
choices=["input:address", "tag:erc721"],
description="Transactions that have been mined into the Polygon blockchain",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/matic-token-inverted-icon.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_whalewatch": SubscriptionTypeResourceData(
id="ethereum_whalewatch",
name="Ethereum whale watch",

0
backend/moonstream/admin/subscriptions.py → backend/moonstreamapi/admin/subscriptions.py
Wyświetl plik

6
backend/moonstream/api.py → backend/moonstreamapi/api.py
Wyświetl plik

@ -20,7 +20,7 @@ from .routes.whales import router as whales_router
from .routes.dashboards import router as dashboards_router
from .middleware import BroodAuthMiddleware, MoonstreamHTTPException
from .settings import DOCS_TARGET_PATH, ORIGINS
from .version import MOONSTREAM_VERSION
from .version import MOONSTREAMAPI_VERSION
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -45,7 +45,7 @@ tags_metadata = [
app = FastAPI(
title=f"Moonstream API",
description="Moonstream API endpoints.",
version=MOONSTREAM_VERSION,
version=MOONSTREAMAPI_VERSION,
openapi_tags=tags_metadata,
openapi_url="/openapi.json",
docs_url=None,
@ -91,7 +91,7 @@ async def version_handler() -> data.VersionResponse:
"""
Get server version.
"""
return data.VersionResponse(version=MOONSTREAM_VERSION)
return data.VersionResponse(version=MOONSTREAMAPI_VERSION)
@app.get("/now", tags=["time"])

3
backend/moonstream/data.py → backend/moonstreamapi/data.py
Wyświetl plik

@ -13,7 +13,6 @@ USER_ONBOARDING_STATE = "onboarding_state"
class TimeScale(Enum):
year = "year"
month = "month"
week = "week"
day = "day"
@ -232,7 +231,7 @@ class OnboardingState(BaseModel):
class DashboardMeta(BaseModel):
subscription_id: Union[UUID, str]
subscription_id: UUID
generic: Optional[List[Dict[str, str]]]
all_methods: bool = False
all_events: bool = False

0
backend/moonstream/middleware.py → backend/moonstreamapi/middleware.py
Wyświetl plik

0
backend/moonstream/providers/__init__.py → backend/moonstreamapi/providers/__init__.py
Wyświetl plik

0
backend/moonstream/providers/bugout.py → backend/moonstreamapi/providers/bugout.py
Wyświetl plik

0
backend/moonstream/providers/ethereum_blockchain.py → backend/moonstreamapi/providers/ethereum_blockchain.py
Wyświetl plik

0
backend/moonstream/py.typed → backend/moonstreamapi/py.typed
Wyświetl plik

2
backend/moonstream/reporter.py → backend/moonstreamapi/reporter.py
Wyświetl plik

@ -9,7 +9,7 @@ session_id = str(uuid.uuid4())
client_id = "moonstream-backend"
reporter = HumbugReporter(
name="moonstream",
name="moonstreamapi",
consent=HumbugConsent(True),
client_id=client_id,
session_id=session_id,

0
backend/moonstream/routes/__init__.py → backend/moonstreamapi/routes/__init__.py
Wyświetl plik

0
backend/moonstream/routes/address_info.py → backend/moonstreamapi/routes/address_info.py
Wyświetl plik

47
backend/moonstream/routes/dashboards.py → backend/moonstreamapi/routes/dashboards.py
Wyświetl plik

@ -16,10 +16,11 @@ from ..reporter import reporter
from ..settings import (
MOONSTREAM_APPLICATION_ID,
bugout_client as bc,
SMARTCONTRACTS_ABI_BUCKET,
BUGOUT_REQUEST_TIMEOUT_SECONDS,
SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
)
import pprint
logger = logging.getLogger(__name__)
@ -32,6 +33,12 @@ BUGOUT_RESOURCE_TYPE_DASHBOARD = "dashboards"
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION = "subscription"
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
}
@router.post("/", tags=["dashboards"], response_model=BugoutResource)
async def add_dashboard_handler(
request: Request, dashboard: data.DashboardCreate
@ -92,10 +99,6 @@ async def add_dashboard_handler(
)
s3_path = f"s3://{bucket}/{key}"
dashboard_subscription.subscription_id = str(
dashboard_subscription.subscription_id
)
try:
response = s3_client.get_object(
@ -105,12 +108,12 @@ async def add_dashboard_handler(
except s3_client.exceptions.NoSuchKey as e:
logger.error(
f"Error getting Abi for subscription {dashboard_subscription.subscription_id} S3 {s3_path} does not exist : {str(e)}"
f"Error getting Abi for subscription {str(dashboard_subscription.subscription_id)} S3 {s3_path} does not exist : {str(e)}"
)
raise MoonstreamHTTPException(
status_code=500,
internal_error=e,
detail=f"We can't access the abi for subscription with id:{dashboard_subscription.subscription_id}.",
detail=f"We can't access the abi for subscription with id:{str(dashboard_subscription.subscription_id)}.",
)
abi = json.loads(response["Body"].read())
@ -121,7 +124,7 @@ async def add_dashboard_handler(
else:
logger.error(
f"Error subscription_id: {dashboard_subscription.subscription_id} not exists."
f"Error subscription_id: {str(dashboard_subscription.subscription_id)} not exists."
)
raise MoonstreamHTTPException(status_code=404)
@ -133,16 +136,18 @@ async def add_dashboard_handler(
)
try:
# json.loads(dashboard_resource.json())
# Necessary because the UUIDs inside dashboard_resources do not get serialized into string if we directly convert to ".dict()"
resource: BugoutResource = bc.create_resource(
token=token,
application_id=MOONSTREAM_APPLICATION_ID,
resource_data=dashboard_resource.dict(),
resource_data=json.loads(dashboard_resource.json()),
)
except BugoutResponseException as e:
logger.error(f"Error creating subscription resource: {str(e)}")
logger.error(f"Error creating dashboard resource: {str(e)}")
raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
except Exception as e:
logger.error(f"Error creating subscription resource: {str(e)}")
logger.error(f"Error creating dashboard resource: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return resource
@ -151,7 +156,7 @@ async def add_dashboard_handler(
@router.delete(
"/{dashboard_id}",
tags=["subscriptions"],
response_model=data.SubscriptionResourceData,
response_model=BugoutResource,
)
async def delete_subscription_handler(request: Request, dashboard_id: str):
"""
@ -338,10 +343,10 @@ async def update_dashboard_handler(
return resource
@router.get("/{dashboard_id}/data_links", tags=["dashboards"])
@router.get("/{dashboard_id}/stats", tags=["dashboards"])
async def get_dashboard_data_links_handler(
request: Request, dashboard_id: str
) -> Dict[str, Any]:
) -> Dict[UUID, Any]:
"""
Update dashboards mainly fully overwrite name and subscription metadata
"""
@ -402,25 +407,29 @@ async def get_dashboard_data_links_handler(
s3_client = boto3.client("s3")
stats: Dict[str, Any] = {}
stats: Dict[UUID, Any] = {}
for subscription in dashboard_subscriptions:
hash = subscription.resource_data["abi_hash"]
available_timescales = [timescale.value for timescale in data.TimeScale]
stats[subscription.id] = {}
for timescale in available_timescales:
try:
result_key = f'contracts_data/{subscription.resource_data["address"]}/v1/{timescale}.json'
result_key = f'{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{blockchain_by_subscription_id[subscription.resource_data["subscription_type_id"]]}/contracts_data/{subscription.resource_data["address"]}/{hash}/v1/{timescale}.json'
stats_presigned_url = s3_client.generate_presigned_url(
"get_object",
Params={"Bucket": SMARTCONTRACTS_ABI_BUCKET, "Key": result_key},
Params={
"Bucket": MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
"Key": result_key,
},
ExpiresIn=300,
HttpMethod="GET",
)
stats[subscription.id][timescale] = stats_presigned_url
except Exception as err:
logger.warning(
f"Can't generate S3 presigned url in stats endpoint for Bucket:{SMARTCONTRACTS_ABI_BUCKET}, Key:{result_key} get error:{err}"
f"Can't generate S3 presigned url in stats endpoint for Bucket:{MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET}, Key:{result_key} get error:{err}"
)
return stats

0
backend/moonstream/routes/nft.py → backend/moonstreamapi/routes/nft.py
Wyświetl plik

0
backend/moonstream/routes/streams.py → backend/moonstreamapi/routes/streams.py
Wyświetl plik

33
backend/moonstream/routes/subscriptions.py → backend/moonstreamapi/routes/subscriptions.py
Wyświetl plik

@ -1,6 +1,7 @@
"""
The Moonstream subscriptions HTTP API
"""
import hashlib
import logging
import json
from typing import List, Optional, Dict, Any
@ -13,7 +14,7 @@ from fastapi import APIRouter, Depends, Request, Form
from web3 import Web3
from ..actions import (
validate_abi_string,
validate_abi_json,
upload_abi_to_s3,
)
from ..admin import subscription_types
@ -23,8 +24,8 @@ from ..reporter import reporter
from ..settings import (
MOONSTREAM_APPLICATION_ID,
bugout_client as bc,
AWS_S3_SMARTCONTRACTS_ABI_BUCKET,
AWS_S3_SMARTCONTRACTS_ABI_PREFIX,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
)
from ..web3_provider import yield_web3_provider
@ -112,10 +113,21 @@ async def add_subscription_handler(
if abi:
validate_abi_string(abi=abi)
try:
json_abi = json.loads(abi)
except json.JSONDecodeError:
raise MoonstreamHTTPException(status_code=400, detail="Malformed abi body.")
validate_abi_json(json_abi)
update_resource = upload_abi_to_s3(resource=resource, abi=abi, update={})
abi_string = json.dumps(json_abi, sort_keys=True, indent=2)
hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest()
update_resource["abi_hash"] = hash
try:
updated_resource: BugoutResource = bc.update_resource(
token=token,
@ -241,7 +253,16 @@ async def update_subscriptions_handler(
if abi:
validate_abi_string(abi=abi)
try:
json_abi = json.loads(abi)
except json.JSONDecodeError:
raise MoonstreamHTTPException(status_code=400, detail="Malformed abi body.")
validate_abi_json(json_abi)
abi_string = json.dumps(json_abi, sort_keys=True, indent=2)
hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest()
try:
subscription_resource: BugoutResource = bc.get_resource(
@ -264,6 +285,8 @@ async def update_subscriptions_handler(
resource=subscription_resource, abi=abi, update=update
)
update["abi_hash"] = hash
try:
resource: BugoutResource = bc.update_resource(
token=token,

0
backend/moonstream/routes/txinfo.py → backend/moonstreamapi/routes/txinfo.py
Wyświetl plik

0
backend/moonstream/routes/users.py → backend/moonstreamapi/routes/users.py
Wyświetl plik

0
backend/moonstream/routes/whales.py → backend/moonstreamapi/routes/whales.py
Wyświetl plik

26
backend/moonstream/settings.py → backend/moonstreamapi/settings.py
Wyświetl plik

@ -32,12 +32,6 @@ if RAW_ORIGINS is None:
raise ValueError(
"MOONSTREAM_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)"
)
SMARTCONTRACTS_ABI_BUCKET = os.environ.get("SMARTCONTRACTS_ABI_BUCKET")
if SMARTCONTRACTS_ABI_BUCKET is None:
raise ValueError("SMARTCONTRACTS_ABI_BUCKET environment variable must be set")
ORIGINS = RAW_ORIGINS.split(",")
# OpenAPI
@ -73,14 +67,20 @@ MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545
)
AWS_S3_SMARTCONTRACTS_ABI_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_BUCKET")
if AWS_S3_SMARTCONTRACTS_ABI_BUCKET is None:
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET = os.environ.get(
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET"
)
if MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET is None:
raise ValueError(
"AWS_S3_SMARTCONTRACTS_ABI_BUCKET environment variable must be set"
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET environment variable must be set"
)
AWS_S3_SMARTCONTRACTS_ABI_PREFIX = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_PREFIX")
if AWS_S3_SMARTCONTRACTS_ABI_PREFIX is None:
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = os.environ.get(
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX"
)
if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None:
raise ValueError(
"AWS_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set"
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set"
)
AWS_S3_SMARTCONTRACTS_ABI_PREFIX = AWS_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/")
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = (
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/")
)

0
backend/moonstream/stream_boundaries.py → backend/moonstreamapi/stream_boundaries.py
Wyświetl plik

0
backend/moonstream/stream_queries.py → backend/moonstreamapi/stream_queries.py
Wyświetl plik

0
backend/moonstream/test_actions_abi_validations.py → backend/moonstreamapi/test_actions_abi_validations.py
Wyświetl plik

0
backend/moonstream/test_stream_boundaries.py → backend/moonstreamapi/test_stream_boundaries.py
Wyświetl plik

0
backend/moonstream/test_stream_queries.py → backend/moonstreamapi/test_stream_queries.py
Wyświetl plik

2
backend/moonstream/version.py → backend/moonstreamapi/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAM_VERSION = "0.0.2"
MOONSTREAMAPI_VERSION = "0.1.0"

0
backend/moonstream/web3_provider.py → backend/moonstreamapi/web3_provider.py
Wyświetl plik

35
backend/requirements.txt
Wyświetl plik

@ -1,35 +0,0 @@
appdirs==1.4.4
asgiref==3.4.1
black==21.7b0
boto3==1.18.1
botocore==1.21.1
bugout==0.1.18
certifi==2021.5.30
charset-normalizer==2.0.3
click==8.0.1
fastapi==0.66.0
h11==0.12.0
idna==3.2
jmespath==0.10.0
humbug==0.2.7
-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3
pathspec==0.9.0
pydantic==1.8.2
pyevmasm==0.2.3
python-dateutil==2.8.2
python-multipart==0.0.5
regex==2021.7.6
requests==2.26.0
s3transfer==0.5.0
six==1.16.0
starlette==0.14.2
toml==0.10.2
tomli==1.0.4
types-python-dateutil==0.1.6
typing-extensions==3.10.0.0
types-requests==2.25.6
urllib3==1.26.6
uvicorn==0.14.0
web3==5.24.0

4
backend/sample.env
Wyświetl plik

@ -7,8 +7,8 @@ export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="<moonstream_internal_hosted_zone_id>"
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="<connection_path_uri_to_ethereum_node>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export AWS_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS S3 bucket to store smart contracts ABI>"
export AWS_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (v1/v2/dev/..)>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS S3 bucket to store smart contracts ABI>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (prod,dev,..)>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout Humbug token for crash reports>"

24
backend/setup.py
Wyświetl plik

@ -1,30 +1,34 @@
from setuptools import find_packages, setup
from moonstream.version import MOONSTREAM_VERSION
from moonstreamapi.version import MOONSTREAMAPI_VERSION
long_description = ""
with open("README.md") as ifp:
long_description = ifp.read()
setup(
name="moonstream",
version=MOONSTREAM_VERSION,
name="moonstreamapi",
version=MOONSTREAMAPI_VERSION,
packages=find_packages(),
install_requires=[
"appdirs",
"boto3",
"bugout >= 0.1.18",
"bugout",
"fastapi",
"humbug>=0.2.7",
"moonstreamdb",
"humbug",
"pydantic",
"pyevmasm",
"python-dateutil",
"python-multipart",
"uvicorn",
"types-python-dateutil",
"types-requests",
"web3",
],
extras_require={
"dev": ["black", "mypy"],
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
"distribute": ["setuptools", "twine", "wheel"],
},
package_data={"moonstream": ["py.typed"]},
package_data={"moonstreamapi": ["py.typed"]},
zip_safe=False,
description="The Bugout blockchain inspector API.",
long_description=long_description,
@ -38,5 +42,5 @@ setup(
"Topic :: Software Development :: Libraries",
],
url="https://github.com/bugout-dev/moonstream",
entry_points={"console_scripts": ["mnstr=moonstream.admin.cli:main"]},
entry_points={"console_scripts": ["mnstr=moonstreamapi.admin.cli:main"]},
)

95
crawlers/deploy/deploy.bash
Wyświetl plik

@ -24,15 +24,27 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
SCRIPT_DIR="$(realpath $(dirname $0))"
# Parameters scripts
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts"
CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash"
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# Ethereum service files
ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
# Polygon service file
POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
set -eu
@ -46,17 +58,13 @@ cd "${EXEC_DIR}"
echo
echo
echo -e "${PREFIX_INFO} Building executable server of moonstreamcrawlers with Go"
EXEC_DIR=$(pwd)
cd "${APP_CRAWLERS_DIR}/server"
HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go"
cd "${EXEC_DIR}"
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
"${PIP}" install --upgrade pip setuptools
echo
echo
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install --upgrade pip
"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
echo -e "${PREFIX_INFO} Installing Python dependencies"
"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/"
echo
echo
@ -67,12 +75,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex
echo
echo
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
@ -84,27 +92,52 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TRENDING_TIMER}"
systemctl restart "${ETHEREUM_TRENDING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
systemctl restart "${ETHEREUM_TXPOOL_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}"
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}"
systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${POLYGON_SYNCHRONIZE_SERVICE}"
systemctl daemon-reload
systemctl restart "${POLYGON_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon missing service and timer with: ${POLYGON_MISSING_SERVICE_FILE}, ${POLYGON_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${POLYGON_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon statistics dashbord service and timer with: ${POLYGON_STATISTICS_SERVICE_FILE}, ${POLYGON_STATISTICS_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_STATISTICS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/etc/systemd/system/${POLYGON_STATISTICS_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${POLYGON_STATISTICS_TIMER_FILE}"

11
crawlers/deploy/ethereum-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Ethereum database
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n

9
crawlers/deploy/ethereum-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Ethereum database
[Timer]
OnBootSec=10s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

2
crawlers/deploy/ethereum-synchronize.service
Wyświetl plik

@ -7,7 +7,7 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler blocks synchronize -c 6 -j 1
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 1
SyslogIdentifier=ethereum-synchronize
[Install]

2
crawlers/deploy/ethereum-trending.service
Wyświetl plik

@ -8,4 +8,4 @@ User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.ethcrawler trending
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler trending

14
crawlers/deploy/moonstreamcrawlers.service
Wyświetl plik

@ -1,14 +0,0 @@
[Unit]
Description=moonstreamcrawlers-service
After=network.target
[Service]
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/server
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream/crawlers/server/moonstreamcrawlers -host 0.0.0.0 -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
SyslogIdentifier=moonstreamcrawlers
[Install]
WantedBy=multi-user.target

89
crawlers/deploy/nodes-connections.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Update nodes connection address environment variables
# from AWS Route53 internal hosted zone
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
echo
echo "CLI to update nodes connection address environment
variables from AWS Route53 internal hosted zone"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -f File path where environment variables update at"
}
file_flag=""
verbose_flag="false"
while getopts 'f:v' flag; do
case "${flag}" in
f) file_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# File flag should be specified
if [ -z "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify file path"
usage
exit 1
fi
if [ ! -f "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Provided file does not exist"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Source environment variables"
. ${file_flag}
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving Polygon node address"
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
exit 1
fi
# TODO(kompotkot): Modify regexp to work with export prefix
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

89
crawlers/deploy/parameters.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and
# opt out as environment variable exports.
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"
echo
echo "CLI to collect secrets from AWS SSM Parameter Store
and output as environment variable exports"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -p Product tag (moonstream, spire, brood, drones)"
echo " -o Output file name environment variables export to"
}
# TODO(kompotkot): Flag for export prefix
product_flag=""
output_flag=""
verbose_flag="false"
while getopts 'p:o:v' flag; do
case "${flag}" in
p) product_flag="${OPTARG}" ;;
o) output_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# Product flag should be specified
# TODO(kompotkot): Extend script to work with few product at once
if [ -z "${product_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify product tag"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=${product_flag} \
| jq -r .Parameters[].Name)
if [ -z "${ENV_PARAMETERS}" ]; then
verbose "${PREFIX_CRIT} There no parameters for provided product tag"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving parameters values"
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
--names ${ENV_PARAMETERS} \
--query "Parameters[*].{Name:Name,Value:Value}")
ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
if [ -z "${output_flag}" ]; then
echo "${param_key}=${param_value}"
else
echo "${param_key}=${param_value}" >> "${output_flag}"
fi
done

11
crawlers/deploy/polygon-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Polygon database
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n

9
crawlers/deploy/polygon-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Polygon database
[Timer]
OnBootSec=10s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

11
crawlers/deploy/polygon-statistics.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Update Polygon statistics dashboard
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard generate --blockchain polygon

9
crawlers/deploy/polygon-statistics.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Update Polygon statistics dashboard each 5 minutes
[Timer]
OnBootSec=10s
OnUnitActiveSec=5m
[Install]
WantedBy=timers.target

14
crawlers/deploy/polygon-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,14 @@
[Unit]
Description=Polygon block with transactions synchronizer
After=network.target
[Service]
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon -c 60 -j 1
SyslogIdentifier=polygon-synchronize
[Install]
WantedBy=multi-user.target

3
crawlers/mooncrawl/.isort.cfg 100644
Wyświetl plik

@ -0,0 +1,3 @@
[settings]
profile = black
multi_line_output = 3

9
crawlers/mooncrawl/dev.sh 100755
Wyświetl plik

@ -0,0 +1,9 @@
#!/usr/bin/env sh
# Expects access to Python environment with the requirements for this project installed.
set -e
MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
MOONSTREAM_PORT="${MOONSTREAM_PORT:-7491}"
uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" mooncrawl.api:app --reload

16
crawlers/mooncrawl/mooncrawl/__init__.py
Wyświetl plik

@ -1,7 +1,11 @@
from .reporter import reporter
from .version import MOONCRAWL_VERSION
try:
from .reporter import reporter
from .version import MOONCRAWL_VERSION
# Reporting
reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
# Reporting
reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
reporter.system_report(publish=True)
reporter.setup_excepthook(publish=True)
except:
# Pass it to be able import MOONCRAWL_VERSION in setup.py with pip
pass

81
crawlers/mooncrawl/mooncrawl/api.py 100644
Wyświetl plik

@ -0,0 +1,81 @@
"""
The Mooncrawl HTTP API
"""
import logging
import time
from typing import Dict
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from . import data
from .middleware import MoonstreamHTTPException
from .settings import DOCS_TARGET_PATH, ORIGINS
from .version import MOONCRAWL_VERSION
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
tags_metadata = [
{"name": "jobs", "description": "Trigger crawler jobs."},
{"name": "time", "description": "Server timestamp endpoints."},
]
app = FastAPI(
title=f"Mooncrawl HTTP API",
description="Mooncrawl API endpoints.",
version=MOONCRAWL_VERSION,
openapi_tags=tags_metadata,
openapi_url="/openapi.json",
docs_url=None,
redoc_url=f"/{DOCS_TARGET_PATH}",
)
app.add_middleware(
CORSMiddleware,
allow_origins=ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/ping", response_model=data.PingResponse)
async def ping_handler() -> data.PingResponse:
"""
Check server status.
"""
return data.PingResponse(status="ok")
@app.get("/version", response_model=data.VersionResponse)
async def version_handler() -> data.VersionResponse:
"""
Get server version.
"""
return data.VersionResponse(version=MOONCRAWL_VERSION)
@app.get("/now", tags=["time"])
async def now_handler() -> data.NowResponse:
"""
Get server current time.
"""
return data.NowResponse(epoch_time=time.time())
@app.get("/jobs/stats_update", tags=["jobs"])
async def status_handler():
"""
Find latest crawlers records with creation timestamp:
- ethereum_txpool
- ethereum_trending
"""
try:
pass
except Exception as e:
logger.error(f"Unhandled status exception, error: {e}")
raise MoonstreamHTTPException(status_code=500)
return

213
crawlers/mooncrawl/mooncrawl/ethereum.py → crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -1,64 +1,138 @@
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from dataclasses import dataclass
from datetime import datetime
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import desc, Column
from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, Query
from tqdm import tqdm
from web3 import Web3, IPCProvider, HTTPProvider
from web3.types import BlockData
from .settings import MOONSTREAM_IPC_PATH, MOONSTREAM_CRAWL_WORKERS
from moonstreamdb.db import yield_db_session, yield_db_session_ctx
from moonstreamdb.models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
)
from psycopg2.errors import UniqueViolation # type: ignore
from sqlalchemy import Column, desc, func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import HTTPProvider, IPCProvider, Web3
from web3.middleware import geth_poa_middleware
from web3.types import BlockData
from .data import AvailableBlockchainType, DateRange
from .settings import (
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_IPC_PATH,
MOONSTREAM_POLYGON_IPC_PATH,
)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class EthereumBlockCrawlError(Exception):
class BlockCrawlError(Exception):
"""
Raised when there is a problem crawling Ethereum blocks.
Raised when there is a problem crawling blocks.
"""
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool
def connect(web3_uri: Optional[str] = MOONSTREAM_IPC_PATH):
def connect(blockchain_type: AvailableBlockchainType, web3_uri: Optional[str] = None):
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
if web3_uri is not None:
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
if web3_uri is None:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
web3_uri = MOONSTREAM_ETHEREUM_IPC_PATH
elif blockchain_type == AvailableBlockchainType.POLYGON:
web3_uri = MOONSTREAM_POLYGON_IPC_PATH
else:
web3_provider = Web3.IPCProvider(web3_uri)
raise Exception("Wrong blockchain type provided for web3 URI")
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
else:
web3_provider = Web3.IPCProvider(web3_uri)
web3_client = Web3(web3_provider)
# Inject --dev middleware if it is not Ethereum mainnet
# Docs: https://web3py.readthedocs.io/en/stable/middleware.html#geth-style-proof-of-authority
if blockchain_type != AvailableBlockchainType.ETHEREUM:
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
return web3_client
def add_block(db_session, block: Any) -> None:
def get_block_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumBlock, PolygonBlock]]:
"""
Depends on provided blockchain type: Ethereum or Polygon,
set proper blocks model: EthereumBlock or PolygonBlock.
"""
block_model: Type[Union[EthereumBlock, PolygonBlock]]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
block_model = EthereumBlock
elif blockchain_type == AvailableBlockchainType.POLYGON:
block_model = PolygonBlock
else:
raise Exception("Unsupported blockchain type provided")
return block_model
def get_label_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumLabel, PolygonLabel]]:
"""
Depends on provided blockchain type: Ethereum or Polygon,
set proper block label model: EthereumLabel or PolygonLabel.
"""
label_model: Type[Union[EthereumLabel, PolygonLabel]]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
label_model = EthereumLabel
elif blockchain_type == AvailableBlockchainType.POLYGON:
label_model = PolygonLabel
else:
raise Exception("Unsupported blockchain type provided")
return label_model
def get_transaction_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumTransaction, PolygonTransaction]]:
"""
Depends on provided blockchain type: Ethereum or Polygon,
set proper block transactions model: EthereumTransaction or PolygonTransaction.
"""
transaction_model: Type[Union[EthereumTransaction, PolygonTransaction]]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
transaction_model = EthereumTransaction
elif blockchain_type == AvailableBlockchainType.POLYGON:
transaction_model = PolygonTransaction
else:
raise Exception("Unsupported blockchain type provided")
return transaction_model
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
"""
Add block if doesn't presented in database.
block: web3.types.BlockData
"""
block_obj = EthereumBlock(
block_model = get_block_model(blockchain_type)
# BlockData.extraData doesn't exist at Polygon mainnet
extra_data = None
if block.get("extraData", None) is not None:
extra_data = block.get("extraData").hex()
block_obj = block_model(
block_number=block.number,
difficulty=block.difficulty,
extra_data=block.extraData.hex(),
extra_data=extra_data,
gas_limit=block.gasLimit,
gas_used=block.gasUsed,
base_fee_per_gas=block.get("baseFeePerGas", None),
@ -78,14 +152,17 @@ def add_block(db_session, block: Any) -> None:
db_session.add(block_obj)
def add_block_transactions(db_session, block: Any) -> None:
def add_block_transactions(
db_session, block: Any, blockchain_type: AvailableBlockchainType
) -> None:
"""
Add block transactions.
block: web3.types.BlockData
"""
transaction_model = get_transaction_model(blockchain_type)
for tx in block.transactions:
tx_obj = EthereumTransaction(
tx_obj = transaction_model(
hash=tx.hash.hex(),
block_number=block.number,
from_address=tx["from"],
@ -103,22 +180,25 @@ def add_block_transactions(db_session, block: Any) -> None:
db_session.add(tx_obj)
def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
def get_latest_blocks(
blockchain_type: AvailableBlockchainType, confirmations: int = 0
) -> Tuple[Optional[int], int]:
"""
Retrieve the latest block from the connected node (connection is created by the connect() method).
Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method).
If confirmations > 0, and the latest block on the node has block number N, this returns the block
with block_number (N - confirmations)
"""
web3_client = connect()
web3_client = connect(blockchain_type)
latest_block_number: int = web3_client.eth.block_number
if confirmations > 0:
latest_block_number -= confirmations
block_model = get_block_model(blockchain_type)
with yield_db_session_ctx() as db_session:
latest_stored_block_row = (
db_session.query(EthereumBlock.block_number)
.order_by(EthereumBlock.block_number.desc())
db_session.query(block_model.block_number)
.order_by(block_model.block_number.desc())
.first()
)
latest_stored_block_number = (
@ -128,11 +208,15 @@ def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
return latest_stored_block_number, latest_block_number
def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> None:
def crawl_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
with_transactions: bool = False,
) -> None:
"""
Open database and geth sessions and fetch block data from blockchain.
"""
web3_client = connect()
web3_client = connect(blockchain_type)
with yield_db_session_ctx() as db_session:
pbar = tqdm(total=len(blocks_numbers))
for block_number in blocks_numbers:
@ -143,10 +227,10 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
block: BlockData = web3_client.eth.get_block(
block_number, full_transactions=with_transactions
)
add_block(db_session, block)
add_block(db_session, block, blockchain_type)
if with_transactions:
add_block_transactions(db_session, block)
add_block_transactions(db_session, block, blockchain_type)
db_session.commit()
except IntegrityError as err:
@ -157,7 +241,7 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
except Exception as err:
db_session.rollback()
message = f"Error adding block (number={block_number}) to database:\n{repr(err)}"
raise EthereumBlockCrawlError(message)
raise BlockCrawlError(message)
except:
db_session.rollback()
logger.error(
@ -168,7 +252,11 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) ->
pbar.close()
def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> List[int]:
def check_missing_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
notransactions=False,
) -> List[int]:
"""
Query block from postgres. If block does not presented in database,
add to missing blocks numbers list.
@ -178,33 +266,35 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
bottom_block = min(blocks_numbers[-1], blocks_numbers[0])
top_block = max(blocks_numbers[-1], blocks_numbers[0])
block_model = get_block_model(blockchain_type)
transaction_model = get_transaction_model(blockchain_type)
with yield_db_session_ctx() as db_session:
if notransactions:
blocks_exist_raw_query = (
db_session.query(EthereumBlock.block_number)
.filter(EthereumBlock.block_number >= bottom_block)
.filter(EthereumBlock.block_number <= top_block)
db_session.query(block_model.block_number)
.filter(block_model.block_number >= bottom_block)
.filter(block_model.block_number <= top_block)
)
blocks_exist = [[block[0]] for block in blocks_exist_raw_query.all()]
else:
corrupted_blocks = []
blocks_exist_raw_query = (
db_session.query(
EthereumBlock.block_number, func.count(EthereumTransaction.hash)
block_model.block_number, func.count(transaction_model.hash)
)
.join(
EthereumTransaction,
EthereumTransaction.block_number == EthereumBlock.block_number,
transaction_model,
transaction_model.block_number == block_model.block_number,
)
.filter(EthereumBlock.block_number >= bottom_block)
.filter(EthereumBlock.block_number <= top_block)
.group_by(EthereumBlock.block_number)
.filter(block_model.block_number >= bottom_block)
.filter(block_model.block_number <= top_block)
.group_by(block_model.block_number)
)
blocks_exist = [
[block[0], block[1]] for block in blocks_exist_raw_query.all()
]
web3_client = connect()
web3_client = connect(blockchain_type)
blocks_exist_len = len(blocks_exist)
pbar = tqdm(total=blocks_exist_len)
@ -218,8 +308,8 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
corrupted_blocks.append(block_in_db[0])
# Delete existing corrupted block and add to missing list
del_block = (
db_session.query(EthereumBlock)
.filter(EthereumBlock.block_number == block_in_db[0])
db_session.query(block_model)
.filter(block_model.block_number == block_in_db[0])
.one()
)
db_session.delete(del_block)
@ -242,6 +332,7 @@ def check_missing_blocks(blocks_numbers: List[int], notransactions=False) -> Lis
def crawl_blocks_executor(
blockchain_type: AvailableBlockchainType,
block_numbers_list: List[int],
with_transactions: bool = False,
num_processes: int = MOONSTREAM_CRAWL_WORKERS,
@ -272,13 +363,15 @@ def crawl_blocks_executor(
results: List[Future] = []
if num_processes == 1:
logger.warning("Executing block crawler in lazy mod")
return crawl_blocks(block_numbers_list, with_transactions)
return crawl_blocks(blockchain_type, block_numbers_list, with_transactions)
else:
with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor:
for worker in worker_indices:
block_chunk = worker_job_lists[worker]
logger.info(f"Spawned process for {len(block_chunk)} blocks")
result = executor.submit(crawl_blocks, block_chunk, with_transactions)
result = executor.submit(
crawl_blocks, blockchain_type, block_chunk, with_transactions
)
result.add_done_callback(record_error)
results.append(result)
@ -286,7 +379,7 @@ def crawl_blocks_executor(
if len(errors) > 0:
error_messages = "\n".join([f"- {error}" for error in errors])
message = f"Error processing blocks in list:\n{error_messages}"
raise EthereumBlockCrawlError(message)
raise BlockCrawlError(message)
def trending(

5
crawlers/mooncrawl/mooncrawl/contract/cli.py
Wyświetl plik

@ -8,7 +8,8 @@ from moonstreamdb.db import yield_db_session_ctx
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from ..blockchain import connect
from ..data import AvailableBlockchainType
from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
logging.basicConfig(level=logging.INFO)
@ -116,7 +117,7 @@ def run_crawler_desc(
def handle_parser(args: argparse.Namespace):
with yield_db_session_ctx() as session:
w3 = connect()
w3 = connect(AvailableBlockchainType.ETHEREUM)
if args.order == "asc":
run_crawler_asc(
w3=w3,

137
crawlers/mooncrawl/mooncrawl/ethcrawler.py → crawlers/mooncrawl/mooncrawl/crawler.py
Wyświetl plik

@ -2,24 +2,25 @@
Moonstream crawlers CLI.
"""
import argparse
from datetime import datetime, timedelta, timezone
from enum import Enum
import json
import logging
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Iterator, List
import dateutil.parser
from .ethereum import (
crawl_blocks_executor,
check_missing_blocks,
get_latest_blocks,
from .blockchain import (
DateRange,
check_missing_blocks,
crawl_blocks_executor,
get_latest_blocks,
trending,
)
from .data import AvailableBlockchainType
from .publish import publish_json
from .settings import MOONSTREAM_CRAWL_WORKERS
from .version import MOONCRAWL_VERSION
@ -86,13 +87,13 @@ def yield_blocks_numbers_lists(
current_block -= block_step
def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
"""
Synchronize latest Ethereum blocks with database.
Synchronize latest Blockchain blocks with database.
"""
while True:
latest_stored_block_number, latest_block_number = get_latest_blocks(
args.confirmations
AvailableBlockchainType(args.blockchain), args.confirmations
)
if latest_stored_block_number is None:
latest_stored_block_number = 0
@ -132,6 +133,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
# TODO(kompotkot): Set num_processes argument based on number of blocks to synchronize.
crawl_blocks_executor(
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=blocks_numbers_list,
with_transactions=True,
num_processes=args.jobs,
@ -141,7 +143,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
"""
Add blocks to moonstream database.
"""
@ -150,7 +152,9 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
logger.info(f"Adding blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]}")
crawl_blocks_executor(
block_numbers_list=blocks_numbers_list, with_transactions=True
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=blocks_numbers_list,
with_transactions=True,
)
logger.info(
@ -158,19 +162,32 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
"""
Check missing blocks and missing transactions in each block.
If block range doesn't provided, get latest block from blockchain minus 50,
and check last 2000 blocks behind.
"""
startTime = time.time()
missing_blocks_numbers_total = []
for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
block_range = args.blocks
if block_range is None:
confirmations = 150
shift = 2000
_, latest_block_number = get_latest_blocks(
AvailableBlockchainType(args.blockchain), confirmations
)
block_range = f"{latest_block_number-shift}-{latest_block_number}"
for blocks_numbers_list in yield_blocks_numbers_lists(block_range):
logger.info(
f"Checking missing blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]} "
f"with comparing transactions: {not args.notransactions}"
)
missing_blocks_numbers = check_missing_blocks(
blockchain_type=AvailableBlockchainType(args.blockchain),
blocks_numbers=blocks_numbers_list,
notransactions=args.notransactions,
)
@ -185,7 +202,8 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
if (len(missing_blocks_numbers_total)) > 0:
time.sleep(5)
crawl_blocks_executor(
missing_blocks_numbers_total,
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=missing_blocks_numbers_total,
with_transactions=True,
num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS,
)
@ -195,7 +213,7 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
)
def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
def crawler_trending_handler(args: argparse.Namespace) -> None:
date_range = DateRange(
start_time=args.start,
end_time=args.end,
@ -228,15 +246,15 @@ def main() -> None:
time_now = datetime.now(timezone.utc)
# Ethereum blocks parser
parser_ethcrawler_blocks = subcommands.add_parser(
"blocks", description="Ethereum blocks commands"
# Blockchain blocks parser
parser_crawler_blocks = subcommands.add_parser(
"blocks", description="Blockchain blocks commands"
)
parser_ethcrawler_blocks.set_defaults(
func=lambda _: parser_ethcrawler_blocks.print_help()
parser_crawler_blocks.set_defaults(
func=lambda _: parser_crawler_blocks.print_help()
)
subcommands_ethcrawler_blocks = parser_ethcrawler_blocks.add_subparsers(
description="Ethereum blocks commands"
subcommands_crawler_blocks = parser_crawler_blocks.add_subparsers(
description="Blockchain blocks commands"
)
valid_processing_orders = {
@ -251,29 +269,29 @@ def main() -> None:
f"Invalid processing order ({raw_order}). Valid choices: {valid_processing_orders.keys()}"
)
parser_ethcrawler_blocks_sync = subcommands_ethcrawler_blocks.add_parser(
"synchronize", description="Synchronize to latest ethereum block commands"
parser_crawler_blocks_sync = subcommands_crawler_blocks.add_parser(
"synchronize", description="Synchronize to latest blockchain block commands"
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-s",
"--start",
type=int,
help="(Optional) Block to start synchronization from. Default: None - current Ethereum block minus confirmations ",
help="(Optional) Block to start synchronization from. Default: None - current Blockchain block minus confirmations ",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-c",
"--confirmations",
type=int,
default=0,
help="Number of confirmations we require before storing a block in the database. (Default: 0)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"--order",
type=processing_order,
default=ProcessingOrder.ASCENDING,
help="Order in which to process blocks (choices: desc, asc; default: asc)",
)
parser_ethcrawler_blocks_sync.add_argument(
parser_crawler_blocks_sync.add_argument(
"-j",
"--jobs",
type=int,
@ -283,72 +301,85 @@ def main() -> None:
" If you set to 1, the main process handles synchronization without spawning subprocesses."
),
)
parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler)
parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser(
"add", description="Add ethereum blocks commands"
parser_crawler_blocks_sync.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_ethcrawler_blocks_add.add_argument(
parser_crawler_blocks_sync.set_defaults(func=crawler_blocks_sync_handler)
parser_crawler_blocks_add = subcommands_crawler_blocks.add_parser(
"add", description="Add blockchain blocks commands"
)
parser_crawler_blocks_add.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler)
parser_ethcrawler_blocks_missing = subcommands_ethcrawler_blocks.add_parser(
"missing", description="Add missing ethereum blocks with transactions commands"
parser_crawler_blocks_add.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_add.set_defaults(func=crawler_blocks_add_handler)
parser_crawler_blocks_missing = subcommands_crawler_blocks.add_parser(
"missing",
description="Add missing Blockchain blocks with transactions commands",
)
parser_crawler_blocks_missing.add_argument(
"-b",
"--blocks",
required=True,
help="List of blocks range in format {bottom_block}-{top_block}",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-n",
"--notransactions",
action="store_true",
help="Skip crawling block transactions",
)
parser_ethcrawler_blocks_missing.add_argument(
parser_crawler_blocks_missing.add_argument(
"-l",
"--lazy",
action="store_true",
help="Lazy block adding one by one",
)
parser_ethcrawler_blocks_missing.set_defaults(
func=ethcrawler_blocks_missing_handler
parser_crawler_blocks_missing.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_crawler_blocks_missing.set_defaults(func=crawler_blocks_missing_handler)
parser_ethcrawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Ethereum blockchain"
parser_crawler_trending = subcommands.add_parser(
"trending", description="Trending addresses on the Blockchain blockchain"
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-s",
"--start",
type=dateutil.parser.parse,
default=(time_now - timedelta(hours=1, minutes=0)).isoformat(),
help=f"Start time for window to calculate trending addresses in (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-start",
action="store_true",
help="Set this flag if range should include start time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-e",
"--end",
type=dateutil.parser.parse,
default=time_now.isoformat(),
help=f"End time for window to calculate trending addresses in (default: {time_now.isoformat()})",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--include-end",
action="store_true",
help="Set this flag if range should include end time",
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"--humbug",
default=None,
help=(
@ -357,14 +388,14 @@ def main() -> None:
"MOONSTREAM_HUMBUG_TOKEN environment variable)"
),
)
parser_ethcrawler_trending.add_argument(
parser_crawler_trending.add_argument(
"-o",
"--outfile",
type=argparse.FileType("w"),
default=sys.stdout,
help="Optional file to write output to. By default, prints to stdout.",
)
parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler)
parser_crawler_trending.set_defaults(func=crawler_trending_handler)
args = parser.parse_args()
args.func(args)

42
crawlers/mooncrawl/mooncrawl/data.py 100644
Wyświetl plik

@ -0,0 +1,42 @@
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from pydantic import BaseModel
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
POLYGON = "polygon"
@dataclass
class DateRange:
start_time: datetime
end_time: datetime
include_start: bool
include_end: bool
class PingResponse(BaseModel):
"""
Schema for ping response
"""
status: str
class VersionResponse(BaseModel):
"""
Schema for responses on /version endpoint
"""
version: str
class NowResponse(BaseModel):
"""
Schema for responses on /now endpoint
"""
epoch_time: float

2
crawlers/mooncrawl/mooncrawl/esd.py
Wyświetl plik

@ -3,10 +3,10 @@ import sys
import time
from typing import Optional, Union
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import ESDEventSignature, ESDFunctionSignature
from sqlalchemy.orm import Session
import requests
CRAWL_URLS = {
"functions": "https://www.4byte.directory/api/v1/signatures/",

10
crawlers/mooncrawl/mooncrawl/etherscan.py
Wyświetl plik

@ -1,23 +1,23 @@
import argparse
import codecs
import csv
from dataclasses import dataclass
from datetime import datetime
import json
import logging
import os
import sys
import time
from typing import Any, List, Optional, Dict
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional
import boto3 # type: ignore
import requests
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
import requests
from sqlalchemy.orm import Session
from .version import MOONCRAWL_VERSION
from .settings import MOONSTREAM_ETHERSCAN_TOKEN
from .version import MOONCRAWL_VERSION
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

3
crawlers/mooncrawl/mooncrawl/identity.py
Wyświetl plik

@ -4,10 +4,9 @@ import os
import time
import requests
from sqlalchemy import text
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumLabel
from sqlalchemy import text
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

26
crawlers/mooncrawl/mooncrawl/middleware.py 100644
Wyświetl plik

@ -0,0 +1,26 @@
import logging
from typing import Any, Dict, Optional
from fastapi import HTTPException
from .reporter import reporter
logger = logging.getLogger(__name__)
class MoonstreamHTTPException(HTTPException):
"""
Extended HTTPException to handle 500 Internal server errors
and send crash reports.
"""
def __init__(
self,
status_code: int,
detail: Any = None,
headers: Optional[Dict[str, Any]] = None,
internal_error: Exception = None,
):
super().__init__(status_code, detail, headers)
if internal_error is not None:
reporter.error_report(internal_error)

41
crawlers/mooncrawl/mooncrawl/nft/cli.py
Wyświetl plik

@ -2,12 +2,12 @@
A command line tool to crawl information about NFTs from various sources.
"""
import argparse
from datetime import datetime, timezone
import json
import logging
import sys
import time
from typing import Any, cast, Dict, Optional
from datetime import datetime, timezone
from typing import Any, Dict, Optional, cast
from bugout.app import Bugout
from bugout.journal import SearchOrder
@ -16,26 +16,25 @@ from moonstreamdb.models import EthereumBlock
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from ..blockchain import connect
from ..data import AvailableBlockchainType
from ..publish import publish_json
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
MOONSTREAM_ETHEREUM_IPC_PATH,
NFT_HUMBUG_TOKEN,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import (
summary as ethereum_summary,
add_labels,
MINT_LABEL,
TRANSFER_LABEL,
SUMMARY_KEY_ARGS,
SUMMARY_KEY_END_BLOCK,
SUMMARY_KEY_ID,
SUMMARY_KEY_NUM_BLOCKS,
SUMMARY_KEY_START_BLOCK,
SUMMARY_KEY_END_BLOCK,
add_labels,
)
from ..publish import publish_json
from ..settings import (
MOONSTREAM_IPC_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
NFT_HUMBUG_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
)
from ..version import MOONCRAWL_VERSION
from .ethereum import summary as ethereum_summary
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -46,17 +45,17 @@ BLOCKS_PER_SUMMARY = 40
def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
"""
Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up
the MOONSTREAM_IPC_PATH environment variable.
the MOONSTREAM_ETHEREUM_IPC_PATH environment variable.
"""
web3_connection_string = MOONSTREAM_IPC_PATH
web3_connection_string = MOONSTREAM_ETHEREUM_IPC_PATH
args_web3 = vars(args).get("web3")
if args_web3 is not None:
web3_connection_string = cast(str, args_web3)
if web3_connection_string is None:
raise ValueError(
"Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable"
"Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_IPC_PATH environment variable"
)
return connect(web3_connection_string)
return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string)
def get_latest_block_from_node(web3_client: Web3):
@ -295,7 +294,7 @@ def main() -> None:
"--web3",
type=str,
default=None,
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.",
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_ETHEREUM_IPC_PATH environment variable.",
)
parser_ethereum_label.set_defaults(func=ethereum_label_handler)

19
crawlers/mooncrawl/mooncrawl/nft/ethereum.py
Wyświetl plik

@ -1,23 +1,18 @@
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta, timezone
import json
import logging
from hexbytes.main import HexBytes
from typing import Any, cast, Dict, List, Optional, Set, Tuple
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set, Tuple, cast
from eth_typing.encoding import HexStr
from moonstreamdb.models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
)
from hexbytes.main import HexBytes
from moonstreamdb.models import EthereumBlock, EthereumLabel, EthereumTransaction
from sqlalchemy import and_, func, text
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Session, Query
from sqlalchemy.orm import Query, Session
from tqdm import tqdm
from web3 import Web3
from web3.types import FilterParams, LogReceipt
from web3._utils.events import get_event_data
from web3.types import FilterParams, LogReceipt
from ..reporter import reporter

2
crawlers/mooncrawl/mooncrawl/publish.py
Wyświetl plik

@ -1,6 +1,6 @@
from datetime import datetime
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
import requests

41
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -1,9 +1,30 @@
import os
from typing import cast
from bugout.app import Bugout
# Bugout
BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
# Origin
RAW_ORIGINS = os.environ.get("MOONSTREAM_CORS_ALLOWED_ORIGINS")
if RAW_ORIGINS is None:
raise ValueError(
"MOONSTREAM_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)"
)
ORIGINS = RAW_ORIGINS.split(",")
# OpenAPI
DOCS_TARGET_PATH = "docs"
# Crawler label
CRAWLER_LABEL = "moonworm"
# Geth connection address
MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1"
@ -11,9 +32,18 @@ MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545
)
MOONSTREAM_IPC_PATH = (
MOONSTREAM_ETHEREUM_IPC_PATH = (
f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}"
)
MOONSTREAM_NODE_POLYGON_IPC_ADDR = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_ADDR", "127.0.0.1"
)
MOONSTREAM_NODE_POLYGON_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_POLYGON_IPC_PORT", 8545
)
MOONSTREAM_POLYGON_IPC_PATH = (
f"http://{MOONSTREAM_NODE_POLYGON_IPC_ADDR}:{MOONSTREAM_NODE_POLYGON_IPC_PORT}"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
@ -40,3 +70,12 @@ if MOONSTREAM_ADMIN_ACCESS_TOKEN == "":
MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "")
if MOONSTREAM_DATA_JOURNAL_ID == "":
raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set")
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = os.environ.get(
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX"
)
if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None:
raise ValueError(
"MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set"
)

0
crawlers/mooncrawl/mooncrawl/stats_worker/__init__.py 100644
Wyświetl plik

493
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py 100644
Wyświetl plik

@ -0,0 +1,493 @@
"""
Generates dashboard.
"""
import argparse
import hashlib
import json
import logging
import time
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Dict, List
import boto3 # type: ignore
from bugout.data import BugoutResources
from moonstreamdb.db import yield_db_session_ctx
from sqlalchemy import Column, Date, and_, func, text
from sqlalchemy.orm import Query, Session
from sqlalchemy.sql.operators import in_op
from ..blockchain import get_block_model, get_label_model, get_transaction_model
from ..data import AvailableBlockchainType
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
CRAWLER_LABEL,
)
from ..settings import bugout_client as bc
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
subscription_id_by_blockchain = {
"ethereum": "ethereum_blockchain",
"polygon": "polygon_blockchain",
}
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
}
class TimeScale(Enum):
# year = "year"
month = "month"
week = "week"
day = "day"
timescales_params: Dict[str, Dict[str, str]] = {
"year": {"timestep": "1 day", "timeformat": "YYYY-MM-DD"},
"month": {"timestep": "1 hours", "timeformat": "YYYY-MM-DD HH24"},
"week": {"timestep": "1 hours", "timeformat": "YYYY-MM-DD HH24"},
"day": {"timestep": "1 hours", "timeformat": "YYYY-MM-DD HH24"},
}
timescales_delta: Dict[str, Dict[str, timedelta]] = {
"year": {"timedelta": timedelta(days=365)},
"month": {"timedelta": timedelta(days=27)},
"week": {"timedelta": timedelta(days=6)},
"day": {"timedelta": timedelta(hours=24)},
}
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION = "subscription"
BUGOUT_RESOURCE_TYPE_DASHBOARD = "dashboards"
def push_statistics(
statistics_data: Dict[str, Any],
subscription: Any,
timescale: str,
bucket: str,
hash: str,
) -> None:
result_bytes = json.dumps(statistics_data).encode("utf-8")
result_key = f'{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{blockchain_by_subscription_id[subscription.resource_data["subscription_type_id"]]}/contracts_data/{subscription.resource_data["address"]}/{hash}/v1/{timescale}.json'
s3 = boto3.client("s3")
s3.put_object(
Body=result_bytes,
Bucket=bucket,
Key=result_key,
ContentType="application/json",
Metadata={"drone": "statistics"},
)
print(f"Statistics push to bucket: s3://{bucket}/{result_key}")
def generate_metrics(
db_session: Session,
blockchain_type: AvailableBlockchainType,
address: str,
timescale: str,
metrics: List[str],
start: Any,
):
"""
Generage metrics
"""
block_model = get_block_model(blockchain_type)
transaction_model = get_transaction_model(blockchain_type)
start = start
end = datetime.utcnow()
start_timestamp = int(start.timestamp())
end_timestamp = int(end.timestamp())
results: Dict[str, Any] = {}
time_step = timescales_params[timescale]["timestep"]
time_format = timescales_params[timescale]["timeformat"]
def make_query(
db_session: Session,
identifying_column: Column,
statistic_column: Column,
) -> Query:
unformated_time_series_subquery = db_session.query(
func.generate_series(
start,
end,
time_step,
).label("timeseries_points")
).subquery(name="unformated_time_series_subquery")
time_series_formated = db_session.query(
func.to_char(
unformated_time_series_subquery.c.timeseries_points, time_format
).label("timeseries_points")
)
time_series_formated_subquery = time_series_formated.subquery(
name="time_series_subquery"
)
metric_count_subquery = (
db_session.query(
func.count(statistic_column).label("count"),
func.to_char(
func.to_timestamp(block_model.timestamp).cast(Date), time_format
).label("timeseries_points"),
)
.join(
block_model,
transaction_model.block_number == block_model.block_number,
)
.filter(identifying_column == address)
.filter(block_model.timestamp >= start_timestamp)
.filter(block_model.timestamp <= end_timestamp)
.group_by(text("timeseries_points"))
).subquery(name="metric_counts")
metrics_time_series = (
db_session.query(
time_series_formated_subquery.c.timeseries_points.label(
"timeseries_points"
),
func.coalesce(metric_count_subquery.c.count.label("count"), 0),
)
.join(
metric_count_subquery,
time_series_formated_subquery.c.timeseries_points
== metric_count_subquery.c.timeseries_points,
isouter=True,
)
.order_by(text("timeseries_points DESC"))
)
response_metric: List[Any] = []
for created_date, count in metrics_time_series:
response_metric.append({"date": created_date, "count": count})
return response_metric
try:
start_time = time.time()
results["transactions_out"] = make_query(
db_session,
transaction_model.from_address,
transaction_model.hash,
)
print("--- transactions_out %s seconds ---" % (time.time() - start_time))
start_time = time.time()
results["transactions_in"] = make_query(
db_session,
transaction_model.to_address,
transaction_model.hash,
)
print("--- transactions_in %s seconds ---" % (time.time() - start_time))
start_time = time.time()
results["value_out"] = make_query(
db_session,
transaction_model.from_address,
transaction_model.value,
)
print("--- value_out %s seconds ---" % (time.time() - start_time))
start_time = time.time()
results["value_in"] = make_query(
db_session,
transaction_model.to_address,
transaction_model.value,
)
print("--- value_in %s seconds ---" % (time.time() - start_time))
except Exception as err:
print(err)
pass
return results
def generate_data(
db_session: Session,
blockchain_type: AvailableBlockchainType,
address: str,
timescale: str,
functions: List[str],
start: Any,
metric_type: str,
):
label_model = get_label_model(blockchain_type)
# create empty time series
time_step = timescales_params[timescale]["timestep"]
time_format = timescales_params[timescale]["timeformat"]
# if end is None:
end = datetime.utcnow()
time_series_subquery = db_session.query(
func.generate_series(
start,
end,
time_step,
).label("timeseries_points")
)
time_series_subquery = time_series_subquery.subquery(name="time_series_subquery")
# get distinct tags labels in that range
label_requested = (
db_session.query(label_model.label_data["name"].astext.label("label"))
.filter(label_model.address == address)
.filter(label_model.label == CRAWLER_LABEL)
.filter(
and_(
label_model.label_data["type"].astext == metric_type,
in_op(label_model.label_data["name"].astext, functions),
)
)
.distinct()
)
if start is not None:
label_requested = label_requested.filter(
func.to_timestamp(label_model.block_timestamp).cast(Date) > start
)
if end is not None:
label_requested = label_requested.filter(
func.to_timestamp(label_model.block_timestamp).cast(Date) < end
)
label_requested = label_requested.subquery(name="label_requested")
# empty timeseries with tags
empty_time_series_subquery = db_session.query(
func.to_char(time_series_subquery.c.timeseries_points, time_format).label(
"timeseries_points"
),
label_requested.c.label.label("label"),
)
empty_time_series_subquery = empty_time_series_subquery.subquery(
name="empty_time_series_subquery"
)
# tags count
label_counts = (
db_session.query(
func.to_char(
func.to_timestamp(label_model.block_timestamp).cast(Date), time_format
).label("timeseries_points"),
func.count(label_model.id).label("count"),
label_model.label_data["name"].astext.label("label"),
)
.filter(label_model.address == address)
.filter(label_model.label == CRAWLER_LABEL)
.filter(
and_(
label_model.label_data["type"].astext == metric_type,
in_op(label_model.label_data["name"].astext, functions),
)
)
)
if start is not None:
label_counts = label_counts.filter(
func.to_timestamp(label_model.block_timestamp).cast(Date) > start
)
if end is not None:
label_counts = label_counts.filter(
func.to_timestamp(label_model.block_timestamp).cast(Date) < end
)
label_counts_subquery = (
label_counts.group_by(
text("timeseries_points"),
label_model.label_data["name"].astext,
)
.order_by(text("timeseries_points desc"))
.subquery(name="label_counts")
)
# Join empty tags_time_series with tags count eg apply tags counts to time series.
labels_time_series = (
db_session.query(
empty_time_series_subquery.c.timeseries_points.label("timeseries_points"),
empty_time_series_subquery.c.label.label("label"),
func.coalesce(label_counts_subquery.c.count.label("count"), 0),
)
.join(
label_counts_subquery,
and_(
empty_time_series_subquery.c.label == label_counts_subquery.c.label,
empty_time_series_subquery.c.timeseries_points
== label_counts_subquery.c.timeseries_points,
),
isouter=True,
)
.order_by(text("timeseries_points DESC"))
)
response_labels: Dict[Any, Any] = {}
for created_date, label, count in labels_time_series:
if not response_labels.get(label):
response_labels[label] = []
response_labels[label].append({"date": created_date, "count": count})
return response_labels
def stats_generate_handler(args: argparse.Namespace):
"""
Start crawler with generate.
"""
blockchain_type = AvailableBlockchainType(args.blockchain)
with yield_db_session_ctx() as db_session:
# read all subscriptions
required_subscriptions: BugoutResources = bc.list_resources(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
params={
"type": BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,
"abi": "true",
"subscription_type_id": subscription_id_by_blockchain[args.blockchain],
},
timeout=10,
)
print(f"Subscriptions for processing: {len(required_subscriptions.resources)}")
s3_client = boto3.client("s3")
# Already processed
already_processed = []
for subscription in required_subscriptions.resources:
bucket = subscription.resource_data["bucket"]
key = subscription.resource_data["s3_path"]
address = subscription.resource_data["address"]
print(f"Expected bucket: s3://{bucket}/{key}")
abi = s3_client.get_object(
Bucket=bucket,
Key=key,
)
abi_json = json.loads(abi["Body"].read())
abi_string = json.dumps(abi_json, sort_keys=True, indent=2)
hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest()
if f"{address}/{hash}" in already_processed:
continue
s3_data_object = {}
abi_functions = [item for item in abi_json if item["type"] == "function"]
abi_events = [item for item in abi_json if item["type"] == "event"]
for timescale in [timescale.value for timescale in TimeScale]:
start_date = (
datetime.utcnow() - timescales_delta[timescale]["timedelta"]
)
print(f"Timescale: {timescale}")
abi_functions_names = [item["name"] for item in abi_functions]
functions_calls_data = generate_data(
db_session=db_session,
blockchain_type=blockchain_type,
address=address,
timescale=timescale,
functions=abi_functions_names,
start=start_date,
metric_type="tx_call",
)
s3_data_object["functions"] = functions_calls_data
# generate data
abi_events_names = [item["name"] for item in abi_events]
events_data = generate_data(
db_session=db_session,
blockchain_type=blockchain_type,
address=address,
timescale=timescale,
functions=abi_events_names,
start=start_date,
metric_type="event",
)
s3_data_object["events"] = events_data
s3_data_object["generic"] = generate_metrics(
db_session=db_session,
blockchain_type=blockchain_type,
address=address,
timescale=timescale,
metrics=abi_events_names,
start=start_date,
)
push_statistics(
statistics_data=s3_data_object,
subscription=subscription,
timescale=timescale,
bucket=bucket,
hash=hash,
)
already_processed.append(f"{address}/{hash}")
def main() -> None:
parser = argparse.ArgumentParser(description="Command Line Interface")
parser.set_defaults(func=lambda _: parser.print_help())
subcommands = parser.add_subparsers(
description="Drone dashboard statistics commands"
)
# Statistics parser
parser_generate = subcommands.add_parser(
"generate", description="Generate statistics"
)
parser_generate.set_defaults(func=lambda _: parser_generate.print_help())
parser_generate.add_argument(
"--blockchain",
required=True,
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
)
parser_generate.set_defaults(func=stats_generate_handler)
args = parser.parse_args()
args.func(args)
if __name__ == "__main__":
main()

42
crawlers/mooncrawl/mooncrawl/test_cli.py
Wyświetl plik

@ -1,13 +1,13 @@
import unittest
from . import ethcrawler
from . import crawler
class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=4
)
]
@ -16,7 +16,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", block_step=3
)
]
@ -25,8 +25,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -34,8 +34,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_10_6_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -43,7 +43,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=4
)
]
@ -52,7 +52,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", block_step=3
)
]
@ -61,8 +61,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 3
)
]
self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@ -70,8 +70,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_descending_6_10_descending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.DESCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.DESCENDING, 10
)
]
self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@ -79,8 +79,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_3(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 3
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 3
)
]
self.assertListEqual(partition, [[6, 7, 8], [9, 10]])
@ -88,8 +88,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_10_6_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"10-6", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"10-6", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])
@ -97,8 +97,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_4(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 4
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 4
)
]
self.assertListEqual(partition, [[6, 7, 8, 9], [10]])
@ -106,8 +106,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
def test_yield_ascending_6_10_ascending_step_10(self):
partition = [
block_numbers_list
for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
"6-10", ethcrawler.ProcessingOrder.ASCENDING, 10
for block_numbers_list in crawler.yield_blocks_numbers_lists(
"6-10", crawler.ProcessingOrder.ASCENDING, 10
)
]
self.assertListEqual(partition, [[6, 7, 8, 9, 10]])

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.0.5"
MOONCRAWL_VERSION = "0.1.1"

59
crawlers/mooncrawl/requirements.txt
Wyświetl plik

@ -1,59 +0,0 @@
aiohttp==3.7.4.post0
async-timeout==3.0.1
attrs==21.2.0
base58==2.1.0
bitarray==1.2.2
black==21.8b0
boto3==1.18.40
botocore==1.21.40
bugout==0.1.17
certifi==2021.5.30
chardet==4.0.0
charset-normalizer==2.0.4
click==8.0.1
cytoolz==0.11.0
-e git+https://git@github.com/bugout-dev/moonstream.git@67fe019f1086c435dd3b58f1ade2778acc2167c7#egg=moonstreamdb&subdirectory=db
eth-abi==2.1.1
eth-account==0.5.5
eth-hash==0.3.2
eth-keyfile==0.5.1
eth-keys==0.3.3
eth-rlp==0.2.1
eth-typing==2.2.2
eth-utils==1.10.0
hexbytes==0.2.2
humbug==0.2.7
idna==3.2
ipfshttpclient==0.8.0a2
jmespath==0.10.0
jsonschema==3.2.0
lru-dict==1.1.7
multiaddr==0.0.9
multidict==5.1.0
mypy==0.910
mypy-extensions==0.4.3
netaddr==0.8.0
parsimonious==0.8.1
pathspec==0.9.0
platformdirs==2.3.0
protobuf==3.17.3
pycryptodome==3.10.1
pyrsistent==0.18.0
python-dateutil==2.8.2
regex==2021.8.28
requests==2.26.0
rlp==2.0.1
s3transfer==0.5.0
six==1.16.0
toml==0.10.2
tomli==1.2.1
toolz==0.11.1
tqdm==4.62.2
types-python-dateutil==2.8.0
types-requests==2.25.6
typing-extensions==3.10.0.2
urllib3==1.26.6
varint==1.0.2
web3==5.23.1
websockets==9.1
yarl==1.6.3

7
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -1,10 +1,15 @@
# Path to IPC socket to use for web3 connections
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1"
export MOONSTREAM_NODE_POLYGON_IPC_PORT="8545"
export MOONSTREAM_CRAWL_WORKERS=4
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (prod,dev,..)>"
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

25
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -1,5 +1,6 @@
from setuptools import find_packages, setup
from mooncrawl.version import MOONCRAWL_VERSION
long_description = ""
with open("README.md") as ifp:
@ -7,7 +8,7 @@ with open("README.md") as ifp:
setup(
name="mooncrawl",
version="0.0.5",
version=MOONCRAWL_VERSION,
author="Bugout.dev",
author_email="engineers@bugout.dev",
license="Apache License 2.0",
@ -33,25 +34,31 @@ setup(
zip_safe=False,
install_requires=[
"boto3",
"bugout >= 0.1.17",
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@0a771ddfbca1254be331149ccf2d162aa09b7bc0#egg=moonstreamdb&subdirectory=db",
"bugout",
"chardet",
"fastapi",
"moonstreamdb",
"humbug",
"pydantic",
"python-dateutil",
"requests",
"tqdm",
"uvicorn",
"web3",
],
extras_require={
"dev": ["black", "mypy", "types-requests", "types-python-dateutil"]
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
"distribute": ["setuptools", "twine", "wheel"],
},
entry_points={
"console_scripts": [
"ethcrawler=mooncrawl.ethcrawler:main",
"esd=mooncrawl.esd:main",
"identity=mooncrawl.identity:main",
"etherscan=mooncrawl.etherscan:main",
"nft=mooncrawl.nft.cli:main",
"crawler=mooncrawl.crawler:main",
"contractcrawler=mooncrawl.contract.cli:main",
"esd=mooncrawl.esd:main",
"etherscan=mooncrawl.etherscan:main",
"identity=mooncrawl.identity:main",
"nft=mooncrawl.nft.cli:main",
"statistics=mooncrawl.stats_worker.dashboard:main",
]
},
)

14
crawlers/server/configs/settings.go
Wyświetl plik

@ -1,14 +0,0 @@
package settings
import (
"fmt"
"os"
)
// Geth configs
var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR")
var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT")
var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT)
// CORS
var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")

3
crawlers/server/go.mod
Wyświetl plik

@ -1,3 +0,0 @@
module github.com/bugout-dev/moonstream/crawlers/server
go 1.17

2
db/setup.py
Wyświetl plik

@ -34,7 +34,7 @@ setup(
zip_safe=False,
install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
extras_require={
"dev": ["black", "mypy"],
"dev": ["black", "isort", "mypy"],
"distribute": ["setuptools", "twine", "wheel"],
},
entry_points={

7
frontend/pages/index.js
Wyświetl plik

@ -19,6 +19,7 @@ import {
Text,
GridItem,
SimpleGrid,
Button,
Image as ChakraImage,
} from "@chakra-ui/react";
import dynamic from "next/dynamic";
@ -777,18 +778,18 @@ const Homepage = () => {
Join our Discord
</RouteButton>
</Flex>
<RouteButton
<Button
placeSelf="center"
isExternal
w={["100%", "100%", "fit-content", null]}
maxW={["250px", null, "fit-content"]}
href={`https://github.com/bugout-dev/moonstream`}
onClick={() => toggleModal("register")}
size="lg"
variant="solid"
colorScheme="orange"
>
Sign up
</RouteButton>
</Button>
</Stack>
</GridItem>
</Grid>

3
frontend/src/components/NewDashboard.js
Wyświetl plik

@ -80,7 +80,8 @@ const NewDashboard = (props) => {
]);
const filterFn = (item, inputValue) =>
item.subscription_type_id === "ethereum_blockchain" &&
(item.subscription_type_id === "ethereum_blockchain" ||
item.subscription_type_id === "polygon_blockchain") &&
(!inputValue ||
item.address.toUpperCase().includes(inputValue.toUpperCase()) ||
item.label.toUpperCase().includes(inputValue.toUpperCase()));

29
frontend/src/components/Report.js
Wyświetl plik

@ -1,7 +1,7 @@
import React from "react";
import { ResponsiveLineCanvas } from "@nivo/line";
const Report = ({ data }) => {
const Report = ({ data, metric }) => {
const commonProperties = {
animate: false,
enableSlices: "x",
@ -10,7 +10,23 @@ const Report = ({ data }) => {
const xyData = data.map((item) => {
return { x: item.date, y: item.count };
});
const plotData = [{ id: "1", data: xyData }];
xyData.reverse();
// Cumulative sum calculation inspired by: https://stackoverflow.com/a/55261098
function generateCumulativeSum(sum) {
function cumulativeSum(item) {
sum += item.y;
return { x: item.x, y: sum };
}
return cumulativeSum;
}
const xyCumulativeData = xyData.map(generateCumulativeSum(0));
console.log(`metric ${metric} \n xyCumulativeData: `, xyCumulativeData);
const plotData = [{ id: "1", data: xyCumulativeData }];
return (
<ResponsiveLineCanvas
@ -19,11 +35,11 @@ const Report = ({ data }) => {
isInteractive={true}
xScale={{
type: "time",
format: "%Y-%m-%d",
format: "%Y-%m-%d %H",
useUTC: false,
precision: "day",
precision: "hour",
}}
xFormat="time:%Y-%m-%d"
xFormat="time:%Y-%m-%d %H"
yScale={{
type: "linear",
}}
@ -40,7 +56,8 @@ const Report = ({ data }) => {
tickValues: "every 7 day",
tickRotation: 90,
}}
curve="step"
curve="linear"
enableArea={true}
enablePointLabel={false}
pointSize={0}
colors="#fd671b"

20
frontend/src/components/SubscriptionReport.js
Wyświetl plik

@ -24,7 +24,7 @@ const SubscriptionReport = ({ url, id, type }) => {
if (!data || isLoading) return <Spinner />;
return (
<Flex w="100%" h="auto" flexGrow={1} flexBasis="420px" direction="column">
{data.data?.events && Object.keys(data.data?.events) && (
{data?.events && Object.keys(data?.events) && (
<Flex
w="100%"
h="auto"
@ -33,7 +33,7 @@ const SubscriptionReport = ({ url, id, type }) => {
direction="column"
>
<Heading size="sm">Events</Heading>
{Object.keys(data.data.events.year).map((key) => {
{Object.keys(data.events).map((key) => {
return (
<Flex
key={v4()}
@ -55,13 +55,13 @@ const SubscriptionReport = ({ url, id, type }) => {
>
{key}
</Text>
<Report data={data.data.events.year[key]} />
<Report data={data.events[key]} metric={key} />
</Flex>
);
})}
</Flex>
)}
{data.data?.methods && Object.keys(data.data?.methods) && (
{data?.functions && Object.keys(data?.functions) && (
<Flex
w="100%"
h="auto"
@ -69,8 +69,8 @@ const SubscriptionReport = ({ url, id, type }) => {
flexBasis="420px"
direction="column"
>
<Heading size="sm">Methods</Heading>
{Object.keys(data.data.methods.year).map((key) => {
<Heading size="sm">functions</Heading>
{Object.keys(data.functions).map((key) => {
return (
<Flex
key={v4()}
@ -92,13 +92,13 @@ const SubscriptionReport = ({ url, id, type }) => {
>
{key}
</Text>
<Report data={data.data.methods.year[key]} />
<Report data={data.functions[key]} metric={key} />
</Flex>
);
})}
</Flex>
)}
{data.data?.generic && Object.keys(data.data?.generic) && (
{data?.generic && Object.keys(data?.generic) && (
<Flex
w="100%"
h="auto"
@ -107,7 +107,7 @@ const SubscriptionReport = ({ url, id, type }) => {
direction="column"
>
<Heading size="sm">Account generic</Heading>
{Object.keys(data.data.generic.year).map((key) => {
{Object.keys(data.generic).map((key) => {
return (
<Flex
key={v4()}
@ -129,7 +129,7 @@ const SubscriptionReport = ({ url, id, type }) => {
>
{key}
</Text>
<Report data={data.data.generic.year[key]} />
<Report data={data.generic[key]} metric={key} />
</Flex>
);
})}

6
frontend/src/core/services/dashboard.service.js
Wyświetl plik

@ -5,7 +5,7 @@ const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
export const createDashboard = (data) => {
return http({
method: "POST",
url: `${API_URL}/dashboards`,
url: `${API_URL}/dashboards/`,
data,
});
};
@ -13,7 +13,7 @@ export const createDashboard = (data) => {
export const getDashboardsList = () => {
return http({
method: "GET",
url: `${API_URL}/dashboards`,
url: `${API_URL}/dashboards/`,
});
};
@ -35,6 +35,6 @@ export const getDashboard = (dashboardId) => {
export const getDashboardLinks = (dashboardId) => {
return http({
method: "GET",
url: `${API_URL}/dashboards/${dashboardId}/data_links`,
url: `${API_URL}/dashboards/${dashboardId}/stats`,
});
};

81
nodes/deploy/ethereum/deploy.bash
Wyświetl plik

@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Deployment script - intended to run on Moonstream node control server
# Deployment script - intended to run on Moonstream Ethereum node control server
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
@ -14,54 +15,72 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Main
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
APP_NODES_DIR="${APP_DIR}/nodes"
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
NODE_PARAMETERS_ENV_PATH="${SECRETS_DIR}/node.env"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
SCRIPT_DIR="$(realpath $(dirname $0))"
BLOCKCHAIN="ethereum"
ETHEREUM_GETH_SERVICE="ethereum-node.service"
# Parameters scripts
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# Nodes server service file
NODES_SERVER_SERVICE_FILE="moonstreamnodes.service"
# Ethereum geth service file
ETHEREUM_GETH_SERVICE_FILE="geth.service"
set -eu
echo
echo
echo -e "${PREFIX_INFO} Building executable server of moonstreamnodes with Go"
EXEC_DIR=$(pwd)
cd "${APP_NODES_DIR}/server"
HOME=/root /usr/local/go/bin/go build -o "${APP_NODES_DIR}/server/moonstreamnodes" "${APP_NODES_DIR}/server/main.go"
cd "${EXEC_DIR}"
echo
echo
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
mkdir -p "${SECRETS_DIR}"
> "${NODE_PARAMETERS_ENV_PATH}"
> "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -vn -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
LOCAL_IP="$(ec2metadata --local-ipv4)"
echo -e "${PREFIX_INFO} Found assign subnet IP ${C_GREEN}${LOCAL_IP}${C_RESET} for machine"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=moonstream Key=tag:Blockchain,Values=$BLOCKCHAIN \
| jq -r .Parameters[].Name)
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
--names $ENV_PARAMETERS \
--query "Parameters[*].{Name:Name,Value:Value}")
ENV_PARAMETERS_VALUES_LENGTH=$(echo $ENV_PARAMETERS_VALUES | jq length)
echo -e "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
for i in $(seq 0 $(($ENV_PARAMETERS_VALUES_LENGTH - 1)))
do
param_key=$(echo $ENV_PARAMETERS_VALUES | jq -r .[$i].Name)
if [ "$param_key" == "MOONSTREAM_NODE_ETHEREUM_IPC_ADDR" ] && [ -n "$LOCAL_IP" ]
then
param_value="\"$LOCAL_IP\""
else
param_value=$(echo $ENV_PARAMETERS_VALUES | jq .[$i].Value)
fi
echo "$param_key=$param_value" >> "${NODE_PARAMETERS_ENV_PATH}"
done
echo -e "${PREFIX_INFO} Replacing current node IP environment variable with local IP ${C_GREEN}${LOCAL_IP}${C_RESET}"
sed -i "s|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$LOCAL_IP\"|" "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing moonstreamnodes service definition with ${NODES_SERVER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${NODES_SERVER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${NODES_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODES_SERVER_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${NODES_SERVER_SERVICE_FILE}"
systemctl status "${NODES_SERVER_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Updating Ethereum Geth service"
if systemctl is-active --quiet "${ETHEREUM_GETH_SERVICE}"
if systemctl is-active --quiet "${ETHEREUM_GETH_SERVICE_FILE}"
then
echo -e "${PREFIX_WARN} Ethereum Geth service ${ETHEREUM_GETH_SERVICE} already running"
echo -e "${PREFIX_WARN} Ethereum Geth service ${ETHEREUM_GETH_SERVICE_FILE} already running"
else
echo -e "${PREFIX_INFO} Restart Geth service ${ETHEREUM_GETH_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_GETH_SERVICE}"
cp "${SCRIPT_DIR}/${ETHEREUM_GETH_SERVICE}" "/etc/systemd/system/${ETHEREUM_GETH_SERVICE}"
echo -e "${PREFIX_INFO} Restart Geth service ${ETHEREUM_GETH_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_GETH_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_GETH_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_GETH_SERVICE_FILE}"
systemctl daemon-reload
systemctl disable "${ETHEREUM_GETH_SERVICE}"
systemctl restart "${ETHEREUM_GETH_SERVICE}"
systemctl disable "${ETHEREUM_GETH_SERVICE_FILE}"
systemctl restart "${ETHEREUM_GETH_SERVICE_FILE}"
sleep 10
fi

9
nodes/deploy/ethereum/ethereum-node.service → nodes/deploy/ethereum/geth.service
Wyświetl plik

@ -5,11 +5,14 @@ After=network.target
[Service]
User=ubuntu
Group=www-data
EnvironmentFile=/home/ubuntu/moonstream-secrets/node.env
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/usr/bin/geth --syncmode snap --cache 4096 \
--port "${MOONSTREAM_NODE_ETHEREUM_LISTENING_PORT}" --datadir /mnt/disks/nodes/ethereum \
--port "${MOONSTREAM_NODE_ETHEREUM_LISTENING_PORT}" \
--datadir /mnt/disks/nodes/ethereum \
--txpool.globalslots 153600 --txpool.globalqueue 3072 \
--http --http.addr "${MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}" --http.port "${MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" --http.api eth,web3,txpool
--http --http.api eth,web3,txpool \
--http.addr "${MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}" \
--http.port "${MOONSTREAM_NODE_ETHEREUM_IPC_PORT}"
ExecStop=/bin/kill -s SIGINT -$MAINPID
TimeoutStopSec=300
SyslogIdentifier=ethereum-node

14
nodes/deploy/ethereum/moonstreamnodes.service 100644
Wyświetl plik

@ -0,0 +1,14 @@
[Unit]
Description=Moonstream node server
After=network.target
[Service]
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/nodes/server
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream/nodes/server/moonstreamnodes -blockchain ethereum -host "${MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}" -port "${MOONSTREAM_NODES_SERVER_PORT}"
SyslogIdentifier=moonstreamnodes
[Install]
WantedBy=multi-user.target

89
nodes/deploy/ethereum/nodes-connections.bash 100755
Wyświetl plik

@ -0,0 +1,89 @@
#!/usr/bin/env bash
#
# Update nodes connection address environment variables
# from AWS Route53 internal hosted zone
VERSION='0.0.1'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
echo
echo "CLI to update nodes connection address environment
variables from AWS Route53 internal hosted zone"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -f File path where environment variables update at"
}
file_flag=""
verbose_flag="false"
while getopts 'f:v' flag; do
case "${flag}" in
f) file_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# File flag should be specified
if [ -z "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify file path"
usage
exit 1
fi
if [ ! -f "${file_flag}" ]; then
verbose "${PREFIX_CRIT} Provided file does not exist"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
verbose "${PREFIX_INFO} Source environment variables"
. ${file_flag}
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving Polygon node address"
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
exit 1
fi
# TODO(kompotkot): Modify regexp to work with export prefix
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

98
nodes/deploy/ethereum/parameters.bash 100755
Wyświetl plik

@ -0,0 +1,98 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and
# opt out as environment variable exports.
VERSION='0.0.2'
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"
echo
echo "CLI to collect secrets from AWS SSM Parameter Store
and output as environment variable exports"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -n Provide true if server is Blockchain node"
echo " -o Output file name environment variables export to"
echo " -p Product tag (moonstream, spire, brood, drones)"
}
# TODO(kompotkot): Flag for export prefix
node_flag=""
output_flag=""
product_flag=""
verbose_flag="false"
while getopts 'no:p:v' flag; do
case "${flag}" in
n) node_flag="true" ;;
o) output_flag="${OPTARG}" ;;
p) product_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# Product flag should be specified
# TODO(kompotkot): Extend script to work with few product at once
if [ -z "${product_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify product tag"
usage
exit 1
fi
verbose "${PREFIX_INFO} Script version: v${VERSION}"
PARAMETER_FILTERS="Key=tag:Product,Values=${product_flag}"
if [ "${node_flag}" == "true" ]; then
verbose "${PREFIX_INFO} Node flag provided, extracting environment variables only for nodes"
PARAMETER_FILTERS="$PARAMETER_FILTERS Key=tag:Node,Values=true"
fi
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters ${PARAMETER_FILTERS} \
| jq -r .Parameters[].Name)
if [ -z "${ENV_PARAMETERS}" ]; then
verbose "${PREFIX_CRIT} There no parameters for provided product tag"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving parameters values"
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
--names ${ENV_PARAMETERS} \
--query "Parameters[*].{Name:Name,Value:Value}")
ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
if [ -z "${output_flag}" ]; then
echo "${param_key}=${param_value}"
else
echo "${param_key}=${param_value}" >> "${output_flag}"
fi
done

73
nodes/deploy/polygon/deploy.bash
Wyświetl plik

@ -1,6 +1,7 @@
#!/usr/bin/env bash
# Deployment script - intended to run on Moonstream node control server
# Deployment script - intended to run on Moonstream Polygon node control server
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
@ -14,26 +15,74 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Main
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
APP_NODES_DIR="${APP_DIR}/nodes"
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
NODE_PARAMETERS_ENV_PATH="${SECRETS_DIR}/node.env"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
SCRIPT_DIR="$(realpath $(dirname $0))"
BLOCKCHAIN="polygon"
HEIMDALL_HOME="/mnt/disks/nodes/${BLOCKCHAIN}/.heimdalld"
# Parameters scripts
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
# Nodes server service file
NODES_SERVER_SERVICE_FILE="moonstreamnodes.service"
# Polygon heimdalld service files
POLYGON_HEIMDALLD_SERVICE_FILE="heimdalld.service"
POLYGON_HEIMDALLD_BRIDGE_SERVICE_FILE="heimdalld-bridge.service"
POLYGON_HEIMDALLD_REST_SERVICE_FILE="heimdalld-rest-server.service"
# Polygon bor service file
POLYGON_BOR_SERVICE_FILE="bor.service"
set -eu
echo
echo
echo -e "${PREFIX_INFO} Building executable server of moonstreamnodes with Go"
EXEC_DIR=$(pwd)
cd "${APP_NODES_DIR}/server"
HOME=/root /usr/local/go/bin/go build -o "${APP_NODES_DIR}/server/moonstreamnodes" "${APP_NODES_DIR}/server/main.go"
cd "${EXEC_DIR}"
echo
echo
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
mkdir -p "${SECRETS_DIR}"
> "${NODE_PARAMETERS_ENV_PATH}"
> "${PARAMETERS_ENV_PATH}"
bash "${CHECKENV_PARAMETERS_SCRIPT}" -vn -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
GETH_NODE_ADDR=$(dig +short ethereum.moonstream.internal)
GETH_NODE_PORT=$(aws ssm get-parameters --names MOONSTREAM_NODE_ETHEREUM_IPC_PORT --query "Parameters[*]" | jq -r .[0].Value)
if [ -n "$GETH_NODE_ADDR" ] && [ -n "$GETH_NODE_PORT" ]
then
MOONSTREAM_NODE_ETHEREUM_IPC_URI="http://$GETH_NODE_ADDR:$GETH_NODE_PORT"
echo "MOONSTREAM_NODE_ETHEREUM_IPC_URI=\"$MOONSTREAM_NODE_ETHEREUM_IPC_URI\"" >> "${NODE_PARAMETERS_ENV_PATH}"
sed -i "s|^eth_rpc_url =.*|eth_rpc_url = \"$MOONSTREAM_NODE_ETHEREUM_IPC_URI\"|" $HEIMDALL_HOME/config/heimdall-config.toml
echo -e "${PREFIX_INFO} Updated ${C_GREEN}eth_rpc_url = $MOONSTREAM_NODE_ETHEREUM_IPC_URI${C_RESET} for Heimdall"
fi
echo
echo
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
echo
echo
LOCAL_IP="$(ec2metadata --local-ipv4)"
echo -e "${PREFIX_INFO} Replacing current node IP environment variable with local IP ${C_GREEN}${LOCAL_IP}${C_RESET}"
sed -i "s|MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$LOCAL_IP\"|" "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing moonstreamnodes service definition with ${NODES_SERVER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${NODES_SERVER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${NODES_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODES_SERVER_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${NODES_SERVER_SERVICE_FILE}"
systemctl status "${NODES_SERVER_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Source extracted parameters"
. "${PARAMETERS_ENV_PATH}"
echo
echo
MOONSTREAM_NODE_ETHEREUM_IPC_URI="http://$MOONSTREAM_NODE_ETHEREUM_IPC_ADDR:$MOONSTREAM_NODE_ETHEREUM_IPC_PORT"
echo -e "${PREFIX_INFO} Update heimdall config file with Ethereum URI ${C_GREEN}${MOONSTREAM_NODE_ETHEREUM_IPC_URI}${C_RESET}"
sed -i "s|^eth_rpc_url =.*|eth_rpc_url = \"$MOONSTREAM_NODE_ETHEREUM_IPC_URI\"|" "${HEIMDALL_HOME}/config/heimdall-config.toml"
echo -e "${PREFIX_INFO} Updated ${C_GREEN}eth_rpc_url = $MOONSTREAM_NODE_ETHEREUM_IPC_URI${C_RESET} for heimdall"

Some files were not shown because too many files have changed in this diff Show More