kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #909 from moonstream-to/zksync-era-mainnet-subscription
Add zksync era smartcontract subscription type.pull/913/head
commit
d6aa376b8e
|
@ -109,6 +109,16 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction
|
|||
WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service"
|
||||
WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer"
|
||||
|
||||
# ZkSync Era
|
||||
ZKSYNC_ERA_SYNCHRONIZE_SERVICE="zksync-era-synchronize.service"
|
||||
ZKSYNC_ERA_MISSING_SERVICE_FILE="zksync-era-missing.service"
|
||||
ZKSYNC_ERA_MISSING_TIMER_FILE="zksync-era-missing.timer"
|
||||
ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-moonworm-crawler.service"
|
||||
ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-crawl-transactions.service"
|
||||
ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer"
|
||||
ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service"
|
||||
ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer"
|
||||
|
||||
# ZkSync Era testnet
|
||||
ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
|
||||
ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service"
|
||||
|
@ -516,6 +526,51 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HIS
|
|||
# ZkSync Era
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_MISSING_TIMER_FILE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_SERVICE_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_TIMER_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MISSING_TIMER_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era moonworm crawler service definition with ${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical transactions crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical events crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
|
||||
|
||||
|
||||
# ZkSync Era testnet
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
|
||||
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
|
||||
|
@ -558,7 +613,6 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
|||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
|
||||
|
||||
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Replacing existing Leaderboards worker service and timer with: ${LEADERBOARDS_WORKER_SERVICE_FILE}, ${LEADERBOARDS_WORKER_TIMER_FILE}"
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=ZkSync Era historical crawler events
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=zksync-era-historical-crawl-events
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=Runs events historical crawler on ZkSync Era
|
||||
|
||||
[Timer]
|
||||
OnBootSec=60s
|
||||
OnUnitActiveSec=10m
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=ZkSync Era historical crawler transactions
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=zksync-era-historical-crawl-transactions
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=Runs transactions historical crawler on ZkSync Era
|
||||
|
||||
[Timer]
|
||||
OnBootSec=60s
|
||||
OnUnitActiveSec=10m
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Fill missing blocks at ZkSync Era database
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n
|
||||
CPUWeight=50
|
||||
SyslogIdentifier=zksync-era-missing
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=Fill missing blocks at ZkSync Era database
|
||||
|
||||
[Timer]
|
||||
OnBootSec=120s
|
||||
OnUnitActiveSec=15m
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=ZkSync Era moonworm crawler
|
||||
After=network.target
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era --confirmations 20 --min-blocks-batch 20
|
||||
CPUWeight=70
|
||||
SyslogIdentifier=zksync-era-moonworm-crawler
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=ZkSync Era block with transactions synchronizer
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=3
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=15s
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2
|
||||
CPUWeight=90
|
||||
SyslogIdentifier=zksync-era-synchronize
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -30,7 +30,6 @@ from .actions import (
|
|||
from .middleware import MoonstreamHTTPException
|
||||
from .settings import (
|
||||
BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
|
||||
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,
|
||||
DOCS_TARGET_PATH,
|
||||
LINKS_EXPIRATION_TIME,
|
||||
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
|
@ -173,7 +172,7 @@ async def status_handler(
|
|||
dashboard_subscription_filters["subscription_id"]
|
||||
]
|
||||
|
||||
for reqired_field in subscription.required_fields:
|
||||
for reqired_field in subscription.required_fields: # type: ignore
|
||||
if "subscription_type_id" in reqired_field:
|
||||
subscriprions_type = reqired_field["subscription_type_id"]
|
||||
|
||||
|
|
|
@ -28,6 +28,7 @@ from .settings import (
|
|||
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI,
|
||||
NB_ACCESS_ID_HEADER,
|
||||
NB_DATA_SOURCE_HEADER,
|
||||
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
|
||||
|
@ -73,6 +74,8 @@ def connect(
|
|||
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
|
||||
else:
|
||||
raise Exception("Wrong blockchain type provided for web3 URI")
|
||||
|
||||
|
@ -126,7 +129,10 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
|
|||
)
|
||||
if blockchain_type == AvailableBlockchainType.XDAI:
|
||||
block_obj.author = block.author
|
||||
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
if (
|
||||
blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
|
||||
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
|
||||
):
|
||||
block_obj.mix_hash = block.get("mixHash", "")
|
||||
block_obj.sha3_uncles = block.get("sha3Uncles", "")
|
||||
block_obj.l1_batch_number = (
|
||||
|
@ -168,7 +174,10 @@ def add_block_transactions(
|
|||
transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None,
|
||||
value=tx.value,
|
||||
)
|
||||
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
if (
|
||||
blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
|
||||
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
|
||||
):
|
||||
tx_obj.l1_batch_number = (
|
||||
int(tx.get("l1BatchNumber"), 0)
|
||||
if tx.get("l1BatchNumber") is not None
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import cast, List
|
||||
import uuid
|
||||
|
||||
|
@ -12,7 +11,6 @@ from .utils import get_results_for_moonstream_query
|
|||
from ..settings import (
|
||||
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
|
||||
BUGOUT_REQUEST_TIMEOUT_SECONDS,
|
||||
MOONSTREAM_API_URL,
|
||||
MOONSTREAM_ENGINE_URL,
|
||||
)
|
||||
|
|
|
@ -3,19 +3,16 @@ import json
|
|||
import logging
|
||||
import random
|
||||
import urllib.request
|
||||
from contextlib import contextmanager
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.error import HTTPError
|
||||
|
||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..db import (
|
||||
yield_db_preping_session_ctx,
|
||||
yield_db_read_only_preping_session_ctx,
|
||||
)
|
||||
from ..settings import MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS
|
||||
from .db import (
|
||||
clean_labels_from_db,
|
||||
get_current_metadata_for_address,
|
||||
|
|
|
@ -9,7 +9,7 @@ from moonstreamdb.blockchain import AvailableBlockchainType
|
|||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||
MoonstreamEthereumStateProvider,
|
||||
)
|
||||
from moonworm.crawler.networks import Network # type: ignore
|
||||
from moonstreamdb.networks import Network
|
||||
from sqlalchemy.orm.session import Session
|
||||
from web3 import Web3
|
||||
|
||||
|
@ -132,6 +132,8 @@ def continuous_crawler(
|
|||
network = Network.wyrm
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
network = Network.zksync_era_testnet
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
network = Network.zksync_era
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
@ -36,6 +36,7 @@ class SubscriptionTypes(Enum):
|
|||
XDAI_BLOCKCHAIN = "xdai_smartcontract"
|
||||
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
|
||||
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
|
||||
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
|
||||
|
||||
|
||||
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
|
||||
|
@ -142,6 +143,8 @@ def blockchain_type_to_subscription_type(
|
|||
return SubscriptionTypes.WYRM_BLOCKCHAIN
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ from moonworm.crawler.function_call_crawler import ( # type: ignore
|
|||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||
MoonstreamEthereumStateProvider,
|
||||
)
|
||||
from moonworm.crawler.networks import Network # type: ignore
|
||||
from moonstreamdb.networks import Network # type: ignore
|
||||
from moonworm.watch import MockState # type: ignore
|
||||
from sqlalchemy.orm import Session
|
||||
from web3 import Web3
|
||||
|
@ -70,6 +70,8 @@ def function_call_crawler(
|
|||
network = Network.wyrm
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
network = Network.zksync_era_testnet
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
network = Network.zksync_era
|
||||
else:
|
||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import logging
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import Dict, List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from eth_typing.evm import ChecksumAddress
|
||||
|
@ -8,7 +8,7 @@ from moonstreamdb.blockchain import AvailableBlockchainType
|
|||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||
MoonstreamEthereumStateProvider,
|
||||
)
|
||||
from moonworm.crawler.networks import Network # type: ignore
|
||||
from moonstreamdb.networks import Network # type: ignore
|
||||
from sqlalchemy.orm.session import Session
|
||||
from web3 import Web3
|
||||
|
||||
|
@ -51,11 +51,23 @@ def historical_crawler(
|
|||
web3.eth.block_number >= start_block
|
||||
), "start_block must be less than current block"
|
||||
|
||||
network = (
|
||||
Network.ethereum
|
||||
if blockchain_type == AvailableBlockchainType.ETHEREUM
|
||||
else Network.polygon
|
||||
)
|
||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||
network = Network.ethereum
|
||||
elif blockchain_type == AvailableBlockchainType.POLYGON:
|
||||
network = Network.polygon
|
||||
elif blockchain_type == AvailableBlockchainType.MUMBAI:
|
||||
network = Network.mumbai
|
||||
elif blockchain_type == AvailableBlockchainType.XDAI:
|
||||
network = Network.xdai
|
||||
elif blockchain_type == AvailableBlockchainType.WYRM:
|
||||
network = Network.wyrm
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
network = Network.zksync_era_testnet
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
network = Network.zksync_era
|
||||
else:
|
||||
raise Exception("Unsupported blockchain type provided")
|
||||
|
||||
ethereum_state_provider = MoonstreamEthereumStateProvider(
|
||||
web3,
|
||||
network,
|
||||
|
|
|
@ -116,6 +116,12 @@ if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
|
|||
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
|
||||
)
|
||||
|
||||
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
|
||||
"MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI", ""
|
||||
)
|
||||
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
|
||||
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
|
||||
|
||||
MOONSTREAM_CRAWL_WORKERS = 4
|
||||
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
|
||||
try:
|
||||
|
|
|
@ -12,7 +12,6 @@ from uuid import UUID
|
|||
|
||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||
from moonstream.client import Moonstream # type: ignore
|
||||
from web3._utils.request import cache_session
|
||||
from web3.middleware import geth_poa_middleware
|
||||
|
||||
from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3
|
||||
|
@ -21,7 +20,6 @@ from ..actions import recive_S3_data_from_query
|
|||
from ..db import PrePing_SessionLocal
|
||||
from ..settings import (
|
||||
INFURA_PROJECT_ID,
|
||||
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
NB_CONTROLLER_ACCESS_ID,
|
||||
infura_networks,
|
||||
multicall_contracts,
|
||||
|
|
|
@ -53,6 +53,7 @@ subscription_id_by_blockchain = {
|
|||
"xdai": "xdai_smartcontract",
|
||||
"wyrm": "wyrm_smartcontract",
|
||||
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
|
||||
"zksync_era": "zksync_era_smartcontract",
|
||||
}
|
||||
|
||||
blockchain_by_subscription_id = {
|
||||
|
@ -68,6 +69,7 @@ blockchain_by_subscription_id = {
|
|||
"xdai_smartcontract": "xdai",
|
||||
"wyrm_smartcontract": "wyrm",
|
||||
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
|
||||
"zksync_era_smartcontract": "zksync_era",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -2,4 +2,4 @@
|
|||
Moonstream crawlers version.
|
||||
"""
|
||||
|
||||
MOONCRAWL_VERSION = "0.3.4"
|
||||
MOONCRAWL_VERSION = "0.3.5"
|
||||
|
|
|
@ -37,7 +37,7 @@ setup(
|
|||
"bugout>=0.2.13",
|
||||
"chardet",
|
||||
"fastapi",
|
||||
"moonstreamdb>=0.3.4",
|
||||
"moonstreamdb>=0.3.5",
|
||||
"moonstream>=0.1.1",
|
||||
"moonworm[moonstream]>=0.6.2",
|
||||
"humbug",
|
||||
|
|
|
@ -43,7 +43,7 @@ from .settings import (
|
|||
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
|
||||
)
|
||||
from .settings import bugout_client as bc
|
||||
from .settings import multicall_contracts, support_interfaces, supportsInterface_abi
|
||||
from .settings import multicall_contracts, supportsInterface_abi
|
||||
from .web3_provider import FunctionSignature, connect, multicall
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -56,12 +56,14 @@ blockchain_by_subscription_id = {
|
|||
"xdai_blockchain": "xdai",
|
||||
"wyrm_blockchain": "wyrm",
|
||||
"zksync_era_testnet_blockchain": "zksync_era_testnet",
|
||||
"zksync_era_blockchain": "zksync_era",
|
||||
"ethereum_smartcontract": "ethereum",
|
||||
"polygon_smartcontract": "polygon",
|
||||
"mumbai_smartcontract": "mumbai",
|
||||
"xdai_smartcontract": "xdai",
|
||||
"wyrm_smartcontract": "wyrm",
|
||||
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
|
||||
"zksync_era_smartcontract": "zksync_era",
|
||||
}
|
||||
|
||||
|
||||
|
@ -486,7 +488,7 @@ def get_all_entries_from_search(
|
|||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
|
||||
if len(results) != existing_methods.total_results:
|
||||
for offset in range(limit, existing_methods.total_results, limit):
|
||||
|
@ -499,7 +501,7 @@ def get_all_entries_from_search(
|
|||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
@ -72,6 +72,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
|
|||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"zksync_era_smartcontract": SubscriptionTypeResourceData(
|
||||
id="zksync_era_smartcontract",
|
||||
name="zkSync Era smartcontract",
|
||||
blockchain="zksync_era",
|
||||
choices=["input:address", "tag:erc721"],
|
||||
description="Contracts events and tx_calls of contract of zkSync Era blockchain.",
|
||||
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-token-logo.png",
|
||||
stripe_product_id=None,
|
||||
stripe_price_id=None,
|
||||
active=True,
|
||||
),
|
||||
"zksync_era_testnet_smartcontract": SubscriptionTypeResourceData(
|
||||
id="zksync_era_testnet_smartcontract",
|
||||
name="zkSync Era testnet smartcontract",
|
||||
|
@ -138,6 +149,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
|
|||
stripe_price_id=None,
|
||||
active=False,
|
||||
),
|
||||
"zksync_era_blockchain": SubscriptionTypeResourceData(
|
||||
id="zksync_era_blockchain",
|
||||
name="zkSync Era transactions",
|
||||
blockchain="zksync_era",
|
||||
choices=["input:address", "tag:erc721"],
|
||||
description="ZkSync Era chain transactions subscription.",
|
||||
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-token-logo.png",
|
||||
stripe_product_id=None,
|
||||
stripe_price_id=None,
|
||||
active=False,
|
||||
),
|
||||
"zksync_era_testnet_blockchain": SubscriptionTypeResourceData(
|
||||
id="zksync_era_testnet_blockchain",
|
||||
name="zkSync Era testnet transactions",
|
||||
|
|
|
@ -4,13 +4,11 @@ Pydantic schemas for the Moonstream HTTP API
|
|||
import json
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from uuid import UUID
|
||||
from xmlrpc.client import Boolean
|
||||
|
||||
from fastapi import Form
|
||||
from pydantic import BaseModel, Field, validator
|
||||
from sqlalchemy import false
|
||||
|
||||
USER_ONBOARDING_STATE = "onboarding_state"
|
||||
|
||||
|
|
|
@ -52,11 +52,13 @@ event_providers: Dict[str, Any] = {
|
|||
moonworm_provider.MumbaiMoonwormProvider.event_type: moonworm_provider.MumbaiMoonwormProvider,
|
||||
moonworm_provider.XDaiMoonwormProvider.event_type: moonworm_provider.XDaiMoonwormProvider,
|
||||
moonworm_provider.ZkSyncEraTestnetMoonwormProvider.event_type: moonworm_provider.ZkSyncEraTestnetMoonwormProvider,
|
||||
moonworm_provider.ZkSyncEraMoonwormProvider.event_type: moonworm_provider.ZkSyncEraMoonwormProvider,
|
||||
transactions.EthereumTransactions.event_type: transactions.EthereumTransactions,
|
||||
transactions.PolygonTransactions.event_type: transactions.PolygonTransactions,
|
||||
transactions.MumbaiTransactions.event_type: transactions.MumbaiTransactions,
|
||||
transactions.XDaiTransactions.event_type: transactions.XDaiTransactions,
|
||||
transactions.ZkSyncEraTestnetTransactions.event_type: transactions.ZkSyncEraTestnetTransactions,
|
||||
transactions.ZkSyncEraTransactions.event_type: transactions.ZkSyncEraTransactions,
|
||||
bugout.polygon_whalewatch_provider.event_type: bugout.polygon_whalewatch_provider,
|
||||
bugout.ethereum_txpool_provider.event_type: bugout.ethereum_txpool_provider,
|
||||
bugout.ethereum_whalewatch_provider.event_type: bugout.ethereum_whalewatch_provider,
|
||||
|
|
|
@ -22,6 +22,7 @@ polygon_event_type = "polygon_blockchain"
|
|||
mumbai_event_type = "mumbai_blockchain"
|
||||
xdai_event_type = "xdai_blockchain"
|
||||
zksync_era_testnet_event_type = "zksync_era_testnet_blockchain"
|
||||
zksync_era_event_type = "zksync_era_blockchain"
|
||||
allowed_tags = ["tag:erc721"]
|
||||
|
||||
description = f"""Event provider for transactions from the Ethereum blockchain.
|
||||
|
@ -421,3 +422,10 @@ ZkSyncEraTestnetMoonwormProvider = MoonwormProvider(
|
|||
description="Provider for reviving transactions from zkSync Era testnet tables.",
|
||||
streamboaundary_range_limit=2 * 60 * 60,
|
||||
)
|
||||
|
||||
ZkSyncEraMoonwormProvider = MoonwormProvider(
|
||||
event_type="zksync_era_smartcontract",
|
||||
blockchain=AvailableBlockchainType("zksync_era"),
|
||||
description="Provider for reviving transactions from zkSync Era tables.",
|
||||
streamboaundary_range_limit=2 * 60 * 60,
|
||||
)
|
||||
|
|
|
@ -482,3 +482,10 @@ ZkSyncEraTestnetTransactions = TransactionsProvider(
|
|||
description="Provider for resiving transactions from ZkSync Era testnet tables.",
|
||||
streamboaundary_range_limit=2 * 60 * 60,
|
||||
)
|
||||
|
||||
ZkSyncEraTransactions = TransactionsProvider(
|
||||
event_type="zksync_era_blockchain",
|
||||
blockchain=AvailableBlockchainType("zksync_era"),
|
||||
description="Provider for resiving transactions from ZkSync Era tables.",
|
||||
streamboaundary_range_limit=2 * 60 * 60,
|
||||
)
|
||||
|
|
|
@ -133,6 +133,12 @@ if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
|
|||
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
|
||||
)
|
||||
|
||||
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
|
||||
"MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI", ""
|
||||
)
|
||||
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
|
||||
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
|
||||
|
||||
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
|
||||
if MOONSTREAM_S3_QUERIES_BUCKET == "":
|
||||
raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set")
|
||||
|
@ -178,22 +184,6 @@ NB_DATA_SOURCE_HEADER_VALUE = os.environ.get(
|
|||
|
||||
THREAD_TIMEOUT_SECONDS = 10
|
||||
|
||||
support_interfaces = [
|
||||
{"name": "_INTERFACE_ID_ERC165", "selector": "0x01ffc9a7"},
|
||||
{"name": "_INTERFACE_ID_ERC20", "selector": "0x36372b07"},
|
||||
{"name": "_INTERFACE_ID_ERC721", "selector": "0x80ac58cd"},
|
||||
{"name": "_INTERFACE_ID_ERC721_METADATA", "selector": "0x5b5e139f"}, # miss
|
||||
{"name": "_INTERFACE_ID_ERC721_ENUMERABLE", "selector": "0x780e9d63"}, # miss
|
||||
{"name": "_INTERFACE_ID_ERC721_RECEIVED", "selector": "0x150b7a02"},
|
||||
{
|
||||
"name": "_INTERFACE_ID_ERC721_METADATA_RECEIVED",
|
||||
"selector": "0x0e89341c",
|
||||
}, # miss
|
||||
{"name": "_INTERFACE_ID_ERC721_ENUMERABLE_RECEIVED", "selector": "0x4e2312e0"},
|
||||
{"name": "_INTERFACE_ID_ERC1155", "selector": "0xd9b67a26"},
|
||||
]
|
||||
|
||||
|
||||
multicall_contracts: Dict[AvailableBlockchainType, str] = {
|
||||
AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6",
|
||||
AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631",
|
||||
|
|
|
@ -2,4 +2,4 @@
|
|||
Moonstream library and API version.
|
||||
"""
|
||||
|
||||
MOONSTREAMAPI_VERSION = "0.2.9"
|
||||
MOONSTREAMAPI_VERSION = "0.3.0"
|
||||
|
|
|
@ -20,6 +20,7 @@ from .settings import (
|
|||
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
|
||||
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI,
|
||||
multicall_contracts,
|
||||
multicall_contract_abi,
|
||||
)
|
||||
|
@ -72,6 +73,8 @@ def connect(
|
|||
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
|
||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
|
||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
|
||||
else:
|
||||
raise Exception("Wrong blockchain type provided for web3 URI")
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ jsonschema==4.17.0
|
|||
lru-dict==1.1.8
|
||||
Mako==1.2.3
|
||||
MarkupSafe==2.1.1
|
||||
moonstreamdb==0.3.4
|
||||
moonstreamdb==0.3.5
|
||||
multiaddr==0.0.9
|
||||
multidict==6.0.2
|
||||
netaddr==0.8.0
|
||||
|
|
|
@ -15,7 +15,7 @@ setup(
|
|||
"boto3",
|
||||
"bugout>=0.2.13",
|
||||
"fastapi",
|
||||
"moonstreamdb>=0.3.4",
|
||||
"moonstreamdb>=0.3.5",
|
||||
"humbug",
|
||||
"pydantic==1.10.2",
|
||||
"pyevmasm",
|
||||
|
|
Ładowanie…
Reference in New Issue