From e64bb7fcad5052b5492b5184b5f21328df3e1d18 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 15 Dec 2022 17:27:51 +0200 Subject: [PATCH] Add deploy services and fix if condition. --- crawlers/deploy/mumbai-state.service | 13 +++++++ crawlers/deploy/mumbai-state.timer | 9 +++++ crawlers/deploy/polygon-state.service | 2 +- crawlers/mooncrawl/mooncrawl/settings.py | 21 +++++++++++ .../mooncrawl/mooncrawl/state_crawler/cli.py | 37 ++++++++++++++----- crawlers/mooncrawl/sample.env | 3 +- 6 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 crawlers/deploy/mumbai-state.service create mode 100644 crawlers/deploy/mumbai-state.timer diff --git a/crawlers/deploy/mumbai-state.service b/crawlers/deploy/mumbai-state.service new file mode 100644 index 00000000..8b11512b --- /dev/null +++ b/crawlers/deploy/mumbai-state.service @@ -0,0 +1,13 @@ +[Unit] +Description=Execute state crawler +After=network.target + +[Service] +Type=oneshot +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain mumbai --infura --jobs-file ./mooncrawl/state_crawler/jobs/mumbai-jobs.json +CPUWeight=60 +SyslogIdentifier=polygon-state diff --git a/crawlers/deploy/mumbai-state.timer b/crawlers/deploy/mumbai-state.timer new file mode 100644 index 00000000..76ea17be --- /dev/null +++ b/crawlers/deploy/mumbai-state.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Execute Polygon state crawler each 10m + +[Timer] +OnBootSec=15s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/polygon-state.service b/crawlers/deploy/polygon-state.service index 46ddd88e..2bdcbadc 100644 --- a/crawlers/deploy/polygon-state.service +++ b/crawlers/deploy/polygon-state.service @@ -8,6 +8,6 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon --infura --jobs-file ./mooncrawl/state_crawler/jobs/polygon-jobs.json CPUWeight=60 SyslogIdentifier=polygon-state diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 66e50dd6..11e654c5 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -210,3 +210,24 @@ if MOONSTREAM_S3_PUBLIC_DATA_BUCKET == "": MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get( "MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev" ) + + +# infura config + + +INFURA_PROJECT_ID = os.environ.get("INFURA_PROJECT_ID") + +infura_networks = { + AvailableBlockchainType.ETHEREUM: { + "name": "mainnet", + "url": f"https://mainnet.infura.io/v3/{INFURA_PROJECT_ID}", + }, + AvailableBlockchainType.POLYGON: { + "name": "polygon", + "url": f"https://polygon-mainnet.infura.io/v3/{INFURA_PROJECT_ID}", + }, + AvailableBlockchainType.MUMBAI: { + "name": "mumbai", + "url": f"https://polygon-mumbai.infura.io/v3/{INFURA_PROJECT_ID}", + }, +} diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py index c79704b0..d4a9ce03 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py @@ -28,9 +28,11 @@ from .Multicall2_interface import Contract as Multicall2 from ..settings import ( NB_CONTROLLER_ACCESS_ID, MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS, + INFURA_PROJECT_ID, multicall_contracts, + infura_networks, ) -from .web3_util import FunctionSignature +from .web3_util import FunctionSignature, connect logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -272,18 +274,13 @@ def parse_jobs( contracts_methods: Dict[str, Any] = {} calls: Dict[int, Any] = {0: []} - if web3_provider_uri is None: - - logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.") - web3_client = _retry_connect_web3( - blockchain_type=blockchain_type, access_id=access_id - ) - else: + if web3_provider_uri is not None: try: logger.info( f"Connecting to blockchain: {blockchain_type} with custom provider!" ) - web3_client = Web3(HTTPProvider(web3_provider_uri)) + + web3_client = connect(web3_provider_uri) if blockchain_type != AvailableBlockchainType.ETHEREUM: web3_client.middleware_onion.inject(geth_poa_middleware, layer=0) @@ -292,6 +289,11 @@ def parse_jobs( f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}" ) raise (e) + else: + logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.") + web3_client = _retry_connect_web3( + blockchain_type=blockchain_type, access_id=access_id + ) logger.info(f"Crawler started connected to blockchain: {blockchain_type}") @@ -459,10 +461,20 @@ def handle_crawl(args: argparse.Namespace) -> None: blockchain_type = AvailableBlockchainType(args.blockchain) + custom_web3_provider = args.custom_web3_provider + + if args.infura and INFURA_PROJECT_ID is not None: + if blockchain_type not in infura_networks: + raise ValueError( + f"Infura is not supported for {blockchain_type} blockchain type" + ) + logger.info(f"Using Infura!") + custom_web3_provider = infura_networks[blockchain_type]["url"] + parse_jobs( jobs, blockchain_type, - args.custom_web3_provider, + custom_web3_provider, args.block_number, args.batch_size, args.access_id, @@ -539,6 +551,11 @@ def main() -> None: help="Type of blovkchain wich writng in database", required=True, ) + view_state_crawler_parser.add_argument( + "--infura", + action="store_true", + help="Use infura as web3 provider", + ) view_state_crawler_parser.add_argument( "--custom-web3-provider", "-w3", diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 5e7e8980..74ab5f8f 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -36,4 +36,5 @@ export COINMARKETCAP_API_KEY="" # Custom crawler export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="" export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev" -export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="" \ No newline at end of file +export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="" +export INFURA_PROJECT_ID="" \ No newline at end of file