Add deploy services and fix if condition.

crawlers/deploy/mumbai-state.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Execute state crawler
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain mumbai --infura --jobs-file ./mooncrawl/state_crawler/jobs/mumbai-jobs.json
+CPUWeight=60
+SyslogIdentifier=polygon-state

crawlers/deploy/mumbai-state.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Execute Polygon state crawler each 10m
+
+[Timer]
+OnBootSec=15s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/polygon-state.service

@@ -8,6 +8,6 @@ User=ubuntu
 Group=www-data
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon --infura --jobs-file ./mooncrawl/state_crawler/jobs/polygon-jobs.json
 CPUWeight=60
 SyslogIdentifier=polygon-state

crawlers/mooncrawl/mooncrawl/settings.py

@@ -210,3 +210,24 @@ if MOONSTREAM_S3_PUBLIC_DATA_BUCKET == "":
 MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get(
     "MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev"
 )
+
+
+# infura config
+
+
+INFURA_PROJECT_ID = os.environ.get("INFURA_PROJECT_ID")
+
+infura_networks = {
+    AvailableBlockchainType.ETHEREUM: {
+        "name": "mainnet",
+        "url": f"https://mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.POLYGON: {
+        "name": "polygon",
+        "url": f"https://polygon-mainnet.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+    AvailableBlockchainType.MUMBAI: {
+        "name": "mumbai",
+        "url": f"https://polygon-mumbai.infura.io/v3/{INFURA_PROJECT_ID}",
+    },
+}

crawlers/mooncrawl/mooncrawl/state_crawler/cli.py

@@ -28,9 +28,11 @@ from .Multicall2_interface import Contract as Multicall2
 from ..settings import (
     NB_CONTROLLER_ACCESS_ID,
     MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
+    INFURA_PROJECT_ID,
     multicall_contracts,
+    infura_networks,
 )
-from .web3_util import FunctionSignature
+from .web3_util import FunctionSignature, connect
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -272,18 +274,13 @@ def parse_jobs(
     contracts_methods: Dict[str, Any] = {}
     calls: Dict[int, Any] = {0: []}
 
-    if web3_provider_uri is None:
-
-        logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
-        web3_client = _retry_connect_web3(
-            blockchain_type=blockchain_type, access_id=access_id
-        )
-    else:
+    if web3_provider_uri is not None:
         try:
             logger.info(
                 f"Connecting to blockchain: {blockchain_type} with custom provider!"
             )
-            web3_client = Web3(HTTPProvider(web3_provider_uri))
+
+            web3_client = connect(web3_provider_uri)
 
             if blockchain_type != AvailableBlockchainType.ETHEREUM:
                 web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
@@ -292,6 +289,11 @@ def parse_jobs(
                 f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
             )
             raise (e)
+    else:
+        logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
+        web3_client = _retry_connect_web3(
+            blockchain_type=blockchain_type, access_id=access_id
+        )
 
     logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
 
@@ -459,10 +461,20 @@ def handle_crawl(args: argparse.Namespace) -> None:
 
     blockchain_type = AvailableBlockchainType(args.blockchain)
 
+    custom_web3_provider = args.custom_web3_provider
+
+    if args.infura and INFURA_PROJECT_ID is not None:
+        if blockchain_type not in infura_networks:
+            raise ValueError(
+                f"Infura is not supported for {blockchain_type} blockchain type"
+            )
+        logger.info(f"Using Infura!")
+        custom_web3_provider = infura_networks[blockchain_type]["url"]
+
     parse_jobs(
         jobs,
         blockchain_type,
-        args.custom_web3_provider,
+        custom_web3_provider,
         args.block_number,
         args.batch_size,
         args.access_id,
@@ -539,6 +551,11 @@ def main() -> None:
         help="Type of blovkchain wich writng in database",
         required=True,
     )
+    view_state_crawler_parser.add_argument(
+        "--infura",
+        action="store_true",
+        help="Use infura as web3 provider",
+    )
     view_state_crawler_parser.add_argument(
         "--custom-web3-provider",
         "-w3",

crawlers/mooncrawl/sample.env

@@ -36,4 +36,5 @@ export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"
 # Custom crawler
 export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="<public_bucket>"
 export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev"
-export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="<access token for run queries for public dashboards>"
+export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="<access token for run queries for public dashboards>"
+export INFURA_PROJECT_ID="<infura_project_id>"