Merge pull request #677 from bugout-dev/add-timeout-to-provider

Add timeout for HTTPProvider.

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -31,6 +31,7 @@ from .settings import (
     MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
     NB_ACCESS_ID_HEADER,
     NB_DATA_SOURCE_HEADER,
+    WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
 )
 
 logger = logging.getLogger(__name__)
@@ -73,6 +74,7 @@ def connect(
             raise Exception("Wrong blockchain type provided for web3 URI")
 
     if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
+        request_kwargs["timeout"] = WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS
         web3_provider = Web3.HTTPProvider(web3_uri, request_kwargs=request_kwargs)
     else:
         web3_provider = Web3.IPCProvider(web3_uri)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -140,7 +140,7 @@ try:
         )
 except:
     raise Exception(
-        f"Could not parse MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}"
+        f"Could not parse MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS_RAW}"
     )
 
 
@@ -171,6 +171,23 @@ except:
     pass
 
 
+# HTTPProvider for web3 client
+
+WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = 600
+WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get(
+    "WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS"
+)
+try:
+    if WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS_RAW is not None:
+        WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = int(
+            WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS_RAW
+        )
+except:
+    raise Exception(
+        f"Could not parse WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS as int: {WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS_RAW}"
+    )
+
+
 multicall_contracts: Dict[AvailableBlockchainType, str] = {
     AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6",
     AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631",