Add new env variable MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS.

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -23,9 +23,7 @@ from .db import (
     metadata_to_label,
     clean_labels_from_db,
 )
-from ..settings import (
+from ..settings import MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS
-    MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
-)
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -104,7 +102,7 @@ def parse_metadata(
         MOONSTREAM_DB_URI,
         pool_pre_ping=True,
         pool_size=MOONSTREAM_POOL_SIZE,
-        statement_timeout=MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
+        statement_timeout=MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS,
     )
     process_session = sessionmaker(bind=engine)
     db_session = process_session()
@@ -117,7 +115,7 @@ def parse_metadata(
     read_only_engine = create_moonstream_engine(
         url=MOONSTREAM_DB_URI_READ_ONLY,
         pool_size=MOONSTREAM_POOL_SIZE,
-        statement_timeout=MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS,
+        statement_timeout=MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS,
         pool_pre_ping=True,
     )
 

crawlers/mooncrawl/mooncrawl/settings.py

@@ -59,6 +59,21 @@ except:
         f"Could not parse MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW}"
     )
 
+
+MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS = 60000
+MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS_RAW = os.environ.get(
+    "MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS"
+)
+try:
+    if MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS_RAW is not None:
+        MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS = int(
+            MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS_RAW
+        )
+except:
+    raise Exception(
+        f"Could not parse MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_CRAWLERS_DB_STATEMENT_TIMEOUT_MILLIS_RAW}"
+    )
+
 # Geth connection address
 MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get(
     "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", ""