Add timeout for bugout in crawlers.

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -16,6 +16,7 @@ from web3.main import Web3
 from ..blockchain import connect
 from ..reporter import reporter
 from ..settings import (
+    BUGOUT_REQUEST_TIMEOUT_SECONDS,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_MOONWORM_TASKS_JOURNAL,
     bugout_client,
@@ -186,6 +187,7 @@ def get_crawl_job_entries(
             query=query,
             offset=current_offset,
             limit=limit,
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS
         )
         entries.extend(search_result.results)
 
@@ -337,6 +339,7 @@ def _get_heartbeat_entry_id(
         journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
         query=f"#{crawler_type} #heartbeat #{blockchain_type.value} !#dead",
         limit=1,
+        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS
     )
     if entries.results:
         return entries.results[0].entry_url.split("/")[-1]
@@ -348,6 +351,7 @@ def _get_heartbeat_entry_id(
             title=f"{crawler_type} Heartbeat - {blockchain_type.value}",
             tags=[crawler_type, "heartbeat", blockchain_type.value],
             content="",
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS
         )
         return str(entry.id)
 
@@ -376,6 +380,7 @@ def heartbeat(
         entry_id=heartbeat_entry_id,
         title=f"{crawler_type} Heartbeat - {blockchain_type.value}. Status: {crawler_status['status']} - {crawler_status['current_time']}",
         content=f"{json.dumps(crawler_status, indent=2)}",
+        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS
     )
     if is_dead:
         bugout_client.update_tags(
@@ -383,4 +388,5 @@ def heartbeat(
             journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
             entry_id=heartbeat_entry_id,
             tags=[crawler_type, "heartbeat", blockchain_type.value, "dead"],
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS
         )

crawlers/mooncrawl/mooncrawl/settings.py

@@ -13,6 +13,9 @@ BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
 BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
 bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
 
+BUGOUT_REQUEST_TIMEOUT_SECONDS = int(os.environ.get("BUGOUT_REQUEST_TIMEOUT_SECONDS", "30"))
+
+
 HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
 
 # Origin

crawlers/mooncrawl/sample.env

@@ -1,6 +1,7 @@
 # Bugout environment variables
 export BUGOUT_BROOD_URL="https://auth.bugout.dev"
 export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
+export BUGOUT_REQUEST_TIMEOUT=30
 export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
 
 # Moonstream environment variables