diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 21fb6ec3..21972a9b 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -16,6 +16,7 @@ from web3.main import Web3 from ..blockchain import connect from ..reporter import reporter from ..settings import ( + BUGOUT_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_MOONWORM_TASKS_JOURNAL, bugout_client, @@ -186,6 +187,7 @@ def get_crawl_job_entries( query=query, offset=current_offset, limit=limit, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS ) entries.extend(search_result.results) @@ -337,6 +339,7 @@ def _get_heartbeat_entry_id( journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, query=f"#{crawler_type} #heartbeat #{blockchain_type.value} !#dead", limit=1, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS ) if entries.results: return entries.results[0].entry_url.split("/")[-1] @@ -348,6 +351,7 @@ def _get_heartbeat_entry_id( title=f"{crawler_type} Heartbeat - {blockchain_type.value}", tags=[crawler_type, "heartbeat", blockchain_type.value], content="", + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS ) return str(entry.id) @@ -376,6 +380,7 @@ def heartbeat( entry_id=heartbeat_entry_id, title=f"{crawler_type} Heartbeat - {blockchain_type.value}. Status: {crawler_status['status']} - {crawler_status['current_time']}", content=f"{json.dumps(crawler_status, indent=2)}", + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS ) if is_dead: bugout_client.update_tags( @@ -383,4 +388,5 @@ def heartbeat( journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, entry_id=heartbeat_entry_id, tags=[crawler_type, "heartbeat", blockchain_type.value, "dead"], + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS ) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 6eddafac..b2e82e8d 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -13,6 +13,9 @@ BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) +BUGOUT_REQUEST_TIMEOUT_SECONDS = int(os.environ.get("BUGOUT_REQUEST_TIMEOUT_SECONDS", "30")) + + HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") # Origin diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index acf2d895..124ea78d 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -1,6 +1,7 @@ # Bugout environment variables export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" +export BUGOUT_REQUEST_TIMEOUT=30 export HUMBUG_REPORTER_CRAWLERS_TOKEN="" # Moonstream environment variables