Humbug reporter for moonstream python crawlers

crawlers/mooncrawl/mooncrawl/__init__.py

@@ -0,0 +1,7 @@
+from .reporter import reporter
+from .version import MOONCRAWL_VERSION
+
+# Reporting
+reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
+reporter.system_report(publish=True)
+reporter.setup_excepthook(publish=True)

crawlers/mooncrawl/mooncrawl/ethcrawler.py

@@ -48,7 +48,7 @@ def yield_blocks_numbers_lists(
         print(
             "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340"
         )
-        return
+        raise Exception
 
     starting_block = max(input_start_block, input_end_block)
     ending_block = min(input_start_block, input_end_block)

crawlers/mooncrawl/mooncrawl/reporter.py

@@ -0,0 +1,18 @@
+import uuid
+
+from humbug.consent import HumbugConsent
+from humbug.report import HumbugReporter
+
+from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN
+
+session_id = str(uuid.uuid4())
+client_id = "moonstream-crawlers"
+
+reporter = HumbugReporter(
+    name="moonstream",
+    consent=HumbugConsent(True),
+    client_id=client_id,
+    session_id=session_id,
+    bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN,
+    tags=[],
+)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -1,5 +1,9 @@
 import os
 
+# Bugout
+HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
+
+# Geth
 MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None)
 
 MOONSTREAM_CRAWL_WORKERS = 4
@@ -12,5 +16,5 @@ except:
         f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}"
     )
 
-
+# Etherscan
 MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN")

crawlers/mooncrawl/sample.env

@@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
 export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
 export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
 export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
+export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

crawlers/mooncrawl/setup.py

@@ -34,6 +34,7 @@ setup(
     zip_safe=False,
     install_requires=[
         "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db",
+        "humbug",
         "python-dateutil",
         "requests",
         "tqdm",