Merge pull request #222 from bugout-dev/humbug-integration-crawlers

Humbug reporter for moonstream crawlers

crawlers/ethtxpool/main.go

@@ -22,12 +22,8 @@ import (
 	"github.com/google/uuid"
 )
 
-// Generate humbug client to be able write data in Bugout journal.
-func humbugClientFromEnv() (*humbug.HumbugReporter, error) {
-	clientID := os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID")
-	humbugToken := os.Getenv("ETHTXPOOL_HUMBUG_TOKEN")
-	sessionID := uuid.New().String()
-
+// Generate humbug client
+func humbugClient(sessionID string, clientID string, humbugToken string) (*humbug.HumbugReporter, error) {
 	consent := humbug.CreateHumbugConsent(humbug.True)
 	reporter, err := humbug.CreateHumbugReporter(consent, clientID, sessionID, humbugToken)
 	return reporter, err
@@ -185,6 +181,23 @@ func main() {
 	flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)")
 	flag.Parse()
 
+	sessionID := uuid.New().String()
+	
+	// Humbug crash client to collect errors
+	crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN"))
+	if err != nil {
+		panic(fmt.Sprintf("Invalid Humbug Crash configuration: %s", err.Error()))
+	}
+	crashReporter.Publish(humbug.SystemReport())
+
+	defer func() {
+		message := recover()
+		if message != nil {
+			fmt.Printf("Error: %s\n", message)
+			crashReporter.Publish(humbug.PanicReport(message))
+		}
+	}()
+
 	// Set connection with Ethereum blockchain via geth
 	gethClient, err := rpc.Dial(gethConnectionString)
 	if err != nil {
@@ -192,7 +205,8 @@ func main() {
 	}
 	defer gethClient.Close()
 
-	reporter, err := humbugClientFromEnv()
+	// Humbug client to be able write data in Bugout journal
+	reporter, err := humbugClient(sessionID, os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID"), os.Getenv("ETHTXPOOL_HUMBUG_TOKEN"))
 	if err != nil {
 		panic(fmt.Sprintf("Invalid Humbug configuration: %s", err.Error()))
 	}

crawlers/ethtxpool/sample.env

@@ -1,2 +1,3 @@
 export ETHTXPOOL_HUMBUG_CLIENT_ID="<client id for the crawling machine>"
 export ETHTXPOOL_HUMBUG_TOKEN="<Generate an integration and a Humbug token from https://bugout.dev/account/teams>"
+export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

crawlers/mooncrawl/mooncrawl/__init__.py

@@ -0,0 +1,7 @@
+from .reporter import reporter
+from .version import MOONCRAWL_VERSION
+
+# Reporting
+reporter.tags.append(f"version:{MOONCRAWL_VERSION}")
+reporter.system_report(publish=True)
+reporter.setup_excepthook(publish=True)

crawlers/mooncrawl/mooncrawl/ethcrawler.py

@@ -48,7 +48,7 @@ def yield_blocks_numbers_lists(
         print(
             "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340"
         )
-        return
+        raise Exception
 
     starting_block = max(input_start_block, input_end_block)
     ending_block = min(input_start_block, input_end_block)

crawlers/mooncrawl/mooncrawl/reporter.py

@@ -0,0 +1,18 @@
+import uuid
+
+from humbug.consent import HumbugConsent
+from humbug.report import HumbugReporter
+
+from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN
+
+session_id = str(uuid.uuid4())
+client_id = "moonstream-crawlers"
+
+reporter = HumbugReporter(
+    name="moonstream-crawlers",
+    consent=HumbugConsent(True),
+    client_id=client_id,
+    session_id=session_id,
+    bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN,
+    tags=[],
+)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -1,5 +1,9 @@
 import os
 
+# Bugout
+HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
+
+# Geth
 MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None)
 
 MOONSTREAM_CRAWL_WORKERS = 4
@@ -12,5 +16,5 @@ except:
         f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}"
     )
 
-
+# Etherscan
 MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN")

crawlers/mooncrawl/sample.env

@@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
 export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
 export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
 export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
+export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

crawlers/mooncrawl/setup.py

@@ -1,6 +1,5 @@
 from setuptools import find_packages, setup
 
-from mooncrawl.version import MOONCRAWL_VERSION
 
 long_description = ""
 with open("README.md") as ifp:
@@ -8,7 +7,7 @@ with open("README.md") as ifp:
 
 setup(
     name="mooncrawl",
-    version=MOONCRAWL_VERSION,
+    version="0.0.3",
     author="Bugout.dev",
     author_email="engineers@bugout.dev",
     license="Apache License 2.0",
@@ -34,6 +33,7 @@ setup(
     zip_safe=False,
     install_requires=[
         "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db",
+        "humbug",
         "python-dateutil",
         "requests",
         "tqdm",