From 44fb2bc8c70f3450cd870da291c0bd0202f7f17b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 13:41:31 +0000 Subject: [PATCH 1/4] Humbug reporter for moonstream python crawlers --- crawlers/mooncrawl/mooncrawl/__init__.py | 7 +++++++ crawlers/mooncrawl/mooncrawl/ethcrawler.py | 2 +- crawlers/mooncrawl/mooncrawl/reporter.py | 18 ++++++++++++++++++ crawlers/mooncrawl/mooncrawl/settings.py | 6 +++++- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 1 + 6 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 crawlers/mooncrawl/mooncrawl/reporter.py diff --git a/crawlers/mooncrawl/mooncrawl/__init__.py b/crawlers/mooncrawl/mooncrawl/__init__.py index e69de29b..9548e297 100644 --- a/crawlers/mooncrawl/mooncrawl/__init__.py +++ b/crawlers/mooncrawl/mooncrawl/__init__.py @@ -0,0 +1,7 @@ +from .reporter import reporter +from .version import MOONCRAWL_VERSION + +# Reporting +reporter.tags.append(f"version:{MOONCRAWL_VERSION}") +reporter.system_report(publish=True) +reporter.setup_excepthook(publish=True) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index 793ab3ea..5e9275c4 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -48,7 +48,7 @@ def yield_blocks_numbers_lists( print( "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340" ) - return + raise Exception starting_block = max(input_start_block, input_end_block) ending_block = min(input_start_block, input_end_block) diff --git a/crawlers/mooncrawl/mooncrawl/reporter.py b/crawlers/mooncrawl/mooncrawl/reporter.py new file mode 100644 index 00000000..96f3de4a --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/reporter.py @@ -0,0 +1,18 @@ +import uuid + +from humbug.consent import HumbugConsent +from humbug.report import HumbugReporter + +from .settings import HUMBUG_REPORTER_CRAWLERS_TOKEN + +session_id = str(uuid.uuid4()) +client_id = "moonstream-crawlers" + +reporter = HumbugReporter( + name="moonstream", + consent=HumbugConsent(True), + client_id=client_id, + session_id=session_id, + bugout_token=HUMBUG_REPORTER_CRAWLERS_TOKEN, + tags=[], +) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 82b13772..307f2e8a 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -1,5 +1,9 @@ import os +# Bugout +HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") + +# Geth MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) MOONSTREAM_CRAWL_WORKERS = 4 @@ -12,5 +16,5 @@ except: f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}" ) - +# Etherscan MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 5ebad6e0..5d4ac548 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -6,3 +6,4 @@ export MOONSTREAM_ETHERSCAN_TOKEN="" export AWS_S3_SMARTCONTRACT_BUCKET="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 1e893bd4..46b397a3 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,6 +34,7 @@ setup( zip_safe=False, install_requires=[ "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@39d2b8e36a49958a9ae085ec2cc1be3fc732b9d0#egg=moonstreamdb&subdirectory=db", + "humbug", "python-dateutil", "requests", "tqdm", From ec19d9e739dfa26f841fe43e7f5180fdff644212 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 13:56:11 +0000 Subject: [PATCH 2/4] Specified a name for report for crawlers --- crawlers/mooncrawl/mooncrawl/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/reporter.py b/crawlers/mooncrawl/mooncrawl/reporter.py index 96f3de4a..0cf170ac 100644 --- a/crawlers/mooncrawl/mooncrawl/reporter.py +++ b/crawlers/mooncrawl/mooncrawl/reporter.py @@ -9,7 +9,7 @@ session_id = str(uuid.uuid4()) client_id = "moonstream-crawlers" reporter = HumbugReporter( - name="moonstream", + name="moonstream-crawlers", consent=HumbugConsent(True), client_id=client_id, session_id=session_id, From 8bbfd1e911d9621c3839fc06fb89fd7130fd8126 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 14:45:05 +0000 Subject: [PATCH 3/4] Humbug crash reporter for moonstream crawlers --- crawlers/ethtxpool/main.go | 28 +++++++++++++++++++++------- crawlers/ethtxpool/sample.env | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index 224ae7dc..217ec8fa 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -21,12 +21,8 @@ import ( "github.com/google/uuid" ) -// Generate humbug client to be able write data in Bugout journal. -func humbugClientFromEnv() (*humbug.HumbugReporter, error) { - clientID := os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID") - humbugToken := os.Getenv("ETHTXPOOL_HUMBUG_TOKEN") - sessionID := uuid.New().String() - +// Generate humbug client +func humbugClient(sessionID string, clientID string, humbugToken string) (*humbug.HumbugReporter, error) { consent := humbug.CreateHumbugConsent(humbug.True) reporter, err := humbug.CreateHumbugReporter(consent, clientID, sessionID, humbugToken) return reporter, err @@ -188,6 +184,23 @@ func main() { flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)") flag.Parse() + sessionID := uuid.New().String() + + // Humbug crash client to collect errors + crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN")) + if err != nil { + panic(fmt.Sprintf("Invalid Humbug Crash configuration: %s", err.Error())) + } + crashReporter.Publish(humbug.SystemReport()) + + defer func() { + message := recover() + if message != nil { + fmt.Printf("Error: %s\n", message) + crashReporter.Publish(humbug.PanicReport(message)) + } + }() + // Set connection with Ethereum blockchain via geth gethClient, err := rpc.Dial(gethConnectionString) if err != nil { @@ -195,7 +208,8 @@ func main() { } defer gethClient.Close() - reporter, err := humbugClientFromEnv() + // Humbug client to be able write data in Bugout journal + reporter, err := humbugClient(sessionID, os.Getenv("ETHTXPOOL_HUMBUG_CLIENT_ID"), os.Getenv("ETHTXPOOL_HUMBUG_TOKEN")) if err != nil { panic(fmt.Sprintf("Invalid Humbug configuration: %s", err.Error())) } diff --git a/crawlers/ethtxpool/sample.env b/crawlers/ethtxpool/sample.env index abd4bc8e..809cc325 100644 --- a/crawlers/ethtxpool/sample.env +++ b/crawlers/ethtxpool/sample.env @@ -1,2 +1,3 @@ export ETHTXPOOL_HUMBUG_CLIENT_ID="" export ETHTXPOOL_HUMBUG_TOKEN="" +export HUMBUG_REPORTER_CRAWLERS_TOKEN="" From 6388716156ff9fb4ade5392584c0d7c85d65de91 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 1 Sep 2021 17:34:02 +0000 Subject: [PATCH 4/4] Hardcoded version in setup.py --- crawlers/mooncrawl/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 46b397a3..f26b82d5 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -1,6 +1,5 @@ from setuptools import find_packages, setup -from mooncrawl.version import MOONCRAWL_VERSION long_description = "" with open("README.md") as ifp: @@ -8,7 +7,7 @@ with open("README.md") as ifp: setup( name="mooncrawl", - version=MOONCRAWL_VERSION, + version="0.0.3", author="Bugout.dev", author_email="engineers@bugout.dev", license="Apache License 2.0",