Merge pull request #94 from zomglings/crawlers-ethereum-trending

Systemd service and timer definition for "ethcrawler trending" crawl

crawlers/deploy/ethereum-trending.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Load trending Ethereum addresses to the database
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers
+EnvironmentFile=/home/ubuntu/mooncrawl-secrets/app.env
+ExecStart=/usr/bin/bash -c '/home/ubuntu/mooncrawl-env/bin/python -m mooncrawl.ethcrawler trending'

crawlers/deploy/ethereum-trending.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Load trending Ethereum addresses to the database every 5 minutes
+
+[Timer]
+OnBootSec=10s
+OnUnitActiveSec=5m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/ethcrawler.py

@@ -2,8 +2,10 @@
 Moonstream crawlers CLI.
 """
 import argparse
+from datetime import datetime, timedelta, timezone
 from enum import Enum
 import json
+import os
 import sys
 import time
 from typing import Iterator, List
@@ -21,6 +23,7 @@ from .ethereum import (
 )
 from .publish import publish_json
 from .settings import MOONSTREAM_CRAWL_WORKERS
+from .version import MOONCRAWL_VERSION
 
 
 class ProcessingOrder(Enum):
@@ -177,11 +180,20 @@ def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
         include_end=args.include_end,
     )
     results = trending(date_range)
-    if args.humbug:
+    humbug_token = args.humbug
+    if humbug_token is None:
+        humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN")
+    if humbug_token:
         opening_bracket = "[" if args.include_start else "("
         closing_bracket = "]" if args.include_end else ")"
         title = f"Ethereum trending addresses: {opening_bracket}{args.start}, {args.end}{closing_bracket}"
-        publish_json("ethereum_trending", args.humbug, title, results)
+        publish_json(
+            "ethereum_trending",
+            humbug_token,
+            title,
+            results,
+            tags=[f"crawler_version:{MOONCRAWL_VERSION}"],
+        )
     with args.outfile as ofp:
         json.dump(results, ofp)
 
@@ -191,6 +203,8 @@ def main() -> None:
     parser.set_defaults(func=lambda _: parser.print_help())
     subcommands = parser.add_subparsers(description="Crawlers commands")
 
+    time_now = datetime.now(timezone.utc)
+
     # Ethereum blocks parser
     parser_ethcrawler_blocks = subcommands.add_parser(
         "blocks", description="Ethereum blocks commands"
@@ -335,8 +349,8 @@ def main() -> None:
         "-s",
         "--start",
         type=dateutil.parser.parse,
-        required=True,
+        default=(time_now - timedelta(hours=1, minutes=0)).isoformat(),
-        help="Start time for window to calculate trending addresses in",
+        help=f"Start time for window to calculate trending addresses in (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})",
     )
     parser_ethcrawler_trending.add_argument(
         "--include-start",
@@ -347,8 +361,8 @@ def main() -> None:
         "-e",
         "--end",
         type=dateutil.parser.parse,
-        required=True,
+        default=time_now.isoformat(),
-        help="End time for window to calculate trending addresses in",
+        help=f"End time for window to calculate trending addresses in (default: {time_now.isoformat()})",
     )
     parser_ethcrawler_trending.add_argument(
         "--include-end",
@@ -358,7 +372,11 @@ def main() -> None:
     parser_ethcrawler_trending.add_argument(
         "--humbug",
         default=None,
-        help="If you would like to write this data to a Moonstream journal, please provide a Humbug token for that here.",
+        help=(
+            "If you would like to write this data to a Moonstream journal, please provide a Humbug "
+            "token for that here. (This argument overrides any value set in the "
+            "MOONSTREAM_HUMBUG_TOKEN environment variable)"
+        ),
     )
     parser_ethcrawler_trending.add_argument(
         "-o",

crawlers/mooncrawl/ethereum.py

@@ -317,7 +317,14 @@ def trending(
 
         return query
 
-    results: Dict[str, Any] = {}
+    results: Dict[str, Any] = {
+        "date_range": {
+            "start_time": date_range.start_time.isoformat(),
+            "end_time": date_range.end_time.isoformat(),
+            "include_start": date_range.include_start,
+            "include_end": date_range.include_end,
+        }
+    }
 
     try:
         transactions_out_query = make_query(