Merge pull request #398 from bugout-dev/missing-timer-crawler

Missing block finder automatization

crawlers/deploy/deploy.bash

@@ -36,11 +36,15 @@ CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
 # Ethereum service files
 ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
 ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
-ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.service"
+ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
 ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
+ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
+ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
 
 # Polygon service file
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
+POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
+POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
 
 
 set -eu
@@ -112,6 +116,15 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${ETHERE
 systemctl daemon-reload
 systemctl restart "${ETHEREUM_TXPOOL_SERVICE_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}"
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}"
@@ -128,3 +141,12 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV
 systemctl daemon-reload
 systemctl restart "${CRAWLERS_SERVICE_FILE}"
 systemctl status "${CRAWLERS_SERVICE_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Polygon missing service and timer with: ${POLYGON_MISSING_SERVICE_FILE}, ${POLYGON_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart "${POLYGON_MISSING_TIMER_FILE}"

crawlers/deploy/ethereum-missing.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at Ethereum database
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler missing --blockchain ethereum -n

crawlers/deploy/ethereum-missing.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at Ethereum database
+
+[Timer]
+OnBootSec=10s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/polygon-missing.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at Polygon database
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler missing --blockchain polygon -n

crawlers/deploy/polygon-missing.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at Polygon database
+
+[Timer]
+OnBootSec=10s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target

crawlers/mooncrawl/mooncrawl/crawler.py

@@ -165,11 +165,23 @@ def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
 def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
     """
     Check missing blocks and missing transactions in each block.
+    If block range doesn't provided, get latest block from blockchain minus 50,
+    and check last 2000 blocks behind.
     """
     startTime = time.time()
 
     missing_blocks_numbers_total = []
-    for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
+
+    block_range = args.blocks
+    if block_range is None:
+        confirmations = 50
+        shift = 2000
+        _, latest_block_number = get_latest_blocks(
+            AvailableBlockchainType(args.blockchain), confirmations
+        )
+        block_range = f"{latest_block_number-shift}-{latest_block_number}"
+
+    for blocks_numbers_list in yield_blocks_numbers_lists(block_range):
         logger.info(
             f"Checking missing blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]} "
             f"with comparing transactions: {not args.notransactions}"
@@ -319,7 +331,6 @@ def main() -> None:
     parser_crawler_blocks_missing.add_argument(
         "-b",
         "--blocks",
-        required=True,
         help="List of blocks range in format {bottom_block}-{top_block}",
     )
     parser_crawler_blocks_missing.add_argument(