From 35b899bf389100546792464e83ec0bd4469abb46 Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Fri, 12 Nov 2021 18:09:55 +0000
Subject: [PATCH 1/3] Missing block finder automatization

---
 crawlers/deploy/deploy.bash             | 22 +++++++++++++++++++++-
 crawlers/mooncrawl/mooncrawl/crawler.py | 15 +++++++++++++--
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash
index 60e50f43..ba40923c 100755
--- a/crawlers/deploy/deploy.bash
+++ b/crawlers/deploy/deploy.bash
@@ -36,8 +36,10 @@ CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
 # Ethereum service files
 ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
 ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
-ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.service"
+ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
 ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
+ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
+ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
 
 # Polygon service file
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
@@ -112,6 +114,15 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${ETHERE
 systemctl daemon-reload
 systemctl restart "${ETHEREUM_TXPOOL_SERVICE_FILE}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}"
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncronizer service definition with ${POLYGON_SYNCHRONIZE_SERVICE}"
@@ -128,3 +139,12 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV
 systemctl daemon-reload
 systemctl restart "${CRAWLERS_SERVICE_FILE}"
 systemctl status "${CRAWLERS_SERVICE_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Polygon missing service and timer with: ${POLYGON_MISSING_SERVICE_FILE}, ${POLYGON_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}"
+systemctl daemon-reload
+systemctl restart "${POLYGON_MISSING_TIMER_FILE}"
diff --git a/crawlers/mooncrawl/mooncrawl/crawler.py b/crawlers/mooncrawl/mooncrawl/crawler.py
index a246e868..b2071553 100644
--- a/crawlers/mooncrawl/mooncrawl/crawler.py
+++ b/crawlers/mooncrawl/mooncrawl/crawler.py
@@ -165,11 +165,23 @@ def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
 def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
     """
     Check missing blocks and missing transactions in each block.
+    If block range doesn't provided, get latest block from blockchain minus 50,
+    and check last 2000 blocks behind.
     """
     startTime = time.time()
 
     missing_blocks_numbers_total = []
-    for blocks_numbers_list in yield_blocks_numbers_lists(args.blocks):
+
+    block_range = args.blocks
+    if block_range is None:
+        confirmations = 50
+        shift = 2000
+        _, latest_block_number = get_latest_blocks(
+            AvailableBlockchainType(args.blockchain), confirmations
+        )
+        block_range = f"{latest_block_number-shift}-{latest_block_number}"
+
+    for blocks_numbers_list in yield_blocks_numbers_lists(block_range):
         logger.info(
             f"Checking missing blocks {blocks_numbers_list[-1]}-{blocks_numbers_list[0]} "
             f"with comparing transactions: {not args.notransactions}"
@@ -319,7 +331,6 @@ def main() -> None:
     parser_crawler_blocks_missing.add_argument(
         "-b",
         "--blocks",
-        required=True,
         help="List of blocks range in format {bottom_block}-{top_block}",
     )
     parser_crawler_blocks_missing.add_argument(

From 273e6f0ef9af2d023b286a81fcbb93c0000191df Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Fri, 12 Nov 2021 18:12:07 +0000
Subject: [PATCH 2/3] Polygon service and timer variables

---
 crawlers/deploy/deploy.bash | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash
index ba40923c..f8c5dc50 100755
--- a/crawlers/deploy/deploy.bash
+++ b/crawlers/deploy/deploy.bash
@@ -43,6 +43,8 @@ ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
 
 # Polygon service file
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
+POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
+POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
 
 
 set -eu

From 1b7ac6b2636c7c6ceff57f84337b11b35cb2bbb2 Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Fri, 12 Nov 2021 18:13:01 +0000
Subject: [PATCH 3/3] Service and timer files

---
 crawlers/deploy/ethereum-missing.service | 11 +++++++++++
 crawlers/deploy/ethereum-missing.timer   |  9 +++++++++
 crawlers/deploy/polygon-missing.service  | 11 +++++++++++
 crawlers/deploy/polygon-missing.timer    |  9 +++++++++
 4 files changed, 40 insertions(+)
 create mode 100644 crawlers/deploy/ethereum-missing.service
 create mode 100644 crawlers/deploy/ethereum-missing.timer
 create mode 100644 crawlers/deploy/polygon-missing.service
 create mode 100644 crawlers/deploy/polygon-missing.timer

diff --git a/crawlers/deploy/ethereum-missing.service b/crawlers/deploy/ethereum-missing.service
new file mode 100644
index 00000000..91c7ce0d
--- /dev/null
+++ b/crawlers/deploy/ethereum-missing.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at Ethereum database
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler missing --blockchain ethereum -n
\ No newline at end of file
diff --git a/crawlers/deploy/ethereum-missing.timer b/crawlers/deploy/ethereum-missing.timer
new file mode 100644
index 00000000..87fc6c16
--- /dev/null
+++ b/crawlers/deploy/ethereum-missing.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at Ethereum database
+
+[Timer]
+OnBootSec=10s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target
diff --git a/crawlers/deploy/polygon-missing.service b/crawlers/deploy/polygon-missing.service
new file mode 100644
index 00000000..0627d6e0
--- /dev/null
+++ b/crawlers/deploy/polygon-missing.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at Polygon database
+After=network.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler missing --blockchain polygon -n
\ No newline at end of file
diff --git a/crawlers/deploy/polygon-missing.timer b/crawlers/deploy/polygon-missing.timer
new file mode 100644
index 00000000..9fd8a52e
--- /dev/null
+++ b/crawlers/deploy/polygon-missing.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at Polygon database
+
+[Timer]
+OnBootSec=10s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target