Fixes for crawler mypy

.github/workflows/lint.crawlers.yml

@@ -19,9 +19,9 @@ jobs:
       - name: Install test requirements
         working-directory: ./crawlers
         run: pip install -e .[dev]
-      # - name: Mypy type check
-      #   working-directory: ./crawlers
-      #   run: mypy mooncrawl/
+      - name: Mypy type check
+        working-directory: ./crawlers
+        run: mypy mooncrawl/
       - name: Black syntax check
         working-directory: ./crawlers
         run: black --check mooncrawl/

crawlers/mooncrawl/ethcrawler.py

@@ -91,6 +91,9 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
     starting_block: int = args.start
     while True:
         bottom_block_number, top_block_number = get_latest_blocks(args.confirmations)
+        if bottom_block_number is None:
+            print("Variable bottom_block_number can't be None")
+            return
         bottom_block_number = max(bottom_block_number + 1, starting_block)
         if bottom_block_number >= top_block_number:
             print(

crawlers/mooncrawl/ethereum.py

@@ -44,9 +44,11 @@ def connect(web3_uri: Optional[str] = MOONSTREAM_IPC_PATH):
     return web3_client
 
 
-def add_block(db_session, block: BlockData) -> None:
+def add_block(db_session, block: Any) -> None:
     """
     Add block if doesn't presented in database.
+
+    block: web3.types.BlockData
     """
     block_obj = EthereumBlock(
         block_number=block.number,
@@ -70,9 +72,11 @@ def add_block(db_session, block: BlockData) -> None:
     db_session.add(block_obj)
 
 
-def add_block_transactions(db_session, block: BlockData) -> None:
+def add_block_transactions(db_session, block: Any) -> None:
     """
     Add block transactions.
+
+    block: web3.types.BlockData
     """
     for tx in block.transactions:
         tx_obj = EthereumTransaction(
@@ -188,7 +192,7 @@ def crawl_blocks_executor(
     Returns nothing, but if there was an error processing the given blocks it raises an EthereumBlocksCrawlError.
     The error message is a list of all the things that went wrong in the crawl.
     """
-    errors: List[Exception] = []
+    errors: List[BaseException] = []
 
     def record_error(f: Future) -> None:
         error = f.exception()
@@ -196,7 +200,7 @@ def crawl_blocks_executor(
             errors.append(error)
 
     worker_indices = range(MOONSTREAM_CRAWL_WORKERS)
-    worker_job_lists = [[] for _ in worker_indices]
+    worker_job_lists: List[List[Any]] = [[] for _ in worker_indices]
     for i, block_number in enumerate(block_numbers_list):
         worker_job_lists[i % MOONSTREAM_CRAWL_WORKERS].append(block_number)
 
@@ -290,6 +294,7 @@ def trending(
     end_timestamp = int(date_range.end_time.timestamp())
 
     def make_query(
+        db_session: Session,
         identifying_column: Column,
         statistic_column: Column,
         aggregate_func: Callable,
@@ -328,6 +333,7 @@ def trending(
 
     try:
         transactions_out_query = make_query(
+            db_session,
             EthereumTransaction.from_address,
             EthereumTransaction.hash,
             func.count,
@@ -339,6 +345,7 @@ def trending(
         ]
 
         transactions_in_query = make_query(
+            db_session,
             EthereumTransaction.to_address,
             EthereumTransaction.hash,
             func.count,
@@ -350,6 +357,7 @@ def trending(
         ]
 
         value_out_query = make_query(
+            db_session,
             EthereumTransaction.from_address,
             EthereumTransaction.value,
             func.sum,
@@ -361,6 +369,7 @@ def trending(
         ]
 
         value_in_query = make_query(
+            db_session,
             EthereumTransaction.to_address,
             EthereumTransaction.value,
             func.sum,

crawlers/mooncrawl/etherscan.py

@@ -1,21 +1,22 @@
 import argparse
-import boto3
+import sys
+import time
+from datetime import datetime
+from typing import Any, List, Optional, Dict
+from dataclasses import dataclass
 import csv
 import codecs
 import json
 import os
-from sqlalchemy.orm import Session
+
+import boto3  # type: ignore
 from moonstreamdb.db import yield_db_session_ctx
-import sys
-import time
-from datetime import datetime
-from typing import Any, List, Optional, Tuple, Dict
-from dataclasses import dataclass
-from sqlalchemy.sql.expression import label, text
-from .version import MOONCRAWL_VERSION
 from moonstreamdb.models import EthereumAddress, EthereumLabel
 import requests
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.expression import text
 
+from .version import MOONCRAWL_VERSION
 from .settings import MOONSTREAM_ETHERSCAN_TOKEN
 
 if MOONSTREAM_ETHERSCAN_TOKEN is None:

crawlers/mooncrawl/test_cli.py

@@ -1,13 +1,13 @@
 import unittest
 
-from . import cli
+from . import ethcrawler
 
 
 class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_10_6_step_4(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
                 "10-6", block_step=4
             )
         ]
@@ -16,7 +16,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_10_6_step_3(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
                 "10-6", block_step=3
             )
         ]
@@ -25,8 +25,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_10_6_descending_step_3(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "10-6", cli.ProcessingOrder.DESCENDING, 3
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "10-6", ethcrawler.ProcessingOrder.DESCENDING, 3
             )
         ]
         self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@@ -34,8 +34,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_10_6_descending_step_10(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "10-6", cli.ProcessingOrder.DESCENDING, 10
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "10-6", ethcrawler.ProcessingOrder.DESCENDING, 10
             )
         ]
         self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@@ -43,7 +43,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_6_10_step_4(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
                 "6-10", block_step=4
             )
         ]
@@ -52,7 +52,7 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_6_10_step_3(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
                 "6-10", block_step=3
             )
         ]
@@ -61,8 +61,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_6_10_descending_step_3(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "6-10", cli.ProcessingOrder.DESCENDING, 3
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "6-10", ethcrawler.ProcessingOrder.DESCENDING, 3
             )
         ]
         self.assertListEqual(partition, [[10, 9, 8], [7, 6]])
@@ -70,8 +70,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_descending_6_10_descending_step_10(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "6-10", cli.ProcessingOrder.DESCENDING, 10
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "6-10", ethcrawler.ProcessingOrder.DESCENDING, 10
             )
         ]
         self.assertListEqual(partition, [[10, 9, 8, 7, 6]])
@@ -79,8 +79,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_ascending_10_6_ascending_step_3(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "10-6", cli.ProcessingOrder.ASCENDING, 3
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "10-6", ethcrawler.ProcessingOrder.ASCENDING, 3
             )
         ]
         self.assertListEqual(partition, [[6, 7, 8], [9, 10]])
@@ -88,8 +88,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_ascending_10_6_ascending_step_10(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "10-6", cli.ProcessingOrder.ASCENDING, 10
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "10-6", ethcrawler.ProcessingOrder.ASCENDING, 10
             )
         ]
         self.assertListEqual(partition, [[6, 7, 8, 9, 10]])
@@ -97,8 +97,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_ascending_6_10_ascending_step_4(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "6-10", cli.ProcessingOrder.ASCENDING, 4
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "6-10", ethcrawler.ProcessingOrder.ASCENDING, 4
             )
         ]
         self.assertListEqual(partition, [[6, 7, 8, 9], [10]])
@@ -106,8 +106,8 @@ class TestYieldBlockNumbersLists(unittest.TestCase):
     def test_yield_ascending_6_10_ascending_step_10(self):
         partition = [
             block_numbers_list
-            for block_numbers_list in cli.yield_blocks_numbers_lists(
-                "6-10", cli.ProcessingOrder.ASCENDING, 10
+            for block_numbers_list in ethcrawler.yield_blocks_numbers_lists(
+                "6-10", ethcrawler.ProcessingOrder.ASCENDING, 10
             )
         ]
         self.assertListEqual(partition, [[6, 7, 8, 9, 10]])

crawlers/setup.py

@@ -40,7 +40,9 @@ setup(
         "web3",
         "boto3",
     ],
-    extras_require={"dev": ["black", "mypy", "types-requests"]},
+    extras_require={
+        "dev": ["black", "mypy", "types-requests", "types-python-dateutil"]
+    },
     entry_points={
         "console_scripts": [
             "ethcrawler=mooncrawl.ethcrawler:main",