From 4b7d0cb35cd36facda611a49f0282d10a511f79d Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 26 Jul 2021 22:16:59 +0300 Subject: [PATCH 1/4] Init of ethereum crawler folder --- crawlers/.gitignore | 165 ++++++++++++++++++++++++ crawlers/README.md | 0 crawlers/moonstreamcrawlers/__init__.py | 0 crawlers/moonstreamcrawlers/cli.py | 0 crawlers/moonstreamcrawlers/ethereum.py | 0 crawlers/requirements.txt | 0 crawlers/setup.py | 0 7 files changed, 165 insertions(+) create mode 100644 crawlers/.gitignore create mode 100644 crawlers/README.md create mode 100644 crawlers/moonstreamcrawlers/__init__.py create mode 100644 crawlers/moonstreamcrawlers/cli.py create mode 100644 crawlers/moonstreamcrawlers/ethereum.py create mode 100644 crawlers/requirements.txt create mode 100644 crawlers/setup.py diff --git a/crawlers/.gitignore b/crawlers/.gitignore new file mode 100644 index 00000000..49a73033 --- /dev/null +++ b/crawlers/.gitignore @@ -0,0 +1,165 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode + +# Custom +dev.env +prod.env +.venv diff --git a/crawlers/README.md b/crawlers/README.md new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/moonstreamcrawlers/__init__.py b/crawlers/moonstreamcrawlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/requirements.txt b/crawlers/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/setup.py b/crawlers/setup.py new file mode 100644 index 00000000..e69de29b From a20e581b52d4f8b54ee375c38ddc0ef4e90c8f26 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 27 Jul 2021 01:15:50 +0300 Subject: [PATCH 2/4] Working crawler to parse blocks and transactions at Ethereum blockchain --- crawlers/README.md | 1 + crawlers/moonstreamcrawlers/cli.py | 96 ++++++++++++++++++++ crawlers/moonstreamcrawlers/ethereum.py | 113 ++++++++++++++++++++++++ crawlers/moonstreamcrawlers/settings.py | 13 +++ crawlers/requirements.txt | Bin 0 -> 1748 bytes crawlers/sample.env | 2 + crawlers/setup.py | 38 ++++++++ 7 files changed, 263 insertions(+) create mode 100644 crawlers/moonstreamcrawlers/settings.py create mode 100644 crawlers/sample.env diff --git a/crawlers/README.md b/crawlers/README.md index e69de29b..ed3b95bd 100644 --- a/crawlers/README.md +++ b/crawlers/README.md @@ -0,0 +1 @@ +# moonstream crawlers \ No newline at end of file diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index e69de29b..40d7e839 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -0,0 +1,96 @@ +""" +Moonstream crawlers CLI. +""" +import argparse +from distutils.util import strtobool +import time + +from .ethereum import crawl +from .settings import MOONSTREAM_CRAWL_WORKERS + + +def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: + """ + Add blocks to moonstream database. + """ + try: + blocks_start_end = args.blocks.split("-") + top_block_number = int(blocks_start_end[1]) + bottom_block_number = int(blocks_start_end[0]) + except Exception: + print( + "Wrong format provided, expected {bottom_block}-{top_block}, as ex. 105-340" + ) + return + + block_step = 1000 + blocks_numbers_list_raw = list(range(top_block_number, bottom_block_number - 1, -1)) + blocks_numbers_list_raw_len = len(blocks_numbers_list_raw) + # Block steps used to prevent long executor tasks and data loss possibility + # Block step 2 convert [1,2,3] -> [[1,2],[3]] + if len(blocks_numbers_list_raw) / block_step > 1: + blocks_numbers_lists = [ + blocks_numbers_list_raw[i : i + block_step] + for i in range(0, blocks_numbers_list_raw_len, block_step) + ] + else: + blocks_numbers_lists = [blocks_numbers_list_raw] + + startTime = time.time() + for blocks_numbers_list in blocks_numbers_lists: + crawl(blocks_numbers_list, bool(strtobool(args.transactions))) + print( + f"Required time: {time.time() - startTime} for: {blocks_numbers_list_raw_len} " + f"blocks with {MOONSTREAM_CRAWL_WORKERS} workers" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Moonstream crawlers CLI") + parser.set_defaults(func=lambda _: parser.print_help()) + subcommands = parser.add_subparsers(description="Crawlers commands") + + parser_ethcrawler = subcommands.add_parser( + "ethcrawler", description="Ethereum crawler" + ) + parser_ethcrawler.set_defaults(func=lambda _: parser_ethcrawler.print_help()) + subcommands_ethcrawler = parser_ethcrawler.add_subparsers( + description="Ethereum crawler commands" + ) + + # Ethereum blocks parser + parser_ethcrawler_blocks = subcommands_ethcrawler.add_parser( + "blocks", description="Ethereum blocks commands" + ) + parser_ethcrawler_blocks.set_defaults( + func=lambda _: parser_ethcrawler_blocks.print_help() + ) + subcommands_ethcrawler_blocks = parser_ethcrawler_blocks.add_subparsers( + description="Ethereum blocks commands" + ) + + parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser( + "add", description="Add ethereum blocks commands" + ) + parser_ethcrawler_blocks_add.add_argument( + "-b", + "--blocks", + required=True, + help="List of blocks range in format {bottom_block}-{top_block}", + ) + parser_ethcrawler_blocks_add.add_argument( + "-t", + "--transactions", + choices=["True", "False"], + default="False", + help="Add or not block transactions", + ) + + parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index e69de29b..4818433d 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -0,0 +1,113 @@ +from concurrent.futures import ProcessPoolExecutor +from typing import List, Optional + +from web3 import Web3 +from web3.types import BlockData + +from .settings import MOONSTREAM_IPC_PATH, MOONSTREAM_CRAWL_WORKERS +from moonstreamdb.db import yield_db_session_ctx +from moonstreamdb.models import EthereumBlock, EthereumTransaction + + +def connect(ipc_path: Optional[str] = MOONSTREAM_IPC_PATH): + web3_client = Web3(Web3.IPCProvider(ipc_path)) + return web3_client + + +def add_block(db_session, block: BlockData, block_number: int) -> None: + """ + Add block if doesn't presented in database. + """ + block_exist = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.block_number == block_number) + .one_or_none() + ) + if block_exist is not None and block_exist.hash == block.hash.hex(): + print(f"Block: {block_number} exists") + return + if block_exist is not None and block_exist.hash != block.hash.hex(): + print(f"Block: {block_number} exists, but incorrect") + db_session.delete(block_exist) + + block_obj = EthereumBlock( + block_number=block.number, + difficulty=block.difficulty, + extra_data=block.extraData.hex(), + gas_limit=block.gasLimit, + gas_used=block.gasUsed, + hash=block.hash.hex(), + logs_bloom=block.logsBloom.hex(), + miner=block.miner, + nonce=block.nonce.hex(), + parent_hash=block.parentHash.hex(), + receipt_root=block.get("receiptRoot", ""), + uncles=block.sha3Uncles.hex(), + size=block.size, + state_root=block.stateRoot.hex(), + timestamp=block.timestamp, + total_difficulty=block.totalDifficulty, + transactions_root=block.transactionsRoot.hex(), + ) + db_session.add(block_obj) + print(f"Added new block: {block_number}") + return + + +def add_block_transaction(db_session, block_number, tx) -> None: + """ + Add block transaction if doesn't presented in database. + """ + tx_exist = ( + db_session.query(EthereumTransaction) + .filter(EthereumTransaction.hash == tx.hash.hex()) + .one_or_none() + ) + if tx_exist is not None: + return + tx_obj = EthereumTransaction( + hash=tx.hash.hex(), + block_number=block_number, + from_address=tx["from"], + to_address=tx.to, + gas=tx.gas, + gas_price=tx.gasPrice, + input=tx.input, + nonce=tx.nonce, + transaction_index=tx.transactionIndex, + value=tx.value, + ) + db_session.add(tx_obj) + + +def process_blocks(blocks_numbers: List[int], with_transactions: bool = False): + """ + Open database and geth sessions and fetch block data from blockchain. + """ + web3_client = connect() + for block_number in blocks_numbers: + with yield_db_session_ctx() as db_session: + block: BlockData = web3_client.eth.get_block( + block_number, full_transactions=with_transactions + ) + add_block(db_session, block, block_number) + if with_transactions: + for tx in block.transactions: + add_block_transaction(db_session, block.number, tx) + db_session.commit() + + +def crawl(block_numbers_list: List[int], with_transactions: bool = False): + """ + Execute crawler. + """ + with ProcessPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor: + for worker in range(1, MOONSTREAM_CRAWL_WORKERS + 1): + print( + f"Added executor for list of blocks with len: {len(block_numbers_list[worker-1::MOONSTREAM_CRAWL_WORKERS])}" + ) + executor.submit( + process_blocks, + block_numbers_list[worker - 1 :: MOONSTREAM_CRAWL_WORKERS], + with_transactions, + ) diff --git a/crawlers/moonstreamcrawlers/settings.py b/crawlers/moonstreamcrawlers/settings.py new file mode 100644 index 00000000..9a511d15 --- /dev/null +++ b/crawlers/moonstreamcrawlers/settings.py @@ -0,0 +1,13 @@ +import os + +MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) + +MOONSTREAM_CRAWL_WORKERS = 4 +MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") +try: + if MOONSTREAM_CRAWL_WORKERS_RAW is not None: + MOONSTREAM_CRAWL_WORKERS = int(MOONSTREAM_CRAWL_WORKERS_RAW) +except: + raise Exception( + f"Could not parse MOONSTREAM_CRAWL_WORKERS as int: {MOONSTREAM_CRAWL_WORKERS_RAW}" + ) diff --git a/crawlers/requirements.txt b/crawlers/requirements.txt index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..3bd0b16c7f5446aa6b107519522d9daefb5c7c6e 100644 GIT binary patch literal 1748 zcmZ`(TW;Gx4CHr#9;JmP$9a78FtY5#YGhf}qlt3(wlkci*8)-qiXD?24u`Y*_fKVw zUF>8Np5A`o+1ht|zmJ#-fXoK1c)dT)6S0>CV9er;tBDKAyYSup3 zKGoMi9-QGbSdv?bh&b$!xxMC{yvqUknt_^d2Jxt&8(EO)gqb@`J7DSx-5dMxE<#t; z*16Rl4Q`JKQL5BHXdR_;99FX*`%2>)^G?WM-iLFSrdb_wIe|@ud1E@Mrye<8Lm}>` zOVbgF^Y5HzKjV5adF}wRJi#;uHGx z=+oJY_tC~<>ybsrkxTYyakI4((B7d9bv&k=b+-l{?=UqdAxC+tLFZ>zLT5#LJg(H^ zxsf-1BJ+aS2#wj1uE=Vucs*h~6FmRKAkut;#7y~u zne^(2nj_|qUQfsnw;o6r?+In{d z?wZ_pu{SX9FmsuZHVI=u-3#7#|2^Lv-2J^P%8Xy&OLt9aayj+>+95lgk<5^~30XeK z6`bu1FN8|+w7eNHo%e8%G-T|v+d_)mo~ZvjKxdvT^>WWMlx*=d*v-~iv2O_tLWke( r1gfaplX2FI^BeSuN_tiIuQ)5l6U2Lxev=dF4DntZTWS!OUFC;=w6*xG literal 0 HcmV?d00001 diff --git a/crawlers/sample.env b/crawlers/sample.env new file mode 100644 index 00000000..7d907c50 --- /dev/null +++ b/crawlers/sample.env @@ -0,0 +1,2 @@ +export MOONSTREAM_IPC_PATH=null +export MOONSTREAM_CRAWL_WORKERS_RAW=4 diff --git a/crawlers/setup.py b/crawlers/setup.py index e69de29b..7be14a58 100644 --- a/crawlers/setup.py +++ b/crawlers/setup.py @@ -0,0 +1,38 @@ +from setuptools import find_packages, setup + +long_description = "" +with open("README.md") as ifp: + long_description = ifp.read() + +setup( + name="moonstreamcrawlers", + version="0.0.1", + author="Bugout.dev", + author_email="engineers@bugout.dev", + license="Apache License 2.0", + description="Moonstream crawlers", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/bugout-dev/moonstream", + platforms="all", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + python_requires=">=3.6", + packages=find_packages(), + package_data={"bugout": ["py.typed"]}, + zip_safe=False, + install_requires=["web3"], + extras_require={"dev": ["black", "mypy"]}, + entry_points={ + "console_scripts": ["moonstreamcrawlers=moonstreamcrawlers.cli:main"] + }, +) From bb16b28a60e3bdae29dde8c18ffa0e24227c0fde Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 27 Jul 2021 17:12:02 +0300 Subject: [PATCH 3/4] Added check argument for crawler --- crawlers/moonstreamcrawlers/cli.py | 13 +++++- crawlers/moonstreamcrawlers/ethereum.py | 53 ++++++++++++++----------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 40d7e839..7991751f 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -38,7 +38,11 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: startTime = time.time() for blocks_numbers_list in blocks_numbers_lists: - crawl(blocks_numbers_list, bool(strtobool(args.transactions))) + crawl( + blocks_numbers=blocks_numbers_list, + with_transactions=bool(strtobool(args.transactions)), + check=bool(strtobool(args.check)), + ) print( f"Required time: {time.time() - startTime} for: {blocks_numbers_list_raw_len} " f"blocks with {MOONSTREAM_CRAWL_WORKERS} workers" @@ -85,6 +89,13 @@ def main() -> None: default="False", help="Add or not block transactions", ) + parser_ethcrawler_blocks_add.add_argument( + "-c", + "--check", + choices=["True", "False"], + default="False", + help="If True, it will check existence of block and transaction before write to database", + ) parser_ethcrawler_blocks_add.set_defaults(func=ethcrawler_blocks_add_handler) diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index 4818433d..a78b3cf8 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -14,21 +14,24 @@ def connect(ipc_path: Optional[str] = MOONSTREAM_IPC_PATH): return web3_client -def add_block(db_session, block: BlockData, block_number: int) -> None: +def add_block( + db_session, block: BlockData, block_number: int, check: bool = False +) -> None: """ Add block if doesn't presented in database. """ - block_exist = ( - db_session.query(EthereumBlock) - .filter(EthereumBlock.block_number == block_number) - .one_or_none() - ) - if block_exist is not None and block_exist.hash == block.hash.hex(): - print(f"Block: {block_number} exists") - return - if block_exist is not None and block_exist.hash != block.hash.hex(): - print(f"Block: {block_number} exists, but incorrect") - db_session.delete(block_exist) + if check: + block_exist = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.block_number == block_number) + .one_or_none() + ) + if block_exist is not None and block_exist.hash == block.hash.hex(): + print(f"Block: {block_number} exists") + return + if block_exist is not None and block_exist.hash != block.hash.hex(): + print(f"Block: {block_number} exists, but incorrect") + db_session.delete(block_exist) block_obj = EthereumBlock( block_number=block.number, @@ -54,17 +57,21 @@ def add_block(db_session, block: BlockData, block_number: int) -> None: return -def add_block_transaction(db_session, block_number, tx) -> None: +def add_block_transaction( + db_session, block_number: int, tx, check: bool = False +) -> None: """ Add block transaction if doesn't presented in database. """ - tx_exist = ( - db_session.query(EthereumTransaction) - .filter(EthereumTransaction.hash == tx.hash.hex()) - .one_or_none() - ) - if tx_exist is not None: - return + if check: + tx_exist = ( + db_session.query(EthereumTransaction) + .filter(EthereumTransaction.hash == tx.hash.hex()) + .one_or_none() + ) + if tx_exist is not None: + return + tx_obj = EthereumTransaction( hash=tx.hash.hex(), block_number=block_number, @@ -80,7 +87,7 @@ def add_block_transaction(db_session, block_number, tx) -> None: db_session.add(tx_obj) -def process_blocks(blocks_numbers: List[int], with_transactions: bool = False): +def process_blocks(blocks_numbers: List[int], with_transactions: bool = False, check: bool = False): """ Open database and geth sessions and fetch block data from blockchain. """ @@ -90,10 +97,10 @@ def process_blocks(blocks_numbers: List[int], with_transactions: bool = False): block: BlockData = web3_client.eth.get_block( block_number, full_transactions=with_transactions ) - add_block(db_session, block, block_number) + add_block(db_session, block, block_number, check=check) if with_transactions: for tx in block.transactions: - add_block_transaction(db_session, block.number, tx) + add_block_transaction(db_session, block.number, tx, check=check) db_session.commit() From 19b2ed19aa31fc5011a3edd070fb822fdfa39344 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 27 Jul 2021 17:46:13 +0300 Subject: [PATCH 4/4] Small fixes in crawler --- crawlers/moonstreamcrawlers/cli.py | 2 +- crawlers/moonstreamcrawlers/ethereum.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 7991751f..0cd2a6f8 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -39,7 +39,7 @@ def ethcrawler_blocks_add_handler(args: argparse.Namespace) -> None: startTime = time.time() for blocks_numbers_list in blocks_numbers_lists: crawl( - blocks_numbers=blocks_numbers_list, + block_numbers_list=blocks_numbers_list, with_transactions=bool(strtobool(args.transactions)), check=bool(strtobool(args.check)), ) diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index a78b3cf8..4250b246 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -87,7 +87,9 @@ def add_block_transaction( db_session.add(tx_obj) -def process_blocks(blocks_numbers: List[int], with_transactions: bool = False, check: bool = False): +def process_blocks( + blocks_numbers: List[int], with_transactions: bool = False, check: bool = False +): """ Open database and geth sessions and fetch block data from blockchain. """ @@ -104,7 +106,9 @@ def process_blocks(blocks_numbers: List[int], with_transactions: bool = False, c db_session.commit() -def crawl(block_numbers_list: List[int], with_transactions: bool = False): +def crawl( + block_numbers_list: List[int], with_transactions: bool = False, check: bool = False +): """ Execute crawler. """ @@ -117,4 +121,5 @@ def crawl(block_numbers_list: List[int], with_transactions: bool = False): process_blocks, block_numbers_list[worker - 1 :: MOONSTREAM_CRAWL_WORKERS], with_transactions, + check, )