diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 6ab869f0..d87b4a38 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -54,11 +54,12 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: """ Synchronize latest Ethereum blocks with database. """ + starting_block: int = args.start while True: bottom_block_number, top_block_number = get_latest_blocks( bool(strtobool(args.transactions)) ) - bottom_block_number = bottom_block_number + 1 + bottom_block_number = max(bottom_block_number + 1, starting_block) if bottom_block_number >= top_block_number: print( f"Synchronization is unnecessary for blocks {bottom_block_number}-{top_block_number - 1}" @@ -178,6 +179,13 @@ def main() -> None: default="True", help="Add or not block transactions", ) + parser_ethcrawler_blocks_sync.add_argument( + "-s", + "--start", + type=int, + default=0, + help="(Optional) Block to start synchronization from. Default: 0", + ) parser_ethcrawler_blocks_sync.set_defaults(func=ethcrawler_blocks_sync_handler) parser_ethcrawler_blocks_add = subcommands_ethcrawler_blocks.add_parser( diff --git a/crawlers/moonstreamcrawlers/esd.py b/crawlers/moonstreamcrawlers/esd.py index e2be8c44..1a2fff6c 100644 --- a/crawlers/moonstreamcrawlers/esd.py +++ b/crawlers/moonstreamcrawlers/esd.py @@ -18,13 +18,18 @@ DB_MODELS = { "events": ESDEventSignature, } -def crawl_step(db_session: Session, crawl_url: str, db_model: Union[ESDEventSignature, ESDFunctionSignature]) -> Optional[str]: + +def crawl_step( + db_session: Session, + crawl_url: str, + db_model: Union[ESDEventSignature, ESDFunctionSignature], +) -> Optional[str]: attempt = 0 current_interval = 2 success = False response: Optional[requests.Response] = None - while (not success) and attempt < 3: + while (not success) and attempt < 3: attempt += 1 try: response = requests.get(crawl_url) @@ -41,12 +46,21 @@ def crawl_step(db_session: Session, crawl_url: str, db_model: Union[ESDEventSign page = response.json() results = page.get("results", []) - rows = [db_model(id=row.get("id"), text_signature=row.get("text_signature"), hex_signature=row.get("hex_signature"), created_at=row.get("created_at")) for row in results] + rows = [ + db_model( + id=row.get("id"), + text_signature=row.get("text_signature"), + hex_signature=row.get("hex_signature"), + created_at=row.get("created_at"), + ) + for row in results + ] db_session.bulk_save_objects(rows) db_session.commit() return page.get("next") + def crawl(crawl_type: str, interval: float) -> None: crawl_url: Optional[str] = CRAWL_URLS[crawl_type] db_model = DB_MODELS[crawl_type] @@ -56,13 +70,26 @@ def crawl(crawl_type: str, interval: float) -> None: crawl_url = crawl_step(db_session, crawl_url, db_model) time.sleep(interval) + def main(): - parser = argparse.ArgumentParser(description="Crawls function and event signatures from the Ethereum Signature Database (https://www.4byte.directory/)") - parser.add_argument("crawl_type", choices=CRAWL_URLS, help="Specifies whether to crawl function signatures or event signatures") - parser.add_argument("--interval", type=float, default=0.1, help="Number of seconds to wait between requests to the Ethereum Signature Database API") + parser = argparse.ArgumentParser( + description="Crawls function and event signatures from the Ethereum Signature Database (https://www.4byte.directory/)" + ) + parser.add_argument( + "crawl_type", + choices=CRAWL_URLS, + help="Specifies whether to crawl function signatures or event signatures", + ) + parser.add_argument( + "--interval", + type=float, + default=0.1, + help="Number of seconds to wait between requests to the Ethereum Signature Database API", + ) args = parser.parse_args() crawl(args.crawl_type, args.interval) + if __name__ == "__main__": - main() \ No newline at end of file + main()