From 31d9ff6f48931aecfe769f6b6859793d9911f969 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 15 Oct 2021 09:03:07 +0000 Subject: [PATCH 1/3] Fix sync blocks logic --- crawlers/mooncrawl/mooncrawl/ethcrawler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index e04d6dae..e89e4781 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -97,6 +97,13 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: ) if latest_stored_block_number is None: latest_stored_block_number = 0 + + if latest_stored_block_number >= latest_block_number: + logger.info( + f"Synchronization is unnecessary for blocks {latest_stored_block_number}-{latest_block_number - 1}" + ) + time.sleep(5) + continue block_number_difference = latest_block_number - 1 - latest_stored_block_number @@ -108,17 +115,10 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: time.sleep(5) continue else: - bottom_block_number = latest_block_number - args.confirmations + bottom_block_number = latest_stored_block_number + 1 else: bottom_block_number = max(latest_stored_block_number + 1, args.start) - if latest_stored_block_number >= latest_block_number: - logger.info( - f"Synchronization is unnecessary for blocks {latest_stored_block_number}-{latest_block_number - 1}" - ) - time.sleep(5) - continue - for blocks_numbers_list in yield_blocks_numbers_lists( f"{bottom_block_number}-{latest_block_number}", order=args.order, From 55c5e89bbed4334ae1245ef72b72f5d7677a0736 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 15 Oct 2021 09:33:56 +0000 Subject: [PATCH 2/3] Logic if block difference to large to catch up with crawler --- crawlers/mooncrawl/mooncrawl/ethcrawler.py | 29 +++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index e89e4781..2317ce6d 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -97,7 +97,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: ) if latest_stored_block_number is None: latest_stored_block_number = 0 - + if latest_stored_block_number >= latest_block_number: logger.info( f"Synchronization is unnecessary for blocks {latest_stored_block_number}-{latest_block_number - 1}" @@ -106,18 +106,23 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: continue block_number_difference = latest_block_number - 1 - latest_stored_block_number - - if args.start is None: - if block_number_difference < args.confirmations: - logger.info( - f"Synchronization is unnecessary for blocks {latest_stored_block_number}-{latest_block_number - 1}" - ) - time.sleep(5) - continue - else: - bottom_block_number = latest_stored_block_number + 1 + if block_number_difference >= 70: + logger.warning( + f"Block difference is too large: {block_number_difference}, crawling {args.confirmations + 1} latest blocks" + ) + bottom_block_number = latest_block_number - args.confirmations + 1 else: - bottom_block_number = max(latest_stored_block_number + 1, args.start) + if args.start is None: + if block_number_difference < args.confirmations: + logger.info( + f"Synchronization is unnecessary for blocks {latest_stored_block_number}-{latest_block_number - 1}" + ) + time.sleep(5) + continue + else: + bottom_block_number = latest_stored_block_number + 1 + else: + bottom_block_number = max(latest_stored_block_number + 1, args.start) for blocks_numbers_list in yield_blocks_numbers_lists( f"{bottom_block_number}-{latest_block_number}", From 68357ec2c4877758ad84aa3cd4bb366b3f22615a Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 15 Oct 2021 09:36:22 +0000 Subject: [PATCH 3/3] Typo fix --- crawlers/mooncrawl/mooncrawl/ethcrawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index 2317ce6d..d684ee63 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -110,7 +110,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None: logger.warning( f"Block difference is too large: {block_number_difference}, crawling {args.confirmations + 1} latest blocks" ) - bottom_block_number = latest_block_number - args.confirmations + 1 + bottom_block_number = latest_block_number - args.confirmations - 1 else: if args.start is None: if block_number_difference < args.confirmations: