diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index 0b0e3fd5..2b271287 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -319,9 +319,9 @@ def label_erc721_addresses( db_session.bulk_save_objects(labels) db_session.commit() except Exception as e: - reporter.error_report(e, ["nft-crawler"], True) db_session.rollback() - logger.error(f"Failed to save labels to db:\n{e}") + logger.error(f"Failed to save erc721 labels to db:\n{e}") + raise e def label_key(label: EthereumLabel) -> Tuple[str, int, int, str, str]: @@ -375,10 +375,10 @@ def label_transfers( db_session.bulk_save_objects(new_labels) db_session.commit() except Exception as e: - reporter.error_report(e, ["nft-crawler"], True) db_session.rollback() logger.error("Could not write transfer/mint labels to database") logger.error(e) + raise e def add_labels( @@ -440,7 +440,11 @@ def add_labels( batch_start = start batch_end = min(start + batch_size - 1, end) - address_ids: Dict[str, int] = {} + # TODO(yhtiyar): Make address_ids as global cache to fast up crawling + # address_ids: Dict[str, int] = {} + # For now quitting this idea because some contracts have unicode escapes + # in their names, and global cache will fuck up not only that batch labeling + # but latet ones as well pbar = tqdm(total=(end - start + 1)) pbar.set_description(f"Labeling blocks {start}-{end}") @@ -453,6 +457,7 @@ def add_labels( ) contract_addresses = {transfer.contract_address for transfer in job} updated_address_ids = ensure_addresses(db_session, contract_addresses) + address_ids: Dict[str, int] = {} for address, address_id in updated_address_ids.items(): address_ids[address] = address_id @@ -472,10 +477,32 @@ def add_labels( ] # Add 'erc721' labels - label_erc721_addresses(w3, db_session, unlabelled_address_ids) + try: + label_erc721_addresses(w3, db_session, unlabelled_address_ids) + except Exception as e: + reporter.error_report( + e, + [ + "nft_crawler", + "erc721_label", + f"batch_start:{batch_start}", + f"batch_end:{batch_end}", + ], + ) # Add mint/transfer labels to (transaction, contract_address) pairs - label_transfers(db_session, job, updated_address_ids) + try: + label_transfers(db_session, job, updated_address_ids) + except Exception as e: + reporter.error_report( + e, + [ + "nft_crawler", + "nft_transfer", + f"batch_start:{batch_start}", + f"batch_end:{batch_end}", + ], + ) # Update batch at end of iteration pbar.update(batch_end - batch_start + 1)