diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/abis/erc20.json b/crawlers/mooncrawl/mooncrawl/generic_crawler/abis/erc20.json index 96654e6a..3c39ee09 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/abis/erc20.json +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/abis/erc20.json @@ -45,25 +45,6 @@ "name": "Approval", "type": "event" }, - { - "anonymous": false, - "inputs": [ - { - "indexed": true, - "internalType": "address", - "name": "previousOwner", - "type": "address" - }, - { - "indexed": true, - "internalType": "address", - "name": "newOwner", - "type": "address" - } - ], - "name": "OwnershipTransferred", - "type": "event" - }, { "anonymous": false, "inputs": [ diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py index 2247e8ac..afc900d3 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py @@ -169,17 +169,18 @@ def process_transaction( function_args = "unknown" transaction_reciept = web3.eth.getTransactionReceipt(transaction["hash"]) + block_timestamp = get_block_timestamp( + db_session, + web3, + blockchain_type, + transaction["blockNumber"], + blocks_cache, + 100, + ) function_call = ExtededFunctionCall( block_number=transaction["blockNumber"], - block_timestamp=get_block_timestamp( - db_session, - web3, - blockchain_type, - transaction["blockNumber"], - blocks_cache, - 100, - ), + block_timestamp=block_timestamp, transaction_hash=transaction["hash"].hex(), contract_address=transaction["to"], caller_address=transaction["from"], @@ -207,8 +208,10 @@ def process_transaction( "blockNumber": raw_event["blockNumber"], "transactionHash": raw_event["transactionHash"].hex(), "logIndex": raw_event["logIndex"], + "blockTimestamp": block_timestamp, } - secondary_logs.append(_processEvent(event)) + processed_event = _processEvent(event) + secondary_logs.append(processed_event) break except: @@ -270,6 +273,7 @@ def crawl( secondary_abi: Dict[str, Any], from_block: int, to_block: int, + crawl_transactions: bool = True, addresses: Optional[List[ChecksumAddress]] = None, batch_size: int = 100, ) -> None: @@ -309,34 +313,35 @@ def crawl( event = _processEvent(raw_event) events.append(event) - transaction_hashes = {event.transaction_hash for event in events} - logger.info(f"Fetched {len(events)} events") - logger.info(f"Fetching {len(transaction_hashes)} transactions") + if crawl_transactions: + transaction_hashes = {event.transaction_hash for event in events} + logger.info(f"Fetched {len(events)} events") + logger.info(f"Fetching {len(transaction_hashes)} transactions") - transactions = _get_transactions( - db_session, web3, blockchain_type, transaction_hashes - ) - logger.info(f"Fetched {len(transactions)} transactions") - - function_calls = [] - for tx in transactions: - processed_tx, secondary_logs = process_transaction( - db_session, - web3, - blockchain_type, - contract, - secondary_abi, - tx, - db_blocks_cache, + transactions = _get_transactions( + db_session, web3, blockchain_type, transaction_hashes + ) + logger.info(f"Fetched {len(transactions)} transactions") + + function_calls = [] + for tx in transactions: + processed_tx, secondary_logs = process_transaction( + db_session, + web3, + blockchain_type, + contract, + secondary_abi, + tx, + db_blocks_cache, + ) + function_calls.append(processed_tx) + events.extend(secondary_logs) + add_function_calls_with_gas_price_to_session( + db_session, + function_calls, + blockchain_type, + label_name, ) - function_calls.append(processed_tx) - events.extend(secondary_logs) - add_function_calls_with_gas_price_to_session( - db_session, - function_calls, - blockchain_type, - label_name, - ) add_events_to_session( db_session, events, diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py index 650a1994..c813dc1f 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py @@ -21,6 +21,9 @@ def handle_nft_crawler(args: argparse.Namespace) -> None: abi = json.load(f) with open("mooncrawl/generic_crawler/abis/erc20.json") as f: erc20_abi = json.load(f) + erc20_abi = [ + abi for abi in erc20_abi if abi.get("name") == "Transfer" + ] # only care about transfer event label = args.label_name from_block = args.start_block @@ -65,11 +68,121 @@ def handle_nft_crawler(args: argparse.Namespace) -> None: ) +def handle_crawl(args: argparse.Namespace) -> None: + logger.info(f"Starting generic crawler") + + label = args.label_name + from_block = args.start_block + to_block = args.end_block + + with open(args.abi) as f: + abi = json.load(f) + + blockchain_type = AvailableBlockchainType(args.blockchain_type) + + logger.info(f"Blockchain type: {blockchain_type.value}") + with yield_db_session_ctx() as db_session: + web3: Optional[Web3] = None + if args.web3 is None: + logger.info( + "No web3 provider URL provided, using default (blockchan.py: connect())" + ) + web3 = connect(blockchain_type) + else: + logger.info(f"Using web3 provider URL: {args.web3}") + web3 = Web3( + Web3.HTTPProvider(args.web3), + ) + if args.poa: + logger.info("Using PoA middleware") + web3.middleware_onion.inject(geth_poa_middleware, layer=0) + last_crawled_block = get_checkpoint( + db_session, blockchain_type, from_block, to_block, label + ) + + logger.info(f"Starting from block: {last_crawled_block}") + crawl_transaction = not args.disable_transactions + crawl( + db_session, + web3, + blockchain_type, + label, + abi, + secondary_abi=[], + from_block=last_crawled_block, + to_block=to_block, + crawl_transactions=crawl_transaction, + batch_size=args.max_blocks_batch, + ) + + def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() + crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi") + crawl_parser.add_argument( + "--blockchain_type", + type=str, + required=True, + choices=[ + "ethereum", + "polygon", + ], + ) + crawl_parser.add_argument( + "--abi", + type=str, + default=None, + help="Abi of the contract", + ) + crawl_parser.add_argument( + "--disable_transactions", + action="store_true", + help="Disable transactions crawling", + ) + crawl_parser.add_argument( + "--web3", + type=str, + default=None, + help="Web3 provider URL", + ) + + crawl_parser.add_argument( + "--poa", + action="store_true", + default=False, + help="Use PoA middleware", + ) + + crawl_parser.add_argument( + "--start_block", + type=int, + default=None, + ) + crawl_parser.add_argument( + "--end_block", + type=int, + default=None, + ) + + crawl_parser.add_argument( + "--max_blocks_batch", + type=int, + default=500, + help="Maximum number of blocks to crawl in a single crawl step", + ) + + crawl_parser.add_argument( + "--label_name", + type=str, + default="erc721", + help="Label name", + ) + + crawl_parser.set_defaults(func=handle_crawl) + nft_crawler_parser = subparsers.add_parser( "nft", help="Run the NFT crawler",