kopia lustrzana https://github.com/bugout-dev/moonstream
added generic crawler
Yhtyyar Sahatov
rodzic
0b8ab06865
commit
c1729dd836
|
|
@ -45,25 +45,6 @@
|
|||
"name": "Approval",
|
||||
"type": "event"
|
||||
},
|
||||
{
|
||||
"anonymous": false,
|
||||
"inputs": [
|
||||
{
|
||||
"indexed": true,
|
||||
"internalType": "address",
|
||||
"name": "previousOwner",
|
||||
"type": "address"
|
||||
},
|
||||
{
|
||||
"indexed": true,
|
||||
"internalType": "address",
|
||||
"name": "newOwner",
|
||||
"type": "address"
|
||||
}
|
||||
],
|
||||
"name": "OwnershipTransferred",
|
||||
"type": "event"
|
||||
},
|
||||
{
|
||||
"anonymous": false,
|
||||
"inputs": [
|
||||
|
|
|
|||
|
|
@ -169,17 +169,18 @@ def process_transaction(
|
|||
function_args = "unknown"
|
||||
|
||||
transaction_reciept = web3.eth.getTransactionReceipt(transaction["hash"])
|
||||
block_timestamp = get_block_timestamp(
|
||||
db_session,
|
||||
web3,
|
||||
blockchain_type,
|
||||
transaction["blockNumber"],
|
||||
blocks_cache,
|
||||
100,
|
||||
)
|
||||
|
||||
function_call = ExtededFunctionCall(
|
||||
block_number=transaction["blockNumber"],
|
||||
block_timestamp=get_block_timestamp(
|
||||
db_session,
|
||||
web3,
|
||||
blockchain_type,
|
||||
transaction["blockNumber"],
|
||||
blocks_cache,
|
||||
100,
|
||||
),
|
||||
block_timestamp=block_timestamp,
|
||||
transaction_hash=transaction["hash"].hex(),
|
||||
contract_address=transaction["to"],
|
||||
caller_address=transaction["from"],
|
||||
|
|
@ -207,8 +208,10 @@ def process_transaction(
|
|||
"blockNumber": raw_event["blockNumber"],
|
||||
"transactionHash": raw_event["transactionHash"].hex(),
|
||||
"logIndex": raw_event["logIndex"],
|
||||
"blockTimestamp": block_timestamp,
|
||||
}
|
||||
secondary_logs.append(_processEvent(event))
|
||||
processed_event = _processEvent(event)
|
||||
secondary_logs.append(processed_event)
|
||||
|
||||
break
|
||||
except:
|
||||
|
|
@ -270,6 +273,7 @@ def crawl(
|
|||
secondary_abi: Dict[str, Any],
|
||||
from_block: int,
|
||||
to_block: int,
|
||||
crawl_transactions: bool = True,
|
||||
addresses: Optional[List[ChecksumAddress]] = None,
|
||||
batch_size: int = 100,
|
||||
) -> None:
|
||||
|
|
@ -309,34 +313,35 @@ def crawl(
|
|||
event = _processEvent(raw_event)
|
||||
events.append(event)
|
||||
|
||||
transaction_hashes = {event.transaction_hash for event in events}
|
||||
logger.info(f"Fetched {len(events)} events")
|
||||
logger.info(f"Fetching {len(transaction_hashes)} transactions")
|
||||
if crawl_transactions:
|
||||
transaction_hashes = {event.transaction_hash for event in events}
|
||||
logger.info(f"Fetched {len(events)} events")
|
||||
logger.info(f"Fetching {len(transaction_hashes)} transactions")
|
||||
|
||||
transactions = _get_transactions(
|
||||
db_session, web3, blockchain_type, transaction_hashes
|
||||
)
|
||||
logger.info(f"Fetched {len(transactions)} transactions")
|
||||
|
||||
function_calls = []
|
||||
for tx in transactions:
|
||||
processed_tx, secondary_logs = process_transaction(
|
||||
db_session,
|
||||
web3,
|
||||
blockchain_type,
|
||||
contract,
|
||||
secondary_abi,
|
||||
tx,
|
||||
db_blocks_cache,
|
||||
transactions = _get_transactions(
|
||||
db_session, web3, blockchain_type, transaction_hashes
|
||||
)
|
||||
logger.info(f"Fetched {len(transactions)} transactions")
|
||||
|
||||
function_calls = []
|
||||
for tx in transactions:
|
||||
processed_tx, secondary_logs = process_transaction(
|
||||
db_session,
|
||||
web3,
|
||||
blockchain_type,
|
||||
contract,
|
||||
secondary_abi,
|
||||
tx,
|
||||
db_blocks_cache,
|
||||
)
|
||||
function_calls.append(processed_tx)
|
||||
events.extend(secondary_logs)
|
||||
add_function_calls_with_gas_price_to_session(
|
||||
db_session,
|
||||
function_calls,
|
||||
blockchain_type,
|
||||
label_name,
|
||||
)
|
||||
function_calls.append(processed_tx)
|
||||
events.extend(secondary_logs)
|
||||
add_function_calls_with_gas_price_to_session(
|
||||
db_session,
|
||||
function_calls,
|
||||
blockchain_type,
|
||||
label_name,
|
||||
)
|
||||
add_events_to_session(
|
||||
db_session,
|
||||
events,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@ def handle_nft_crawler(args: argparse.Namespace) -> None:
|
|||
abi = json.load(f)
|
||||
with open("mooncrawl/generic_crawler/abis/erc20.json") as f:
|
||||
erc20_abi = json.load(f)
|
||||
erc20_abi = [
|
||||
abi for abi in erc20_abi if abi.get("name") == "Transfer"
|
||||
] # only care about transfer event
|
||||
|
||||
label = args.label_name
|
||||
from_block = args.start_block
|
||||
|
|
@ -65,11 +68,121 @@ def handle_nft_crawler(args: argparse.Namespace) -> None:
|
|||
)
|
||||
|
||||
|
||||
def handle_crawl(args: argparse.Namespace) -> None:
|
||||
logger.info(f"Starting generic crawler")
|
||||
|
||||
label = args.label_name
|
||||
from_block = args.start_block
|
||||
to_block = args.end_block
|
||||
|
||||
with open(args.abi) as f:
|
||||
abi = json.load(f)
|
||||
|
||||
blockchain_type = AvailableBlockchainType(args.blockchain_type)
|
||||
|
||||
logger.info(f"Blockchain type: {blockchain_type.value}")
|
||||
with yield_db_session_ctx() as db_session:
|
||||
web3: Optional[Web3] = None
|
||||
if args.web3 is None:
|
||||
logger.info(
|
||||
"No web3 provider URL provided, using default (blockchan.py: connect())"
|
||||
)
|
||||
web3 = connect(blockchain_type)
|
||||
else:
|
||||
logger.info(f"Using web3 provider URL: {args.web3}")
|
||||
web3 = Web3(
|
||||
Web3.HTTPProvider(args.web3),
|
||||
)
|
||||
if args.poa:
|
||||
logger.info("Using PoA middleware")
|
||||
web3.middleware_onion.inject(geth_poa_middleware, layer=0)
|
||||
last_crawled_block = get_checkpoint(
|
||||
db_session, blockchain_type, from_block, to_block, label
|
||||
)
|
||||
|
||||
logger.info(f"Starting from block: {last_crawled_block}")
|
||||
crawl_transaction = not args.disable_transactions
|
||||
crawl(
|
||||
db_session,
|
||||
web3,
|
||||
blockchain_type,
|
||||
label,
|
||||
abi,
|
||||
secondary_abi=[],
|
||||
from_block=last_crawled_block,
|
||||
to_block=to_block,
|
||||
crawl_transactions=crawl_transaction,
|
||||
batch_size=args.max_blocks_batch,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi")
|
||||
crawl_parser.add_argument(
|
||||
"--blockchain_type",
|
||||
type=str,
|
||||
required=True,
|
||||
choices=[
|
||||
"ethereum",
|
||||
"polygon",
|
||||
],
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--abi",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Abi of the contract",
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--disable_transactions",
|
||||
action="store_true",
|
||||
help="Disable transactions crawling",
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--web3",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Web3 provider URL",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--poa",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Use PoA middleware",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--start_block",
|
||||
type=int,
|
||||
default=None,
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--end_block",
|
||||
type=int,
|
||||
default=None,
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--max_blocks_batch",
|
||||
type=int,
|
||||
default=500,
|
||||
help="Maximum number of blocks to crawl in a single crawl step",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--label_name",
|
||||
type=str,
|
||||
default="erc721",
|
||||
help="Label name",
|
||||
)
|
||||
|
||||
crawl_parser.set_defaults(func=handle_crawl)
|
||||
|
||||
nft_crawler_parser = subparsers.add_parser(
|
||||
"nft",
|
||||
help="Run the NFT crawler",
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue