kopia lustrzana https://github.com/bugout-dev/moonstream
Add inital version.
Andrey
rodzic
77c3e7e2f8
commit
54870630c4
|
|
@ -29,6 +29,22 @@ DOCS_TARGET_PATH = "docs"
|
|||
|
||||
# Crawler label
|
||||
CRAWLER_LABEL = "moonworm-alpha"
|
||||
VIEW_STATE_CRAWLER_LABEL = "view-state-alpha"
|
||||
|
||||
|
||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS = 30000
|
||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW = os.environ.get(
|
||||
"MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS"
|
||||
)
|
||||
try:
|
||||
if MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW is not None:
|
||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS = int(
|
||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW
|
||||
)
|
||||
except:
|
||||
raise Exception(
|
||||
f"Could not parse MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS as int: {MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW}"
|
||||
)
|
||||
|
||||
# Geth connection address
|
||||
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,487 @@
|
|||
# Code generated by moonworm : https://github.com/bugout-dev/moonworm
|
||||
# Moonworm version : 0.2.4
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from brownie import Contract, network, project
|
||||
from brownie.network.contract import ContractContainer
|
||||
from eth_typing.evm import ChecksumAddress
|
||||
|
||||
|
||||
PROJECT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
|
||||
print("PROJECT_DIRECTORY", PROJECT_DIRECTORY)
|
||||
BUILD_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "build", "contracts")
|
||||
|
||||
|
||||
def boolean_argument_type(raw_value: str) -> bool:
|
||||
TRUE_VALUES = ["1", "t", "y", "true", "yes"]
|
||||
FALSE_VALUES = ["0", "f", "n", "false", "no"]
|
||||
|
||||
if raw_value.lower() in TRUE_VALUES:
|
||||
return True
|
||||
elif raw_value.lower() in FALSE_VALUES:
|
||||
return False
|
||||
|
||||
raise ValueError(
|
||||
f"Invalid boolean argument: {raw_value}. Value must be one of: {','.join(TRUE_VALUES + FALSE_VALUES)}"
|
||||
)
|
||||
|
||||
|
||||
def bytes_argument_type(raw_value: str) -> str:
|
||||
return raw_value
|
||||
|
||||
|
||||
def get_abi_json(abi_name: str) -> List[Dict[str, Any]]:
|
||||
abi_full_path = os.path.join(BUILD_DIRECTORY, f"{abi_name}.json")
|
||||
if not os.path.isfile(abi_full_path):
|
||||
raise IOError(
|
||||
f"File does not exist: {abi_full_path}. Maybe you have to compile the smart contracts?"
|
||||
)
|
||||
|
||||
with open(abi_full_path, "r") as ifp:
|
||||
build = json.load(ifp)
|
||||
|
||||
abi_json = build.get("abi")
|
||||
if abi_json is None:
|
||||
raise ValueError(f"Could not find ABI definition in: {abi_full_path}")
|
||||
|
||||
return abi_json
|
||||
|
||||
|
||||
def contract_from_build(abi_name: str) -> ContractContainer:
|
||||
# This is workaround because brownie currently doesn't support loading the same project multiple
|
||||
# times. This causes problems when using multiple contracts from the same project in the same
|
||||
# python project.
|
||||
PROJECT = project.main.Project("moonworm", Path(PROJECT_DIRECTORY))
|
||||
|
||||
abi_full_path = os.path.join(BUILD_DIRECTORY, f"{abi_name}.json")
|
||||
if not os.path.isfile(abi_full_path):
|
||||
raise IOError(
|
||||
f"File does not exist: {abi_full_path}. Maybe you have to compile the smart contracts?"
|
||||
)
|
||||
|
||||
with open(abi_full_path, "r") as ifp:
|
||||
build = json.load(ifp)
|
||||
|
||||
return ContractContainer(PROJECT, build)
|
||||
|
||||
|
||||
class Multicall2:
|
||||
def __init__(self, contract_address: Optional[ChecksumAddress]):
|
||||
print("contract_address", contract_address)
|
||||
self.contract_name = "Multicall2"
|
||||
self.address = contract_address
|
||||
self.contract = None
|
||||
self.abi = get_abi_json("Multicall2")
|
||||
if self.address is not None:
|
||||
print(f"Using contract at {self.address}")
|
||||
print(f"Contract ABI: {self.abi}")
|
||||
self.contract: Optional[Contract] = Contract.from_abi(
|
||||
self.contract_name, self.address, self.abi
|
||||
)
|
||||
|
||||
def deploy(self, transaction_config):
|
||||
contract_class = contract_from_build(self.contract_name)
|
||||
deployed_contract = contract_class.deploy(transaction_config)
|
||||
self.address = deployed_contract.address
|
||||
self.contract = deployed_contract
|
||||
return deployed_contract.tx
|
||||
|
||||
def assert_contract_is_instantiated(self) -> None:
|
||||
if self.contract is None:
|
||||
raise Exception("contract has not been instantiated")
|
||||
|
||||
def verify_contract(self):
|
||||
self.assert_contract_is_instantiated()
|
||||
contract_class = contract_from_build(self.contract_name)
|
||||
contract_class.publish_source(self.contract)
|
||||
|
||||
def aggregate(self, calls: List, transaction_config) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.aggregate(calls, transaction_config)
|
||||
|
||||
def block_and_aggregate(self, calls: List, transaction_config) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.blockAndAggregate(calls, transaction_config)
|
||||
|
||||
def get_block_hash(
|
||||
self, block_number_arg: int, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getBlockHash.call(
|
||||
block_number_arg, block_identifier=block_number
|
||||
)
|
||||
|
||||
def get_block_number(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getBlockNumber.call(block_identifier=block_number)
|
||||
|
||||
def get_current_block_coinbase(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getCurrentBlockCoinbase.call(block_identifier=block_number)
|
||||
|
||||
def get_current_block_difficulty(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getCurrentBlockDifficulty.call(
|
||||
block_identifier=block_number
|
||||
)
|
||||
|
||||
def get_current_block_gas_limit(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getCurrentBlockGasLimit.call(block_identifier=block_number)
|
||||
|
||||
def get_current_block_timestamp(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getCurrentBlockTimestamp.call(
|
||||
block_identifier=block_number
|
||||
)
|
||||
|
||||
def get_eth_balance(
|
||||
self, addr: ChecksumAddress, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getEthBalance.call(addr, block_identifier=block_number)
|
||||
|
||||
def get_last_block_hash(
|
||||
self, block_number: Optional[Union[str, int]] = "latest"
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.getLastBlockHash.call(block_identifier=block_number)
|
||||
|
||||
def try_aggregate(
|
||||
self, require_success: bool, calls: List, transaction_config
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.tryAggregate(require_success, calls, transaction_config)
|
||||
|
||||
def try_block_and_aggregate(
|
||||
self, require_success: bool, calls: List, transaction_config
|
||||
) -> Any:
|
||||
self.assert_contract_is_instantiated()
|
||||
return self.contract.tryBlockAndAggregate(
|
||||
require_success, calls, transaction_config
|
||||
)
|
||||
|
||||
|
||||
def get_transaction_config(args: argparse.Namespace) -> Dict[str, Any]:
|
||||
signer = network.accounts.load(args.sender, args.password)
|
||||
transaction_config: Dict[str, Any] = {"from": signer}
|
||||
if args.gas_price is not None:
|
||||
transaction_config["gas_price"] = args.gas_price
|
||||
if args.max_fee_per_gas is not None:
|
||||
transaction_config["max_fee"] = args.max_fee_per_gas
|
||||
if args.max_priority_fee_per_gas is not None:
|
||||
transaction_config["priority_fee"] = args.max_priority_fee_per_gas
|
||||
if args.confirmations is not None:
|
||||
transaction_config["required_confs"] = args.confirmations
|
||||
if args.nonce is not None:
|
||||
transaction_config["nonce"] = args.nonce
|
||||
return transaction_config
|
||||
|
||||
|
||||
def add_default_arguments(parser: argparse.ArgumentParser, transact: bool) -> None:
|
||||
parser.add_argument(
|
||||
"--network", required=True, help="Name of brownie network to connect to"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--address", required=False, help="Address of deployed contract to connect to"
|
||||
)
|
||||
if not transact:
|
||||
parser.add_argument(
|
||||
"--block-number",
|
||||
required=False,
|
||||
type=int,
|
||||
help="Call at the given block number, defaults to latest",
|
||||
)
|
||||
return
|
||||
parser.add_argument(
|
||||
"--sender", required=True, help="Path to keystore file for transaction sender"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--password",
|
||||
required=False,
|
||||
help="Password to keystore file (if you do not provide it, you will be prompted for it)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gas-price", default=None, help="Gas price at which to submit transaction"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-fee-per-gas",
|
||||
default=None,
|
||||
help="Max fee per gas for EIP1559 transactions",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-priority-fee-per-gas",
|
||||
default=None,
|
||||
help="Max priority fee per gas for EIP1559 transactions",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirmations",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Number of confirmations to await before considering a transaction completed",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--nonce", type=int, default=None, help="Nonce for the transaction (optional)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--value", default=None, help="Value of the transaction in wei(optional)"
|
||||
)
|
||||
parser.add_argument("--verbose", action="store_true", help="Print verbose output")
|
||||
|
||||
|
||||
def handle_deploy(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
transaction_config = get_transaction_config(args)
|
||||
contract = Multicall2(None)
|
||||
result = contract.deploy(transaction_config=transaction_config)
|
||||
print(result)
|
||||
if args.verbose:
|
||||
print(result.info())
|
||||
|
||||
|
||||
def handle_verify_contract(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.verify_contract()
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_aggregate(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
transaction_config = get_transaction_config(args)
|
||||
result = contract.aggregate(calls=args.calls, transaction_config=transaction_config)
|
||||
print(result)
|
||||
if args.verbose:
|
||||
print(result.info())
|
||||
|
||||
|
||||
def handle_block_and_aggregate(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
transaction_config = get_transaction_config(args)
|
||||
result = contract.block_and_aggregate(
|
||||
calls=args.calls, transaction_config=transaction_config
|
||||
)
|
||||
print(result)
|
||||
if args.verbose:
|
||||
print(result.info())
|
||||
|
||||
|
||||
def handle_get_block_hash(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_block_hash(
|
||||
block_number_arg=args.block_number_arg, block_number=args.block_number
|
||||
)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_block_number(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_block_number(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_current_block_coinbase(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_current_block_coinbase(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_current_block_difficulty(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_current_block_difficulty(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_current_block_gas_limit(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_current_block_gas_limit(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_current_block_timestamp(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_current_block_timestamp(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_eth_balance(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_eth_balance(addr=args.addr, block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_get_last_block_hash(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
result = contract.get_last_block_hash(block_number=args.block_number)
|
||||
print(result)
|
||||
|
||||
|
||||
def handle_try_aggregate(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
transaction_config = get_transaction_config(args)
|
||||
result = contract.try_aggregate(
|
||||
require_success=args.require_success,
|
||||
calls=args.calls,
|
||||
transaction_config=transaction_config,
|
||||
)
|
||||
print(result)
|
||||
if args.verbose:
|
||||
print(result.info())
|
||||
|
||||
|
||||
def handle_try_block_and_aggregate(args: argparse.Namespace) -> None:
|
||||
network.connect(args.network)
|
||||
contract = Multicall2(args.address)
|
||||
transaction_config = get_transaction_config(args)
|
||||
result = contract.try_block_and_aggregate(
|
||||
require_success=args.require_success,
|
||||
calls=args.calls,
|
||||
transaction_config=transaction_config,
|
||||
)
|
||||
print(result)
|
||||
if args.verbose:
|
||||
print(result.info())
|
||||
|
||||
|
||||
def generate_cli() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="CLI for Multicall2")
|
||||
parser.set_defaults(func=lambda _: parser.print_help())
|
||||
subcommands = parser.add_subparsers()
|
||||
|
||||
deploy_parser = subcommands.add_parser("deploy")
|
||||
add_default_arguments(deploy_parser, True)
|
||||
deploy_parser.set_defaults(func=handle_deploy)
|
||||
|
||||
verify_contract_parser = subcommands.add_parser("verify-contract")
|
||||
add_default_arguments(verify_contract_parser, False)
|
||||
verify_contract_parser.set_defaults(func=handle_verify_contract)
|
||||
|
||||
aggregate_parser = subcommands.add_parser("aggregate")
|
||||
add_default_arguments(aggregate_parser, True)
|
||||
aggregate_parser.add_argument(
|
||||
"--calls", required=True, help="Type: tuple[]", nargs="+"
|
||||
)
|
||||
aggregate_parser.set_defaults(func=handle_aggregate)
|
||||
|
||||
block_and_aggregate_parser = subcommands.add_parser("block-and-aggregate")
|
||||
add_default_arguments(block_and_aggregate_parser, True)
|
||||
block_and_aggregate_parser.add_argument(
|
||||
"--calls", required=True, help="Type: tuple[]", nargs="+"
|
||||
)
|
||||
block_and_aggregate_parser.set_defaults(func=handle_block_and_aggregate)
|
||||
|
||||
get_block_hash_parser = subcommands.add_parser("get-block-hash")
|
||||
add_default_arguments(get_block_hash_parser, False)
|
||||
get_block_hash_parser.add_argument(
|
||||
"--block-number", required=True, help="Type: uint256", type=int
|
||||
)
|
||||
get_block_hash_parser.set_defaults(func=handle_get_block_hash)
|
||||
|
||||
get_block_number_parser = subcommands.add_parser("get-block-number")
|
||||
add_default_arguments(get_block_number_parser, False)
|
||||
get_block_number_parser.set_defaults(func=handle_get_block_number)
|
||||
|
||||
get_current_block_coinbase_parser = subcommands.add_parser(
|
||||
"get-current-block-coinbase"
|
||||
)
|
||||
add_default_arguments(get_current_block_coinbase_parser, False)
|
||||
get_current_block_coinbase_parser.set_defaults(
|
||||
func=handle_get_current_block_coinbase
|
||||
)
|
||||
|
||||
get_current_block_difficulty_parser = subcommands.add_parser(
|
||||
"get-current-block-difficulty"
|
||||
)
|
||||
add_default_arguments(get_current_block_difficulty_parser, False)
|
||||
get_current_block_difficulty_parser.set_defaults(
|
||||
func=handle_get_current_block_difficulty
|
||||
)
|
||||
|
||||
get_current_block_gas_limit_parser = subcommands.add_parser(
|
||||
"get-current-block-gas-limit"
|
||||
)
|
||||
add_default_arguments(get_current_block_gas_limit_parser, False)
|
||||
get_current_block_gas_limit_parser.set_defaults(
|
||||
func=handle_get_current_block_gas_limit
|
||||
)
|
||||
|
||||
get_current_block_timestamp_parser = subcommands.add_parser(
|
||||
"get-current-block-timestamp"
|
||||
)
|
||||
add_default_arguments(get_current_block_timestamp_parser, False)
|
||||
get_current_block_timestamp_parser.set_defaults(
|
||||
func=handle_get_current_block_timestamp
|
||||
)
|
||||
|
||||
get_eth_balance_parser = subcommands.add_parser("get-eth-balance")
|
||||
add_default_arguments(get_eth_balance_parser, False)
|
||||
get_eth_balance_parser.add_argument("--addr", required=True, help="Type: address")
|
||||
get_eth_balance_parser.set_defaults(func=handle_get_eth_balance)
|
||||
|
||||
get_last_block_hash_parser = subcommands.add_parser("get-last-block-hash")
|
||||
add_default_arguments(get_last_block_hash_parser, False)
|
||||
get_last_block_hash_parser.set_defaults(func=handle_get_last_block_hash)
|
||||
|
||||
try_aggregate_parser = subcommands.add_parser("try-aggregate")
|
||||
add_default_arguments(try_aggregate_parser, True)
|
||||
try_aggregate_parser.add_argument(
|
||||
"--require-success",
|
||||
required=True,
|
||||
help="Type: bool",
|
||||
type=boolean_argument_type,
|
||||
)
|
||||
try_aggregate_parser.add_argument(
|
||||
"--calls", required=True, help="Type: tuple[]", nargs="+"
|
||||
)
|
||||
try_aggregate_parser.set_defaults(func=handle_try_aggregate)
|
||||
|
||||
try_block_and_aggregate_parser = subcommands.add_parser("try-block-and-aggregate")
|
||||
add_default_arguments(try_block_and_aggregate_parser, True)
|
||||
try_block_and_aggregate_parser.add_argument(
|
||||
"--require-success",
|
||||
required=True,
|
||||
help="Type: bool",
|
||||
type=boolean_argument_type,
|
||||
)
|
||||
try_block_and_aggregate_parser.add_argument(
|
||||
"--calls", required=True, help="Type: tuple[]", nargs="+"
|
||||
)
|
||||
try_block_and_aggregate_parser.set_defaults(func=handle_try_block_and_aggregate)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = generate_cli()
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -2,22 +2,28 @@ import argparse
|
|||
import json
|
||||
import hashlib
|
||||
import itertools
|
||||
from pickle import TRUE
|
||||
from pprint import pprint
|
||||
import logging
|
||||
from random import random
|
||||
from typing import Optional, List, Any, Union, Dict, Tuple, Callable, Iterable, cast
|
||||
from typing import List, Any
|
||||
from uuid import UUID
|
||||
|
||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||
from moonstreamdb.db import yield_db_session_ctx
|
||||
import web3
|
||||
from web3 import Web3
|
||||
from web3.middleware import geth_poa_middleware
|
||||
from moonstreamdb.db import (
|
||||
MOONSTREAM_DB_URI_READ_ONLY,
|
||||
MOONSTREAM_POOL_SIZE,
|
||||
create_moonstream_engine,
|
||||
)
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from .db import view_call_to_label, commit_session
|
||||
from ..settings import (
|
||||
NB_CONTROLLER_ACCESS_ID,
|
||||
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
|
||||
)
|
||||
|
||||
from ..settings import MOONSTREAM_MOONWORM_TASKS_JOURNAL, NB_CONTROLLER_ACCESS_ID
|
||||
from .db import get_first_labeled_block_number, get_last_labeled_block_number
|
||||
from .historical_crawler import historical_crawler
|
||||
from brownie import Contract, network, chain
|
||||
# from .db import get_first_labeled_block_number, get_last_labeled_block_number
|
||||
from brownie import Contract, network, chain, web3
|
||||
import Multicall2
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
|
@ -30,19 +36,21 @@ Multicall2_address = "0xc8E51042792d7405184DfCa245F2d27B94D013b6"
|
|||
def make_multicall(
|
||||
multicall_method: Any,
|
||||
calls: List[Any],
|
||||
block_timestamp: int,
|
||||
block_number: str = "latest",
|
||||
) -> Any:
|
||||
# print("calls")
|
||||
# pprint(calls)
|
||||
|
||||
multicall_calls = [
|
||||
(
|
||||
call["address"],
|
||||
call["method"].encode_input(*call["inputs"]),
|
||||
)
|
||||
for call in calls
|
||||
]
|
||||
|
||||
multicall_result = multicall_method.call(
|
||||
False, # success not required
|
||||
[
|
||||
(
|
||||
call["address"],
|
||||
call["method"].encode_input(*call["inputs"]),
|
||||
)
|
||||
for call in calls
|
||||
],
|
||||
multicall_calls,
|
||||
block_identifier=block_number,
|
||||
)
|
||||
|
||||
|
|
@ -51,65 +59,143 @@ def make_multicall(
|
|||
# Handle the case with not successful calls
|
||||
for index, encoded_data in enumerate(multicall_result):
|
||||
if encoded_data[0]:
|
||||
# print(dir(calls[index]["method"]))
|
||||
results.append(
|
||||
{
|
||||
"result": calls[index]["method"].decode_output(encoded_data[1]),
|
||||
"hash": calls[index]["hash"],
|
||||
"method": calls[index]["method"],
|
||||
"address": calls[index]["address"],
|
||||
"name": calls[index]["method"]._name,
|
||||
"name": calls[index]["method"].abi["name"],
|
||||
"inputs": calls[index]["inputs"],
|
||||
"call_data": multicall_calls[index][1],
|
||||
"block_number": block_number,
|
||||
"block_timestamp": block_timestamp,
|
||||
"status": encoded_data[0],
|
||||
}
|
||||
)
|
||||
else:
|
||||
# print(encoded_data)
|
||||
results.append(
|
||||
{
|
||||
"result": None, # calls[index]["method"].decode_output(encoded_data[1]),
|
||||
"hash": calls[index]["hash"],
|
||||
"method": calls[index]["method"],
|
||||
"address": calls[index]["address"],
|
||||
"name": calls[index]["method"]._name,
|
||||
"name": calls[index]["method"].abi["name"],
|
||||
"inputs": calls[index]["inputs"],
|
||||
"call_data": multicall_calls[index][1],
|
||||
"block_number": block_number,
|
||||
"block_timestamp": block_timestamp,
|
||||
"status": encoded_data[0],
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
# def crawl_job(job):
|
||||
# print(job)
|
||||
# pass
|
||||
def crawl_calls_level(
|
||||
db_session,
|
||||
calls,
|
||||
responces,
|
||||
contracts_ABIs,
|
||||
interfaces,
|
||||
batch_size,
|
||||
multicall_method,
|
||||
block_number,
|
||||
blockchain_type,
|
||||
block_timestamp,
|
||||
):
|
||||
|
||||
calls_of_level = []
|
||||
|
||||
for call in calls:
|
||||
parameters = []
|
||||
|
||||
for input in call["inputs"]:
|
||||
|
||||
if type(input["value"]) in (str, int):
|
||||
if input["value"] not in responces:
|
||||
parameters.append([input["value"]])
|
||||
else:
|
||||
if (
|
||||
contracts_ABIs[call["address"]][input["value"]]["name"]
|
||||
== "totalSupply"
|
||||
):
|
||||
parameters.append(
|
||||
list(range(1, responces[input["value"]][0] + 1))
|
||||
)
|
||||
else:
|
||||
parameters.append(responces[input["value"]])
|
||||
elif type(input["value"]) == list:
|
||||
parameters.append(input["value"])
|
||||
else:
|
||||
raise
|
||||
|
||||
for call_parameters in itertools.product(*parameters):
|
||||
calls_of_level.append(
|
||||
{
|
||||
"address": call["address"],
|
||||
"method": interfaces[call["address"]].get_method_object(
|
||||
interfaces[call["address"]].signatures[call["name"]]
|
||||
),
|
||||
"hash": call["generated_hash"],
|
||||
"inputs": call_parameters,
|
||||
}
|
||||
)
|
||||
|
||||
for call_chunk in [
|
||||
calls_of_level[i : i + batch_size]
|
||||
for i in range(0, len(calls_of_level), batch_size)
|
||||
]:
|
||||
|
||||
while True:
|
||||
try:
|
||||
make_multicall_result = make_multicall(
|
||||
multicall_method=multicall_method,
|
||||
calls=call_chunk,
|
||||
block_number=block_number,
|
||||
block_timestamp=block_timestamp,
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
# results parsing and writing to database
|
||||
|
||||
for result in make_multicall_result:
|
||||
|
||||
db_view = view_call_to_label(blockchain_type, result)
|
||||
db_session.add(db_view)
|
||||
|
||||
if result["hash"] not in responces:
|
||||
responces[result["hash"]] = []
|
||||
responces[result["hash"]].append(result["result"])
|
||||
commit_session(db_session)
|
||||
|
||||
|
||||
# return generated_hash
|
||||
|
||||
|
||||
def generate_call_tree(jobs):
|
||||
def parse_jobs(jobs, blockchain_type, block_number):
|
||||
|
||||
contracts_ABIs = {}
|
||||
contracts_methods = {}
|
||||
calls = {"WithoutSubcalls": [], "WithSubcalls": {}}
|
||||
calls = {0: []}
|
||||
|
||||
# client = web3.Web3()
|
||||
blockchain_type_to_brownie_network = {
|
||||
AvailableBlockchainType.POLYGON: "polygon-main",
|
||||
AvailableBlockchainType.MUMBAI: "polygon-test",
|
||||
}
|
||||
|
||||
network.connect("polygon-main")
|
||||
network.connect(blockchain_type_to_brownie_network[blockchain_type])
|
||||
|
||||
if block_number is None:
|
||||
block_number = len(chain) - 1
|
||||
|
||||
block_timestamp = web3.eth.get_block(block_number).timestamp
|
||||
|
||||
print(len(chain))
|
||||
multicaller = Multicall2.Multicall2(Multicall2_address)
|
||||
|
||||
multicall_method = multicaller.contract.tryAggregate
|
||||
|
||||
# test call
|
||||
|
||||
def recursive_unpack(method_abi: Any, level: int = 0) -> Any:
|
||||
have_subcalls = False
|
||||
|
||||
print(level)
|
||||
print(method_abi)
|
||||
|
||||
abi = {
|
||||
"level": level,
|
||||
"inputs": [],
|
||||
"outputs": method_abi["outputs"],
|
||||
"name": method_abi["name"],
|
||||
|
|
@ -129,41 +215,22 @@ def generate_call_tree(jobs):
|
|||
input["value"] = hash_link
|
||||
have_subcalls = True
|
||||
abi["inputs"].append(input)
|
||||
|
||||
abi["address"] = method_abi["address"]
|
||||
generated_hash = hashlib.md5(json.dumps(abi).encode("utf-8")).hexdigest()
|
||||
|
||||
abi["generated_hash"] = generated_hash
|
||||
if have_subcalls:
|
||||
abi["generated_hash"] = generated_hash
|
||||
abi["address"] = method_abi["address"]
|
||||
if not calls["WithSubcalls"].get(level):
|
||||
calls["WithSubcalls"][level] = []
|
||||
calls["WithSubcalls"][level].append(abi)
|
||||
level += 1
|
||||
if not calls.get(level):
|
||||
calls[level] = []
|
||||
calls[level].append(abi)
|
||||
else:
|
||||
# generate calls
|
||||
|
||||
# generate parameters list of parameters lists
|
||||
level = 0
|
||||
|
||||
parameters = []
|
||||
|
||||
for input in abi["inputs"]:
|
||||
if type(input["value"]) in (str, int):
|
||||
parameters.append([input["value"]])
|
||||
elif type(input["value"]) == list:
|
||||
parameters.append(input["value"])
|
||||
else:
|
||||
raise Exception(f"Unknown type {type(input['value'])}")
|
||||
|
||||
for call_parameters in itertools.product(*parameters):
|
||||
# if not calls["WithoutSubcalls"].get(generated_hash):
|
||||
# calls["WithoutSubcalls"][generated_hash] = []
|
||||
calls["WithoutSubcalls"].append(
|
||||
{
|
||||
"address": job["address"],
|
||||
"inputs": call_parameters,
|
||||
"generated_hash": generated_hash,
|
||||
"name": abi["name"],
|
||||
}
|
||||
)
|
||||
if not calls.get(level):
|
||||
calls[level] = []
|
||||
calls[level].append(abi)
|
||||
|
||||
if not contracts_methods.get(job["address"]):
|
||||
contracts_methods[job["address"]] = []
|
||||
|
|
@ -175,9 +242,6 @@ def generate_call_tree(jobs):
|
|||
|
||||
return generated_hash
|
||||
|
||||
# selector
|
||||
# hashlib.md5(json.dumps(method).encode("utf-8")).hexdigest()
|
||||
|
||||
for job in jobs:
|
||||
if job["address"] not in contracts_ABIs:
|
||||
contracts_ABIs[job["address"]] = []
|
||||
|
|
@ -188,32 +252,6 @@ def generate_call_tree(jobs):
|
|||
|
||||
interfaces = {}
|
||||
|
||||
# web3 general interface workflow
|
||||
# for contract_address in contracts_ABIs:
|
||||
|
||||
# # collect abis for each contract
|
||||
# abis = []
|
||||
|
||||
# for method_hash in contracts_methods[contract_address]:
|
||||
# abis.append(contracts_ABIs[contract_address][method_hash])
|
||||
|
||||
# # generate interface
|
||||
# interfaces[contract_address] = client.eth.contract(abi=abis,address=contract_address)
|
||||
|
||||
# batch_calls = []
|
||||
|
||||
# for call in calls["WithoutSubcalls"]:
|
||||
# print(dir(interfaces[call["address"]].functions[call["name"]](*call["inputs"])))
|
||||
# batch_calls.append(
|
||||
# {
|
||||
# "to": call["address"],
|
||||
# "call": interfaces[call["address"]].encodeABI(fn_name=call["name"], args=call["inputs"]),
|
||||
# "hash": call["generated_hash"]
|
||||
# }
|
||||
# )
|
||||
|
||||
# Brownie interface generating
|
||||
|
||||
for contract_address in contracts_ABIs:
|
||||
|
||||
# collect abis for each contract
|
||||
|
|
@ -227,134 +265,60 @@ def generate_call_tree(jobs):
|
|||
random(), contract_address, abis
|
||||
)
|
||||
|
||||
batch_calls = []
|
||||
responces = {}
|
||||
|
||||
for call in calls["WithoutSubcalls"]:
|
||||
# print(dir(interfaces[call["address"]]))
|
||||
# print(
|
||||
# interfaces[call["address"]].get_method_object(
|
||||
# interfaces[call["address"]].signatures[call["name"]]
|
||||
# )
|
||||
# )
|
||||
batch_calls.append(
|
||||
{
|
||||
"address": call["address"],
|
||||
"method": interfaces[call["address"]].get_method_object(
|
||||
interfaces[call["address"]].signatures[call["name"]]
|
||||
),
|
||||
"hash": call["generated_hash"],
|
||||
"inputs": call["inputs"],
|
||||
}
|
||||
)
|
||||
|
||||
make_multicall_result = make_multicall(
|
||||
multicall_method=multicall_method, calls=batch_calls
|
||||
)
|
||||
|
||||
# responces dict
|
||||
|
||||
responces = {} # {generated_hash: []}
|
||||
|
||||
# create chunks of calls
|
||||
# # create chunks of calls
|
||||
batch_size = 500
|
||||
|
||||
for call_chunk in [
|
||||
batch_calls[i : i + batch_size] for i in range(0, len(batch_calls), batch_size)
|
||||
]:
|
||||
print("batch_step", len(call_chunk))
|
||||
|
||||
while True:
|
||||
try:
|
||||
make_multicall_result = make_multicall(
|
||||
multicall_method=multicall_method, calls=call_chunk
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
# results parsing and writing to database
|
||||
|
||||
for result in make_multicall_result:
|
||||
if result["hash"] not in responces:
|
||||
responces[result["hash"]] = []
|
||||
responces[result["hash"]].append(result["result"])
|
||||
|
||||
# proccessing call_tree
|
||||
|
||||
# reverse call_tree
|
||||
call_tree_levels = sorted(calls["WithSubcalls"].keys(), reverse=True)
|
||||
call_tree_levels = sorted(calls.keys(), reverse=True)[:-1]
|
||||
|
||||
for level in call_tree_levels:
|
||||
|
||||
calls_of_level = []
|
||||
|
||||
for call in calls["WithSubcalls"][level]:
|
||||
parameters = []
|
||||
|
||||
for input in call["inputs"]:
|
||||
|
||||
if type(input["value"]) in (str, int):
|
||||
if input["value"] not in responces:
|
||||
parameters.append([input["value"]])
|
||||
else:
|
||||
# print(responces[input["value"]])
|
||||
if (
|
||||
contracts_ABIs[call["address"]][input["value"]]["name"]
|
||||
== "totalSupply"
|
||||
):
|
||||
parameters.append(
|
||||
list(range(0, responces[input["value"]][0]))
|
||||
)
|
||||
print(len(list(range(0, responces[input["value"]][0]))))
|
||||
else:
|
||||
parameters.append(responces[input["value"]])
|
||||
elif type(input["value"]) == list:
|
||||
parameters.append(input["value"])
|
||||
else:
|
||||
raise
|
||||
|
||||
for call_parameters in itertools.product(*parameters):
|
||||
calls_of_level.append(
|
||||
{
|
||||
"address": call["address"],
|
||||
"method": interfaces[call["address"]].get_method_object(
|
||||
interfaces[call["address"]].signatures[call["name"]]
|
||||
),
|
||||
"hash": call["generated_hash"],
|
||||
"inputs": call_parameters,
|
||||
}
|
||||
)
|
||||
print(len(call_parameters))
|
||||
|
||||
for call_chunk in [
|
||||
calls_of_level[i : i + batch_size]
|
||||
for i in range(0, len(calls_of_level), batch_size)
|
||||
]:
|
||||
|
||||
print("batch_step", len(call_chunk))
|
||||
|
||||
while True:
|
||||
try:
|
||||
make_multicall_result = make_multicall(
|
||||
multicall_method=multicall_method, calls=call_chunk
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
# results parsing and writing to database
|
||||
|
||||
for result in make_multicall_result:
|
||||
if result["hash"] not in responces:
|
||||
responces[result["hash"]] = []
|
||||
responces[result["hash"]].append(result["result"])
|
||||
|
||||
print("contracts_methods")
|
||||
pprint(contracts_methods)
|
||||
print("contracts_ABIs")
|
||||
pprint(contracts_ABIs)
|
||||
print("calls")
|
||||
pprint(calls)
|
||||
print("responses")
|
||||
pprint(responces)
|
||||
|
||||
engine = create_moonstream_engine(
|
||||
MOONSTREAM_DB_URI_READ_ONLY,
|
||||
pool_pre_ping=True,
|
||||
pool_size=MOONSTREAM_POOL_SIZE,
|
||||
statement_timeout=MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
|
||||
)
|
||||
process_session = sessionmaker(bind=engine)
|
||||
db_session = process_session()
|
||||
|
||||
# run crawling of levels
|
||||
try:
|
||||
# initial call
|
||||
crawl_calls_level(
|
||||
db_session,
|
||||
calls[0],
|
||||
responces,
|
||||
contracts_ABIs,
|
||||
interfaces,
|
||||
batch_size,
|
||||
multicall_method,
|
||||
block_number,
|
||||
blockchain_type,
|
||||
block_timestamp,
|
||||
)
|
||||
|
||||
for level in call_tree_levels:
|
||||
|
||||
crawl_calls_level(
|
||||
db_session,
|
||||
calls[level],
|
||||
responces,
|
||||
contracts_ABIs,
|
||||
interfaces,
|
||||
batch_size,
|
||||
multicall_method,
|
||||
block_number,
|
||||
blockchain_type,
|
||||
block_timestamp,
|
||||
)
|
||||
|
||||
finally:
|
||||
db_session.close()
|
||||
|
||||
print(responces)
|
||||
|
||||
|
||||
def handle_crawl(args: argparse.Namespace) -> None:
|
||||
|
|
@ -363,195 +327,39 @@ def handle_crawl(args: argparse.Namespace) -> None:
|
|||
Ability to track states of the contracts.
|
||||
|
||||
Read all view methods of the contracts and crawl
|
||||
|
||||
The querstion is input patameters of method
|
||||
|
||||
tags:
|
||||
address:0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f
|
||||
|
||||
type:function
|
||||
|
||||
abi_method_hash:5fbaec05ae19ac90eea75b676a20e495
|
||||
|
||||
subscription_type:polygon_smartcontract
|
||||
|
||||
abi_name:transfer
|
||||
|
||||
status:active
|
||||
|
||||
contract_address:0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f
|
||||
|
||||
content:
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"internalType": "address",
|
||||
"name": "owner",
|
||||
"type": "address"
|
||||
}
|
||||
],
|
||||
"name": "balanceOf",
|
||||
"outputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "",
|
||||
"type": "uint256"
|
||||
}
|
||||
],
|
||||
"stateMutability": "view",
|
||||
"type": "function"
|
||||
},
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "_tokenId",
|
||||
"type": "uint256"
|
||||
}
|
||||
],
|
||||
"name": "getDNA",
|
||||
"outputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "",
|
||||
"type": "uint256"
|
||||
}
|
||||
],
|
||||
"stateMutability": "view",
|
||||
"type": "function"
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"name": "_tokenId",
|
||||
"type": "uint256"
|
||||
"position":0,
|
||||
"value": {
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [],
|
||||
"name": "totalSupply"
|
||||
}
|
||||
}
|
||||
],
|
||||
"name": "getDNA",
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
"inputs": [
|
||||
{
|
||||
"name": "_tokenId",
|
||||
"type": "uint256"
|
||||
"value": {
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [],
|
||||
"name": "totalSupply"
|
||||
}
|
||||
}
|
||||
],
|
||||
"name": "getDNA",
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
"name": "getUnicornBodyParts",
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [ {
|
||||
"name": "_dna",
|
||||
"type": "uint256"
|
||||
"value":{
|
||||
"inputs": [
|
||||
{
|
||||
"name": "_tokenId",
|
||||
"type": "uint256"
|
||||
"value": {
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [],
|
||||
"name": "totalSupply"
|
||||
}
|
||||
}
|
||||
],
|
||||
"name": "getDNA",
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
"""
|
||||
|
||||
my_job = {
|
||||
"name": "getUnicornBodyParts",
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"type": "function",
|
||||
"stateMutability": "view",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "_dna",
|
||||
"type": "uint256",
|
||||
"internalType": "uint256",
|
||||
"name": "tokenId",
|
||||
"type": "uint256",
|
||||
"value": {
|
||||
"type": "function",
|
||||
"inputs": [
|
||||
"name": "totalSupply",
|
||||
"outputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"modify": "range",
|
||||
"name": "_tokenId",
|
||||
"name": "",
|
||||
"type": "uint256",
|
||||
"value": {
|
||||
"type": "function",
|
||||
"name": "totalSupply",
|
||||
"outputs": [
|
||||
{
|
||||
"internalType": "uint256",
|
||||
"name": "",
|
||||
"type": "uint256",
|
||||
}
|
||||
],
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [],
|
||||
},
|
||||
}
|
||||
],
|
||||
"name": "getDNA",
|
||||
"outputs": [
|
||||
{"internalType": "uint256", "name": "", "type": "uint256"}
|
||||
],
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
"inputs": [],
|
||||
},
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{"internalType": "uint256", "name": "bodyPartId", "type": "uint256"},
|
||||
{"internalType": "uint256", "name": "facePartId", "type": "uint256"},
|
||||
{"internalType": "uint256", "name": "hornPartId", "type": "uint256"},
|
||||
{"internalType": "uint256", "name": "hoovesPartId", "type": "uint256"},
|
||||
{"internalType": "uint256", "name": "manePartId", "type": "uint256"},
|
||||
{"internalType": "uint256", "name": "tailPartId", "type": "uint256"},
|
||||
{"internalType": "uint8", "name": "mythicCount", "type": "uint8"},
|
||||
],
|
||||
"name": "tokenURI",
|
||||
"outputs": [{"internalType": "string", "name": "", "type": "string"}],
|
||||
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
|
||||
}
|
||||
|
||||
# blockchain_type = AvailableBlockchainType(args.blockchain_type)
|
||||
blockchain_type = AvailableBlockchainType(args.blockchain_type)
|
||||
|
||||
# if args.web3 is None:
|
||||
# raise ValueError("Web3 provider URL is required")
|
||||
|
||||
# web3 = Web3(Web3.HTTPProvider(args.web3))
|
||||
|
||||
# if args.poa:
|
||||
# web3.middleware_stack.inject(geth_poa_middleware, layer=0)
|
||||
|
||||
# Generate call tree
|
||||
call_tree = generate_call_tree([my_job])
|
||||
print(call_tree)
|
||||
parse_jobs([my_job], blockchain_type, args.block_number)
|
||||
|
||||
|
||||
def parse_abi(args: argparse.Namespace) -> None:
|
||||
|
|
@ -586,99 +394,21 @@ def main() -> None:
|
|||
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
parser_parse_abi = subparsers.add_parser(
|
||||
"parse_job",
|
||||
view_state_crawler_parser = subparsers.add_parser(
|
||||
"crawl_jobs",
|
||||
help="continuous crawling the event/function call jobs from bugout journal",
|
||||
)
|
||||
parser_parse_abi.set_defaults(func=handle_crawl)
|
||||
|
||||
crawl_parser = subparsers.add_parser(
|
||||
"crawl",
|
||||
help="continuous crawling the event/function call jobs from bugout journal",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--start",
|
||||
"-s",
|
||||
type=int,
|
||||
default=None,
|
||||
help="start block number",
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
view_state_crawler_parser.add_argument(
|
||||
"--blockchain-type",
|
||||
"-b",
|
||||
type=str,
|
||||
help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
|
||||
help="Type of blovkchain wich writng in database",
|
||||
required=True,
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--web3",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Web3 provider URL",
|
||||
view_state_crawler_parser.add_argument(
|
||||
"--block-number", "-N", type=str, help="Block number."
|
||||
)
|
||||
crawl_parser.add_argument(
|
||||
"--poa",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Use PoA middleware",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--max-blocks-batch",
|
||||
"-m",
|
||||
type=int,
|
||||
default=80,
|
||||
help="Maximum number of blocks to crawl in a single batch",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--min-blocks-batch",
|
||||
"-n",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Minimum number of blocks to crawl in a single batch",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--confirmations",
|
||||
"-c",
|
||||
type=int,
|
||||
default=175,
|
||||
help="Number of confirmations to wait for",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--min-sleep-time",
|
||||
"-t",
|
||||
type=float,
|
||||
default=0.1,
|
||||
help="Minimum time to sleep between crawl step",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--heartbeat-interval",
|
||||
"-i",
|
||||
type=float,
|
||||
default=60,
|
||||
help="Heartbeat interval in seconds",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--new-jobs-refetch-interval",
|
||||
"-r",
|
||||
type=float,
|
||||
default=180,
|
||||
help="Time to wait before refetching new jobs",
|
||||
)
|
||||
|
||||
crawl_parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force start from the start block",
|
||||
)
|
||||
|
||||
crawl_parser.set_defaults(func=handle_crawl)
|
||||
view_state_crawler_parser.set_defaults(func=handle_crawl)
|
||||
|
||||
generate_view_parser = subparsers.add_parser(
|
||||
"parse-abi",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
import logging
|
||||
from typing import Dict, Any
|
||||
|
||||
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..settings import VIEW_STATE_CRAWLER_LABEL
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def view_call_to_label(
|
||||
blockchain_type: AvailableBlockchainType,
|
||||
call: Dict[str, Any],
|
||||
label_name=VIEW_STATE_CRAWLER_LABEL,
|
||||
):
|
||||
|
||||
"""
|
||||
Creates a label model.
|
||||
|
||||
"result": calls[index]["method"].decode_output(encoded_data[1]),
|
||||
"hash": calls[index]["hash"],
|
||||
"address": calls[index]["address"],
|
||||
"name": calls[index]["method"].name,
|
||||
"inputs": calls[index]["inputs"],
|
||||
"block_number": block_number,
|
||||
"""
|
||||
label_model = get_label_model(blockchain_type)
|
||||
label = label_model(
|
||||
label=label_name,
|
||||
label_data={
|
||||
"type": "view",
|
||||
"name": call["name"],
|
||||
"result": call["result"],
|
||||
"inputs": call["inputs"],
|
||||
"call_data": call["call_data"],
|
||||
"status": call["status"],
|
||||
},
|
||||
address=call["address"],
|
||||
block_number=call["block_number"],
|
||||
transaction_hash=None,
|
||||
block_timestamp=call["block_timestamp"],
|
||||
)
|
||||
|
||||
return label
|
||||
|
||||
|
||||
def commit_session(db_session: Session) -> None:
|
||||
"""
|
||||
Save labels in the database.
|
||||
"""
|
||||
try:
|
||||
logger.info("Committing session to database")
|
||||
db_session.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save labels: {e}")
|
||||
db_session.rollback()
|
||||
raise e
|
||||
|
|
@ -62,6 +62,7 @@ setup(
|
|||
"moonworm-crawler=mooncrawl.moonworm_crawler.cli:main",
|
||||
"nft=mooncrawl.nft.cli:main",
|
||||
"statistics=mooncrawl.stats_worker.dashboard:main",
|
||||
"state-crawler=mooncrawl.state_crawler.cli:main"
|
||||
]
|
||||
},
|
||||
)
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue