From 454b5d77545916abc17b14a7a051a5bc0775fe9e Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 19 Sep 2022 18:15:40 -0700 Subject: [PATCH] Removed "watch-cu" command and all related code --- moonworm/cli.py | 73 -------- moonworm/cu_current_cuts.py | 130 -------------- moonworm/cu_watch.py | 344 ------------------------------------ 3 files changed, 547 deletions(-) delete mode 100644 moonworm/cu_current_cuts.py delete mode 100644 moonworm/cu_watch.py diff --git a/moonworm/cli.py b/moonworm/cli.py index bbbc100..6424df4 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -177,34 +177,6 @@ def handle_watch(args: argparse.Namespace) -> None: ) -def handle_watch_cu(args: argparse.Namespace) -> None: - from moonworm.cu_watch import watch_cu_contract - - MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") - if not MOONSTREAM_DB_URI: - print("Please set MOONSTREAM_DB_URI environment variable") - return - - if args.abi is not None: - with open(args.abi, "r") as ifp: - contract_abi = json.load(ifp) - else: - print("Using CUContract abi since no abi is specified") - contract_abi = CU.abi() - - web3 = Web3(Web3.HTTPProvider(args.web3)) - web3.middleware_onion.inject(geth_poa_middleware, layer=0) - - watch_cu_contract( - web3, - web3.toChecksumAddress(args.contract), - contract_abi, - args.confirmations, - start_block=args.deployment_block, - force_start=args.force, - ) - - def handle_find_deployment(args: argparse.Namespace) -> None: web3_client = Web3(Web3.HTTPProvider(args.web3)) result = find_deployment_block(web3_client, args.contract, args.interval) @@ -327,51 +299,6 @@ def generate_argument_parser() -> argparse.ArgumentParser: watch_parser.set_defaults(func=handle_watch) - watch_cu_parser = subcommands.add_parser( - "watch-cu", help="Watch a Crypto Unicorns contract" - ) - watch_cu_parser.add_argument( - "-i", - "--abi", - default=None, - help="ABI file path, default is abi in fixtures/abis/CU.json", - ) - watch_cu_parser.add_argument( - "-f", - "--force", - action="store_true", - help="Force start from given block", - ) - watch_cu_parser.add_argument( - "-c", - "--contract", - required=True, - help="Contract address", - ) - - watch_cu_parser.add_argument( - "-w", - "--web3", - required=True, - help="Web3 provider", - ) - - watch_cu_parser.add_argument( - "--confirmations", - default=10, - type=int, - help="Number of confirmations to wait for. Default=12", - ) - - watch_cu_parser.add_argument( - "--deployment-block", - "-d", - type=int, - help="Block number of the deployment", - ) - - watch_cu_parser.set_defaults(func=handle_watch_cu) - generate_brownie_parser = subcommands.add_parser( "generate-brownie", description="Moonworm code generator for brownie projects" ) diff --git a/moonworm/cu_current_cuts.py b/moonworm/cu_current_cuts.py deleted file mode 100644 index 164c8e0..0000000 --- a/moonworm/cu_current_cuts.py +++ /dev/null @@ -1,130 +0,0 @@ -import json -from dataclasses import dataclass -from re import L - -from moonstreamdb.db import yield_db_session_ctx -from moonstreamdb.models import PolygonLabel -from sqlalchemy.orm.session import Session -from web3 import Web3 - -from .contracts import CU - -ADDRESS = "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f" -MUMBAI_ADDRESS = "0xA993c4759B731f650dfA011765a6aedaC91a4a88" - - -def get_all_DNA_events(session: Session): - labels = ( - session.query(PolygonLabel) - .filter(PolygonLabel.label == "moonworm") - .filter(PolygonLabel.address == ADDRESS) - .filter(PolygonLabel.label_data["name"].astext == "DNAUpdated") - .all() - ) - max_token = -1 - token_ids = [] - for label in labels: - token_ids.append(label.label_data["args"]["tokenId"]) - max_token = max(max_token, label.label_data["args"]["tokenId"]) - - print(len(token_ids)) - return token_ids - - -def filter_ids(_id, labeled_ids): - with open(f"unicorn-classes-{_id}.json", "r") as ifp: - original_ids = json.load(ifp) - - token_ids = [item["tokenId"] for item in original_ids] - result = [] - for token in token_ids: - if token not in labeled_ids: - result.append({"tokenId": token, "class": _id}) - - with open(f"processes-{_id}.json", "w") as ofp: - json.dump(result, ofp) - - -action_map = { - 0: "add", - 1: "replace", - 2: "remove", -} - -cu_contract = Web3().eth.contract(abi=CU.abi()) - - -def get_all_diamond_cuts(session: Session): - labels = ( - session.query(PolygonLabel.label_data) - .filter(PolygonLabel.label == "moonworm") - .filter(PolygonLabel.address == ADDRESS) - .filter(PolygonLabel.label_data["name"].astext == "diamondCut") - .filter(PolygonLabel.label_data["status"].astext == "1") - .order_by(PolygonLabel.block_number.asc()) - .all() - ) - - return labels - - -def get_function_name_by_selector(selector: str): - try: - - name = cu_contract.get_function_by_selector(selector).function_identifier - return name - except Exception as e: - print(e) - print(selector) - return "UNKNOWN" - - -def run(): - - with yield_db_session_ctx() as session: - diamond_cuts_events = get_all_diamond_cuts(session) - - selector_actions = {} - current_index = 0 - for event in diamond_cuts_events: - diamond_cuts = event[0]["args"]["_diamondCut"] - for diamond_cut in diamond_cuts: - for selector in diamond_cut[2]: - if selector not in selector_actions: - selector_actions[selector] = [] - selector_actions[selector].append( - { - "name": get_function_name_by_selector(selector), - "action": action_map[diamond_cut[1]], - "address": diamond_cut[0], - "action_index": current_index, - } - ) - current_index += 1 - - current_cuts = {} - for selector, actions in selector_actions.items(): - last_cut = None - last_cut_index = -1 - for action in actions: - if action["action_index"] > last_cut_index: - last_cut_index = action["action_index"] - if action["action"] == "remove": - last_cut = {"name": action["name"], "address": None} - else: - last_cut = {"name": action["name"], "address": action["address"]} - current_cuts[selector] = last_cut - - grouped_by_name = {} - for selector, cut in current_cuts.items(): - if grouped_by_name.get(cut["name"]) is None: - grouped_by_name[cut["name"]] = [] - grouped_by_name[cut["name"]].append( - {"selector": selector, "address": cut["address"]} - ) - - with open("current_cuts.json", "w") as ofp: - json.dump(grouped_by_name, ofp) - - -run() diff --git a/moonworm/cu_watch.py b/moonworm/cu_watch.py deleted file mode 100644 index c7058a7..0000000 --- a/moonworm/cu_watch.py +++ /dev/null @@ -1,344 +0,0 @@ -import json -import logging -import os -import pprint as pp -import re -import time -from os import stat -from re import S -from typing import Any, Dict, List, Optional, cast - -import web3 -from eth_typing.evm import ChecksumAddress -from moonstreamdb.db import yield_db_session_ctx -from moonstreamdb.models import EthereumLabel, PolygonLabel -from sqlalchemy.orm import Query, Session -from sqlalchemy.sql.expression import delete -from tqdm import tqdm -from web3 import Web3 -from web3.middleware import geth_poa_middleware - -from moonworm.crawler.moonstream_ethereum_state_provider import ( - MoonstreamEthereumStateProvider, -) -from moonworm.crawler.utils import Network - -from .contracts import CU, ERC721 -from .crawler.ethereum_state_provider import EthereumStateProvider -from .crawler.function_call_crawler import ( - ContractFunctionCall, - FunctionCallCrawler, - FunctionCallCrawlerState, - Web3StateProvider, - utfy_dict, -) -from .crawler.log_scanner import _fetch_events_chunk - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def _get_last_crawled_block( - session: Session, contract_address: ChecksumAddress -) -> Optional[int]: - """ - Gets the last block that was crawled. - """ - - query = ( - session.query(PolygonLabel) - .filter( - PolygonLabel.label == "moonworm", - PolygonLabel.address == contract_address, - ) - .order_by(PolygonLabel.block_number.desc()) - ) - if query.count(): - return query.first().block_number - return None - - -def _add_function_call_labels( - session: Session, - function_calls: List[ContractFunctionCall], - address: ChecksumAddress, -) -> None: - """ - Adds a label to a function call. - """ - - existing_function_call_labels = ( - session.query(PolygonLabel) - .filter( - PolygonLabel.label == "moonworm", - PolygonLabel.log_index == None, - PolygonLabel.address == address, - PolygonLabel.transaction_hash.in_( - [call.transaction_hash for call in function_calls] - ), - ) - .all() - ) - # deletin existing labels - for label in existing_function_call_labels: - session.delete(label) - - try: - if existing_function_call_labels: - logger.info( - f"Deleting {len(existing_function_call_labels)} existing tx labels" - ) - session.commit() - except Exception as e: - try: - session.commit() - except: - logger.error(f"Failed!!!\n{e}") - session.rollback() - - for function_call in function_calls: - label = PolygonLabel( - label="moonworm", - label_data={ - "type": "tx_call", - "name": function_call.function_name, - "caller": function_call.caller_address, - "args": function_call.function_args, - "status": function_call.status, - "gasUsed": function_call.gas_used, - }, - address=function_call.contract_address, - block_number=function_call.block_number, - transaction_hash=function_call.transaction_hash, - block_timestamp=function_call.block_timestamp, - ) - session.add(label) - try: - session.commit() - except Exception as e: - try: - session.commit() - except: - logger.error(f"Failed!!!\n{e}") - session.rollback() - - -def _add_event_labels( - session: Session, events: List[Dict[str, Any]], address: ChecksumAddress -) -> None: - """ - Adds events to database. - """ - - transactions = [event["transactionHash"] for event in events] - log_indexes = [event["logIndex"] for event in events] - for ev in events: - print(ev) - existing_event_labels = ( - session.query(PolygonLabel) - .filter( - PolygonLabel.label == "moonworm", - PolygonLabel.address == address, - PolygonLabel.transaction_hash.in_(transactions), - PolygonLabel.log_index != None, - ) - .all() - ) - - # deletin existing labels - deleted = 0 - for label in existing_event_labels: - if label.log_index in log_indexes: - deleted += 1 - session.delete(label) - - try: - if deleted > 0: - logger.error(f"Deleting {deleted} existing event labels") - session.commit() - except Exception as e: - try: - session.commit() - except: - logger.error(f"Failed!!!\n{e}") - session.rollback() - - for event in events: - label = PolygonLabel( - label="moonworm", - label_data={ - "type": "event", - "name": event["event"], - "args": event["args"], - }, - address=event["address"], - block_number=event["blockNumber"], - transaction_hash=event["transactionHash"], - block_timestamp=event["blockTimestamp"], - log_index=event["logIndex"], - ) - session.add(label) - try: - session.commit() - except Exception as e: - try: - session.commit() - except: - logger.error(f"Failed!!!\n{e}") - session.rollback() - - -class MockState(FunctionCallCrawlerState): - def __init__(self) -> None: - self.state: List[ContractFunctionCall] = [] - - def get_last_crawled_block(self) -> int: - """ - Returns the last block number that was crawled. - """ - return 0 - - def register_call(self, function_call: ContractFunctionCall) -> None: - """ - Processes the given function call (store it, etc.). - """ - self.state.append(function_call) - - def flush(self) -> None: - """ - Flushes cached state to storage layer. - """ - self.state = [] - - -def watch_cu_contract( - web3: Web3, - contract_address: ChecksumAddress, - contract_abi: List[Dict[str, Any]], - num_confirmations: int = 60, - min_blocks_to_crawl: int = 10, - sleep_time: float = 1, - start_block: Optional[int] = None, - force_start: bool = False, - use_moonstream_web3_provider: bool = False, -) -> None: - """ - Watches a contract for events and calls. - """ - if force_start and start_block is None: - raise ValueError("start_block must be specified if force_start is True") - - with yield_db_session_ctx() as session: - function_call_state = MockState() - - eth_state_provider: Optional[EthereumStateProvider] = None - if use_moonstream_web3_provider: - eth_state_provider = MoonstreamEthereumStateProvider( - web3, network=Network.polygon, db_session=session - ) - else: - eth_state_provider = Web3StateProvider(web3) - - crawler = FunctionCallCrawler( - function_call_state, - eth_state_provider, - contract_abi, - [web3.toChecksumAddress(contract_address)], - ) - - last_crawled_block = _get_last_crawled_block(session, contract_address) - if start_block is None: - if last_crawled_block is not None: - current_block = last_crawled_block - logger.info(f"Starting from block {current_block}, last crawled block") - else: - current_block = web3.eth.blockNumber - num_confirmations * 2 - logger.info(f"Starting from block {current_block}, current block") - else: - current_block = start_block - if not force_start and last_crawled_block is not None: - if start_block > last_crawled_block: - logger.info( - f"Starting from block {start_block}, last crawled block {last_crawled_block}" - ) - else: - current_block = last_crawled_block - logger.info(f"Starting from last crawled block {current_block}") - - event_abis = [item for item in contract_abi if item["type"] == "event"] - - while True: - try: - session.execute("select 1") - time.sleep(sleep_time) - end_block = min( - web3.eth.blockNumber - num_confirmations, current_block + 100 - ) - if end_block < current_block + min_blocks_to_crawl: - logger.info( - f"Sleeping crawling, end_block {end_block} < current_block {current_block} + min_blocks_to_crawl {min_blocks_to_crawl}" - ) - sleep_time += 1 - continue - - sleep_time /= 2 - - logger.info("Getting txs") - crawler.crawl(current_block, end_block) - if function_call_state.state: - _add_function_call_labels( - session, function_call_state.state, contract_address - ) - logger.info( - f"Got {len(function_call_state.state)} transaction calls:" - ) - function_call_state.flush() - - logger.info("Getting events") - all_events = [] - for event_abi in event_abis: - raw_events = _fetch_events_chunk( - web3, - event_abi, - current_block, - end_block, - [contract_address], - ) - - for raw_event in raw_events: - raw_event["blockTimestamp"] = ( - crawler.ethereum_state_provider.get_block_timestamp( - raw_event["blockNumber"] - ), - ) - all_events.append(raw_event) - - if all_events: - print(f"Got {len(all_events)} events:") - _add_event_labels(session, all_events, contract_address) - logger.info(f"Current block {end_block + 1}") - current_block = end_block + 1 - except Exception as e: - logger.error(f"Something went wrong: {e}") - logger.info(f"Trying to recover from error") - for i in range(10): - logger.info(f"Attempt {i}:") - try: - time.sleep(10) - logger.info("Trying to reconnect to database") - session.rollback() - session.execute("select 1") - logger.info("Trying to reconnect to web3") - web3.eth.block_number - break - except Exception as e: - logger.error(f"Failed: {e}") - continue - - try: - session.execute("select 1") - web3.eth.block_number - continue - except Exception as e: - logger.error("Moonworm is going to die") - raise e