From 1213ad874de6a37b7dac0d1eaf0373d59b0b1e7b Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 15 Nov 2021 15:42:20 +0200 Subject: [PATCH 01/10] Ad init state. --- .../mooncrawl/mooncrawl/stats_worker/dashboard.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index d5f5d83f..32f4f236 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -17,7 +17,7 @@ from sqlalchemy import Column, Date, and_, func, text from sqlalchemy.orm import Query, Session from sqlalchemy.sql.operators import in_op -from ..blockchain import get_block_model, get_label_model, get_transaction_model +from ..blockchain import get_block_model, get_label_model, get_transaction_model, connect from ..data import AvailableBlockchainType from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -26,6 +26,10 @@ from ..settings import ( ) from ..settings import bugout_client as bc +from web3 import HTTPProvider, IPCProvider, Web3 +from web3.middleware import geth_poa_middleware +from web3.types import BlockData + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -382,6 +386,8 @@ def stats_generate_handler(args: argparse.Namespace): s3_client = boto3.client("s3") + + # Already processed already_processed = [] @@ -409,6 +415,11 @@ def stats_generate_handler(args: argparse.Namespace): abi_functions = [item for item in abi_json if item["type"] == "function"] abi_events = [item for item in abi_json if item["type"] == "event"] + abi_extentions = [item for item in abi_json if item["type"] == "extention"] + + web3_client = connect(blockchain_type) + + extention = web3_client for timescale in [timescale.value for timescale in TimeScale]: @@ -418,6 +429,8 @@ def stats_generate_handler(args: argparse.Namespace): print(f"Timescale: {timescale}") + s3_data_object["web3_metric"] = extention + abi_functions_names = [item["name"] for item in abi_functions] functions_calls_data = generate_data( From 14f0f572d703ce354bbb380bceda9a33cc16a38f Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Mon, 15 Nov 2021 17:29:19 +0300 Subject: [PATCH 02/10] added calls to web3 --- .../mooncrawl/stats_worker/dashboard.py | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 32f4f236..2bccb91e 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -8,7 +8,9 @@ import logging import time from datetime import datetime, timedelta from enum import Enum -from typing import Any, Dict, List +from typing import Any, Callable, Dict, List + +from web3.datastructures import T import boto3 # type: ignore from bugout.data import BugoutResources @@ -363,6 +365,20 @@ def generate_data( return response_labels +def cast_to_python_type(evm_type: str) -> Callable: + if evm_type.startswith(("uint", "int")): + return int + elif evm_type.startswith("bytes"): + return bytes + elif evm_type == "string": + return str + elif evm_type == "address": + return Web3.toChecksumAddress + elif evm_type == "bool": + return bool + else: + raise ValueError(f"Cannot convert to python type {evm_type}") + def stats_generate_handler(args: argparse.Namespace): """ @@ -415,11 +431,61 @@ def stats_generate_handler(args: argparse.Namespace): abi_functions = [item for item in abi_json if item["type"] == "function"] abi_events = [item for item in abi_json if item["type"] == "event"] - abi_extentions = [item for item in abi_json if item["type"] == "extention"] - web3_client = connect(blockchain_type) + abi_external_calls = [item for item in abi_json if item["type"] == "external_call"] + + external_calls = [] + + for external_call in abi_external_calls: + try: + func_abi = [] + input_args = [] + for func_input in external_call["inputs"]: + func_abi.append({"name":func_input["name"], "input": func_input["type"]}) + input_args.append(cast_to_python_type(func_input["type"])(func_input["value"])) + func_abi["outputs"] = external_call["outputs"] + external_calls.append( + { + "display_name": external_call["display_name"], + "address": Web3.toChecksumAddress(external_call["address"]), + "name": external_call["name"], + "abi": func_abi, + "input_args": input_args, + } + ) + except Exception as e: + print(f"Error processing external call: {e}") - extention = web3_client + web3_client = connect(blockchain_type) + # { + # "type": "external_call" + # "display_name": "Total weth earned" + # "address": "0x0d1d4e623D10F9FBA5Db95830F7d3839406C6AF2", + # "name": "balanceOf", + # "inputs": [ + # { + # "name": "owner", + # "type": "address" + # "value": "0x123fsdaf9432jrejfr9u9" + # } + # ], + # "outputs": [ + # { + # "internalType": "uint256", + # "name": "", + # "type": "uint256" + # } + # } + + extention_data = [] + for extcall in external_calls: + try: + contract = web3_client.eth.contract(address=extcall['address'], abi=extcall['abi']) + response = contract.functions[extcall['name']](*extcall['inputs']).call() + extention_data.append({"display_name" : extcall['display_name'], "value": response}) + except Exception as e: + print(f"Failed to call {extcall['name']}") + for timescale in [timescale.value for timescale in TimeScale]: @@ -429,7 +495,7 @@ def stats_generate_handler(args: argparse.Namespace): print(f"Timescale: {timescale}") - s3_data_object["web3_metric"] = extention + s3_data_object["web3_metric"] = extention_data abi_functions_names = [item["name"] for item in abi_functions] From 8bab2340c6459ea70610afb3d656d8e4638b76bb Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 15 Nov 2021 17:00:29 +0200 Subject: [PATCH 03/10] Fixes --- .../mooncrawl/stats_worker/dashboard.py | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 2bccb91e..9c49b769 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -9,6 +9,8 @@ import time from datetime import datetime, timedelta from enum import Enum from typing import Any, Callable, Dict, List +from pprint import pprint +import traceback from web3.datastructures import T @@ -19,7 +21,12 @@ from sqlalchemy import Column, Date, and_, func, text from sqlalchemy.orm import Query, Session from sqlalchemy.sql.operators import in_op -from ..blockchain import get_block_model, get_label_model, get_transaction_model, connect +from ..blockchain import ( + get_block_model, + get_label_model, + get_transaction_model, + connect, +) from ..data import AvailableBlockchainType from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -365,6 +372,7 @@ def generate_data( return response_labels + def cast_to_python_type(evm_type: str) -> Callable: if evm_type.startswith(("uint", "int")): return int @@ -402,8 +410,6 @@ def stats_generate_handler(args: argparse.Namespace): s3_client = boto3.client("s3") - - # Already processed already_processed = [] @@ -432,30 +438,37 @@ def stats_generate_handler(args: argparse.Namespace): abi_functions = [item for item in abi_json if item["type"] == "function"] abi_events = [item for item in abi_json if item["type"] == "event"] - abi_external_calls = [item for item in abi_json if item["type"] == "external_call"] + abi_external_calls = [ + item for item in abi_json if item["type"] == "external_call" + ] + pprint(abi_external_calls) external_calls = [] for external_call in abi_external_calls: try: - func_abi = [] + func_input_abi = [] input_args = [] for func_input in external_call["inputs"]: - func_abi.append({"name":func_input["name"], "input": func_input["type"]}) - input_args.append(cast_to_python_type(func_input["type"])(func_input["value"])) - func_abi["outputs"] = external_call["outputs"] + func_input_abi.append( + {"name": func_input["name"], "input": func_input["type"]} + ) + input_args.append( + cast_to_python_type(func_input["type"])(func_input["value"]) + ) + func_input_abi.extend(external_call["outputs"]) external_calls.append( - { + { "display_name": external_call["display_name"], "address": Web3.toChecksumAddress(external_call["address"]), "name": external_call["name"], - "abi": func_abi, + "abi": func_input_abi, "input_args": input_args, } ) except Exception as e: print(f"Error processing external call: {e}") - + web3_client = connect(blockchain_type) # { # "type": "external_call" @@ -477,15 +490,23 @@ def stats_generate_handler(args: argparse.Namespace): # } # } - extention_data = [] + extention_data = [] + pprint(external_calls) for extcall in external_calls: try: - contract = web3_client.eth.contract(address=extcall['address'], abi=extcall['abi']) - response = contract.functions[extcall['name']](*extcall['inputs']).call() - extention_data.append({"display_name" : extcall['display_name'], "value": response}) - except Exception as e: - print(f"Failed to call {extcall['name']}") + contract = web3_client.eth.contract( + address=extcall["address"], abi=extcall["abi"] + ) + response = contract.functions[extcall["name"]]( + *extcall["input_args"] + ).call() + extention_data.append( + {"display_name": extcall["display_name"], "value": response} + ) + except Exception as e: + traceback.print_exc() + print(f"Failed to call {extcall['name']} error: {e}") for timescale in [timescale.value for timescale in TimeScale]: From edcde4f930f1d268b5d08e3f4ce5e5d197e99794 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Mon, 15 Nov 2021 18:01:14 +0300 Subject: [PATCH 04/10] fix of comment --- crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 2bccb91e..784a185e 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -460,13 +460,13 @@ def stats_generate_handler(args: argparse.Namespace): # { # "type": "external_call" # "display_name": "Total weth earned" - # "address": "0x0d1d4e623D10F9FBA5Db95830F7d3839406C6AF2", + # "address": "0xdf2811b6432cae65212528f0a7186b71adaec03a", # "name": "balanceOf", # "inputs": [ # { # "name": "owner", # "type": "address" - # "value": "0x123fsdaf9432jrejfr9u9" + # "value": "0xA993c4759B731f650dfA011765a6aedaC91a4a88" # } # ], # "outputs": [ From 44cc1562bb167a611e27f0a79fc0de0af8d738d7 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Mon, 15 Nov 2021 18:04:56 +0300 Subject: [PATCH 05/10] fixed func_abi --- .../mooncrawl/mooncrawl/stats_worker/dashboard.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index d5961ec3..1580a23f 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -456,13 +456,21 @@ def stats_generate_handler(args: argparse.Namespace): input_args.append( cast_to_python_type(func_input["type"])(func_input["value"]) ) - func_input_abi.extend(external_call["outputs"]) + + func_abi = [ + { + "name": external_call["name"], + "inputs": func_input_abi, + "outputs": external_call["outputs"], + } + ] + external_calls.append( { "display_name": external_call["display_name"], "address": Web3.toChecksumAddress(external_call["address"]), "name": external_call["name"], - "abi": func_input_abi, + "abi": func_abi, "input_args": input_args, } ) From 10a7ab874813cf91090ea14d370a196824661480 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Mon, 15 Nov 2021 18:07:28 +0300 Subject: [PATCH 06/10] fixed func_abi --- crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 1580a23f..f69e6a7a 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -451,7 +451,7 @@ def stats_generate_handler(args: argparse.Namespace): input_args = [] for func_input in external_call["inputs"]: func_input_abi.append( - {"name": func_input["name"], "input": func_input["type"]} + {"name": func_input["name"], "type": func_input["type"]} ) input_args.append( cast_to_python_type(func_input["type"])(func_input["value"]) From 2bc565cebdea5426b7bb05de5c448a8aae94b871 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Mon, 15 Nov 2021 18:14:28 +0300 Subject: [PATCH 07/10] fixed func_abi --- crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index f69e6a7a..c5c97bdb 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -462,6 +462,8 @@ def stats_generate_handler(args: argparse.Namespace): "name": external_call["name"], "inputs": func_input_abi, "outputs": external_call["outputs"], + "type": "function", + "stateMutability": "view", } ] From 8f6e7ea01f62f8cc3ae3e541f945c6752ca5146d Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 15 Nov 2021 18:39:33 +0200 Subject: [PATCH 08/10] Remove debuging prints. --- crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index c5c97bdb..c9dc1966 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -441,7 +441,6 @@ def stats_generate_handler(args: argparse.Namespace): abi_external_calls = [ item for item in abi_json if item["type"] == "external_call" ] - pprint(abi_external_calls) external_calls = [] @@ -501,7 +500,6 @@ def stats_generate_handler(args: argparse.Namespace): # } extention_data = [] - pprint(external_calls) for extcall in external_calls: try: contract = web3_client.eth.contract( @@ -515,7 +513,6 @@ def stats_generate_handler(args: argparse.Namespace): {"display_name": extcall["display_name"], "value": response} ) except Exception as e: - traceback.print_exc() print(f"Failed to call {extcall['name']} error: {e}") for timescale in [timescale.value for timescale in TimeScale]: From ee5ee332858866433fc8770be2b93e6785417630 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 15 Nov 2021 20:41:18 +0200 Subject: [PATCH 09/10] Add unique users metrics. --- .../mooncrawl/stats_worker/dashboard.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index c9dc1966..cd7ef74d 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -9,8 +9,6 @@ import time from datetime import datetime, timedelta from enum import Enum from typing import Any, Callable, Dict, List -from pprint import pprint -import traceback from web3.datastructures import T @@ -35,9 +33,7 @@ from ..settings import ( ) from ..settings import bugout_client as bc -from web3 import HTTPProvider, IPCProvider, Web3 -from web3.middleware import geth_poa_middleware -from web3.types import BlockData +from web3 import Web3 logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -132,6 +128,7 @@ def generate_metrics( db_session: Session, identifying_column: Column, statistic_column: Column, + aggregate_func: Callable, ) -> Query: unformated_time_series_subquery = db_session.query( @@ -154,7 +151,7 @@ def generate_metrics( metric_count_subquery = ( db_session.query( - func.count(statistic_column).label("count"), + aggregate_func(statistic_column).label("count"), func.to_char( func.to_timestamp(block_model.timestamp).cast(Date), time_format ).label("timeseries_points"), @@ -189,6 +186,8 @@ def generate_metrics( for created_date, count in metrics_time_series: + if not isinstance(count, int): + count = int(count) response_metric.append({"date": created_date, "count": count}) return response_metric @@ -200,6 +199,7 @@ def generate_metrics( db_session, transaction_model.from_address, transaction_model.hash, + func.count, ) print("--- transactions_out %s seconds ---" % (time.time() - start_time)) @@ -209,6 +209,7 @@ def generate_metrics( db_session, transaction_model.to_address, transaction_model.hash, + func.count, ) print("--- transactions_in %s seconds ---" % (time.time() - start_time)) @@ -218,6 +219,7 @@ def generate_metrics( db_session, transaction_model.from_address, transaction_model.value, + func.sum, ) print("--- value_out %s seconds ---" % (time.time() - start_time)) @@ -226,6 +228,7 @@ def generate_metrics( db_session, transaction_model.to_address, transaction_model.value, + func.sum, ) print("--- value_in %s seconds ---" % (time.time() - start_time)) @@ -388,6 +391,22 @@ def cast_to_python_type(evm_type: str) -> Callable: raise ValueError(f"Cannot convert to python type {evm_type}") +def get_unique_address( + db_session: Session, blockchain_type: AvailableBlockchainType, address: str +): + label_model = get_label_model(blockchain_type) + + return ( + db_session.query(label_model.label_data["args"]["to"]) + .filter(label_model.address == address) + .filter(label_model.label == CRAWLER_LABEL) + .filter(label_model.label_data["type"].astext == "event") + .filter(label_model.label_data["name"].astext == "Transfer") + .distinct() + .count() + ) + + def stats_generate_handler(args: argparse.Namespace): """ Start crawler with generate. @@ -515,6 +534,17 @@ def stats_generate_handler(args: argparse.Namespace): except Exception as e: print(f"Failed to call {extcall['name']} error: {e}") + extention_data.append( + { + "display_name": "Overall unique token owners.", + "value": get_unique_address( + db_session=db_session, + blockchain_type=blockchain_type, + address=address, + ), + } + ) + for timescale in [timescale.value for timescale in TimeScale]: start_date = ( From f4e3c0addd4d5f15b6163a659a8feeab19db1b86 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 15 Nov 2021 20:46:49 +0200 Subject: [PATCH 10/10] Fix missing import. --- crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index cd7ef74d..667aa6a5 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -10,8 +10,6 @@ from datetime import datetime, timedelta from enum import Enum from typing import Any, Callable, Dict, List -from web3.datastructures import T - import boto3 # type: ignore from bugout.data import BugoutResources from moonstreamdb.db import yield_db_session_ctx