mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #922 from moonstream-to/add-state-task-SC 

Add Shadowcorns.
pull/925/head
Andrey Dolgolev 2023-09-11 19:39:57 +03:00 zatwierdzone przez GitHub
rodzic 6292aecd3a 21a0fc4157
commit e69d81d1fb
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
4 zmienionych plików z 74 dodań i 35 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

5
crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py
Wyświetl plik

@ -55,7 +55,10 @@ def crawl_uri(metadata_uri: str) -> Any:
try: try:
response = urllib.request.urlopen(metadata_uri, timeout=10) response = urllib.request.urlopen(metadata_uri, timeout=10)
if response.status == 200: if (
metadata_uri.startswith("data:application/json")
or response.status == 200
):
result = json.loads(response.read()) result = json.loads(response.read())
break break
retry += 1 retry += 1

1
crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py
Wyświetl plik

@ -132,6 +132,7 @@ def get_current_metadata_for_address(
WHERE WHERE
address = :address address = :address
AND label = :label AND label = :label
AND label_data ->>'metadata' != 'null'
ORDER BY ORDER BY
label_data ->> 'token_id' ASC, label_data ->> 'token_id' ASC,
block_number :: INT DESC; block_number :: INT DESC;

31
crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
Wyświetl plik

@ -188,5 +188,36 @@
} }
], ],
"address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb"
},
{
"type": "function",
"stateMutability": "view",
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256",
"value": {
"type": "queryAPI",
"query_url": "template_erc721_minting",
"blockchain": "polygon",
"params": {
"address": "0xa7D50EE3D7485288107664cf758E877a0D351725"
},
"keys": [
"token_id"
]
}
}
],
"name": "tokenURI",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"address": "0xa7D50EE3D7485288107664cf758E877a0D351725"
} }
] ]

72
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -160,7 +160,7 @@ def generate_data(
.filter(label_model.address == address) .filter(label_model.address == address)
.filter(label_model.label == crawler_label) .filter(label_model.label == crawler_label)
.filter(label_model.label_data["type"].astext == metric_type) .filter(label_model.label_data["type"].astext == metric_type)
.filter(in_op(label_model.label_data["name"].astext, functions)) .filter(in_op(label_model.label_data["name"].astext, functions)) # type: ignore
.filter( .filter(
label_model.block_timestamp label_model.block_timestamp
>= sqlalchemy_cast( >= sqlalchemy_cast(
@ -225,8 +225,8 @@ def generate_data(
.join( .join(
with_empty_times_series_with_tags, with_empty_times_series_with_tags,
and_( and_(
with_empty_times_series_with_tags.c.label with_empty_times_series_with_tags.c.label # type: ignore
== with_timetrashold_data.c.label, == with_timetrashold_data.c.label, # type: ignore
with_empty_times_series_with_tags.c.timeseries_points with_empty_times_series_with_tags.c.timeseries_points
== with_timetrashold_data.c.timeseries_points, == with_timetrashold_data.c.timeseries_points,
), ),
@ -405,7 +405,7 @@ def process_external_merged(
for extcall in external_calls_normalized: for extcall in external_calls_normalized:
try: try:
contract = web3_client.eth.contract( contract = web3_client.eth.contract( # type: ignore
address=extcall["address"], abi=extcall["abi"] address=extcall["address"], abi=extcall["abi"]
) )
response = contract.functions[extcall["name"]]( response = contract.functions[extcall["name"]](
@ -472,7 +472,7 @@ def process_external(
for extcall in external_calls: for extcall in external_calls:
try: try:
contract = web3_client.eth.contract( contract = web3_client.eth.contract( # type: ignore
address=extcall["address"], abi=extcall["abi"] address=extcall["address"], abi=extcall["abi"]
) )
response = contract.functions[extcall["name"]]( response = contract.functions[extcall["name"]](
@ -709,25 +709,29 @@ def stats_generate_handler(args: argparse.Namespace):
address = subscription.address address = subscription.address
if address not in address_dashboard_id_subscription_id_tree: if address not in address_dashboard_id_subscription_id_tree:
address_dashboard_id_subscription_id_tree[address] = {} address_dashboard_id_subscription_id_tree[address] = {} # type: ignore
if ( if (
str(dashboard.id) str(dashboard.id)
not in address_dashboard_id_subscription_id_tree not in address_dashboard_id_subscription_id_tree
): ):
address_dashboard_id_subscription_id_tree[address][ address_dashboard_id_subscription_id_tree[address][ # type: ignore
str(dashboard.id) str(dashboard.id)
] = [] ] = []
if ( if (
subscription_id subscription_id
not in address_dashboard_id_subscription_id_tree[ not in address_dashboard_id_subscription_id_tree[ # type: ignore
address address # type: ignore
][str(dashboard.id)] ][
):
address_dashboard_id_subscription_id_tree[address][
str(dashboard.id) str(dashboard.id)
].append(subscription_id) ]
): # type: ignore
address_dashboard_id_subscription_id_tree[address][ # type: ignore
str(dashboard.id)
].append(
subscription_id
)
abi = None abi = None
if "abi" in subscription.secondary_fields: if "abi" in subscription.secondary_fields:
@ -761,23 +765,23 @@ def stats_generate_handler(args: argparse.Namespace):
) )
if address not in merged_events: if address not in merged_events:
merged_events[address] = {} merged_events[address] = {} # type: ignore
merged_events[address]["merged"] = set() merged_events[address]["merged"] = set() # type: ignore
if address not in merged_functions: if address not in merged_functions:
merged_functions[address] = {} merged_functions[address] = {} # type: ignore
merged_functions[address]["merged"] = set() merged_functions[address]["merged"] = set() # type: ignore
if str(dashboard.id) not in merged_events[address]: if str(dashboard.id) not in merged_events[address]: # type: ignore
merged_events[address][str(dashboard.id)] = {} merged_events[address][str(dashboard.id)] = {} # type: ignore
if str(dashboard.id) not in merged_functions[address]: if str(dashboard.id) not in merged_functions[address]: # type: ignore
merged_functions[address][str(dashboard.id)] = {} merged_functions[address][str(dashboard.id)] = {} # type: ignore
merged_events[address][str(dashboard.id)][ merged_events[address][str(dashboard.id)][ # type: ignore
subscription_id subscription_id
] = events ] = events
merged_functions[address][str(dashboard.id)][ merged_functions[address][str(dashboard.id)][ # type: ignore
subscription_id subscription_id
] = methods ] = methods
@ -844,10 +848,10 @@ def stats_generate_handler(args: argparse.Namespace):
# Fill merged events and functions calls for all subscriptions # Fill merged events and functions calls for all subscriptions
for event in events: for event in events:
merged_events[address]["merged"].add(event) merged_events[address]["merged"].add(event) # type: ignore
for method in methods: for method in methods:
merged_functions[address]["merged"].add(method) merged_functions[address]["merged"].add(method) # type: ignore
except Exception as e: except Exception as e:
logger.error(f"Error while merging subscriptions: {e}") logger.error(f"Error while merging subscriptions: {e}")
@ -996,7 +1000,7 @@ def stats_generate_handler(args: argparse.Namespace):
"statistics", "statistics",
f"blockchain:{args.blockchain}" f"blockchain:{args.blockchain}"
f"subscriptions:{subscription_id}", f"subscriptions:{subscription_id}",
f"dashboard:{dashboard}", f"dashboard:{dashboard}", # type: ignore
], ],
) )
logger.error(err) logger.error(err)
@ -1040,7 +1044,7 @@ def stats_generate_api_task(
subscription_id = dashboard_subscription_filters["subscription_id"] subscription_id = dashboard_subscription_filters["subscription_id"]
subscription_type_id = None subscription_type_id = None
for required_field in subscription_by_id[ for required_field in subscription_by_id[ # type: ignore
subscription_id subscription_id
].required_fields: ].required_fields:
if "subscription_type_id" in required_field: if "subscription_type_id" in required_field:
@ -1065,8 +1069,8 @@ def stats_generate_api_task(
crawler_label = CRAWLER_LABEL crawler_label = CRAWLER_LABEL
abi = None abi = None
if "abi" in subscription_by_id[subscription_id].secondary_fields: if "abi" in subscription_by_id[subscription_id].secondary_fields: # type: ignore
abi = subscription_by_id[subscription_id].secondary_fields["abi"] abi = subscription_by_id[subscription_id].secondary_fields["abi"] # type: ignore
# Read required events, functions and web3_call form ABI # Read required events, functions and web3_call form ABI
if abi is None: if abi is None:
@ -1097,7 +1101,7 @@ def stats_generate_api_task(
db_session=db_session, db_session=db_session,
events=events, events=events,
blockchain_type=blockchain_type, blockchain_type=blockchain_type,
address=address, address=address, # type: ignore
crawler_label=crawler_label, crawler_label=crawler_label,
abi_json=abi_json, abi_json=abi_json,
access_id=access_id, access_id=access_id,
@ -1127,7 +1131,7 @@ def stats_generate_api_task(
functions_calls_data = generate_data( functions_calls_data = generate_data(
db_session=db_session, db_session=db_session,
blockchain_type=blockchain_type, blockchain_type=blockchain_type,
address=address, address=address, # type: ignore
timescale=timescale, timescale=timescale,
functions=methods, functions=methods,
start=start_date, start=start_date,
@ -1140,7 +1144,7 @@ def stats_generate_api_task(
events_data = generate_data( events_data = generate_data(
db_session=db_session, db_session=db_session,
blockchain_type=blockchain_type, blockchain_type=blockchain_type,
address=address, address=address, # type: ignore
timescale=timescale, timescale=timescale,
functions=events, functions=events,
start=start_date, start=start_date,
@ -1154,7 +1158,7 @@ def stats_generate_api_task(
push_statistics( push_statistics(
statistics_data=s3_data_object, statistics_data=s3_data_object,
subscription_type_id=subscription_type_id, subscription_type_id=subscription_type_id,
address=address, address=address, # type: ignore
timescale=timescale, timescale=timescale,
bucket=MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, # type: ignore bucket=MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, # type: ignore
dashboard_id=dashboard.id, dashboard_id=dashboard.id,
@ -1165,7 +1169,7 @@ def stats_generate_api_task(
[ [
"dashboard", "dashboard",
"statistics", "statistics",
f"subscriptions:{subscription_id}", f"subscriptions:{subscription_id}", # type: ignore
f"dashboard:{str(dashboard.id)}", f"dashboard:{str(dashboard.id)}",
], ],
) )