mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #814 from moonstream-to/support-interfaces-endpoint 

Support interfaces endpoint
pull/828/head
Andrey Dolgolev 2023-07-04 13:29:57 +03:00 zatwierdzone przez GitHub
rodzic 068bebb586 42849691ff
commit 4fa396d365
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
6 zmienionych plików z 689 dodań i 39 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

166
moonstreamapi/moonstreamapi/actions.py
Wyświetl plik

@ -22,6 +22,7 @@ from entity.exceptions import EntityUnexpectedResponse # type: ignore
from ens.utils import is_valid_ens_name # type: ignore
from eth_utils.address import is_address # type: ignore
from moonstreamdb.models import EthereumLabel
from moonstreamdb.blockchain import AvailableBlockchainType
from slugify import slugify # type: ignore
from sqlalchemy import text
from sqlalchemy.orm import Session
@ -41,8 +42,13 @@ from .settings import (
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
support_interfaces,
supportsInterface_abi,
)
from .settings import bugout_client as bc, entity_client as ec
from .web3_provider import multicall, FunctionSignature, connect
from .selectors_storage import selectors
logger = logging.getLogger(__name__)
@ -467,20 +473,16 @@ def get_all_entries_from_search(
results: List[BugoutSearchResult] = []
try:
existing_metods = bc.search(
token=token,
journal_id=journal_id,
query=search_query,
content=False,
timeout=10.0,
limit=limit,
offset=offset,
)
results.extend(existing_metods.results)
except Exception as e:
reporter.error_report(e)
existing_metods = bc.search(
token=token,
journal_id=journal_id,
query=search_query,
content=False,
timeout=10.0,
limit=limit,
offset=offset,
)
results.extend(existing_metods.results)
if len(results) != existing_metods.total_results:
for offset in range(limit, existing_metods.total_results, limit):
@ -780,16 +782,134 @@ def query_parameter_hash(params: Dict[str, Any]) -> str:
return hash
def get_moonworm_jobs(
address: str,
subscription_type_id: str,
entries_limit: int = 100,
):
def parse_abi_to_name_tags(user_abi: List[Dict[str, Any]]):
return [
f"abi_name:{method['name']}"
for method in user_abi
if method["type"] in ("event", "function")
]
def filter_tasks(entries, tag_filters):
return [entry for entry in entries if any(tag in tag_filters for tag in entry.tags)]
def fetch_and_filter_tasks(
journal_id, address, subscription_type_id, token, user_abi, limit=100
) -> List[BugoutSearchResult]:
"""
Fetch tasks from journal and filter them by user abi
"""
entries = get_all_entries_from_search(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
journal_id=journal_id,
search_query=f"tag:address:{address} tag:subscription_type:{subscription_type_id}",
limit=entries_limit, # load per request
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
limit=limit,
token=token,
)
return entries
user_loaded_abi_tags = parse_abi_to_name_tags(json.loads(user_abi))
moonworm_tasks = filter_tasks(entries, user_loaded_abi_tags)
return moonworm_tasks
def get_moonworm_tasks(
subscription_type_id: str,
address: str,
user_abi: List[Dict[str, Any]],
) -> List[BugoutSearchResult]:
"""
Get moonworm tasks from journal and filter them by user abi
"""
try:
moonworm_tasks = fetch_and_filter_tasks(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
address=address,
subscription_type_id=subscription_type_id,
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
user_abi=user_abi,
)
except Exception as e:
logger.error(f"Error get moonworm tasks: {str(e)}")
MoonstreamHTTPException(status_code=500, internal_error=e)
return moonworm_tasks
def get_list_of_support_interfaces(
blockchain_type: AvailableBlockchainType,
address: str,
user_token: uuid.UUID,
multicall_method: str = "tryAggregate",
):
"""
Returns list of interfaces supported by given address
"""
web3_client = connect(blockchain_type, user_token=user_token)
contract = web3_client.eth.contract(
address=Web3.toChecksumAddress(address),
abi=supportsInterface_abi,
)
calls = []
list_of_interfaces = list(selectors.keys())
list_of_interfaces.sort()
for interaface in list_of_interfaces:
calls.append(
(
contract.address,
FunctionSignature(contract.get_function_by_name("supportsInterface"))
.encode_data([bytes.fromhex(interaface)])
.hex(),
)
)
try:
multicall_result = multicall(
web3_client=web3_client,
blockchain_type=blockchain_type,
calls=calls,
method=multicall_method,
)
except Exception as e:
logger.error(f"Error while getting list of support interfaces: {e}")
result = {}
for i, selector in enumerate(list_of_interfaces):
if multicall_result[i][0]:
supported = FunctionSignature(
contract.get_function_by_name("supportsInterface")
).decode_data(multicall_result[i][1])
if supported[0]:
result[selectors[selector]["name"]] = { # type: ignore
"selector": selector,
"abi": selectors[selector]["abi"], # type: ignore
}
return result
def check_if_smartcontract(
blockchain_type: AvailableBlockchainType,
address: str,
user_token: uuid.UUID,
):
"""
Checks if address is a smart contract on blockchain
"""
web3_client = connect(blockchain_type, user_token=user_token)
is_contract = False
code = web3_client.eth.getCode(address)
if code != b"":
is_contract = True
return blockchain_type, address, is_contract

18
moonstreamapi/moonstreamapi/data.py
Wyświetl plik

@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Union, Literal
from uuid import UUID
from xmlrpc.client import Boolean
from pydantic import BaseModel, Field, validator
from pydantic import BaseModel, Field, validator
from sqlalchemy import false
USER_ONBOARDING_STATE = "onboarding_state"
@ -295,10 +295,18 @@ class QueryInfoResponse(BaseModel):
preapprove: bool = False
approved: bool = False
parameters: Dict[str, Any] = Field(default_factory=dict)
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
created_at: Optional[datetime]
updated_at: Optional[datetime]
class SuggestedQueriesResponse(BaseModel):
interfaces: Dict[str, Any] = Field(default_factory=dict)
queries: List[Any] = Field(default_factory=list)
queries: List[Any] = Field(default_factory=list)
class ContractInfoResponse(BaseModel):
contract_info: Dict[str, Any] = Field(default_factory=dict)
class ContractInterfacesResponse(BaseModel):
interfaces: Dict[str, Any] = Field(default_factory=dict)

133
moonstreamapi/moonstreamapi/routes/subscriptions.py
Wyświetl plik

@ -1,13 +1,17 @@
"""
The Moonstream subscriptions HTTP API
"""
from concurrent.futures import as_completed, ProcessPoolExecutor, ThreadPoolExecutor
import hashlib
import json
import logging
from typing import Any, Dict, List, Optional
import traceback
from bugout.exceptions import BugoutResponseException
from bugout.data import BugoutSearchResult
from fastapi import APIRouter, Depends, Request, Form, BackgroundTasks
from moonstreamdb.blockchain import AvailableBlockchainType
from web3 import Web3
from ..actions import (
@ -15,7 +19,9 @@ from ..actions import (
apply_moonworm_tasks,
get_entity_subscription_collection_id,
EntityCollectionNotFoundException,
get_moonworm_jobs,
get_moonworm_tasks,
check_if_smartcontract,
get_list_of_support_interfaces,
)
from ..admin import subscription_types
from .. import data
@ -23,8 +29,10 @@ from ..admin import subscription_types
from ..middleware import MoonstreamHTTPException
from ..reporter import reporter
from ..settings import bugout_client as bc, entity_client as ec
from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_MOONWORM_TASKS_JOURNAL
from ..web3_provider import yield_web3_provider
from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, THREAD_TIMEOUT_SECONDS
from ..web3_provider import (
yield_web3_provider,
)
logger = logging.getLogger(__name__)
@ -488,7 +496,7 @@ async def get_subscription_abi_handler(
@router.get(
"/{subscription_id}/jobs",
tags=["subscriptions"],
response_model=data.SubdcriptionsAbiResponse,
response_model=List[BugoutSearchResult],
)
async def get_subscription_jobs_handler(
request: Request,
@ -527,12 +535,12 @@ async def get_subscription_jobs_handler(
if "subscription_type_id" in field:
subscription_type_id = field["subscription_type_id"]
if "address" in field:
subscription_address = field["address"]
subscription_address = subscription_resource.address
get_moonworm_jobs_response = get_moonworm_jobs(
get_moonworm_jobs_response = get_moonworm_tasks(
subscription_type_id=subscription_type_id,
address=subscription_address,
user_abi=subscription_resource.secondary_fields.get("abi") or [],
)
return get_moonworm_jobs_response
@ -559,3 +567,114 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse:
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.SubscriptionTypesListResponse(subscription_types=results)
@router.get(
"/is_contract",
tags=["subscriptions"],
response_model=data.ContractInfoResponse,
)
async def address_info(request: Request, address: str):
"""
Looking if address is contract
"""
user_token = request.state.token
try:
Web3.toChecksumAddress(address)
except ValueError as e:
raise MoonstreamHTTPException(
status_code=400,
detail=str(e),
internal_error=e,
)
contract_info = {}
for blockchain_type in AvailableBlockchainType:
try:
# connnect to blockchain
futures = []
with ThreadPoolExecutor(max_workers=5) as executor:
futures.append(
executor.submit(
check_if_smartcontract,
address=address,
blockchain_type=blockchain_type,
user_token=user_token,
)
)
for future in as_completed(futures):
blockchain_type, address, is_contract = future.result(
timeout=THREAD_TIMEOUT_SECONDS
)
if is_contract:
contract_info[blockchain_type.value] = is_contract
except Exception as e:
logger.error(f"Error reading contract info from web3: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
if len(contract_info) == 0:
raise MoonstreamHTTPException(
status_code=404,
detail="Not found contract on chains. EOA address or not used valid address.",
)
return data.ContractInfoResponse(
contract_info=contract_info,
)
@router.get(
"/supported_interfaces",
tags=["subscriptions"],
response_model=data.ContractInterfacesResponse,
)
def get_contract_interfaces(
request: Request,
address: str,
blockchain: str,
):
"""
Request contract interfaces from web3
"""
user_token = request.state.token
try:
Web3.toChecksumAddress(address)
except ValueError as e:
raise MoonstreamHTTPException(
status_code=400,
detail=str(e),
internal_error=e,
)
try:
blockchain_type = AvailableBlockchainType(blockchain)
except ValueError as e:
raise MoonstreamHTTPException(
status_code=400,
detail=str(e),
internal_error=e,
)
try:
interfaces = get_list_of_support_interfaces(
blockchain_type=blockchain_type,
address=address,
user_token=user_token,
)
except Exception as e:
logger.error(f"Error reading contract info from web3: {str(e)}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return data.ContractInterfacesResponse(
interfaces=interfaces,
)

129
moonstreamapi/moonstreamapi/selectors_storage.py 100644
Wyświetl plik

File diff suppressed because one or more lines are too long

132
moonstreamapi/moonstreamapi/settings.py
Wyświetl plik

@ -1,7 +1,10 @@
import os
from typing import Optional, Dict
from uuid import UUID
from bugout.app import Bugout
from entity.client import Entity # type: ignore
from moonstreamdb.blockchain import AvailableBlockchainType
# Bugout
@ -110,6 +113,30 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "":
raise ValueError(
"MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable must be set"
)
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_POLYGON_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_POLYGON_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_POLYGON_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_XDAI_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_WYRM_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
if MOONSTREAM_S3_QUERIES_BUCKET == "":
@ -129,7 +156,106 @@ if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "":
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION = "subscription"
BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION = "entity_subscription"
BUGOUT_RESOURCE_TYPE_DASHBOARD = "dashboards"
### Moonstream queries
MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE = "moonstream_query_template"
# Node balancer
NB_ACCESS_ID_HEADER = os.environ.get("NB_ACCESS_ID_HEADER", "x-node-balancer-access-id")
NB_DATA_SOURCE_HEADER = os.environ.get(
"NB_DATA_SOURCE_HEADER", "x-node-balancer-data-source"
)
NB_DATA_SOURCE_HEADER_VALUE = os.environ.get(
"NB_DATA_SOURCE_HEADER_VALUE", "blockchain"
)
# Thread timeout
THREAD_TIMEOUT_SECONDS = 10
support_interfaces = [
{"name": "_INTERFACE_ID_ERC165", "selector": "0x01ffc9a7"},
{"name": "_INTERFACE_ID_ERC20", "selector": "0x36372b07"},
{"name": "_INTERFACE_ID_ERC721", "selector": "0x80ac58cd"},
{"name": "_INTERFACE_ID_ERC721_METADATA", "selector": "0x5b5e139f"}, # miss
{"name": "_INTERFACE_ID_ERC721_ENUMERABLE", "selector": "0x780e9d63"}, # miss
{"name": "_INTERFACE_ID_ERC721_RECEIVED", "selector": "0x150b7a02"},
{
"name": "_INTERFACE_ID_ERC721_METADATA_RECEIVED",
"selector": "0x0e89341c",
}, # miss
{"name": "_INTERFACE_ID_ERC721_ENUMERABLE_RECEIVED", "selector": "0x4e2312e0"},
{"name": "_INTERFACE_ID_ERC1155", "selector": "0xd9b67a26"},
]
multicall_contracts: Dict[AvailableBlockchainType, str] = {
AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6",
AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631",
AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696",
}
multicall_contract_abi = [
{
"inputs": [
{"internalType": "bool", "name": "requireSuccess", "type": "bool"},
{
"components": [
{
"internalType": "address",
"name": "target",
"type": "address",
},
{
"internalType": "bytes",
"name": "callData",
"type": "bytes",
},
],
"internalType": "struct Multicall2.Call[]",
"name": "calls",
"type": "tuple[]",
},
],
"name": "tryAggregate",
"outputs": [
{
"components": [
{"internalType": "bool", "name": "success", "type": "bool"},
{
"internalType": "bytes",
"name": "returnData",
"type": "bytes",
},
],
"internalType": "struct Multicall2.Result[]",
"name": "returnData",
"type": "tuple[]",
}
],
"stateMutability": "nonpayable",
"type": "function",
},
{
"inputs": [{"internalType": "bytes", "name": "data", "type": "bytes"}],
"name": "aggregate",
"outputs": [
{"internalType": "bool", "name": "success", "type": "bool"},
{"internalType": "bytes", "name": "result", "type": "bytes"},
],
"stateMutability": "payable",
"type": "function",
},
]
supportsInterface_abi = [
{
"inputs": [{"internalType": "bytes4", "name": "interfaceId", "type": "bytes4"}],
"name": "supportsInterface",
"outputs": [{"internalType": "bool", "name": "", "type": "bool"}],
"stateMutability": "view",
"type": "function",
}
]

150
moonstreamapi/moonstreamapi/web3_provider.py
Wyświetl plik

@ -1,8 +1,28 @@
import logging
from uuid import UUID
from typing import Any, Optional, Union, Callable
from web3 import Web3
from web3.middleware import geth_poa_middleware
from eth_abi import encode_single, decode_single
from eth_utils import function_signature_to_4byte_selector
from web3 import Web3
from web3.contract import ContractFunction
from web3.providers.rpc import HTTPProvider
from web3._utils.abi import normalize_event_input_types
from .settings import MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI
from .settings import (
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
NB_ACCESS_ID_HEADER,
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
multicall_contracts,
multicall_contract_abi,
)
from moonstreamdb.blockchain import AvailableBlockchainType
logger = logging.getLogger(__name__)
@ -13,3 +33,131 @@ moonstream_web3_provider = Web3(
def yield_web3_provider() -> Web3:
return moonstream_web3_provider
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = 10
def connect(
blockchain_type: AvailableBlockchainType,
web3_uri: Optional[str] = None,
access_id: Optional[UUID] = None,
user_token: Optional[UUID] = None,
) -> Web3:
request_kwargs: D = {}
if blockchain_type != AvailableBlockchainType.WYRM:
request_kwargs = {
"headers": {
"Content-Type": "application/json",
}
}
if access_id is not None:
request_kwargs["headers"][NB_ACCESS_ID_HEADER] = str(access_id)
elif user_token is not None:
request_kwargs["headers"]["Authorization"] = f"Bearer {user_token}"
if web3_uri is None:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.POLYGON:
web3_uri = MOONSTREAM_POLYGON_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.MUMBAI:
web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.XDAI:
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.WYRM:
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
request_kwargs["timeout"] = WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS
web3_client = Web3(HTTPProvider(web3_uri, request_kwargs=request_kwargs)) # type: ignore
else:
web3_client = Web3(Web3.IPCProvider(web3_uri))
if blockchain_type != AvailableBlockchainType.ETHEREUM:
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
return web3_client
def multicall(
web3_client: Web3,
blockchain_type: AvailableBlockchainType,
calls: list,
method: str = "tryAggregate",
block_identifier: Union[str, int, bytes, None] = "latest",
) -> list:
"""
Calls multicall contract with given calls and returns list of results
"""
multicall_contract = web3_client.eth.contract(
address=Web3.toChecksumAddress(multicall_contracts[blockchain_type]),
abi=multicall_contract_abi,
)
return multicall_contract.get_function_by_name(method)(False, calls).call(
block_identifier=block_identifier
)
def cast_to_python_type(evm_type: str) -> Callable:
if evm_type.startswith(("uint", "int")):
return int
elif evm_type.startswith("bytes"):
return bytes
elif evm_type == "string":
return str
elif evm_type == "address":
return Web3.toChecksumAddress
elif evm_type == "bool":
return bool
else:
raise ValueError(f"Cannot convert to python type {evm_type}")
class FunctionInput:
def __init__(self, name: str, value: Any, solidity_type: str):
self.name = name
self.value = value
self.solidity_type = solidity_type
class FunctionSignature:
def __init__(self, function: ContractFunction):
self.name = function.abi["name"]
self.inputs = [
{"name": arg["name"], "type": arg["type"]}
for arg in normalize_event_input_types(function.abi.get("inputs", []))
]
self.input_types_signature = "({})".format(
",".join([inp["type"] for inp in self.inputs])
)
self.output_types_signature = "({})".format(
",".join(
[
arg["type"]
for arg in normalize_event_input_types(
function.abi.get("outputs", [])
)
]
)
)
self.signature = "{}{}".format(self.name, self.input_types_signature)
self.fourbyte = function_signature_to_4byte_selector(self.signature)
def encode_data(self, args=None) -> bytes:
return (
self.fourbyte + encode_single(self.input_types_signature, args)
if args
else self.fourbyte
)
def decode_data(self, output):
return decode_single(self.output_types_signature, output)