moonstream/moonstreamapi/moonstreamapi/actions.py

from collections import OrderedDict
import hashlib
import json
from itertools import chain
import logging
from typing import List, Optional, Dict, Any, Union
from enum import Enum
import uuid

import boto3  # type: ignore

from bugout.data import (
    BugoutSearchResults,
    BugoutSearchResult,
    BugoutResource,
    BugoutResources,
)
from bugout.journal import SearchOrder
from bugout.exceptions import BugoutResponseException
from entity.data import EntityCollectionsResponse, EntityCollectionResponse  # type: ignore
from entity.exceptions import EntityUnexpectedResponse  # type: ignore
from ens.utils import is_valid_ens_name  # type: ignore
from eth_utils.address import is_address  # type: ignore
from moonstreamdb.models import EthereumLabel
from slugify import slugify  # type: ignore
from sqlalchemy import text
from sqlalchemy.orm import Session
from web3 import Web3
from web3._utils.validation import validate_abi

from . import data
from .middleware import MoonstreamHTTPException
from .reporter import reporter
from .settings import (
    BUGOUT_REQUEST_TIMEOUT_SECONDS,
    ETHERSCAN_SMARTCONTRACTS_BUCKET,
    MOONSTREAM_ADMIN_ACCESS_TOKEN,
    MOONSTREAM_APPLICATION_ID,
    MOONSTREAM_DATA_JOURNAL_ID,
    MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
    MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
    MOONSTREAM_MOONWORM_TASKS_JOURNAL,
    MOONSTREAM_ADMIN_ACCESS_TOKEN,
)
from .settings import bugout_client as bc, entity_client as ec

logger = logging.getLogger(__name__)


blockchain_by_subscription_id = {
    "ethereum_blockchain": "ethereum",
    "polygon_blockchain": "polygon",
    "mumbai_blockchain": "mumbai",
    "xdai_blockchain": "xdai",
    "wyrm_blockchain": "wyrm",
    "ethereum_smartcontract": "ethereum",
    "polygon_smartcontract": "polygon",
    "mumbai_smartcontract": "mumbai",
    "xdai_smartcontract": "xdai",
    "wyrm_smartcontract": "wyrm",
}


class StatusAPIException(Exception):
    """
    Raised during checking Moonstream API statuses.
    """


class NameNormalizationException(Exception):
    """
    Raised on actions when slugify can't normalize name.
    """


class ResourceQueryFetchException(Exception):
    """
    Exception in queries API
    """


class EntityCollectionNotFoundException(Exception):
    """
    Raised when entity collection is not found
    """


class LabelNames(Enum):
    ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract"
    COINMARKETCAP_TOKEN = "coinmarketcap_token"
    ERC721 = "erc721"


def get_contract_source_info(
    db_session: Session, contract_address: str
) -> Optional[data.EthereumSmartContractSourceInfo]:
    label = (
        db_session.query(EthereumLabel)
        .filter(EthereumLabel.address == contract_address)
        .filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value)
        .one_or_none()
    )
    if label is None:
        return None

    object_uri = label.label_data["object_uri"]
    key = object_uri.split("s3://etherscan-smart-contracts/")[1]
    s3 = boto3.client("s3")
    bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET
    try:
        raw_obj = s3.get_object(Bucket=bucket, Key=key)
        obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"]
        contract_source_info = data.EthereumSmartContractSourceInfo(
            name=obj_data["ContractName"],
            source_code=obj_data["SourceCode"],
            compiler_version=obj_data["CompilerVersion"],
            abi=obj_data["ABI"],
        )
        return contract_source_info
    except Exception as e:
        logger.error(f"Failed to load smart contract {object_uri}")
        reporter.error_report(e)

    return None


def get_ens_name(web3: Web3, address: str) -> Optional[str]:
    try:
        checksum_address = web3.toChecksumAddress(address)
    except:
        raise ValueError(f"{address} is invalid ethereum address is passed")
    try:
        ens_name = web3.ens.name(checksum_address)
        return ens_name
    except Exception as e:
        reporter.error_report(e, ["web3", "ens"])
        logger.error(
            f"Cannot get ens name for address {checksum_address}. Probably node is down"
        )
        raise e


def get_ens_address(web3: Web3, name: str) -> Optional[str]:
    if not is_valid_ens_name(name):
        raise ValueError(f"{name} is not valid ens name")

    try:
        ens_checksum_address = web3.ens.address(name)
        if ens_checksum_address is not None:
            ordinary_address = ens_checksum_address.lower()
            return ordinary_address
        return None
    except Exception as e:
        reporter.error_report(e, ["web3", "ens"])
        logger.error(f"Cannot get ens address for name {name}. Probably node is down")
        raise e


def get_ethereum_address_info(
    db_session: Session, web3: Web3, address: str
) -> Optional[data.EthereumAddressInfo]:
    if not is_address(address):
        raise ValueError(f"Invalid ethereum address : {address}")

    address_info = data.EthereumAddressInfo(address=address)

    try:
        address_info.ens_name = get_ens_name(web3, address)
    except:
        pass

    etherscan_address_url = f"https://etherscan.io/address/{address}"
    etherscan_token_url = f"https://etherscan.io/token/{address}"
    blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}"

    coinmarketcap_label: Optional[EthereumLabel] = (
        db_session.query(EthereumLabel)
        .filter(EthereumLabel.address == address)
        .filter(EthereumLabel.label == LabelNames.COINMARKETCAP_TOKEN.value)
        .order_by(text("created_at desc"))
        .limit(1)
        .one_or_none()
    )

    if coinmarketcap_label is not None:
        address_info.token = data.EthereumTokenDetails(
            name=coinmarketcap_label.label_data["name"],
            symbol=coinmarketcap_label.label_data["symbol"],
            external_url=[
                coinmarketcap_label.label_data["coinmarketcap_url"],
                etherscan_token_url,
                blockchain_com_url,
            ],
        )

    # Checking for smart contract
    etherscan_label: Optional[EthereumLabel] = (
        db_session.query(EthereumLabel)
        .filter(EthereumLabel.address == address)
        .filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value)
        .order_by(text("created_at desc"))
        .limit(1)
        .one_or_none()
    )
    if etherscan_label is not None:
        address_info.smart_contract = data.EthereumSmartContractDetails(
            name=etherscan_label.label_data["name"],
            external_url=[etherscan_address_url, blockchain_com_url],
        )

    # Checking for NFT
    # Checking for smart contract
    erc721_label: Optional[EthereumLabel] = (
        db_session.query(EthereumLabel)
        .filter(EthereumLabel.address == address)
        .filter(EthereumLabel.label == LabelNames.ERC721.value)
        .order_by(text("created_at desc"))
        .limit(1)
        .one_or_none()
    )
    if erc721_label is not None:
        address_info.nft = data.EthereumNFTDetails(
            name=erc721_label.label_data.get("name"),
            symbol=erc721_label.label_data.get("symbol"),
            total_supply=erc721_label.label_data.get("totalSupply"),
            external_url=[etherscan_token_url, blockchain_com_url],
        )
    return address_info


def get_address_labels(
    db_session: Session, start: int, limit: int, addresses: Optional[str] = None
) -> data.AddressListLabelsResponse:
    """
    Attach labels to addresses.
    """
    if addresses is not None:
        addresses_list = addresses.split(",")
        addresses_obj = addresses_list[start : start + limit]
    else:
        addresses_obj = []

    addresses_response = data.AddressListLabelsResponse(addresses=[])

    for address in addresses_obj:
        labels_obj = (
            db_session.query(EthereumLabel)
            .filter(EthereumLabel.address == address)
            .all()
        )
        addresses_response.addresses.append(
            data.AddressLabelsResponse(
                address=address,
                labels=[
                    data.AddressLabelResponse(
                        label=label.label, label_data=label.label_data
                    )
                    for label in labels_obj
                ],
            )
        )

    return addresses_response


def create_onboarding_resource(
    token: uuid.UUID,
    resource_data: Dict[str, Any] = {
        "type": data.USER_ONBOARDING_STATE,
        "steps": {
            "welcome": 0,
            "subscriptions": 0,
            "stream": 0,
        },
        "is_complete": False,
    },
) -> BugoutResource:
    resource = bc.create_resource(
        token=token,
        application_id=MOONSTREAM_APPLICATION_ID,
        resource_data=resource_data,
        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
    )
    return resource


def check_api_status():
    crawl_types_timestamp: Dict[str, Any] = {
        "ethereum_txpool": None,
        "ethereum_trending": None,
    }
    for crawl_type in crawl_types_timestamp.keys():
        try:
            search_results: BugoutSearchResults = bc.search(
                token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
                journal_id=MOONSTREAM_DATA_JOURNAL_ID,
                query=f"tag:crawl_type:{crawl_type}",
                limit=1,
                content=False,
                timeout=10.0,
                order=SearchOrder.DESCENDING,
            )
            if len(search_results.results) == 1:
                crawl_types_timestamp[crawl_type] = search_results.results[0].created_at
        except Exception:
            raise StatusAPIException(
                f"Unable to get status for crawler with type: {crawl_type}"
            )

    return crawl_types_timestamp


def json_type(evm_type: str) -> type:
    if evm_type.startswith(("uint", "int")):
        return int
    elif evm_type.startswith("bytes") or evm_type == "string" or evm_type == "address":
        return str
    elif evm_type == "bool":
        return bool
    else:
        raise ValueError(f"Cannot convert to python type {evm_type}")


def dashboards_abi_validation(
    dashboard_subscription: data.DashboardMeta,
    abi: Any,
):
    """
    Validate current dashboard subscription : https://github.com/bugout-dev/moonstream/issues/345#issuecomment-953052444
    with contract abi on S3

    """

    # maybe its over but not found beter way
    abi_functions = {
        item["name"]: {inputs["name"]: inputs["type"] for inputs in item["inputs"]}
        for item in abi
        if item["type"] == "function"
    }
    if not dashboard_subscription.all_methods:
        for method in dashboard_subscription.methods:
            if method["name"] not in abi_functions:
                # Method not exists
                logger.error(
                    f"Error on dashboard resource validation method:{method['name']}"
                    f" of subscription: {dashboard_subscription.subscription_id}"
                    f"does not exists in Abi "
                )
                raise MoonstreamHTTPException(status_code=400)
            if method.get("filters") and isinstance(method["filters"], dict):
                for input_argument_name, value in method["filters"].items():
                    if input_argument_name not in abi_functions[method["name"]]:
                        # Argument not exists
                        logger.error(
                            f"Error on dashboard resource validation type argument: {input_argument_name} of method:{method['name']} "
                            f" of subscription: {dashboard_subscription.subscription_id} has incorrect"
                            f"does not exists in Abi"
                        )
                        raise MoonstreamHTTPException(status_code=400)

                    if not isinstance(
                        value,
                        json_type(abi_functions[method["name"]][input_argument_name]),
                    ):
                        # Argument has incorrect type
                        logger.error(
                            f"Error on dashboard resource validation type argument: {input_argument_name} of method:{method['name']} "
                            f" of subscription: {dashboard_subscription.subscription_id} has incorrect type {type(value)}"
                            f" when {abi_functions[method['name']][input_argument_name]} required."
                        )
                        raise MoonstreamHTTPException(status_code=400)
    abi_events = {
        item["name"]: {inputs["name"]: inputs["type"] for inputs in item["inputs"]}
        for item in abi
        if item["type"] == "event"
    }

    if not dashboard_subscription.all_events:
        for event in dashboard_subscription.events:
            if event["name"] not in abi_events:
                logger.error(
                    f"Error on dashboard resource validation event:{event['name']}"
                    f" of subscription: {dashboard_subscription.subscription_id}"
                    f"does not exists in Abi"
                )
                raise MoonstreamHTTPException(status_code=400)

            if event.get("filters") and isinstance(event["filters"], dict):
                for input_argument_name, value in event["filters"].items():
                    if input_argument_name not in abi_events[event["name"]]:
                        # Argument not exists
                        logger.error(
                            f"Error on dashboard resource validation type argument: {input_argument_name} of method:{event['name']} "
                            f" of subscription: {dashboard_subscription.subscription_id} has incorrect"
                            f"does not exists in Abi"
                        )
                        raise MoonstreamHTTPException(status_code=400)

                    if not isinstance(
                        value,
                        json_type(abi_events[event["name"]][input_argument_name]),
                    ):
                        logger.error(
                            f"Error on dashboard resource validation type argument: {input_argument_name} of method:{event['name']} "
                            f" of subscription: {dashboard_subscription.subscription_id} has incorrect type {type(value)}"
                            f" when {abi_events[event['name']][input_argument_name]} required."
                        )
                        raise MoonstreamHTTPException(status_code=400)
    return True


def validate_abi_json(abi: Any) -> None:
    """
    Transform string to json and run validation
    """

    try:
        validate_abi(abi)
    except ValueError as e:
        raise MoonstreamHTTPException(status_code=400, detail=e)
    except:
        raise MoonstreamHTTPException(
            status_code=400, detail="Error on abi valiadation."
        )


def upload_abi_to_s3(
    resource: BugoutResource,
    abi: str,
    update: Dict[str, Any],
) -> Dict[str, Any]:
    """
    Uploading ABI to s3 bucket. Return object for updating resource.

    """

    s3 = boto3.client("s3")

    bucket = MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET

    result_bytes = abi.encode("utf-8")
    result_key = f"{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{blockchain_by_subscription_id[resource.resource_data['subscription_type_id']]}/abi/{resource.resource_data['address']}/{resource.id}/abi.json"

    s3.put_object(
        Body=result_bytes,
        Bucket=bucket,
        Key=result_key,
        ContentType="application/json",
        Metadata={"Moonstream": "Abi data"},
    )

    update["abi"] = True

    update["bucket"] = MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET
    update["s3_path"] = result_key

    return update


def get_all_entries_from_search(
    journal_id: str, search_query: str, limit: int, token: str
) -> List[BugoutSearchResult]:
    """
    Get all required entries from journal using search interface
    """
    offset = 0

    results: List[BugoutSearchResult] = []

    try:
        existing_metods = bc.search(
            token=token,
            journal_id=journal_id,
            query=search_query,
            content=False,
            timeout=10.0,
            limit=limit,
            offset=offset,
        )
        results.extend(existing_metods.results)

    except Exception as e:
        reporter.error_report(e)

    if len(results) != existing_metods.total_results:
        for offset in range(limit, existing_metods.total_results, limit):
            existing_metods = bc.search(
                token=token,
                journal_id=journal_id,
                query=search_query,
                content=False,
                timeout=10.0,
                limit=limit,
                offset=offset,
            )
        results.extend(existing_metods.results)

    return results


def apply_moonworm_tasks(
    subscription_type: str,
    abi: Any,
    address: str,
    entries_limit: int = 100,
) -> None:
    """
    Get list of subscriptions loads abi and apply them as moonworm tasks if it not exist
    """

    moonworm_abi_tasks_entries_pack = []

    try:
        entries = get_all_entries_from_search(
            journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
            search_query=f"tag:address:{address} tag:subscription_type:{subscription_type}",
            limit=entries_limit,  # load per request
            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
        )

        # create historical crawl task in journal

        # will use create_entries_pack for creating entries in journal

        existing_tags = [entry.tags for entry in entries]

        existing_hashes = [
            tag.split(":")[-1]
            for tag in chain(*existing_tags)
            if "abi_method_hash" in tag
        ]

        abi_hashes_dict = {
            hashlib.md5(json.dumps(method).encode("utf-8")).hexdigest(): method
            for method in abi
            if (method["type"] in ("event", "function"))
            and (method.get("stateMutability", "") != "view")
        }

        for hash in abi_hashes_dict:
            if hash not in existing_hashes:
                abi_selector = Web3.keccak(
                    text=abi_hashes_dict[hash]["name"]
                    + "("
                    + ",".join(
                        map(lambda x: x["type"], abi_hashes_dict[hash]["inputs"])
                    )
                    + ")"
                )[:4].hex()

                moonworm_abi_tasks_entries_pack.append(
                    {
                        "title": address,
                        "content": json.dumps(abi_hashes_dict[hash], indent=4),
                        "tags": [
                            f"address:{address}",
                            f"type:{abi_hashes_dict[hash]['type']}",
                            f"abi_method_hash:{hash}",
                            f"abi_selector:{abi_selector}",
                            f"subscription_type:{subscription_type}",
                            f"abi_name:{abi_hashes_dict[hash]['name']}",
                            f"status:active",
                            f"task_type:moonworm",
                            f"moonworm_task_pickedup:False",  # True if task picked up by moonworm-crawler(default each 120 sec)
                            f"historical_crawl_status:pending",  # pending, in_progress, done
                            f"progress:0",  # 0-100 %
                        ],
                    }
                )
    except Exception as e:
        logger.error(f"Error get moonworm tasks: {str(e)}")
        reporter.error_report(e)

    if len(moonworm_abi_tasks_entries_pack) > 0:
        try:
            bc.create_entries_pack(
                token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
                journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
                entries=moonworm_abi_tasks_entries_pack,
                timeout=25,
            )
        except Exception as e:
            logger.error(f"Error create moonworm tasks: {str(e)}")
            reporter.error_report(e)


def name_normalization(query_name: str) -> str:
    """
    Sanitize provided query name.
    """
    try:
        normalized_query_name = slugify(
            query_name, max_length=50, lowercase=False, separator="_"
        )
    except Exception as e:
        logger.error(f"Error in query normalization. Error: {e}")
        raise NameNormalizationException(f"Can't normalize name:{query_name}")

    return normalized_query_name


def get_query_by_name(query_name: str, token: uuid.UUID) -> str:
    """
    Fetch query_id from Brood resources.
    """
    try:
        query_name = name_normalization(query_name)
    except Exception:
        raise NameNormalizationException("Unable to normalize query name")

    params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name}
    try:
        resources: BugoutResources = bc.list_resources(token=token, params=params)
    except BugoutResponseException as e:
        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
    except Exception as e:
        logger.error(f"Error get query, error: {str(e)}")
        raise MoonstreamHTTPException(status_code=500, internal_error=e)

    available_queries: Dict[str, str] = {
        resource.resource_data["name"]: resource.resource_data["entry_id"]
        for resource in resources.resources
    }

    if query_name not in available_queries:
        raise MoonstreamHTTPException(status_code=404, detail="Query not found.")

    query_id = available_queries[query_name]

    return query_id


def get_entity_subscription_collection_id(
    resource_type: str,
    token: Union[uuid.UUID, str],
    user_id: uuid.UUID,
    create_if_not_exist: bool = False,
) -> Optional[str]:
    """
    Get collection_id from brood resources. If collection not exist and create_if_not_exist is True
    """

    params = {
        "type": resource_type,
        "user_id": str(user_id),
    }
    try:
        resources: BugoutResources = bc.list_resources(token=token, params=params)
    except BugoutResponseException as e:
        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
    except Exception as e:
        logger.error(
            f"Error listing subscriptions for user ({user_id}) with token ({token}), error: {str(e)}"
        )
        reporter.error_report(e)
        raise MoonstreamHTTPException(status_code=500, internal_error=e)

    if len(resources.resources) == 0:
        if not create_if_not_exist:
            raise EntityCollectionNotFoundException(
                "Subscription collection not found."
            )
        collection_id = generate_collection_for_user(resource_type, token, user_id)

        return collection_id

    else:
        resource = resources.resources[0]
    return resource.resource_data["collection_id"]


def generate_collection_for_user(
    resource_type: str,
    token: Union[uuid.UUID, str],
    user_id: uuid.UUID,
) -> str:
    try:
        # try get collection

        collections: EntityCollectionsResponse = ec.list_collections(token=token)

        available_collections: Dict[str, str] = {
            collection.name: collection.collection_id
            for collection in collections.collections
        }

        subscription_collection_name = f"subscriptions_{user_id}"

        if subscription_collection_name not in available_collections:
            collection: EntityCollectionResponse = ec.add_collection(
                token=token, name=subscription_collection_name
            )
            collection_id = collection.collection_id
        else:
            collection_id = available_collections[subscription_collection_name]
    except EntityUnexpectedResponse as e:
        logger.error(f"Error create collection, error: {str(e)}")
        raise MoonstreamHTTPException(
            status_code=500, detail="Can't create collection for subscriptions"
        )

    resource_data = {
        "type": resource_type,
        "user_id": str(user_id),
        "collection_id": str(collection_id),
    }

    try:
        bc.create_resource(
            token=token,
            application_id=MOONSTREAM_APPLICATION_ID,
            resource_data=resource_data,
        )
    except BugoutResponseException as e:
        raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail)
    except Exception as e:
        logger.error(f"Error creating subscription resource: {str(e)}")
        logger.error(
            f"Required create resource data: {resource_data}, and grand access to journal: {collection_id}, for user: {user_id}"
        )
        raise MoonstreamHTTPException(status_code=500, internal_error=e)

    try:
        bc.update_journal_scopes(
            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
            journal_id=collection_id,
            holder_type="user",
            holder_id=user_id,
            permission_list=[
                "journals.read",
                "journals.entries.read",
                "journals.entries.create",
                "journals.entries.update",
                "journals.entries.delete",
            ],
        )
        logger.info(
            f"Grand access to journal: {collection_id}, for user: {user_id} successfully"
        )
    except Exception as e:
        logger.error(f"Error updating journal scopes: {str(e)}")
        logger.error(
            f"Required grand access to journal: {collection_id}, for user: {user_id}"
        )
        raise MoonstreamHTTPException(status_code=500, internal_error=e)

    return collection_id


def generate_s3_access_links(
    method_name: str,
    bucket: str,
    key: str,
    http_method: str,
    expiration: int = 300,
) -> str:
    s3 = boto3.client("s3")
    stats_presigned_url = s3.generate_presigned_url(
        method_name,
        Params={
            "Bucket": bucket,
            "Key": key,
        },
        ExpiresIn=expiration,
        HttpMethod=http_method,
    )

    return stats_presigned_url


def query_parameter_hash(params: Dict[str, Any]) -> str:
    """
    Generate a hash of the query parameters
    """

    hash = hashlib.md5(
        json.dumps(OrderedDict(params), sort_keys=True).encode("utf-8")
    ).hexdigest()

    return hash


def get_moonworm_jobs(
    address: str,
    subscription_type_id: str,
    entries_limit: int = 100,
):
    entries = get_all_entries_from_search(
        journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
        search_query=f"tag:address:{address} tag:subscription_type:{subscription_type_id}",
        limit=entries_limit,  # load per request
        token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
    )

    return entries