diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index 77311354..bb113a64 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -2,6 +2,17 @@ # Deployment script - intended to run on Moonstream servers +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + # Main APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" APP_BACKEND_DIR="${APP_DIR}/backend" @@ -11,6 +22,7 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" @@ -20,23 +32,28 @@ set -eu echo echo -echo "Updating pip and setuptools" +echo -e "${PREFIX_INFO} Updating pip and setuptools" "${PIP}" install -U pip setuptools echo echo -echo "Updating Python dependencies" +echo -e "${PREFIX_INFO} Updating Python dependencies" "${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt" echo echo -echo "Retrieving deployment parameters" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" mkdir -p "${SECRETS_DIR}" AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}" echo echo -echo "Replacing existing Moonstream service definition with ${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" +bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}" chmod 644 "${SERVICE_FILE}" cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service systemctl daemon-reload diff --git a/backend/deploy/parameters.bash b/backend/deploy/parameters.bash new file mode 100755 index 00000000..cb6ea2e9 --- /dev/null +++ b/backend/deploy/parameters.bash @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Collect secrets from AWS SSM Parameter Store and output as environment variable exports. + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT" + echo + echo "CLI to collect secrets from AWS SSM Parameter Store +and output as environment variable exports" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -p Product tag (moonstream, spire, brood, drones)" + echo " -o Output file name environment variables export to" +} + +product_flag="" +output_flag="" +verbose_flag="false" + +while getopts 'p:o:v' flag; do + case "${flag}" in + p) product_flag="${OPTARG}" ;; + o) output_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# Product flag should be specified +# TODO(kompotkot): Extend script to work with few product at once +if [ -z "${product_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify product tag" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=${product_flag} \ + | jq -r .Parameters[].Name) +if [ -z "${ENV_PARAMETERS}" ]; then + verbose "${PREFIX_CRIT} There no parameters for provided product tag" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving parameters values" +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names ${ENV_PARAMETERS} \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length) +verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do + param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name) + param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value) + if [ -z "${output_flag}" ]; then + echo "${param_key}=${param_value}" + else + echo "${param_key}=${param_value}" >> "${output_flag}" + fi +done diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 0950ce3c..b750a64f 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -7,6 +7,8 @@ import uuid import boto3 # type: ignore from bugout.data import BugoutSearchResults from bugout.journal import SearchOrder +from ens.utils import is_valid_ens_name # type: ignore +from eth_utils.address import is_address # type: ignore from moonstreamdb.models import ( EthereumLabel, ) @@ -24,9 +26,9 @@ from .settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_DATA_JOURNAL_ID, ) +from web3 import Web3 logger = logging.getLogger(__name__) -ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract" class StatusAPIException(Exception): @@ -35,54 +37,96 @@ class StatusAPIException(Exception): """ -def get_contract_source_info( - db_session: Session, contract_address: str -) -> Optional[data.EthereumSmartContractSourceInfo]: - labels = ( - db_session.query(EthereumLabel) - .filter(EthereumLabel.address == contract_address) - .all() - ) - if not labels: - return None - - for label in labels: - if label.label == ETHERSCAN_SMARTCONTRACT_LABEL_NAME: - object_uri = label.label_data["object_uri"] - key = object_uri.split("s3://etherscan-smart-contracts/")[1] - s3 = boto3.client("s3") - bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET - try: - raw_obj = s3.get_object(Bucket=bucket, Key=key) - obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"] - contract_source_info = data.EthereumSmartContractSourceInfo( - name=obj_data["ContractName"], - source_code=obj_data["SourceCode"], - compiler_version=obj_data["CompilerVersion"], - abi=obj_data["ABI"], - ) - return contract_source_info - except Exception as e: - logger.error(f"Failed to load smart contract {object_uri}") - reporter.error_report(e) - return None - - class LabelNames(Enum): ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" COINMARKETCAP_TOKEN = "coinmarketcap_token" ERC721 = "erc721" +def get_contract_source_info( + db_session: Session, contract_address: str +) -> Optional[data.EthereumSmartContractSourceInfo]: + label = ( + db_session.query(EthereumLabel) + .filter(EthereumLabel.address == contract_address) + .filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value) + .one_or_none() + ) + if label is None: + return None + + object_uri = label.label_data["object_uri"] + key = object_uri.split("s3://etherscan-smart-contracts/")[1] + s3 = boto3.client("s3") + bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET + try: + raw_obj = s3.get_object(Bucket=bucket, Key=key) + obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"] + contract_source_info = data.EthereumSmartContractSourceInfo( + name=obj_data["ContractName"], + source_code=obj_data["SourceCode"], + compiler_version=obj_data["CompilerVersion"], + abi=obj_data["ABI"], + ) + return contract_source_info + except Exception as e: + logger.error(f"Failed to load smart contract {object_uri}") + reporter.error_report(e) + + return None + + +def get_ens_name(web3: Web3, address: str) -> Optional[str]: + try: + checksum_address = web3.toChecksumAddress(address) + except: + raise ValueError(f"{address} is invalid ethereum address is passed") + try: + ens_name = web3.ens.name(checksum_address) + return ens_name + except Exception as e: + reporter.error_report(e, ["web3", "ens"]) + logger.error( + f"Cannot get ens name for address {checksum_address}. Probably node is down" + ) + raise e + + +def get_ens_address(web3: Web3, name: str) -> Optional[str]: + + if not is_valid_ens_name(name): + raise ValueError(f"{name} is not valid ens name") + + try: + ens_checksum_address = web3.ens.address(name) + if ens_checksum_address is not None: + ordinary_address = ens_checksum_address.lower() + return ordinary_address + return None + except Exception as e: + reporter.error_report(e, ["web3", "ens"]) + logger.error(f"Cannot get ens address for name {name}. Probably node is down") + raise e + + def get_ethereum_address_info( - db_session: Session, address: str + db_session: Session, web3: Web3, address: str ) -> Optional[data.EthereumAddressInfo]: + if not is_address(address): + raise ValueError(f"Invalid ethereum address : {address}") + address_info = data.EthereumAddressInfo(address=address) + + try: + address_info.ens_name = get_ens_name(web3, address) + except: + pass + etherscan_address_url = f"https://etherscan.io/address/{address}" etherscan_token_url = f"https://etherscan.io/token/{address}" blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}" - # Checking for token: + coinmarketcap_label: Optional[EthereumLabel] = ( db_session.query(EthereumLabel) .filter(EthereumLabel.address == address) @@ -91,6 +135,7 @@ def get_ethereum_address_info( .limit(1) .one_or_none() ) + if coinmarketcap_label is not None: address_info.token = data.EthereumTokenDetails( name=coinmarketcap_label.label_data["name"], diff --git a/backend/moonstream/data.py b/backend/moonstream/data.py index 7a7089a2..a1c7b385 100644 --- a/backend/moonstream/data.py +++ b/backend/moonstream/data.py @@ -186,6 +186,7 @@ class EthereumNFTDetails(EthereumTokenDetails): class EthereumAddressInfo(BaseModel): address: str + ens_name: Optional[str] = None token: Optional[EthereumTokenDetails] = None smart_contract: Optional[EthereumSmartContractDetails] = None nft: Optional[EthereumNFTDetails] = None diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index 49c0fb78..c861b1c1 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -4,10 +4,12 @@ from typing import Optional from fastapi import APIRouter, Depends, Query from moonstreamdb.db import yield_db_session from sqlalchemy.orm import Session +from web3 import Web3 from .. import actions from .. import data from ..middleware import MoonstreamHTTPException +from ..web3_provider import yield_web3_provider logger = logging.getLogger(__name__) @@ -17,16 +19,19 @@ router = APIRouter( @router.get( - "/ethereum_blockchain", + "/ethereum", tags=["addressinfo"], response_model=data.EthereumAddressInfo, ) async def addressinfo_handler( address: str, db_session: Session = Depends(yield_db_session), + web3: Web3 = Depends(yield_web3_provider), ) -> Optional[data.EthereumAddressInfo]: try: - response = actions.get_ethereum_address_info(db_session, address) + response = actions.get_ethereum_address_info(db_session, web3, address) + except ValueError as e: + raise MoonstreamHTTPException(status_code=400, detail=str(e), internal_error=e) except Exception as e: logger.error(f"Unable to get info about Ethereum address {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -34,7 +39,61 @@ async def addressinfo_handler( @router.get( - "/labels/ethereum_blockchain", + "/ethereum/ens_name", + tags=["ens_name"], + response_model=str, +) +async def ens_name_handler( + address: str, + web3: Web3 = Depends(yield_web3_provider), +) -> Optional[str]: + try: + response = actions.get_ens_name(web3, address) + except ValueError as e: + raise MoonstreamHTTPException( + status_code=400, + detail=str(e), + internal_error=e, + ) + except Exception as e: + logger.error(f"Failed to get ens name: {e}") + raise MoonstreamHTTPException( + status_code=500, + internal_error=e, + detail="Currently unable to get ens name", + ) + return response + + +@router.get( + "/ethereum/ens_address", + tags=["ens_address"], + response_model=str, +) +async def ens_address_handler( + name: str, + web3: Web3 = Depends(yield_web3_provider), +) -> Optional[str]: + try: + response = actions.get_ens_address(web3, name) + except ValueError as e: + raise MoonstreamHTTPException( + status_code=400, + detail=str(e), + internal_error=e, + ) + except Exception as e: + logger.error(f"Failed to get ens address: {e}") + raise MoonstreamHTTPException( + status_code=500, + internal_error=e, + detail="Currently unable to get ens address", + ) + return response + + +@router.get( + "/labels/ethereum", tags=["labels"], response_model=data.AddressListLabelsResponse, ) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 982229e4..e57c7124 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -45,3 +45,20 @@ ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get( ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") + +# Web3 provider +MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get( + "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", "" +) +if MOONSTREAM_INTERNAL_HOSTED_ZONE_ID == "": + raise ValueError( + "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID environment variable must be set" + ) +MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": + raise ValueError("MOONSTREAM_WEB3_PROVIDER_URI environment variable must be set") +MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 +) diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py new file mode 100644 index 00000000..719de079 --- /dev/null +++ b/backend/moonstream/web3_provider.py @@ -0,0 +1,48 @@ +import logging + +import boto3 # type: ignore +from web3 import Web3 + +from .settings import ( + MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, + MOONSTREAM_INTERNAL_HOSTED_ZONE_ID, + MOONSTREAM_NODE_ETHEREUM_IPC_PORT, +) + +logger = logging.getLogger(__name__) + + +def fetch_web3_provider_ip(): + r53 = boto3.client("route53") + r53_response = r53.list_resource_record_sets( + HostedZoneId=MOONSTREAM_INTERNAL_HOSTED_ZONE_ID, + StartRecordName=f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.", + StartRecordType="A", + ) + try: + r53_records = r53_response["ResourceRecordSets"] + if r53_records[0]["Name"] != f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.": + return None + + record_value = r53_records[0]["ResourceRecords"][0]["Value"] + except Exception as e: + logger.error(e) + return None + + return record_value + + +if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric(): + web3_provider_ip = fetch_web3_provider_ip() + if web3_provider_ip is None: + raise ValueError("Unable to extract web3 provider IP") +else: + web3_provider_ip = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI + +moonstream_web3_provider = Web3( + Web3.HTTPProvider(f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}") +) + + +def yield_web3_provider() -> Web3: + return moonstream_web3_provider diff --git a/backend/requirements.txt b/backend/requirements.txt index 3c2d65e9..e4c0b435 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,7 +12,7 @@ h11==0.12.0 idna==3.2 jmespath==0.10.0 humbug==0.2.7 --e git+https://git@github.com/bugout-dev/moonstream.git@6b5b6049b58b1edf0e5de261614c616e8e034b6e#egg=moonstreamdb&subdirectory=db +-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db mypy==0.910 mypy-extensions==0.4.3 pathspec==0.9.0 @@ -32,3 +32,4 @@ typing-extensions==3.10.0.0 types-requests==2.25.6 urllib3==1.26.6 uvicorn==0.14.0 +web3==5.24.0 \ No newline at end of file diff --git a/backend/sample.env b/backend/sample.env index 2f994334..8f1723ea 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -4,8 +4,10 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="" export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" export HUMBUG_REPORTER_BACKEND_TOKEN="" -export ETHTXPOOL_HUMBUG_CLIENT_ID="" \ No newline at end of file +export ETHTXPOOL_HUMBUG_CLIENT_ID=""