Merge pull request #340 from bugout-dev/ens-resolver

Ens resolver
pull/352/head
Neeraj Kashyap 2021-10-28 09:07:50 -07:00 zatwierdzone przez GitHub
commit 19ed617633
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
9 zmienionych plików z 318 dodań i 45 usunięć

Wyświetl plik

@ -2,6 +2,17 @@
# Deployment script - intended to run on Moonstream servers
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Main
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
APP_BACKEND_DIR="${APP_DIR}/backend"
@ -11,6 +22,7 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python"
PIP="${PYTHON_ENV_DIR}/bin/pip"
SCRIPT_DIR="$(realpath $(dirname $0))"
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
@ -20,23 +32,28 @@ set -eu
echo
echo
echo "Updating pip and setuptools"
echo -e "${PREFIX_INFO} Updating pip and setuptools"
"${PIP}" install -U pip setuptools
echo
echo
echo "Updating Python dependencies"
echo -e "${PREFIX_INFO} Updating Python dependencies"
"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
echo
echo
echo "Retrieving deployment parameters"
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
mkdir -p "${SECRETS_DIR}"
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo "Replacing existing Moonstream service definition with ${SERVICE_FILE}"
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}"
chmod 644 "${SERVICE_FILE}"
cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service
systemctl daemon-reload

Wyświetl plik

@ -0,0 +1,83 @@
#!/usr/bin/env bash
#
# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
# Colors
C_RESET='\033[0m'
C_RED='\033[1;31m'
C_GREEN='\033[1;32m'
C_YELLOW='\033[1;33m'
# Logs
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
# Print help message
function usage {
echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"
echo
echo "CLI to collect secrets from AWS SSM Parameter Store
and output as environment variable exports"
echo
echo "Optional arguments:"
echo " -h Show this help message and exit"
echo " -p Product tag (moonstream, spire, brood, drones)"
echo " -o Output file name environment variables export to"
}
product_flag=""
output_flag=""
verbose_flag="false"
while getopts 'p:o:v' flag; do
case "${flag}" in
p) product_flag="${OPTARG}" ;;
o) output_flag="${OPTARG}" ;;
h) usage
exit 1 ;;
v) verbose_flag="true" ;;
*) usage
exit 1 ;;
esac
done
# Log messages
function verbose {
if [ "${verbose_flag}" == "true" ]; then
echo -e "$1"
fi
}
# Product flag should be specified
# TODO(kompotkot): Extend script to work with few product at once
if [ -z "${product_flag}" ]; then
verbose "${PREFIX_CRIT} Please specify product tag"
usage
exit 1
fi
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
ENV_PARAMETERS=$(aws ssm describe-parameters \
--parameter-filters Key=tag:Product,Values=${product_flag} \
| jq -r .Parameters[].Name)
if [ -z "${ENV_PARAMETERS}" ]; then
verbose "${PREFIX_CRIT} There no parameters for provided product tag"
exit 1
fi
verbose "${PREFIX_INFO} Retrieving parameters values"
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
--names ${ENV_PARAMETERS} \
--query "Parameters[*].{Name:Name,Value:Value}")
ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
if [ -z "${output_flag}" ]; then
echo "${param_key}=${param_value}"
else
echo "${param_key}=${param_value}" >> "${output_flag}"
fi
done

Wyświetl plik

@ -7,6 +7,8 @@ import uuid
import boto3 # type: ignore
from bugout.data import BugoutSearchResults
from bugout.journal import SearchOrder
from ens.utils import is_valid_ens_name # type: ignore
from eth_utils.address import is_address # type: ignore
from moonstreamdb.models import (
EthereumLabel,
)
@ -24,9 +26,9 @@ from .settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
)
from web3 import Web3
logger = logging.getLogger(__name__)
ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract"
class StatusAPIException(Exception):
@ -35,54 +37,96 @@ class StatusAPIException(Exception):
"""
def get_contract_source_info(
db_session: Session, contract_address: str
) -> Optional[data.EthereumSmartContractSourceInfo]:
labels = (
db_session.query(EthereumLabel)
.filter(EthereumLabel.address == contract_address)
.all()
)
if not labels:
return None
for label in labels:
if label.label == ETHERSCAN_SMARTCONTRACT_LABEL_NAME:
object_uri = label.label_data["object_uri"]
key = object_uri.split("s3://etherscan-smart-contracts/")[1]
s3 = boto3.client("s3")
bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET
try:
raw_obj = s3.get_object(Bucket=bucket, Key=key)
obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"]
contract_source_info = data.EthereumSmartContractSourceInfo(
name=obj_data["ContractName"],
source_code=obj_data["SourceCode"],
compiler_version=obj_data["CompilerVersion"],
abi=obj_data["ABI"],
)
return contract_source_info
except Exception as e:
logger.error(f"Failed to load smart contract {object_uri}")
reporter.error_report(e)
return None
class LabelNames(Enum):
ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract"
COINMARKETCAP_TOKEN = "coinmarketcap_token"
ERC721 = "erc721"
def get_contract_source_info(
db_session: Session, contract_address: str
) -> Optional[data.EthereumSmartContractSourceInfo]:
label = (
db_session.query(EthereumLabel)
.filter(EthereumLabel.address == contract_address)
.filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value)
.one_or_none()
)
if label is None:
return None
object_uri = label.label_data["object_uri"]
key = object_uri.split("s3://etherscan-smart-contracts/")[1]
s3 = boto3.client("s3")
bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET
try:
raw_obj = s3.get_object(Bucket=bucket, Key=key)
obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"]
contract_source_info = data.EthereumSmartContractSourceInfo(
name=obj_data["ContractName"],
source_code=obj_data["SourceCode"],
compiler_version=obj_data["CompilerVersion"],
abi=obj_data["ABI"],
)
return contract_source_info
except Exception as e:
logger.error(f"Failed to load smart contract {object_uri}")
reporter.error_report(e)
return None
def get_ens_name(web3: Web3, address: str) -> Optional[str]:
try:
checksum_address = web3.toChecksumAddress(address)
except:
raise ValueError(f"{address} is invalid ethereum address is passed")
try:
ens_name = web3.ens.name(checksum_address)
return ens_name
except Exception as e:
reporter.error_report(e, ["web3", "ens"])
logger.error(
f"Cannot get ens name for address {checksum_address}. Probably node is down"
)
raise e
def get_ens_address(web3: Web3, name: str) -> Optional[str]:
if not is_valid_ens_name(name):
raise ValueError(f"{name} is not valid ens name")
try:
ens_checksum_address = web3.ens.address(name)
if ens_checksum_address is not None:
ordinary_address = ens_checksum_address.lower()
return ordinary_address
return None
except Exception as e:
reporter.error_report(e, ["web3", "ens"])
logger.error(f"Cannot get ens address for name {name}. Probably node is down")
raise e
def get_ethereum_address_info(
db_session: Session, address: str
db_session: Session, web3: Web3, address: str
) -> Optional[data.EthereumAddressInfo]:
if not is_address(address):
raise ValueError(f"Invalid ethereum address : {address}")
address_info = data.EthereumAddressInfo(address=address)
try:
address_info.ens_name = get_ens_name(web3, address)
except:
pass
etherscan_address_url = f"https://etherscan.io/address/{address}"
etherscan_token_url = f"https://etherscan.io/token/{address}"
blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}"
# Checking for token:
coinmarketcap_label: Optional[EthereumLabel] = (
db_session.query(EthereumLabel)
.filter(EthereumLabel.address == address)
@ -91,6 +135,7 @@ def get_ethereum_address_info(
.limit(1)
.one_or_none()
)
if coinmarketcap_label is not None:
address_info.token = data.EthereumTokenDetails(
name=coinmarketcap_label.label_data["name"],

Wyświetl plik

@ -186,6 +186,7 @@ class EthereumNFTDetails(EthereumTokenDetails):
class EthereumAddressInfo(BaseModel):
address: str
ens_name: Optional[str] = None
token: Optional[EthereumTokenDetails] = None
smart_contract: Optional[EthereumSmartContractDetails] = None
nft: Optional[EthereumNFTDetails] = None

Wyświetl plik

@ -4,10 +4,12 @@ from typing import Optional
from fastapi import APIRouter, Depends, Query
from moonstreamdb.db import yield_db_session
from sqlalchemy.orm import Session
from web3 import Web3
from .. import actions
from .. import data
from ..middleware import MoonstreamHTTPException
from ..web3_provider import yield_web3_provider
logger = logging.getLogger(__name__)
@ -17,16 +19,19 @@ router = APIRouter(
@router.get(
"/ethereum_blockchain",
"/ethereum",
tags=["addressinfo"],
response_model=data.EthereumAddressInfo,
)
async def addressinfo_handler(
address: str,
db_session: Session = Depends(yield_db_session),
web3: Web3 = Depends(yield_web3_provider),
) -> Optional[data.EthereumAddressInfo]:
try:
response = actions.get_ethereum_address_info(db_session, address)
response = actions.get_ethereum_address_info(db_session, web3, address)
except ValueError as e:
raise MoonstreamHTTPException(status_code=400, detail=str(e), internal_error=e)
except Exception as e:
logger.error(f"Unable to get info about Ethereum address {e}")
raise MoonstreamHTTPException(status_code=500, internal_error=e)
@ -34,7 +39,61 @@ async def addressinfo_handler(
@router.get(
"/labels/ethereum_blockchain",
"/ethereum/ens_name",
tags=["ens_name"],
response_model=str,
)
async def ens_name_handler(
address: str,
web3: Web3 = Depends(yield_web3_provider),
) -> Optional[str]:
try:
response = actions.get_ens_name(web3, address)
except ValueError as e:
raise MoonstreamHTTPException(
status_code=400,
detail=str(e),
internal_error=e,
)
except Exception as e:
logger.error(f"Failed to get ens name: {e}")
raise MoonstreamHTTPException(
status_code=500,
internal_error=e,
detail="Currently unable to get ens name",
)
return response
@router.get(
"/ethereum/ens_address",
tags=["ens_address"],
response_model=str,
)
async def ens_address_handler(
name: str,
web3: Web3 = Depends(yield_web3_provider),
) -> Optional[str]:
try:
response = actions.get_ens_address(web3, name)
except ValueError as e:
raise MoonstreamHTTPException(
status_code=400,
detail=str(e),
internal_error=e,
)
except Exception as e:
logger.error(f"Failed to get ens address: {e}")
raise MoonstreamHTTPException(
status_code=500,
internal_error=e,
detail="Currently unable to get ens address",
)
return response
@router.get(
"/labels/ethereum",
tags=["labels"],
response_model=data.AddressListLabelsResponse,
)

Wyświetl plik

@ -45,3 +45,20 @@ ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get(
ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET")
if ETHERSCAN_SMARTCONTRACTS_BUCKET is None:
raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set")
# Web3 provider
MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get(
"MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", ""
)
if MOONSTREAM_INTERNAL_HOSTED_ZONE_ID == "":
raise ValueError(
"MOONSTREAM_INTERNAL_HOSTED_ZONE_ID environment variable must be set"
)
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "":
raise ValueError("MOONSTREAM_WEB3_PROVIDER_URI environment variable must be set")
MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545
)

Wyświetl plik

@ -0,0 +1,48 @@
import logging
import boto3 # type: ignore
from web3 import Web3
from .settings import (
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
MOONSTREAM_INTERNAL_HOSTED_ZONE_ID,
MOONSTREAM_NODE_ETHEREUM_IPC_PORT,
)
logger = logging.getLogger(__name__)
def fetch_web3_provider_ip():
r53 = boto3.client("route53")
r53_response = r53.list_resource_record_sets(
HostedZoneId=MOONSTREAM_INTERNAL_HOSTED_ZONE_ID,
StartRecordName=f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.",
StartRecordType="A",
)
try:
r53_records = r53_response["ResourceRecordSets"]
if r53_records[0]["Name"] != f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.":
return None
record_value = r53_records[0]["ResourceRecords"][0]["Value"]
except Exception as e:
logger.error(e)
return None
return record_value
if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric():
web3_provider_ip = fetch_web3_provider_ip()
if web3_provider_ip is None:
raise ValueError("Unable to extract web3 provider IP")
else:
web3_provider_ip = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI
moonstream_web3_provider = Web3(
Web3.HTTPProvider(f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}")
)
def yield_web3_provider() -> Web3:
return moonstream_web3_provider

Wyświetl plik

@ -12,7 +12,7 @@ h11==0.12.0
idna==3.2
jmespath==0.10.0
humbug==0.2.7
-e git+https://git@github.com/bugout-dev/moonstream.git@6b5b6049b58b1edf0e5de261614c616e8e034b6e#egg=moonstreamdb&subdirectory=db
-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db
mypy==0.910
mypy-extensions==0.4.3
pathspec==0.9.0
@ -32,3 +32,4 @@ typing-extensions==3.10.0.0
types-requests==2.25.6
urllib3==1.26.6
uvicorn==0.14.0
web3==5.24.0

Wyświetl plik

@ -4,8 +4,10 @@ export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access token to application resources>"
export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="<moonstream_internal_hosted_zone_id>"
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="<connection_path_uri_to_ethereum_node>"
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket to store smart contracts>"
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout Humbug token for crash reports>"
export ETHTXPOOL_HUMBUG_CLIENT_ID="<Bugout Humbug client id for txpool transactions in journal>"
export ETHTXPOOL_HUMBUG_CLIENT_ID="<Bugout Humbug client id for txpool transactions in journal>"