From 9e02e969ba74f8ceaf63b17ae970366b3bef0770 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 11:12:09 +0300 Subject: [PATCH 01/12] added ens name for address --- backend/moonstream/actions.py | 84 +++++++++++++---------- backend/moonstream/api.py | 2 +- backend/moonstream/data.py | 1 + backend/moonstream/routes/address_info.py | 11 ++- backend/moonstream/settings.py | 5 ++ backend/moonstream/web3_provider.py | 8 +++ backend/requirements.txt | 3 +- 7 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 backend/moonstream/web3_provider.py diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 0950ce3c..bffe1a6b 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -24,9 +24,9 @@ from .settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_DATA_JOURNAL_ID, ) +from web3 import Web3 logger = logging.getLogger(__name__) -ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract" class StatusAPIException(Exception): @@ -35,54 +35,65 @@ class StatusAPIException(Exception): """ -def get_contract_source_info( - db_session: Session, contract_address: str -) -> Optional[data.EthereumSmartContractSourceInfo]: - labels = ( - db_session.query(EthereumLabel) - .filter(EthereumLabel.address == contract_address) - .all() - ) - if not labels: - return None - - for label in labels: - if label.label == ETHERSCAN_SMARTCONTRACT_LABEL_NAME: - object_uri = label.label_data["object_uri"] - key = object_uri.split("s3://etherscan-smart-contracts/")[1] - s3 = boto3.client("s3") - bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET - try: - raw_obj = s3.get_object(Bucket=bucket, Key=key) - obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"] - contract_source_info = data.EthereumSmartContractSourceInfo( - name=obj_data["ContractName"], - source_code=obj_data["SourceCode"], - compiler_version=obj_data["CompilerVersion"], - abi=obj_data["ABI"], - ) - return contract_source_info - except Exception as e: - logger.error(f"Failed to load smart contract {object_uri}") - reporter.error_report(e) - return None - - class LabelNames(Enum): ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" COINMARKETCAP_TOKEN = "coinmarketcap_token" ERC721 = "erc721" +def get_contract_source_info( + db_session: Session, contract_address: str +) -> Optional[data.EthereumSmartContractSourceInfo]: + label = ( + db_session.query(EthereumLabel) + .filter(EthereumLabel.address == contract_address) + .filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value) + .one_or_none() + ) + if label is None: + return None + + object_uri = label.label_data["object_uri"] + key = object_uri.split("s3://etherscan-smart-contracts/")[1] + s3 = boto3.client("s3") + bucket = ETHERSCAN_SMARTCONTRACTS_BUCKET + try: + raw_obj = s3.get_object(Bucket=bucket, Key=key) + obj_data = json.loads(raw_obj["Body"].read().decode("utf-8"))["data"] + contract_source_info = data.EthereumSmartContractSourceInfo( + name=obj_data["ContractName"], + source_code=obj_data["SourceCode"], + compiler_version=obj_data["CompilerVersion"], + abi=obj_data["ABI"], + ) + return contract_source_info + except Exception as e: + logger.error(f"Failed to load smart contract {object_uri}") + reporter.error_report(e) + + return None + + def get_ethereum_address_info( - db_session: Session, address: str + db_session: Session, web3: Web3, address: str ) -> Optional[data.EthereumAddressInfo]: address_info = data.EthereumAddressInfo(address=address) etherscan_address_url = f"https://etherscan.io/address/{address}" etherscan_token_url = f"https://etherscan.io/token/{address}" blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}" - # Checking for token: + try: + checksum_address = web3.toChecksumAddress(address) + try: + ens_name = web3.ens.name(checksum_address) + address_info.ens_name = ens_name + except: + logger.warning( + f"Cannot get ens name for address {ens_name}. Probably node is down" + ) + except: + raise ValueError("Invalid address is passed") + coinmarketcap_label: Optional[EthereumLabel] = ( db_session.query(EthereumLabel) .filter(EthereumLabel.address == address) @@ -91,6 +102,7 @@ def get_ethereum_address_info( .limit(1) .one_or_none() ) + if coinmarketcap_label is not None: address_info.token = data.EthereumTokenDetails( name=coinmarketcap_label.label_data["name"], diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 15bf4f4a..21685c66 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -67,7 +67,7 @@ whitelist_paths.update( "/users/password/reset_complete": "POST", } ) -app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) +# app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) app.add_middleware( CORSMiddleware, allow_origins=ORIGINS, diff --git a/backend/moonstream/data.py b/backend/moonstream/data.py index 7a7089a2..a1c7b385 100644 --- a/backend/moonstream/data.py +++ b/backend/moonstream/data.py @@ -186,6 +186,7 @@ class EthereumNFTDetails(EthereumTokenDetails): class EthereumAddressInfo(BaseModel): address: str + ens_name: Optional[str] = None token: Optional[EthereumTokenDetails] = None smart_contract: Optional[EthereumSmartContractDetails] = None nft: Optional[EthereumNFTDetails] = None diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index 49c0fb78..c67f8150 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -4,10 +4,12 @@ from typing import Optional from fastapi import APIRouter, Depends, Query from moonstreamdb.db import yield_db_session from sqlalchemy.orm import Session +from web3 import Web3 from .. import actions from .. import data from ..middleware import MoonstreamHTTPException +from ..web3_provider import yield_web3_provider logger = logging.getLogger(__name__) @@ -17,16 +19,19 @@ router = APIRouter( @router.get( - "/ethereum_blockchain", + "/ethereum", tags=["addressinfo"], response_model=data.EthereumAddressInfo, ) async def addressinfo_handler( address: str, db_session: Session = Depends(yield_db_session), + web3: Web3 = Depends(yield_web3_provider), ) -> Optional[data.EthereumAddressInfo]: try: - response = actions.get_ethereum_address_info(db_session, address) + response = actions.get_ethereum_address_info(db_session, web3, address) + except ValueError as e: + raise MoonstreamHTTPException(status_code=400, detail=str(e), internal_error=e) except Exception as e: logger.error(f"Unable to get info about Ethereum address {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -34,7 +39,7 @@ async def addressinfo_handler( @router.get( - "/labels/ethereum_blockchain", + "/labels/ethereum", tags=["labels"], response_model=data.AddressListLabelsResponse, ) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 982229e4..04b60407 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -45,3 +45,8 @@ ETHTXPOOL_HUMBUG_CLIENT_ID = os.environ.get( ETHERSCAN_SMARTCONTRACTS_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACT_BUCKET") if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") + +# Web3 provider +MOONSTREAM_WEB3_PROVIDER = os.environ.get("MOONSTREAM_WEB3_PROVIDER", "") +if MOONSTREAM_WEB3_PROVIDER == "": + raise ValueError("MOONSTREAM_WEB3_PROVIDER environment variable must be set") diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py new file mode 100644 index 00000000..efc60ee2 --- /dev/null +++ b/backend/moonstream/web3_provider.py @@ -0,0 +1,8 @@ +from web3 import Web3 +from .settings import MOONSTREAM_WEB3_PROVIDER + +moonstream_web3_provider = Web3(Web3.HTTPProvider(MOONSTREAM_WEB3_PROVIDER)) + + +def yield_web3_provider() -> Web3: + return moonstream_web3_provider diff --git a/backend/requirements.txt b/backend/requirements.txt index 3c2d65e9..e4c0b435 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,7 +12,7 @@ h11==0.12.0 idna==3.2 jmespath==0.10.0 humbug==0.2.7 --e git+https://git@github.com/bugout-dev/moonstream.git@6b5b6049b58b1edf0e5de261614c616e8e034b6e#egg=moonstreamdb&subdirectory=db +-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db mypy==0.910 mypy-extensions==0.4.3 pathspec==0.9.0 @@ -32,3 +32,4 @@ typing-extensions==3.10.0.0 types-requests==2.25.6 urllib3==1.26.6 uvicorn==0.14.0 +web3==5.24.0 \ No newline at end of file From 8be1b7948c0a0ebb61c9b49755a9e36230a7b03a Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 12:27:42 +0300 Subject: [PATCH 02/12] added ens name and ens address resolver --- backend/moonstream/actions.py | 55 ++++++++++++++++++----- backend/moonstream/routes/address_info.py | 54 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index bffe1a6b..72c63ec3 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -7,6 +7,8 @@ import uuid import boto3 # type: ignore from bugout.data import BugoutSearchResults from bugout.journal import SearchOrder +from ens.utils import is_valid_ens_name +from eth_utils.address import is_address from moonstreamdb.models import ( EthereumLabel, ) @@ -74,25 +76,56 @@ def get_contract_source_info( return None +def get_ens_name(web3: Web3, address: str) -> Optional[str]: + try: + checksum_address = web3.toChecksumAddress(address) + except: + raise ValueError(f"{address} is invalid ethereum address is passed") + try: + ens_name = web3.ens.name(checksum_address) + return ens_name + except Exception as e: + reporter.error_report(e, ["web3", "ens"]) + logger.error( + f"Cannot get ens name for address {checksum_address}. Probably node is down" + ) + raise e + + +def get_ens_address(web3: Web3, name: str) -> Optional[str]: + + if not is_valid_ens_name(name): + raise ValueError(f"{name} is not valid ens name") + + try: + ens_checksum_address = web3.ens.address(name) + if ens_checksum_address is not None: + ordinary_address = ens_checksum_address.lower() + return ordinary_address + return None + except Exception as e: + reporter.error_report(e, ["web3", "ens"]) + logger.error(f"Cannot get ens address for name {name}. Probably node is down") + raise e + + def get_ethereum_address_info( db_session: Session, web3: Web3, address: str ) -> Optional[data.EthereumAddressInfo]: + if not is_address(address): + raise ValueError(f"Invalid ethereum address : {address}") + address_info = data.EthereumAddressInfo(address=address) + + try: + address_info.ens_name = get_ens_name(web3, address) + except: + pass + etherscan_address_url = f"https://etherscan.io/address/{address}" etherscan_token_url = f"https://etherscan.io/token/{address}" blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}" - try: - checksum_address = web3.toChecksumAddress(address) - try: - ens_name = web3.ens.name(checksum_address) - address_info.ens_name = ens_name - except: - logger.warning( - f"Cannot get ens name for address {ens_name}. Probably node is down" - ) - except: - raise ValueError("Invalid address is passed") coinmarketcap_label: Optional[EthereumLabel] = ( db_session.query(EthereumLabel) diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py index c67f8150..c861b1c1 100644 --- a/backend/moonstream/routes/address_info.py +++ b/backend/moonstream/routes/address_info.py @@ -38,6 +38,60 @@ async def addressinfo_handler( return response +@router.get( + "/ethereum/ens_name", + tags=["ens_name"], + response_model=str, +) +async def ens_name_handler( + address: str, + web3: Web3 = Depends(yield_web3_provider), +) -> Optional[str]: + try: + response = actions.get_ens_name(web3, address) + except ValueError as e: + raise MoonstreamHTTPException( + status_code=400, + detail=str(e), + internal_error=e, + ) + except Exception as e: + logger.error(f"Failed to get ens name: {e}") + raise MoonstreamHTTPException( + status_code=500, + internal_error=e, + detail="Currently unable to get ens name", + ) + return response + + +@router.get( + "/ethereum/ens_address", + tags=["ens_address"], + response_model=str, +) +async def ens_address_handler( + name: str, + web3: Web3 = Depends(yield_web3_provider), +) -> Optional[str]: + try: + response = actions.get_ens_address(web3, name) + except ValueError as e: + raise MoonstreamHTTPException( + status_code=400, + detail=str(e), + internal_error=e, + ) + except Exception as e: + logger.error(f"Failed to get ens address: {e}") + raise MoonstreamHTTPException( + status_code=500, + internal_error=e, + detail="Currently unable to get ens address", + ) + return response + + @router.get( "/labels/ethereum", tags=["labels"], From 1b311a9e8f7ebf2f9df53de36a80aeb19b8cffa4 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 12:46:41 +0300 Subject: [PATCH 03/12] added env variable to sample.env --- backend/moonstream/settings.py | 4 ++-- backend/moonstream/web3_provider.py | 4 ++-- backend/sample.env | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 04b60407..6abb6c6f 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -47,6 +47,6 @@ if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") # Web3 provider -MOONSTREAM_WEB3_PROVIDER = os.environ.get("MOONSTREAM_WEB3_PROVIDER", "") -if MOONSTREAM_WEB3_PROVIDER == "": +MOONSTREAM_ETHEREUM_WEB3_PROVIDER = os.environ.get("MOONSTREAM_WEB3_PROVIDER", "") +if MOONSTREAM_ETHEREUM_WEB3_PROVIDER == "": raise ValueError("MOONSTREAM_WEB3_PROVIDER environment variable must be set") diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py index efc60ee2..f74ab3d2 100644 --- a/backend/moonstream/web3_provider.py +++ b/backend/moonstream/web3_provider.py @@ -1,7 +1,7 @@ from web3 import Web3 -from .settings import MOONSTREAM_WEB3_PROVIDER +from .settings import MOONSTREAM_ETHEREUM_WEB3_PROVIDER -moonstream_web3_provider = Web3(Web3.HTTPProvider(MOONSTREAM_WEB3_PROVIDER)) +moonstream_web3_provider = Web3(Web3.HTTPProvider(MOONSTREAM_ETHEREUM_WEB3_PROVIDER)) def yield_web3_provider() -> Web3: diff --git a/backend/sample.env b/backend/sample.env index 2f994334..20beae3a 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -4,8 +4,9 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER="" export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" export HUMBUG_REPORTER_BACKEND_TOKEN="" -export ETHTXPOOL_HUMBUG_CLIENT_ID="" \ No newline at end of file +export ETHTXPOOL_HUMBUG_CLIENT_ID="" From 4a0bcd05e4bf4bf065cf0957d4311af83b0dae5d Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 12:50:55 +0300 Subject: [PATCH 04/12] renamed env variable --- backend/moonstream/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index 6abb6c6f..ff4a5f6e 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -47,6 +47,8 @@ if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") # Web3 provider -MOONSTREAM_ETHEREUM_WEB3_PROVIDER = os.environ.get("MOONSTREAM_WEB3_PROVIDER", "") +MOONSTREAM_ETHEREUM_WEB3_PROVIDER = os.environ.get( + "MOONSTREAM_ETHEREUM_WEB3_PROVIDER", "" +) if MOONSTREAM_ETHEREUM_WEB3_PROVIDER == "": raise ValueError("MOONSTREAM_WEB3_PROVIDER environment variable must be set") From a5141d109457503077a32291f60b1782b9e4c453 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 12:56:02 +0300 Subject: [PATCH 05/12] remove of unneeded commit --- backend/moonstream/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 21685c66..15bf4f4a 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -67,7 +67,7 @@ whitelist_paths.update( "/users/password/reset_complete": "POST", } ) -# app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) +app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) app.add_middleware( CORSMiddleware, allow_origins=ORIGINS, From 73b7392962359f9fbbab97488202eeb72114c8db Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 26 Oct 2021 13:06:25 +0300 Subject: [PATCH 06/12] fixed mypy error --- backend/moonstream/actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 72c63ec3..b750a64f 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -7,8 +7,8 @@ import uuid import boto3 # type: ignore from bugout.data import BugoutSearchResults from bugout.journal import SearchOrder -from ens.utils import is_valid_ens_name -from eth_utils.address import is_address +from ens.utils import is_valid_ens_name # type: ignore +from eth_utils.address import is_address # type: ignore from moonstreamdb.models import ( EthereumLabel, ) From 915c51d830cff051d753e0cd21712c214c1f11e8 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 27 Oct 2021 13:44:44 +0000 Subject: [PATCH 07/12] Retrieving ip from route53 --- backend/deploy/deploy.bash | 38 +++++++++++++++++++++++--- backend/moonstream/settings.py | 20 +++++++++++--- backend/moonstream/web3_provider.py | 41 ++++++++++++++++++++++++++--- backend/sample.env | 3 ++- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index 77311354..7dd6bec0 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -2,6 +2,17 @@ # Deployment script - intended to run on Moonstream servers +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + # Main APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" APP_BACKEND_DIR="${APP_DIR}/backend" @@ -20,23 +31,42 @@ set -eu echo echo -echo "Updating pip and setuptools" +echo -e "${PREFIX_INFO} Updating pip and setuptools" "${PIP}" install -U pip setuptools echo echo -echo "Updating Python dependencies" +echo -e "${PREFIX_INFO} Updating Python dependencies" "${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt" echo echo -echo "Retrieving deployment parameters" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" mkdir -p "${SECRETS_DIR}" AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}" echo echo -echo "Replacing existing Moonstream service definition with ${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=moonstream \ + | jq -r .Parameters[].Name) +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names $ENV_PARAMETERS \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo $ENV_PARAMETERS_VALUES | jq length) +echo -e "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $(($ENV_PARAMETERS_VALUES_LENGTH - 1))) +do + param_key=$(echo $ENV_PARAMETERS_VALUES | jq -r .[$i].Name) + param_value=$(echo $ENV_PARAMETERS_VALUES | jq .[$i].Value) + echo "$param_key=$param_value" >> "${PARAMETERS_ENV_PATH}" +done + + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}" chmod 644 "${SERVICE_FILE}" cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service systemctl daemon-reload diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index ff4a5f6e..c412c8d3 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -47,8 +47,20 @@ if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: raise ValueError("AWS_S3_SMARTCONTRACT_BUCKET is not set") # Web3 provider -MOONSTREAM_ETHEREUM_WEB3_PROVIDER = os.environ.get( - "MOONSTREAM_ETHEREUM_WEB3_PROVIDER", "" +MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get( + "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID" ) -if MOONSTREAM_ETHEREUM_WEB3_PROVIDER == "": - raise ValueError("MOONSTREAM_WEB3_PROVIDER environment variable must be set") +if MOONSTREAM_INTERNAL_HOSTED_ZONE_ID is None: + raise ValueError( + "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID environment variable must be set" + ) +MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI" +) +if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI is None: + raise ValueError("MOONSTREAM_WEB3_PROVIDER_URI environment variable must be set") +MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") +if MOONSTREAM_NODE_ETHEREUM_IPC_PORT is None: + raise ValueError( + "MOONSTREAM_NODE_ETHEREUM_IPC_PORT environment variable must be set" + ) diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py index f74ab3d2..71ca6229 100644 --- a/backend/moonstream/web3_provider.py +++ b/backend/moonstream/web3_provider.py @@ -1,7 +1,42 @@ -from web3 import Web3 -from .settings import MOONSTREAM_ETHEREUM_WEB3_PROVIDER +import logging -moonstream_web3_provider = Web3(Web3.HTTPProvider(MOONSTREAM_ETHEREUM_WEB3_PROVIDER)) +import boto3 # type: ignore +from web3 import Web3 + +from .settings import ( + MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, + MOONSTREAM_INTERNAL_HOSTED_ZONE_ID, + MOONSTREAM_NODE_ETHEREUM_IPC_PORT +) + +logger = logging.getLogger(__name__) + + +def fetch_web3_provider_ip(): + r53 = boto3.client("route53") + r53_response = r53.list_resource_record_sets( + HostedZoneId=MOONSTREAM_INTERNAL_HOSTED_ZONE_ID, + StartRecordName=f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.", + StartRecordType="A", + ) + try: + r53_records = r53_response["ResourceRecordSets"] + if r53_records[0]["Name"] != f"{MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.": + return None + + record_value = r53_records[0]["ResourceRecords"][0]["Value"] + except Exception as e: + logger.error(e) + return None + + return record_value + + +if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".","").isnumeric(): + web3_provider_ip = fetch_web3_provider_ip() + if web3_provider_ip is None: + raise ValueError("Unable to extract web3 provider IP") + moonstream_web3_provider = Web3(Web3.HTTPProvider(f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}")) def yield_web3_provider() -> Web3: diff --git a/backend/sample.env b/backend/sample.env index 20beae3a..80289693 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -4,7 +4,8 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export MOONSTREAM_ETHEREUM_WEB3_PROVIDER="" +export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" +export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="" export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" From c0ca537a27c757385da193ddc49838aa970f20ce Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 27 Oct 2021 13:51:09 +0000 Subject: [PATCH 08/12] Lint fix --- backend/moonstream/settings.py | 16 +++++++--------- backend/moonstream/web3_provider.py | 10 +++++++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/backend/moonstream/settings.py b/backend/moonstream/settings.py index c412c8d3..e57c7124 100644 --- a/backend/moonstream/settings.py +++ b/backend/moonstream/settings.py @@ -48,19 +48,17 @@ if ETHERSCAN_SMARTCONTRACTS_BUCKET is None: # Web3 provider MOONSTREAM_INTERNAL_HOSTED_ZONE_ID = os.environ.get( - "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID" + "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID", "" ) -if MOONSTREAM_INTERNAL_HOSTED_ZONE_ID is None: +if MOONSTREAM_INTERNAL_HOSTED_ZONE_ID == "": raise ValueError( "MOONSTREAM_INTERNAL_HOSTED_ZONE_ID environment variable must be set" ) MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI" + "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", "" ) -if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI is None: +if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": raise ValueError("MOONSTREAM_WEB3_PROVIDER_URI environment variable must be set") -MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") -if MOONSTREAM_NODE_ETHEREUM_IPC_PORT is None: - raise ValueError( - "MOONSTREAM_NODE_ETHEREUM_IPC_PORT environment variable must be set" - ) +MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 +) diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py index 71ca6229..0c1a93c4 100644 --- a/backend/moonstream/web3_provider.py +++ b/backend/moonstream/web3_provider.py @@ -6,7 +6,7 @@ from web3 import Web3 from .settings import ( MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, MOONSTREAM_INTERNAL_HOSTED_ZONE_ID, - MOONSTREAM_NODE_ETHEREUM_IPC_PORT + MOONSTREAM_NODE_ETHEREUM_IPC_PORT, ) logger = logging.getLogger(__name__) @@ -32,11 +32,15 @@ def fetch_web3_provider_ip(): return record_value -if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".","").isnumeric(): +if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric(): web3_provider_ip = fetch_web3_provider_ip() if web3_provider_ip is None: raise ValueError("Unable to extract web3 provider IP") - moonstream_web3_provider = Web3(Web3.HTTPProvider(f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}")) + moonstream_web3_provider = Web3( + Web3.HTTPProvider( + f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" + ) + ) def yield_web3_provider() -> Web3: From 3b89a2343279783c664c095a01437d2d4a624982 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 11:50:49 +0000 Subject: [PATCH 09/12] Parameters bash script --- backend/deploy/deploy.bash | 16 +------ backend/deploy/parameters.bash | 83 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) create mode 100755 backend/deploy/parameters.bash diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index 7dd6bec0..935a2875 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -48,21 +48,7 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$ echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -ENV_PARAMETERS=$(aws ssm describe-parameters \ - --parameter-filters Key=tag:Product,Values=moonstream \ - | jq -r .Parameters[].Name) -ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ - --names $ENV_PARAMETERS \ - --query "Parameters[*].{Name:Name,Value:Value}") -ENV_PARAMETERS_VALUES_LENGTH=$(echo $ENV_PARAMETERS_VALUES | jq length) -echo -e "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" -for i in $(seq 0 $(($ENV_PARAMETERS_VALUES_LENGTH - 1))) -do - param_key=$(echo $ENV_PARAMETERS_VALUES | jq -r .[$i].Name) - param_value=$(echo $ENV_PARAMETERS_VALUES | jq .[$i].Value) - echo "$param_key=$param_value" >> "${PARAMETERS_ENV_PATH}" -done - +bash parameters.bash -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo diff --git a/backend/deploy/parameters.bash b/backend/deploy/parameters.bash new file mode 100755 index 00000000..cb6ea2e9 --- /dev/null +++ b/backend/deploy/parameters.bash @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Collect secrets from AWS SSM Parameter Store and output as environment variable exports. + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT" + echo + echo "CLI to collect secrets from AWS SSM Parameter Store +and output as environment variable exports" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -p Product tag (moonstream, spire, brood, drones)" + echo " -o Output file name environment variables export to" +} + +product_flag="" +output_flag="" +verbose_flag="false" + +while getopts 'p:o:v' flag; do + case "${flag}" in + p) product_flag="${OPTARG}" ;; + o) output_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# Product flag should be specified +# TODO(kompotkot): Extend script to work with few product at once +if [ -z "${product_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify product tag" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=${product_flag} \ + | jq -r .Parameters[].Name) +if [ -z "${ENV_PARAMETERS}" ]; then + verbose "${PREFIX_CRIT} There no parameters for provided product tag" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving parameters values" +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names ${ENV_PARAMETERS} \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length) +verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do + param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name) + param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value) + if [ -z "${output_flag}" ]; then + echo "${param_key}=${param_value}" + else + echo "${param_key}=${param_value}" >> "${output_flag}" + fi +done From 8ed4f57c369b23f3f00c0b74a88b688534f4f9f4 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 11:54:01 +0000 Subject: [PATCH 10/12] Parameters script with full path --- backend/deploy/deploy.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index 935a2875..bb113a64 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -22,6 +22,7 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" @@ -48,7 +49,7 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$ echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -bash parameters.bash -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo From 3120f39c59ac9f74bb599218b2ce72d041b5cfe2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 14:34:25 +0000 Subject: [PATCH 11/12] Resolve ip and domain correctly for web3 --- backend/moonstream/web3_provider.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py index 0c1a93c4..d3457849 100644 --- a/backend/moonstream/web3_provider.py +++ b/backend/moonstream/web3_provider.py @@ -36,11 +36,14 @@ if not MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI.replace(".", "").isnumeric(): web3_provider_ip = fetch_web3_provider_ip() if web3_provider_ip is None: raise ValueError("Unable to extract web3 provider IP") - moonstream_web3_provider = Web3( - Web3.HTTPProvider( - f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" - ) +else: + web3_provider_ip = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI + +moonstream_web3_provider = Web3( + Web3.HTTPProvider( + f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" ) +) def yield_web3_provider() -> Web3: From b3197b9c8991d9ea843826bde49e00c7fb05613f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Oct 2021 14:58:45 +0000 Subject: [PATCH 12/12] Lint fix --- backend/moonstream/web3_provider.py | 4 +--- backend/sample.env | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/moonstream/web3_provider.py b/backend/moonstream/web3_provider.py index d3457849..719de079 100644 --- a/backend/moonstream/web3_provider.py +++ b/backend/moonstream/web3_provider.py @@ -40,9 +40,7 @@ else: web3_provider_ip = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI moonstream_web3_provider = Web3( - Web3.HTTPProvider( - f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" - ) + Web3.HTTPProvider(f"http://{web3_provider_ip}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}") ) diff --git a/backend/sample.env b/backend/sample.env index 80289693..8f1723ea 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -4,7 +4,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" +export MOONSTREAM_INTERNAL_HOSTED_ZONE_ID="" export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="" export AWS_S3_SMARTCONTRACT_BUCKET="" export BUGOUT_BROOD_URL="https://auth.bugout.dev"