kopia lustrzana https://github.com/bugout-dev/moonstream
Removed requirements.txt and used setup.py instead
kompotkot
rodzic
d9f965dc1d
commit
dd8e8d3db2
|
|
@ -16,6 +16,8 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: Upgrade pip and setuptools
|
||||
run: pip install --upgrade pip setuptools
|
||||
- name: Install test requirements
|
||||
working-directory: ./backend
|
||||
run: pip install -r requirements.txt
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: Upgrade pip and setuptools
|
||||
run: pip install --upgrade pip setuptools
|
||||
- name: Install test requirements
|
||||
working-directory: ./crawlers/mooncrawl
|
||||
run: pip install -e .[dev]
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: Upgrade pip and setuptools
|
||||
run: pip install --upgrade pip setuptools
|
||||
- name: Install test requirements
|
||||
working-directory: ./db
|
||||
run: pip install -e .[dev]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Deployment script - intended to run on Moonstream servers
|
||||
# Deployment script - intended to run on Moonstream API server
|
||||
|
||||
# Colors
|
||||
C_RESET='\033[0m'
|
||||
|
|
@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
|
|||
PYTHON="${PYTHON_ENV_DIR}/bin/python"
|
||||
PIP="${PYTHON_ENV_DIR}/bin/pip"
|
||||
SCRIPT_DIR="$(realpath $(dirname $0))"
|
||||
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
|
||||
PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
|
||||
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
||||
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
||||
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
|
||||
|
||||
# Parameters scripts
|
||||
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
|
||||
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
|
||||
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
|
||||
|
||||
# API server service file
|
||||
SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
|
||||
|
||||
set -eu
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Updating pip and setuptools"
|
||||
"${PIP}" install -U pip setuptools
|
||||
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
|
||||
"${PIP}" install --upgrade pip setuptools
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Updating Python dependencies"
|
||||
"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
|
||||
echo -e "${PREFIX_INFO} Installing Python dependencies"
|
||||
"${PIP}" install -e "${APP_BACKEND_DIR}/"
|
||||
|
||||
echo
|
||||
echo
|
||||
|
|
@ -49,7 +54,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$
|
|||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
|
||||
bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
|
||||
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
|
||||
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
|
||||
|
||||
echo
|
||||
echo
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Update nodes connection address environment variables
|
||||
# from AWS Route53 internal hosted zone
|
||||
|
||||
VERSION='0.0.1'
|
||||
|
||||
# Colors
|
||||
C_RESET='\033[0m'
|
||||
C_RED='\033[1;31m'
|
||||
C_GREEN='\033[1;32m'
|
||||
C_YELLOW='\033[1;33m'
|
||||
|
||||
# Logs
|
||||
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
|
||||
# Print help message
|
||||
function usage {
|
||||
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
|
||||
echo
|
||||
echo "CLI to update nodes connection address environment
|
||||
variables from AWS Route53 internal hosted zone"
|
||||
echo
|
||||
echo "Optional arguments:"
|
||||
echo " -h Show this help message and exit"
|
||||
echo " -f File path where environment variables update at"
|
||||
}
|
||||
|
||||
file_flag=""
|
||||
verbose_flag="false"
|
||||
|
||||
while getopts 'f:v' flag; do
|
||||
case "${flag}" in
|
||||
f) file_flag="${OPTARG}" ;;
|
||||
h) usage
|
||||
exit 1 ;;
|
||||
v) verbose_flag="true" ;;
|
||||
*) usage
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Log messages
|
||||
function verbose {
|
||||
if [ "${verbose_flag}" == "true" ]; then
|
||||
echo -e "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# File flag should be specified
|
||||
if [ -z "${file_flag}" ]; then
|
||||
verbose "${PREFIX_CRIT} Please specify file path"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${file_flag}" ]; then
|
||||
verbose "${PREFIX_CRIT} Provided file does not exist"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Script version: v${VERSION}"
|
||||
|
||||
verbose "${PREFIX_INFO} Source environment variables"
|
||||
. ${file_flag}
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
|
||||
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
|
||||
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
|
||||
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving Polygon node address"
|
||||
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
|
||||
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
|
||||
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO(kompotkot): Modify regexp to work with export prefix
|
||||
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
|
||||
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
|
||||
|
||||
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
|
||||
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
|
||||
# Collect secrets from AWS SSM Parameter Store and
|
||||
# opt out as environment variable exports.
|
||||
|
||||
VERSION='0.0.1'
|
||||
|
||||
# Colors
|
||||
C_RESET='\033[0m'
|
||||
|
|
@ -26,6 +29,7 @@ and output as environment variable exports"
|
|||
echo " -o Output file name environment variables export to"
|
||||
}
|
||||
|
||||
# TODO(kompotkot): Flag for export prefix
|
||||
product_flag=""
|
||||
output_flag=""
|
||||
verbose_flag="false"
|
||||
|
|
@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Script version: v${VERSION}"
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
|
||||
ENV_PARAMETERS=$(aws ssm describe-parameters \
|
||||
--parameter-filters Key=tag:Product,Values=${product_flag} \
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
from .reporter import reporter
|
||||
from .version import MOONSTREAM_VERSION
|
||||
try:
|
||||
from .reporter import reporter
|
||||
from .version import MOONSTREAM_VERSION
|
||||
|
||||
# Reporting
|
||||
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
|
||||
reporter.system_report(publish=True)
|
||||
reporter.setup_excepthook(publish=True)
|
||||
# Reporting
|
||||
reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
|
||||
reporter.system_report(publish=True)
|
||||
reporter.setup_excepthook(publish=True)
|
||||
except:
|
||||
# Pass it to be able import MOONSTREAM_VERSION in setup.py with pip
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
appdirs==1.4.4
|
||||
asgiref==3.4.1
|
||||
black==21.7b0
|
||||
boto3==1.18.1
|
||||
botocore==1.21.1
|
||||
bugout==0.1.18
|
||||
certifi==2021.5.30
|
||||
charset-normalizer==2.0.3
|
||||
click==8.0.1
|
||||
fastapi==0.66.0
|
||||
h11==0.12.0
|
||||
idna==3.2
|
||||
jmespath==0.10.0
|
||||
humbug==0.2.7
|
||||
-e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db
|
||||
mypy==0.910
|
||||
mypy-extensions==0.4.3
|
||||
pathspec==0.9.0
|
||||
pydantic==1.8.2
|
||||
pyevmasm==0.2.3
|
||||
python-dateutil==2.8.2
|
||||
python-multipart==0.0.5
|
||||
regex==2021.7.6
|
||||
requests==2.26.0
|
||||
s3transfer==0.5.0
|
||||
six==1.16.0
|
||||
starlette==0.14.2
|
||||
toml==0.10.2
|
||||
tomli==1.0.4
|
||||
types-python-dateutil==0.1.6
|
||||
typing-extensions==3.10.0.0
|
||||
types-requests==2.25.6
|
||||
urllib3==1.26.6
|
||||
uvicorn==0.14.0
|
||||
web3==5.24.0
|
||||
|
|
@ -11,17 +11,20 @@ setup(
|
|||
version=MOONSTREAM_VERSION,
|
||||
packages=find_packages(),
|
||||
install_requires=[
|
||||
"boto3",
|
||||
"bugout >= 0.1.18",
|
||||
"fastapi",
|
||||
"humbug>=0.2.7",
|
||||
"python-dateutil",
|
||||
"uvicorn",
|
||||
"types-python-dateutil",
|
||||
"types-requests",
|
||||
"appdirs~=1.4.4",
|
||||
"boto3~=1.20.2",
|
||||
"bugout~=0.1.18",
|
||||
"fastapi~=0.70.0",
|
||||
"moonstreamdb==0.2.0",
|
||||
"humbug~=0.2.7",
|
||||
"pydantic~=1.8.2",
|
||||
"python-dateutil~=2.8.2",
|
||||
"python-multipart~=0.0.5"
|
||||
"uvicorn~=0.15.0",
|
||||
"web3~=5.24.0",
|
||||
],
|
||||
extras_require={
|
||||
"dev": ["black", "mypy"],
|
||||
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
|
||||
"distribute": ["setuptools", "twine", "wheel"],
|
||||
},
|
||||
package_data={"moonstream": ["py.typed"]},
|
||||
|
|
|
|||
|
|
@ -24,10 +24,14 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
|||
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
||||
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
|
||||
SCRIPT_DIR="$(realpath $(dirname $0))"
|
||||
|
||||
# Parameters scripts
|
||||
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
|
||||
CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts"
|
||||
CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash"
|
||||
CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash"
|
||||
CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
|
||||
CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
|
||||
|
||||
# Crawlers server service file
|
||||
CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
|
||||
|
||||
# Ethereum service files
|
||||
ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
|
||||
|
|
@ -38,9 +42,6 @@ ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
|
|||
# Polygon service file
|
||||
POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
|
||||
|
||||
CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
|
||||
|
||||
|
||||
|
||||
set -eu
|
||||
|
||||
|
|
@ -62,9 +63,13 @@ cd "${EXEC_DIR}"
|
|||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Updating Python dependencies"
|
||||
"${PIP}" install --upgrade pip
|
||||
"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
|
||||
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
|
||||
"${PIP}" install --upgrade pip setuptools
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Installing Python dependencies"
|
||||
"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/"
|
||||
|
||||
echo
|
||||
echo
|
||||
|
|
@ -75,12 +80,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex
|
|||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
|
||||
curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
|
||||
bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo -e "${PREFIX_INFO} Updating nodes connection parameters"
|
||||
curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}"
|
||||
bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
|
||||
|
||||
echo
|
||||
echo
|
||||
|
|
@ -123,4 +128,3 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV
|
|||
systemctl daemon-reload
|
||||
systemctl restart "${CRAWLERS_SERVICE_FILE}"
|
||||
systemctl status "${CRAWLERS_SERVICE_FILE}"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Update nodes connection address environment variables
|
||||
# from AWS Route53 internal hosted zone
|
||||
|
||||
VERSION='0.0.1'
|
||||
|
||||
# Colors
|
||||
C_RESET='\033[0m'
|
||||
C_RED='\033[1;31m'
|
||||
C_GREEN='\033[1;32m'
|
||||
C_YELLOW='\033[1;33m'
|
||||
|
||||
# Logs
|
||||
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
|
||||
# Print help message
|
||||
function usage {
|
||||
echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"
|
||||
echo
|
||||
echo "CLI to update nodes connection address environment
|
||||
variables from AWS Route53 internal hosted zone"
|
||||
echo
|
||||
echo "Optional arguments:"
|
||||
echo " -h Show this help message and exit"
|
||||
echo " -f File path where environment variables update at"
|
||||
}
|
||||
|
||||
file_flag=""
|
||||
verbose_flag="false"
|
||||
|
||||
while getopts 'f:v' flag; do
|
||||
case "${flag}" in
|
||||
f) file_flag="${OPTARG}" ;;
|
||||
h) usage
|
||||
exit 1 ;;
|
||||
v) verbose_flag="true" ;;
|
||||
*) usage
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Log messages
|
||||
function verbose {
|
||||
if [ "${verbose_flag}" == "true" ]; then
|
||||
echo -e "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# File flag should be specified
|
||||
if [ -z "${file_flag}" ]; then
|
||||
verbose "${PREFIX_CRIT} Please specify file path"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${file_flag}" ]; then
|
||||
verbose "${PREFIX_CRIT} Provided file does not exist"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Script version: v${VERSION}"
|
||||
|
||||
verbose "${PREFIX_INFO} Source environment variables"
|
||||
. ${file_flag}
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving Ethereum node address"
|
||||
RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
|
||||
if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
|
||||
verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving Polygon node address"
|
||||
RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
|
||||
if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
|
||||
verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO(kompotkot): Modify regexp to work with export prefix
|
||||
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
|
||||
sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
|
||||
|
||||
verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
|
||||
sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Collect secrets from AWS SSM Parameter Store and
|
||||
# opt out as environment variable exports.
|
||||
|
||||
VERSION='0.0.1'
|
||||
|
||||
# Colors
|
||||
C_RESET='\033[0m'
|
||||
C_RED='\033[1;31m'
|
||||
C_GREEN='\033[1;32m'
|
||||
C_YELLOW='\033[1;33m'
|
||||
|
||||
# Logs
|
||||
PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||
|
||||
# Print help message
|
||||
function usage {
|
||||
echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"
|
||||
echo
|
||||
echo "CLI to collect secrets from AWS SSM Parameter Store
|
||||
and output as environment variable exports"
|
||||
echo
|
||||
echo "Optional arguments:"
|
||||
echo " -h Show this help message and exit"
|
||||
echo " -p Product tag (moonstream, spire, brood, drones)"
|
||||
echo " -o Output file name environment variables export to"
|
||||
}
|
||||
|
||||
# TODO(kompotkot): Flag for export prefix
|
||||
product_flag=""
|
||||
output_flag=""
|
||||
verbose_flag="false"
|
||||
|
||||
while getopts 'p:o:v' flag; do
|
||||
case "${flag}" in
|
||||
p) product_flag="${OPTARG}" ;;
|
||||
o) output_flag="${OPTARG}" ;;
|
||||
h) usage
|
||||
exit 1 ;;
|
||||
v) verbose_flag="true" ;;
|
||||
*) usage
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Log messages
|
||||
function verbose {
|
||||
if [ "${verbose_flag}" == "true" ]; then
|
||||
echo -e "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Product flag should be specified
|
||||
# TODO(kompotkot): Extend script to work with few product at once
|
||||
if [ -z "${product_flag}" ]; then
|
||||
verbose "${PREFIX_CRIT} Please specify product tag"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Script version: v${VERSION}"
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
|
||||
ENV_PARAMETERS=$(aws ssm describe-parameters \
|
||||
--parameter-filters Key=tag:Product,Values=${product_flag} \
|
||||
| jq -r .Parameters[].Name)
|
||||
if [ -z "${ENV_PARAMETERS}" ]; then
|
||||
verbose "${PREFIX_CRIT} There no parameters for provided product tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "${PREFIX_INFO} Retrieving parameters values"
|
||||
ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
|
||||
--names ${ENV_PARAMETERS} \
|
||||
--query "Parameters[*].{Name:Name,Value:Value}")
|
||||
ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
|
||||
verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
|
||||
for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
|
||||
param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
|
||||
param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
|
||||
if [ -z "${output_flag}" ]; then
|
||||
echo "${param_key}=${param_value}"
|
||||
else
|
||||
echo "${param_key}=${param_value}" >> "${output_flag}"
|
||||
fi
|
||||
done
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
aiohttp==3.7.4.post0
|
||||
async-timeout==3.0.1
|
||||
attrs==21.2.0
|
||||
base58==2.1.0
|
||||
bitarray==1.2.2
|
||||
black==21.8b0
|
||||
boto3==1.18.40
|
||||
botocore==1.21.40
|
||||
bugout==0.1.17
|
||||
certifi==2021.5.30
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.4
|
||||
click==8.0.1
|
||||
cytoolz==0.11.0
|
||||
eth-abi==2.1.1
|
||||
eth-account==0.5.5
|
||||
eth-hash==0.3.2
|
||||
eth-keyfile==0.5.1
|
||||
eth-keys==0.3.3
|
||||
eth-rlp==0.2.1
|
||||
eth-typing==2.2.2
|
||||
eth-utils==1.10.0
|
||||
hexbytes==0.2.2
|
||||
humbug==0.2.7
|
||||
idna==3.2
|
||||
ipfshttpclient==0.8.0a2
|
||||
jmespath==0.10.0
|
||||
jsonschema==3.2.0
|
||||
isort==5.10.0
|
||||
lru-dict==1.1.7
|
||||
moonstreamdb==0.2.0
|
||||
multiaddr==0.0.9
|
||||
multidict==5.1.0
|
||||
mypy==0.910
|
||||
mypy-extensions==0.4.3
|
||||
netaddr==0.8.0
|
||||
parsimonious==0.8.1
|
||||
pathspec==0.9.0
|
||||
platformdirs==2.3.0
|
||||
protobuf==3.17.3
|
||||
pycryptodome==3.10.1
|
||||
pyrsistent==0.18.0
|
||||
python-dateutil==2.8.2
|
||||
regex==2021.8.28
|
||||
requests==2.26.0
|
||||
rlp==2.0.1
|
||||
s3transfer==0.5.0
|
||||
six==1.16.0
|
||||
toml==0.10.2
|
||||
tomli==1.2.1
|
||||
toolz==0.11.1
|
||||
tqdm==4.62.2
|
||||
types-python-dateutil==2.8.0
|
||||
types-requests==2.25.6
|
||||
typing-extensions==3.10.0.2
|
||||
urllib3==1.26.6
|
||||
varint==1.0.2
|
||||
web3==5.23.1
|
||||
websockets==9.1
|
||||
yarl==1.6.3
|
||||
|
|
@ -33,14 +33,14 @@ setup(
|
|||
package_data={"mooncrawl": ["py.typed"]},
|
||||
zip_safe=False,
|
||||
install_requires=[
|
||||
"boto3",
|
||||
"bugout >= 0.1.17",
|
||||
"moonstreamdb == 0.2.0",
|
||||
"humbug",
|
||||
"python-dateutil",
|
||||
"requests",
|
||||
"tqdm",
|
||||
"web3",
|
||||
"boto3~=1.20.2",
|
||||
"bugout~=0.1.17",
|
||||
"moonstreamdb==0.2.0",
|
||||
"humbug~=0.2.7",
|
||||
"python-dateutil~=2.8.2",
|
||||
"requests~=2.26.0",
|
||||
"tqdm~=4.62.3",
|
||||
"web3~=5.24.0",
|
||||
],
|
||||
extras_require={
|
||||
"dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"]
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ setup(
|
|||
packages=find_packages(),
|
||||
package_data={"moonstreamdb": ["py.typed"]},
|
||||
zip_safe=False,
|
||||
install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
|
||||
install_requires=["alembic~=1.7.4", "psycopg2-binary~=2.9.1", "sqlalchemy~=1.4.26"],
|
||||
extras_require={
|
||||
"dev": ["black", "mypy"],
|
||||
"dev": ["black", "isort", "mypy"],
|
||||
"distribute": ["setuptools", "twine", "wheel"],
|
||||
},
|
||||
entry_points={
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue