diff --git a/.github/workflows/lint.backend.yml b/.github/workflows/lint.backend.yml index 8aae8ed9..ed3fc503 100644 --- a/.github/workflows/lint.backend.yml +++ b/.github/workflows/lint.backend.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./backend run: pip install -r requirements.txt diff --git a/.github/workflows/lint.crawlers.yml b/.github/workflows/lint.crawlers.yml index a1d563d8..c8d202c6 100644 --- a/.github/workflows/lint.crawlers.yml +++ b/.github/workflows/lint.crawlers.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./crawlers/mooncrawl run: pip install -e .[dev] diff --git a/.github/workflows/lint.db.yml b/.github/workflows/lint.db.yml index 62937a51..99c26d72 100644 --- a/.github/workflows/lint.db.yml +++ b/.github/workflows/lint.db.yml @@ -16,6 +16,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Upgrade pip and setuptools + run: pip install --upgrade pip setuptools - name: Install test requirements working-directory: ./db run: pip install -e .[dev] diff --git a/backend/deploy/deploy.bash b/backend/deploy/deploy.bash index bb113a64..d7ff3735 100755 --- a/backend/deploy/deploy.bash +++ b/backend/deploy/deploy.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Deployment script - intended to run on Moonstream servers +# Deployment script - intended to run on Moonstream API server # Colors C_RESET='\033[0m' @@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}" PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SCRIPT_DIR="$(realpath $(dirname $0))" -PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" + +# Parameters scripts +PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# API server service file SERVICE_FILE="${SCRIPT_DIR}/moonstream.service" set -eu echo echo -echo -e "${PREFIX_INFO} Updating pip and setuptools" -"${PIP}" install -U pip setuptools +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt" +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_BACKEND_DIR}/" echo echo @@ -49,7 +54,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$ echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Updating nodes connection parameters" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" echo echo diff --git a/backend/deploy/nodes-connections.bash b/backend/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/backend/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/backend/deploy/parameters.bash b/backend/deploy/parameters.bash index cb6ea2e9..dc401951 100755 --- a/backend/deploy/parameters.bash +++ b/backend/deploy/parameters.bash @@ -1,6 +1,9 @@ #!/usr/bin/env bash # -# Collect secrets from AWS SSM Parameter Store and output as environment variable exports. +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' # Colors C_RESET='\033[0m' @@ -26,6 +29,7 @@ and output as environment variable exports" echo " -o Output file name environment variables export to" } +# TODO(kompotkot): Flag for export prefix product_flag="" output_flag="" verbose_flag="false" @@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then exit 1 fi +verbose "${PREFIX_INFO} Script version: v${VERSION}" + verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" ENV_PARAMETERS=$(aws ssm describe-parameters \ --parameter-filters Key=tag:Product,Values=${product_flag} \ diff --git a/backend/moonstream/__init__.py b/backend/moonstream/__init__.py index 5df6f535..14174c60 100644 --- a/backend/moonstream/__init__.py +++ b/backend/moonstream/__init__.py @@ -1,7 +1,11 @@ -from .reporter import reporter -from .version import MOONSTREAM_VERSION +try: + from .reporter import reporter + from .version import MOONSTREAM_VERSION -# Reporting -reporter.tags.append(f"version:{MOONSTREAM_VERSION}") -reporter.system_report(publish=True) -reporter.setup_excepthook(publish=True) + # Reporting + reporter.tags.append(f"version:{MOONSTREAM_VERSION}") + reporter.system_report(publish=True) + reporter.setup_excepthook(publish=True) +except: + # Pass it to be able import MOONSTREAM_VERSION in setup.py with pip + pass diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index e4c0b435..00000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,35 +0,0 @@ -appdirs==1.4.4 -asgiref==3.4.1 -black==21.7b0 -boto3==1.18.1 -botocore==1.21.1 -bugout==0.1.18 -certifi==2021.5.30 -charset-normalizer==2.0.3 -click==8.0.1 -fastapi==0.66.0 -h11==0.12.0 -idna==3.2 -jmespath==0.10.0 -humbug==0.2.7 --e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db -mypy==0.910 -mypy-extensions==0.4.3 -pathspec==0.9.0 -pydantic==1.8.2 -pyevmasm==0.2.3 -python-dateutil==2.8.2 -python-multipart==0.0.5 -regex==2021.7.6 -requests==2.26.0 -s3transfer==0.5.0 -six==1.16.0 -starlette==0.14.2 -toml==0.10.2 -tomli==1.0.4 -types-python-dateutil==0.1.6 -typing-extensions==3.10.0.0 -types-requests==2.25.6 -urllib3==1.26.6 -uvicorn==0.14.0 -web3==5.24.0 \ No newline at end of file diff --git a/backend/setup.py b/backend/setup.py index c86bdfef..506ca956 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -11,17 +11,20 @@ setup( version=MOONSTREAM_VERSION, packages=find_packages(), install_requires=[ - "boto3", - "bugout >= 0.1.18", - "fastapi", - "humbug>=0.2.7", - "python-dateutil", - "uvicorn", - "types-python-dateutil", - "types-requests", + "appdirs~=1.4.4", + "boto3~=1.20.2", + "bugout~=0.1.18", + "fastapi~=0.70.0", + "moonstreamdb==0.2.0", + "humbug~=0.2.7", + "pydantic~=1.8.2", + "python-dateutil~=2.8.2", + "python-multipart~=0.0.5" + "uvicorn~=0.15.0", + "web3~=5.24.0", ], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"], "distribute": ["setuptools", "twine", "wheel"], }, package_data={"moonstream": ["py.typed"]}, diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index f38a61a4..60e50f43 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -24,10 +24,14 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" + +# Parameters scripts PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts" -CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash" -CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash" +CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash" + +# Crawlers server service file +CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" @@ -38,9 +42,6 @@ ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" # Polygon service file POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service" -CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" - - set -eu @@ -62,9 +63,13 @@ cd "${EXEC_DIR}" echo echo -echo -e "${PREFIX_INFO} Updating Python dependencies" -"${PIP}" install --upgrade pip -"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt" +echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" +"${PIP}" install --upgrade pip setuptools + +echo +echo +echo -e "${PREFIX_INFO} Installing Python dependencies" +"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/" echo echo @@ -75,12 +80,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex echo echo echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" echo echo echo -e "${PREFIX_INFO} Updating nodes connection parameters" -curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}" +bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}" echo echo @@ -123,4 +128,3 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV systemctl daemon-reload systemctl restart "${CRAWLERS_SERVICE_FILE}" systemctl status "${CRAWLERS_SERVICE_FILE}" - diff --git a/crawlers/deploy/nodes-connections.bash b/crawlers/deploy/nodes-connections.bash new file mode 100755 index 00000000..3718b5c9 --- /dev/null +++ b/crawlers/deploy/nodes-connections.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Update nodes connection address environment variables +# from AWS Route53 internal hosted zone + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH" + echo + echo "CLI to update nodes connection address environment + variables from AWS Route53 internal hosted zone" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -f File path where environment variables update at" +} + +file_flag="" +verbose_flag="false" + +while getopts 'f:v' flag; do + case "${flag}" in + f) file_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# File flag should be specified +if [ -z "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify file path" + usage + exit 1 +fi + +if [ ! -f "${file_flag}" ]; then + verbose "${PREFIX_CRIT} Provided file does not exist" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Source environment variables" +. ${file_flag} + +verbose "${PREFIX_INFO} Retrieving Ethereum node address" +RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving Polygon node address" +RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0]) +if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then + verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null" + exit 1 +fi + +# TODO(kompotkot): Modify regexp to work with export prefix +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag} + +verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}" +sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag} diff --git a/crawlers/deploy/parameters.bash b/crawlers/deploy/parameters.bash new file mode 100755 index 00000000..dc401951 --- /dev/null +++ b/crawlers/deploy/parameters.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Collect secrets from AWS SSM Parameter Store and +# opt out as environment variable exports. + +VERSION='0.0.1' + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Print help message +function usage { + echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT" + echo + echo "CLI to collect secrets from AWS SSM Parameter Store +and output as environment variable exports" + echo + echo "Optional arguments:" + echo " -h Show this help message and exit" + echo " -p Product tag (moonstream, spire, brood, drones)" + echo " -o Output file name environment variables export to" +} + +# TODO(kompotkot): Flag for export prefix +product_flag="" +output_flag="" +verbose_flag="false" + +while getopts 'p:o:v' flag; do + case "${flag}" in + p) product_flag="${OPTARG}" ;; + o) output_flag="${OPTARG}" ;; + h) usage + exit 1 ;; + v) verbose_flag="true" ;; + *) usage + exit 1 ;; + esac +done + +# Log messages +function verbose { + if [ "${verbose_flag}" == "true" ]; then + echo -e "$1" + fi +} + +# Product flag should be specified +# TODO(kompotkot): Extend script to work with few product at once +if [ -z "${product_flag}" ]; then + verbose "${PREFIX_CRIT} Please specify product tag" + usage + exit 1 +fi + +verbose "${PREFIX_INFO} Script version: v${VERSION}" + +verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}" +ENV_PARAMETERS=$(aws ssm describe-parameters \ + --parameter-filters Key=tag:Product,Values=${product_flag} \ + | jq -r .Parameters[].Name) +if [ -z "${ENV_PARAMETERS}" ]; then + verbose "${PREFIX_CRIT} There no parameters for provided product tag" + exit 1 +fi + +verbose "${PREFIX_INFO} Retrieving parameters values" +ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \ + --names ${ENV_PARAMETERS} \ + --query "Parameters[*].{Name:Name,Value:Value}") +ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length) +verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters" +for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do + param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name) + param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value) + if [ -z "${output_flag}" ]; then + echo "${param_key}=${param_value}" + else + echo "${param_key}=${param_value}" >> "${output_flag}" + fi +done diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt deleted file mode 100644 index 79549178..00000000 --- a/crawlers/mooncrawl/requirements.txt +++ /dev/null @@ -1,60 +0,0 @@ -aiohttp==3.7.4.post0 -async-timeout==3.0.1 -attrs==21.2.0 -base58==2.1.0 -bitarray==1.2.2 -black==21.8b0 -boto3==1.18.40 -botocore==1.21.40 -bugout==0.1.17 -certifi==2021.5.30 -chardet==4.0.0 -charset-normalizer==2.0.4 -click==8.0.1 -cytoolz==0.11.0 -eth-abi==2.1.1 -eth-account==0.5.5 -eth-hash==0.3.2 -eth-keyfile==0.5.1 -eth-keys==0.3.3 -eth-rlp==0.2.1 -eth-typing==2.2.2 -eth-utils==1.10.0 -hexbytes==0.2.2 -humbug==0.2.7 -idna==3.2 -ipfshttpclient==0.8.0a2 -jmespath==0.10.0 -jsonschema==3.2.0 -isort==5.10.0 -lru-dict==1.1.7 -moonstreamdb==0.2.0 -multiaddr==0.0.9 -multidict==5.1.0 -mypy==0.910 -mypy-extensions==0.4.3 -netaddr==0.8.0 -parsimonious==0.8.1 -pathspec==0.9.0 -platformdirs==2.3.0 -protobuf==3.17.3 -pycryptodome==3.10.1 -pyrsistent==0.18.0 -python-dateutil==2.8.2 -regex==2021.8.28 -requests==2.26.0 -rlp==2.0.1 -s3transfer==0.5.0 -six==1.16.0 -toml==0.10.2 -tomli==1.2.1 -toolz==0.11.1 -tqdm==4.62.2 -types-python-dateutil==2.8.0 -types-requests==2.25.6 -typing-extensions==3.10.0.2 -urllib3==1.26.6 -varint==1.0.2 -web3==5.23.1 -websockets==9.1 -yarl==1.6.3 diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index d6e0f305..07ccd3ed 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -33,14 +33,14 @@ setup( package_data={"mooncrawl": ["py.typed"]}, zip_safe=False, install_requires=[ - "boto3", - "bugout >= 0.1.17", - "moonstreamdb == 0.2.0", - "humbug", - "python-dateutil", - "requests", - "tqdm", - "web3", + "boto3~=1.20.2", + "bugout~=0.1.17", + "moonstreamdb==0.2.0", + "humbug~=0.2.7", + "python-dateutil~=2.8.2", + "requests~=2.26.0", + "tqdm~=4.62.3", + "web3~=5.24.0", ], extras_require={ "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"] diff --git a/db/setup.py b/db/setup.py index 601c4c14..c6396a6d 100644 --- a/db/setup.py +++ b/db/setup.py @@ -32,9 +32,9 @@ setup( packages=find_packages(), package_data={"moonstreamdb": ["py.typed"]}, zip_safe=False, - install_requires=["alembic", "psycopg2-binary", "sqlalchemy"], + install_requires=["alembic~=1.7.4", "psycopg2-binary~=2.9.1", "sqlalchemy~=1.4.26"], extras_require={ - "dev": ["black", "mypy"], + "dev": ["black", "isort", "mypy"], "distribute": ["setuptools", "twine", "wheel"], }, entry_points={