Removed requirements.txt and used setup.py instead

.github/workflows/lint.backend.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./backend
         run: pip install -r requirements.txt

.github/workflows/lint.crawlers.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./crawlers/mooncrawl
         run: pip install -e .[dev]

.github/workflows/lint.db.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./db
         run: pip install -e .[dev]

backend/deploy/deploy.bash

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Deployment script - intended to run on Moonstream servers
+# Deployment script - intended to run on Moonstream API server
 
 # Colors
 C_RESET='\033[0m'
@@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
 PYTHON="${PYTHON_ENV_DIR}/bin/python"
 PIP="${PYTHON_ENV_DIR}/bin/pip"
 SCRIPT_DIR="$(realpath $(dirname $0))"
-PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
+
+# Parameters scripts
+PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
+CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
+CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
+
+# API server service file
 SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
 
 set -eu
 
 echo
 echo
-echo -e "${PREFIX_INFO} Updating pip and setuptools"
+echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
-"${PIP}" install -U pip setuptools
+"${PIP}" install --upgrade pip setuptools
 
 echo
 echo
-echo -e "${PREFIX_INFO} Updating Python dependencies"
+echo -e "${PREFIX_INFO} Installing Python dependencies"
-"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
+"${PIP}" install -e "${APP_BACKEND_DIR}/"
 
 echo
 echo
@@ -49,7 +54,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$
 echo
 echo
 echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
-bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
+bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Updating nodes connection parameters"
+bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
 
 echo
 echo

backend/deploy/nodes-connections.bash

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+#
+# Update nodes connection address environment variables 
+# from AWS Route53 internal hosted zone
+
+VERSION='0.0.1'
+
+# Colors
+C_RESET='\033[0m'
+C_RED='\033[1;31m'
+C_GREEN='\033[1;32m'
+C_YELLOW='\033[1;33m'
+
+# Logs
+PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
+
+# Print help message
+function usage {
+  echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"  
+  echo
+  echo "CLI to update nodes connection address environment 
+  variables from AWS Route53 internal hosted zone"
+  echo
+  echo "Optional arguments:"
+  echo "  -h  Show this help message and exit"
+  echo "  -f  File path where environment variables update at"
+}
+
+file_flag=""
+verbose_flag="false"
+
+while getopts 'f:v' flag; do
+  case "${flag}" in
+    f) file_flag="${OPTARG}" ;;
+    h) usage
+      exit 1 ;;
+    v) verbose_flag="true" ;;
+    *) usage
+      exit 1 ;;
+  esac
+done
+
+# Log messages
+function verbose {
+  if [ "${verbose_flag}" == "true" ]; then
+    echo -e "$1"
+  fi
+}
+
+# File flag should be specified
+if [ -z "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Please specify file path"
+  usage
+  exit 1
+fi
+
+if [ ! -f "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Provided file does not exist"
+  usage
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
+verbose "${PREFIX_INFO} Source environment variables"
+. ${file_flag}
+
+verbose "${PREFIX_INFO} Retrieving Ethereum node address"
+RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Retrieving Polygon node address"
+RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
+  exit 1
+fi
+
+# TODO(kompotkot): Modify regexp to work with export prefix
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
+
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

backend/deploy/parameters.bash

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 #
-# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
+# Collect secrets from AWS SSM Parameter Store and 
+# opt out as environment variable exports.
+
+VERSION='0.0.1'
 
 # Colors
 C_RESET='\033[0m'
@@ -26,6 +29,7 @@ and output as environment variable exports"
   echo "  -o  Output file name environment variables export to"
 }
 
+# TODO(kompotkot): Flag for export prefix
 product_flag=""
 output_flag=""
 verbose_flag="false"
@@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then
   exit 1
 fi
 
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
 verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
 ENV_PARAMETERS=$(aws ssm describe-parameters \
   --parameter-filters Key=tag:Product,Values=${product_flag} \

backend/moonstream/__init__.py

@@ -1,7 +1,11 @@
-from .reporter import reporter
+try:
-from .version import MOONSTREAM_VERSION
+    from .reporter import reporter
+    from .version import MOONSTREAM_VERSION
 
-# Reporting
+    # Reporting
-reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
+    reporter.tags.append(f"version:{MOONSTREAM_VERSION}")
-reporter.system_report(publish=True)
+    reporter.system_report(publish=True)
-reporter.setup_excepthook(publish=True)
+    reporter.setup_excepthook(publish=True)
+except:
+    # Pass it to be able import MOONSTREAM_VERSION in setup.py with pip
+    pass

backend/requirements.txt

@@ -1,35 +0,0 @@
-appdirs==1.4.4
-asgiref==3.4.1
-black==21.7b0
-boto3==1.18.1
-botocore==1.21.1
-bugout==0.1.18
-certifi==2021.5.30
-charset-normalizer==2.0.3
-click==8.0.1
-fastapi==0.66.0
-h11==0.12.0
-idna==3.2
-jmespath==0.10.0
-humbug==0.2.7
--e git+https://git@github.com/bugout-dev/moonstream.git@5dad139d311920c943d842673003312fa6cb2bdb#egg=moonstreamdb&subdirectory=db
-mypy==0.910
-mypy-extensions==0.4.3
-pathspec==0.9.0
-pydantic==1.8.2
-pyevmasm==0.2.3
-python-dateutil==2.8.2
-python-multipart==0.0.5
-regex==2021.7.6
-requests==2.26.0
-s3transfer==0.5.0
-six==1.16.0
-starlette==0.14.2
-toml==0.10.2
-tomli==1.0.4
-types-python-dateutil==0.1.6
-typing-extensions==3.10.0.0
-types-requests==2.25.6
-urllib3==1.26.6
-uvicorn==0.14.0
-web3==5.24.0

backend/setup.py

@@ -11,17 +11,20 @@ setup(
     version=MOONSTREAM_VERSION,
     packages=find_packages(),
     install_requires=[
-        "boto3",
+        "appdirs~=1.4.4",
-        "bugout >= 0.1.18",
+        "boto3~=1.20.2",
-        "fastapi",
+        "bugout~=0.1.18",
-        "humbug>=0.2.7",
+        "fastapi~=0.70.0",
-        "python-dateutil",
+        "moonstreamdb==0.2.0",
-        "uvicorn",
+        "humbug~=0.2.7",
-        "types-python-dateutil",
+        "pydantic~=1.8.2",
-        "types-requests",
+        "python-dateutil~=2.8.2",
+        "python-multipart~=0.0.5"
+        "uvicorn~=0.15.0",
+        "web3~=5.24.0",
     ],
     extras_require={
-        "dev": ["black", "mypy"],
+        "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"],
         "distribute": ["setuptools", "twine", "wheel"],
     },
     package_data={"moonstream": ["py.typed"]},

crawlers/deploy/deploy.bash

@@ -24,10 +24,14 @@ SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
 SCRIPT_DIR="$(realpath $(dirname $0))"
+
+# Parameters scripts
 PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts"
+CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
-CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash"
+CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
-CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash"
+
+# Crawlers server service file
+CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
 
 # Ethereum service files
 ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
@@ -38,9 +42,6 @@ ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
 # Polygon service file
 POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service"
 
-CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service"
-
-
 
 set -eu
 
@@ -62,9 +63,13 @@ cd "${EXEC_DIR}"
 
 echo
 echo
-echo -e "${PREFIX_INFO} Updating Python dependencies"
+echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
-"${PIP}" install --upgrade pip
+"${PIP}" install --upgrade pip setuptools
-"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Installing Python dependencies"
+"${PIP}" install -e "${APP_CRAWLERS_DIR}/mooncrawl/"
 
 echo
 echo
@@ -75,12 +80,12 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" ex
 echo
 echo
 echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
-curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
+bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
 
 echo
 echo
 echo -e "${PREFIX_INFO} Updating nodes connection parameters"
-curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}"
+bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
 
 echo
 echo
@@ -123,4 +128,3 @@ cp "${SCRIPT_DIR}/${CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${CRAWLERS_SERV
 systemctl daemon-reload
 systemctl restart "${CRAWLERS_SERVICE_FILE}"
 systemctl status "${CRAWLERS_SERVICE_FILE}"
-

crawlers/deploy/nodes-connections.bash

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+#
+# Update nodes connection address environment variables 
+# from AWS Route53 internal hosted zone
+
+VERSION='0.0.1'
+
+# Colors
+C_RESET='\033[0m'
+C_RED='\033[1;31m'
+C_GREEN='\033[1;32m'
+C_YELLOW='\033[1;33m'
+
+# Logs
+PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
+
+# Print help message
+function usage {
+  echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"  
+  echo
+  echo "CLI to update nodes connection address environment 
+  variables from AWS Route53 internal hosted zone"
+  echo
+  echo "Optional arguments:"
+  echo "  -h  Show this help message and exit"
+  echo "  -f  File path where environment variables update at"
+}
+
+file_flag=""
+verbose_flag="false"
+
+while getopts 'f:v' flag; do
+  case "${flag}" in
+    f) file_flag="${OPTARG}" ;;
+    h) usage
+      exit 1 ;;
+    v) verbose_flag="true" ;;
+    *) usage
+      exit 1 ;;
+  esac
+done
+
+# Log messages
+function verbose {
+  if [ "${verbose_flag}" == "true" ]; then
+    echo -e "$1"
+  fi
+}
+
+# File flag should be specified
+if [ -z "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Please specify file path"
+  usage
+  exit 1
+fi
+
+if [ ! -f "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Provided file does not exist"
+  usage
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
+verbose "${PREFIX_INFO} Source environment variables"
+. ${file_flag}
+
+verbose "${PREFIX_INFO} Retrieving Ethereum node address"
+RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Retrieving Polygon node address"
+RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
+  exit 1
+fi
+
+# TODO(kompotkot): Modify regexp to work with export prefix
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
+
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

crawlers/deploy/parameters.bash

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+#
+# Collect secrets from AWS SSM Parameter Store and 
+# opt out as environment variable exports.
+
+VERSION='0.0.1'
+
+# Colors
+C_RESET='\033[0m'
+C_RED='\033[1;31m'
+C_GREEN='\033[1;32m'
+C_YELLOW='\033[1;33m'
+
+# Logs
+PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
+
+# Print help message
+function usage {
+  echo "Usage: $0 [-h] -p PRODUCT -o OUTPUT"  
+  echo
+  echo "CLI to collect secrets from AWS SSM Parameter Store 
+and output as environment variable exports"
+  echo
+  echo "Optional arguments:"
+  echo "  -h  Show this help message and exit"
+  echo "  -p  Product tag (moonstream, spire, brood, drones)"
+  echo "  -o  Output file name environment variables export to"
+}
+
+# TODO(kompotkot): Flag for export prefix
+product_flag=""
+output_flag=""
+verbose_flag="false"
+
+while getopts 'p:o:v' flag; do
+  case "${flag}" in
+    p) product_flag="${OPTARG}" ;;
+    o) output_flag="${OPTARG}" ;;
+    h) usage
+      exit 1 ;;
+    v) verbose_flag="true" ;;
+    *) usage
+      exit 1 ;;
+  esac
+done
+
+# Log messages
+function verbose {
+  if [ "${verbose_flag}" == "true" ]; then
+    echo -e "$1"
+  fi
+}
+
+# Product flag should be specified
+# TODO(kompotkot): Extend script to work with few product at once
+if [ -z "${product_flag}" ]; then
+  verbose "${PREFIX_CRIT} Please specify product tag"
+  usage
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
+verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
+ENV_PARAMETERS=$(aws ssm describe-parameters \
+  --parameter-filters Key=tag:Product,Values=${product_flag} \
+  | jq -r .Parameters[].Name)
+if [ -z "${ENV_PARAMETERS}" ]; then
+  verbose "${PREFIX_CRIT} There no parameters for provided product tag"
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Retrieving parameters values"
+ENV_PARAMETERS_VALUES=$(aws ssm get-parameters \
+  --names ${ENV_PARAMETERS} \
+  --query "Parameters[*].{Name:Name,Value:Value}")
+ENV_PARAMETERS_VALUES_LENGTH=$(echo ${ENV_PARAMETERS_VALUES} | jq length)
+verbose "${PREFIX_INFO} Extracted ${ENV_PARAMETERS_VALUES_LENGTH} parameters"
+for i in $(seq 0 $((${ENV_PARAMETERS_VALUES_LENGTH} - 1))); do
+  param_key=$(echo ${ENV_PARAMETERS_VALUES} | jq -r .[$i].Name)
+  param_value=$(echo ${ENV_PARAMETERS_VALUES} | jq .[$i].Value)
+  if [ -z "${output_flag}" ]; then
+    echo "${param_key}=${param_value}"
+  else
+    echo "${param_key}=${param_value}" >> "${output_flag}"
+  fi
+done

crawlers/mooncrawl/requirements.txt

@@ -1,60 +0,0 @@
-aiohttp==3.7.4.post0
-async-timeout==3.0.1
-attrs==21.2.0
-base58==2.1.0
-bitarray==1.2.2
-black==21.8b0
-boto3==1.18.40
-botocore==1.21.40
-bugout==0.1.17
-certifi==2021.5.30
-chardet==4.0.0
-charset-normalizer==2.0.4
-click==8.0.1
-cytoolz==0.11.0
-eth-abi==2.1.1
-eth-account==0.5.5
-eth-hash==0.3.2
-eth-keyfile==0.5.1
-eth-keys==0.3.3
-eth-rlp==0.2.1
-eth-typing==2.2.2
-eth-utils==1.10.0
-hexbytes==0.2.2
-humbug==0.2.7
-idna==3.2
-ipfshttpclient==0.8.0a2
-jmespath==0.10.0
-jsonschema==3.2.0
-isort==5.10.0
-lru-dict==1.1.7
-moonstreamdb==0.2.0
-multiaddr==0.0.9
-multidict==5.1.0
-mypy==0.910
-mypy-extensions==0.4.3
-netaddr==0.8.0
-parsimonious==0.8.1
-pathspec==0.9.0
-platformdirs==2.3.0
-protobuf==3.17.3
-pycryptodome==3.10.1
-pyrsistent==0.18.0
-python-dateutil==2.8.2
-regex==2021.8.28
-requests==2.26.0
-rlp==2.0.1
-s3transfer==0.5.0
-six==1.16.0
-toml==0.10.2
-tomli==1.2.1
-toolz==0.11.1
-tqdm==4.62.2
-types-python-dateutil==2.8.0
-types-requests==2.25.6
-typing-extensions==3.10.0.2
-urllib3==1.26.6
-varint==1.0.2
-web3==5.23.1
-websockets==9.1
-yarl==1.6.3

crawlers/mooncrawl/setup.py

@@ -33,14 +33,14 @@ setup(
     package_data={"mooncrawl": ["py.typed"]},
     zip_safe=False,
     install_requires=[
-        "boto3",
+        "boto3~=1.20.2",
-        "bugout >= 0.1.17",
+        "bugout~=0.1.17",
-        "moonstreamdb == 0.2.0",
+        "moonstreamdb==0.2.0",
-        "humbug",
+        "humbug~=0.2.7",
-        "python-dateutil",
+        "python-dateutil~=2.8.2",
-        "requests",
+        "requests~=2.26.0",
-        "tqdm",
+        "tqdm~=4.62.3",
-        "web3",
+        "web3~=5.24.0",
     ],
     extras_require={
         "dev": ["black", "isort", "mypy", "types-requests", "types-python-dateutil"]

db/setup.py

@@ -32,9 +32,9 @@ setup(
     packages=find_packages(),
     package_data={"moonstreamdb": ["py.typed"]},
     zip_safe=False,
-    install_requires=["alembic", "psycopg2-binary", "sqlalchemy"],
+    install_requires=["alembic~=1.7.4", "psycopg2-binary~=2.9.1", "sqlalchemy~=1.4.26"],
     extras_require={
-        "dev": ["black", "mypy"],
+        "dev": ["black", "isort", "mypy"],
         "distribute": ["setuptools", "twine", "wheel"],
     },
     entry_points={