kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #249 from bugout-dev/crawlers-deploy
Deployment script for crawlerspull/261/head
Sergei Sumarokov
GitHub
commit
dc4bfd8f71
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Deployment script - intended to run on Moonstream crawlers server
|
||||
|
||||
# Main
|
||||
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
|
||||
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
|
||||
APP_CRAWLERS_DIR="${APP_DIR}/crawlers"
|
||||
PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
|
||||
PYTHON="${PYTHON_ENV_DIR}/bin/python"
|
||||
PIP="${PYTHON_ENV_DIR}/bin/pip"
|
||||
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
||||
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
||||
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
|
||||
SCRIPT_DIR="$(realpath $(dirname $0))"
|
||||
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
|
||||
ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
|
||||
ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
|
||||
ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
|
||||
|
||||
set -eu
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "Building executable Ethereum transaction pool crawler script with Go"
|
||||
HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool" "${APP_CRAWLERS_DIR}/main.go"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "Updating Python dependencies"
|
||||
"${PIP}" install --upgrade pip
|
||||
"${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "Retrieving deployment parameters"
|
||||
mkdir -p "${SECRETS_DIR}"
|
||||
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}"
|
||||
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}"
|
||||
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}"
|
||||
systemctl daemon-reload
|
||||
systemctl restart "${ETHEREUM_TRENDING_TIMER}"
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}"
|
||||
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
|
||||
cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
|
||||
systemctl daemon-reload
|
||||
systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Ethereum txpool crawler
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=ubuntu
|
||||
Group=www-data
|
||||
WorkingDirectory=/home/ubuntu/moonstream/crawlers/ethtxpool
|
||||
ExecStart=/home/ubuntu/moonstream/crawlers/ethtxpool/ethtxpool
|
||||
SyslogIdentifier=ethereum-txpool
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
"""
|
||||
Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
|
||||
"""
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
import boto3
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnvironmentVariable:
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
def get_parameters(path: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
|
||||
|
||||
Relies on the appropriate environment variables to authenticate against AWS:
|
||||
https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
|
||||
"""
|
||||
ssm = boto3.client("ssm")
|
||||
next_token: Optional[bool] = True
|
||||
parameters: List[Dict[str, Any]] = []
|
||||
while next_token is not None:
|
||||
kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
|
||||
if next_token is not True:
|
||||
kwargs["NextToken"] = next_token
|
||||
response = ssm.get_parameters_by_path(**kwargs)
|
||||
new_parameters = response.get("Parameters", [])
|
||||
parameters.extend(new_parameters)
|
||||
next_token = response.get("NextToken")
|
||||
|
||||
return parameters
|
||||
|
||||
|
||||
def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
|
||||
"""
|
||||
Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
|
||||
"""
|
||||
parameter_path = parameter_object.get("Name")
|
||||
if parameter_path is None:
|
||||
raise ValueError('Did not find "Name" in parameter object')
|
||||
name = parameter_path.split("/")[-1].upper()
|
||||
|
||||
value = parameter_object.get("Value")
|
||||
if value is None:
|
||||
raise ValueError('Did not find "Value" in parameter object')
|
||||
|
||||
return EnvironmentVariable(name, value)
|
||||
|
||||
|
||||
def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
|
||||
"""
|
||||
Produces a string which, when executed in a shell, exports the desired environment variables as
|
||||
specified by env_vars.
|
||||
"""
|
||||
prefix = "export " if with_export else ""
|
||||
return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
|
||||
|
||||
|
||||
def extract_handler(args: argparse.Namespace) -> None:
|
||||
"""
|
||||
Save environment variables to file.
|
||||
"""
|
||||
result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
|
||||
with args.outfile as ofp:
|
||||
print(result, file=ofp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Materialize environment variables from AWS SSM Parameter Store"
|
||||
)
|
||||
parser.set_defaults(func=lambda _: parser.print_help())
|
||||
subcommands = parser.add_subparsers(description="Parameters commands")
|
||||
|
||||
parser_extract = subcommands.add_parser(
|
||||
"extract", description="Parameters extract commands"
|
||||
)
|
||||
parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
|
||||
parser_extract.add_argument(
|
||||
"-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
|
||||
)
|
||||
parser_extract.add_argument(
|
||||
"--export",
|
||||
action="store_true",
|
||||
help="Set to output environment strings with export statements",
|
||||
)
|
||||
parser_extract.add_argument(
|
||||
"-p",
|
||||
"--path",
|
||||
default=None,
|
||||
help="SSM path from which to pull environment variables (pull is NOT recursive)",
|
||||
)
|
||||
parser_extract.set_defaults(func=extract_handler)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
aiohttp==3.7.4.post0
|
||||
async-timeout==3.0.1
|
||||
attrs==21.2.0
|
||||
base58==2.1.0
|
||||
bitarray==1.2.2
|
||||
black==21.8b0
|
||||
boto3==1.18.40
|
||||
botocore==1.21.40
|
||||
certifi==2021.5.30
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.4
|
||||
click==8.0.1
|
||||
cytoolz==0.11.0
|
||||
-e git+https://git@github.com/bugout-dev/moonstream.git@a4fff6498f66789934d4af26fd42a8cfb6e5eed5#egg=moonstreamdb&subdirectory=db
|
||||
eth-abi==2.1.1
|
||||
eth-account==0.5.5
|
||||
eth-hash==0.3.2
|
||||
eth-keyfile==0.5.1
|
||||
eth-keys==0.3.3
|
||||
eth-rlp==0.2.1
|
||||
eth-typing==2.2.2
|
||||
eth-utils==1.10.0
|
||||
hexbytes==0.2.2
|
||||
humbug==0.2.7
|
||||
idna==3.2
|
||||
ipfshttpclient==0.8.0a2
|
||||
jmespath==0.10.0
|
||||
jsonschema==3.2.0
|
||||
lru-dict==1.1.7
|
||||
multiaddr==0.0.9
|
||||
multidict==5.1.0
|
||||
mypy==0.910
|
||||
mypy-extensions==0.4.3
|
||||
netaddr==0.8.0
|
||||
parsimonious==0.8.1
|
||||
pathspec==0.9.0
|
||||
platformdirs==2.3.0
|
||||
protobuf==3.17.3
|
||||
pycryptodome==3.10.1
|
||||
pyrsistent==0.18.0
|
||||
python-dateutil==2.8.2
|
||||
regex==2021.8.28
|
||||
requests==2.26.0
|
||||
rlp==2.0.1
|
||||
s3transfer==0.5.0
|
||||
six==1.16.0
|
||||
toml==0.10.2
|
||||
tomli==1.2.1
|
||||
toolz==0.11.1
|
||||
tqdm==4.62.2
|
||||
types-python-dateutil==2.8.0
|
||||
types-requests==2.25.6
|
||||
typing-extensions==3.10.0.2
|
||||
urllib3==1.26.6
|
||||
varint==1.0.2
|
||||
web3==5.23.1
|
||||
websockets==9.1
|
||||
yarl==1.6.3
|
||||
Ładowanie…
Reference in New Issue