kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #803 from moonstream-to/struct-nodebalancer
Updated nodebalancer placementpull/812/head
commit
8df6ba410d
|
@ -0,0 +1,65 @@
|
||||||
|
|
||||||
|
# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,go
|
||||||
|
# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,go
|
||||||
|
|
||||||
|
### Go ###
|
||||||
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
|
#
|
||||||
|
# Binaries for programs and plugins
|
||||||
|
*.exe
|
||||||
|
*.exe~
|
||||||
|
*.dll
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
|
||||||
|
# Test binary, built with `go test -c`
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||||
|
*.out
|
||||||
|
|
||||||
|
# Dependency directories (remove the comment below to include it)
|
||||||
|
# vendor/
|
||||||
|
|
||||||
|
# Go workspace file
|
||||||
|
go.work
|
||||||
|
|
||||||
|
### Go Patch ###
|
||||||
|
/vendor/
|
||||||
|
/Godeps/
|
||||||
|
|
||||||
|
### VisualStudioCode ###
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
!.vscode/*.code-snippets
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
# Built Visual Studio Code Extensions
|
||||||
|
*.vsix
|
||||||
|
|
||||||
|
### VisualStudioCode Patch ###
|
||||||
|
# Ignore all local history of files
|
||||||
|
.history
|
||||||
|
.ionide
|
||||||
|
|
||||||
|
# Support for Project snippet scope
|
||||||
|
.vscode/*.code-snippets
|
||||||
|
|
||||||
|
# Ignore code-workspaces
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,go
|
||||||
|
|
||||||
|
# Custom
|
||||||
|
.secrets/*
|
||||||
|
dev.env
|
||||||
|
prod.env
|
||||||
|
test.env
|
||||||
|
.venv
|
||||||
|
fakenode
|
|
@ -16,27 +16,15 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
|
||||||
# Main
|
# Main
|
||||||
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
|
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
|
||||||
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
|
APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
|
||||||
APP_NODES_DIR="${APP_DIR}/nodes"
|
|
||||||
PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
|
|
||||||
PYTHON="${PYTHON_ENV_DIR}/bin/python"
|
|
||||||
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
||||||
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
||||||
AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
|
|
||||||
SCRIPT_DIR="$(realpath $(dirname $0))"
|
SCRIPT_DIR="$(realpath $(dirname $0))"
|
||||||
PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
|
|
||||||
NODE_BALANCER_CONFIG_PATH="${NODE_BALANCER_CONFIG_PATH:-/home/ubuntu/.nodebalancer/config.json}"
|
|
||||||
|
|
||||||
# Service file
|
# Service file
|
||||||
NODE_BALANCER_SERVICE_FILE="nodebalancer.service"
|
NODE_BALANCER_SERVICE_FILE="nodebalancer.service"
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
|
|
||||||
mkdir -p "${SECRETS_DIR}"
|
|
||||||
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Install checkenv"
|
echo -e "${PREFIX_INFO} Install checkenv"
|
||||||
|
@ -56,8 +44,8 @@ echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Building executable load balancer for nodes script with Go"
|
echo -e "${PREFIX_INFO} Building executable load balancer for nodes script with Go"
|
||||||
EXEC_DIR=$(pwd)
|
EXEC_DIR=$(pwd)
|
||||||
cd "${APP_NODES_DIR}/node_balancer"
|
cd "${APP_DIR}/nodebalancer"
|
||||||
HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_NODES_DIR}/node_balancer/nodebalancer" "${APP_NODES_DIR}/node_balancer/cmd/nodebalancer/"
|
HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/nodebalancer/nodebalancer" "${APP_DIR}/nodebalancer/cmd/nodebalancer/*.go"
|
||||||
cd "${EXEC_DIR}"
|
cd "${EXEC_DIR}"
|
||||||
|
|
||||||
echo
|
echo
|
|
@ -5,11 +5,11 @@ StartLimitIntervalSec=300
|
||||||
StartLimitBurst=3
|
StartLimitBurst=3
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
WorkingDirectory=/home/ubuntu/moonstream/nodes/node_balancer
|
WorkingDirectory=/home/ubuntu/moonstream/nodebalancer
|
||||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=15s
|
RestartSec=15s
|
||||||
ExecStart=/home/ubuntu/moonstream/nodes/node_balancer/nodebalancer server \
|
ExecStart=/home/ubuntu/moonstream/nodebalancer/nodebalancer server \
|
||||||
-host "${AWS_LOCAL_IPV4}" \
|
-host "${AWS_LOCAL_IPV4}" \
|
||||||
-port 8544 \
|
-port 8544 \
|
||||||
-healthcheck \
|
-healthcheck \
|
|
@ -1,102 +0,0 @@
|
||||||
"""
|
|
||||||
Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
|
|
||||||
"""
|
|
||||||
import argparse
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import sys
|
|
||||||
from typing import Any, Dict, Iterable, List, Optional
|
|
||||||
|
|
||||||
import boto3
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class EnvironmentVariable:
|
|
||||||
name: str
|
|
||||||
value: str
|
|
||||||
|
|
||||||
|
|
||||||
def get_parameters(path: str) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
|
|
||||||
|
|
||||||
Relies on the appropriate environment variables to authenticate against AWS:
|
|
||||||
https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
|
|
||||||
"""
|
|
||||||
ssm = boto3.client("ssm")
|
|
||||||
next_token: Optional[bool] = True
|
|
||||||
parameters: List[Dict[str, Any]] = []
|
|
||||||
while next_token is not None:
|
|
||||||
kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
|
|
||||||
if next_token is not True:
|
|
||||||
kwargs["NextToken"] = next_token
|
|
||||||
response = ssm.get_parameters_by_path(**kwargs)
|
|
||||||
new_parameters = response.get("Parameters", [])
|
|
||||||
parameters.extend(new_parameters)
|
|
||||||
next_token = response.get("NextToken")
|
|
||||||
|
|
||||||
return parameters
|
|
||||||
|
|
||||||
|
|
||||||
def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
|
|
||||||
"""
|
|
||||||
Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
|
|
||||||
"""
|
|
||||||
parameter_path = parameter_object.get("Name")
|
|
||||||
if parameter_path is None:
|
|
||||||
raise ValueError('Did not find "Name" in parameter object')
|
|
||||||
name = parameter_path.split("/")[-1].upper()
|
|
||||||
|
|
||||||
value = parameter_object.get("Value")
|
|
||||||
if value is None:
|
|
||||||
raise ValueError('Did not find "Value" in parameter object')
|
|
||||||
|
|
||||||
return EnvironmentVariable(name, value)
|
|
||||||
|
|
||||||
|
|
||||||
def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
|
|
||||||
"""
|
|
||||||
Produces a string which, when executed in a shell, exports the desired environment variables as
|
|
||||||
specified by env_vars.
|
|
||||||
"""
|
|
||||||
prefix = "export " if with_export else ""
|
|
||||||
return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
|
|
||||||
|
|
||||||
|
|
||||||
def extract_handler(args: argparse.Namespace) -> None:
|
|
||||||
"""
|
|
||||||
Save environment variables to file.
|
|
||||||
"""
|
|
||||||
result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
|
|
||||||
with args.outfile as ofp:
|
|
||||||
print(result, file=ofp)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Materialize environment variables from AWS SSM Parameter Store"
|
|
||||||
)
|
|
||||||
parser.set_defaults(func=lambda _: parser.print_help())
|
|
||||||
subcommands = parser.add_subparsers(description="Parameters commands")
|
|
||||||
|
|
||||||
parser_extract = subcommands.add_parser(
|
|
||||||
"extract", description="Parameters extract commands"
|
|
||||||
)
|
|
||||||
parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
|
|
||||||
parser_extract.add_argument(
|
|
||||||
"-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
|
|
||||||
)
|
|
||||||
parser_extract.add_argument(
|
|
||||||
"--export",
|
|
||||||
action="store_true",
|
|
||||||
help="Set to output environment strings with export statements",
|
|
||||||
)
|
|
||||||
parser_extract.add_argument(
|
|
||||||
"-p",
|
|
||||||
"--path",
|
|
||||||
default=None,
|
|
||||||
help="SSM path from which to pull environment variables (pull is NOT recursive)",
|
|
||||||
)
|
|
||||||
parser_extract.set_defaults(func=extract_handler)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
args.func(args)
|
|
Ładowanie…
Reference in New Issue