From 4647ce74267c401a6decb1d911dffb0208b3e4e4 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 6 Jun 2023 11:35:51 +0000 Subject: [PATCH] Updated nodebalancer placement --- nodebalancer/.gitignore | 65 +++++++++++ .../node_balancer => nodebalancer}/README.md | 0 .../cmd/nodebalancer/access_cache_test.go | 0 .../cmd/nodebalancer/balancer.go | 0 .../cmd/nodebalancer/blockchain.go | 0 .../cmd/nodebalancer/cli.go | 0 .../cmd/nodebalancer/clients.go | 0 .../cmd/nodebalancer/clients_test.go | 0 .../cmd/nodebalancer/configs.go | 0 .../cmd/nodebalancer/main.go | 0 .../cmd/nodebalancer/middleware.go | 0 .../cmd/nodebalancer/routes.go | 0 .../cmd/nodebalancer/server.go | 0 .../cmd/nodebalancer/version.go | 0 {nodes => nodebalancer}/deploy/deploy.bash | 16 +-- .../deploy/nodebalancer.service | 4 +- {nodes/node_balancer => nodebalancer}/dev.sh | 0 {nodes/node_balancer => nodebalancer}/go.mod | 0 {nodes/node_balancer => nodebalancer}/go.sum | 0 .../migrations/migrations.py | 0 .../migrations/requirements.txt | 0 .../node_balancer => nodebalancer}/sample.env | 0 nodes/deploy/parameters.py | 102 ------------------ 23 files changed, 69 insertions(+), 118 deletions(-) create mode 100644 nodebalancer/.gitignore rename {nodes/node_balancer => nodebalancer}/README.md (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/access_cache_test.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/balancer.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/blockchain.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/cli.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/clients.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/clients_test.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/configs.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/main.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/middleware.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/routes.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/server.go (100%) rename {nodes/node_balancer => nodebalancer}/cmd/nodebalancer/version.go (100%) rename {nodes => nodebalancer}/deploy/deploy.bash (69%) rename {nodes => nodebalancer}/deploy/nodebalancer.service (69%) rename {nodes/node_balancer => nodebalancer}/dev.sh (100%) rename {nodes/node_balancer => nodebalancer}/go.mod (100%) rename {nodes/node_balancer => nodebalancer}/go.sum (100%) rename {nodes/node_balancer => nodebalancer}/migrations/migrations.py (100%) rename {nodes/node_balancer => nodebalancer}/migrations/requirements.txt (100%) rename {nodes/node_balancer => nodebalancer}/sample.env (100%) delete mode 100644 nodes/deploy/parameters.py diff --git a/nodebalancer/.gitignore b/nodebalancer/.gitignore new file mode 100644 index 00000000..b64683fb --- /dev/null +++ b/nodebalancer/.gitignore @@ -0,0 +1,65 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,go +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,go + +### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +### Go Patch ### +/vendor/ +/Godeps/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# Support for Project snippet scope +.vscode/*.code-snippets + +# Ignore code-workspaces +*.code-workspace + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,go + +# Custom +.secrets/* +dev.env +prod.env +test.env +.venv +fakenode diff --git a/nodes/node_balancer/README.md b/nodebalancer/README.md similarity index 100% rename from nodes/node_balancer/README.md rename to nodebalancer/README.md diff --git a/nodes/node_balancer/cmd/nodebalancer/access_cache_test.go b/nodebalancer/cmd/nodebalancer/access_cache_test.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/access_cache_test.go rename to nodebalancer/cmd/nodebalancer/access_cache_test.go diff --git a/nodes/node_balancer/cmd/nodebalancer/balancer.go b/nodebalancer/cmd/nodebalancer/balancer.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/balancer.go rename to nodebalancer/cmd/nodebalancer/balancer.go diff --git a/nodes/node_balancer/cmd/nodebalancer/blockchain.go b/nodebalancer/cmd/nodebalancer/blockchain.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/blockchain.go rename to nodebalancer/cmd/nodebalancer/blockchain.go diff --git a/nodes/node_balancer/cmd/nodebalancer/cli.go b/nodebalancer/cmd/nodebalancer/cli.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/cli.go rename to nodebalancer/cmd/nodebalancer/cli.go diff --git a/nodes/node_balancer/cmd/nodebalancer/clients.go b/nodebalancer/cmd/nodebalancer/clients.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/clients.go rename to nodebalancer/cmd/nodebalancer/clients.go diff --git a/nodes/node_balancer/cmd/nodebalancer/clients_test.go b/nodebalancer/cmd/nodebalancer/clients_test.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/clients_test.go rename to nodebalancer/cmd/nodebalancer/clients_test.go diff --git a/nodes/node_balancer/cmd/nodebalancer/configs.go b/nodebalancer/cmd/nodebalancer/configs.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/configs.go rename to nodebalancer/cmd/nodebalancer/configs.go diff --git a/nodes/node_balancer/cmd/nodebalancer/main.go b/nodebalancer/cmd/nodebalancer/main.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/main.go rename to nodebalancer/cmd/nodebalancer/main.go diff --git a/nodes/node_balancer/cmd/nodebalancer/middleware.go b/nodebalancer/cmd/nodebalancer/middleware.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/middleware.go rename to nodebalancer/cmd/nodebalancer/middleware.go diff --git a/nodes/node_balancer/cmd/nodebalancer/routes.go b/nodebalancer/cmd/nodebalancer/routes.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/routes.go rename to nodebalancer/cmd/nodebalancer/routes.go diff --git a/nodes/node_balancer/cmd/nodebalancer/server.go b/nodebalancer/cmd/nodebalancer/server.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/server.go rename to nodebalancer/cmd/nodebalancer/server.go diff --git a/nodes/node_balancer/cmd/nodebalancer/version.go b/nodebalancer/cmd/nodebalancer/version.go similarity index 100% rename from nodes/node_balancer/cmd/nodebalancer/version.go rename to nodebalancer/cmd/nodebalancer/version.go diff --git a/nodes/deploy/deploy.bash b/nodebalancer/deploy/deploy.bash similarity index 69% rename from nodes/deploy/deploy.bash rename to nodebalancer/deploy/deploy.bash index 673d3123..f110d509 100755 --- a/nodes/deploy/deploy.bash +++ b/nodebalancer/deploy/deploy.bash @@ -16,27 +16,15 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" # Main AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" -APP_NODES_DIR="${APP_DIR}/nodes" -PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}" -PYTHON="${PYTHON_ENV_DIR}/bin/python" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" -AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" -PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" -NODE_BALANCER_CONFIG_PATH="${NODE_BALANCER_CONFIG_PATH:-/home/ubuntu/.nodebalancer/config.json}" # Service file NODE_BALANCER_SERVICE_FILE="nodebalancer.service" set -eu -echo -echo -echo -e "${PREFIX_INFO} Retrieving deployment parameters" -mkdir -p "${SECRETS_DIR}" -AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}" - echo echo echo -e "${PREFIX_INFO} Install checkenv" @@ -56,8 +44,8 @@ echo echo echo -e "${PREFIX_INFO} Building executable load balancer for nodes script with Go" EXEC_DIR=$(pwd) -cd "${APP_NODES_DIR}/node_balancer" -HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_NODES_DIR}/node_balancer/nodebalancer" "${APP_NODES_DIR}/node_balancer/cmd/nodebalancer/" +cd "${APP_DIR}/nodebalancer" +HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/nodebalancer/nodebalancer" "${APP_DIR}/nodebalancer/cmd/nodebalancer/*.go" cd "${EXEC_DIR}" echo diff --git a/nodes/deploy/nodebalancer.service b/nodebalancer/deploy/nodebalancer.service similarity index 69% rename from nodes/deploy/nodebalancer.service rename to nodebalancer/deploy/nodebalancer.service index 1bd9faab..43c855fe 100644 --- a/nodes/deploy/nodebalancer.service +++ b/nodebalancer/deploy/nodebalancer.service @@ -5,11 +5,11 @@ StartLimitIntervalSec=300 StartLimitBurst=3 [Service] -WorkingDirectory=/home/ubuntu/moonstream/nodes/node_balancer +WorkingDirectory=/home/ubuntu/moonstream/nodebalancer EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream/nodes/node_balancer/nodebalancer server \ +ExecStart=/home/ubuntu/moonstream/nodebalancer/nodebalancer server \ -host "${AWS_LOCAL_IPV4}" \ -port 8544 \ -healthcheck \ diff --git a/nodes/node_balancer/dev.sh b/nodebalancer/dev.sh similarity index 100% rename from nodes/node_balancer/dev.sh rename to nodebalancer/dev.sh diff --git a/nodes/node_balancer/go.mod b/nodebalancer/go.mod similarity index 100% rename from nodes/node_balancer/go.mod rename to nodebalancer/go.mod diff --git a/nodes/node_balancer/go.sum b/nodebalancer/go.sum similarity index 100% rename from nodes/node_balancer/go.sum rename to nodebalancer/go.sum diff --git a/nodes/node_balancer/migrations/migrations.py b/nodebalancer/migrations/migrations.py similarity index 100% rename from nodes/node_balancer/migrations/migrations.py rename to nodebalancer/migrations/migrations.py diff --git a/nodes/node_balancer/migrations/requirements.txt b/nodebalancer/migrations/requirements.txt similarity index 100% rename from nodes/node_balancer/migrations/requirements.txt rename to nodebalancer/migrations/requirements.txt diff --git a/nodes/node_balancer/sample.env b/nodebalancer/sample.env similarity index 100% rename from nodes/node_balancer/sample.env rename to nodebalancer/sample.env diff --git a/nodes/deploy/parameters.py b/nodes/deploy/parameters.py deleted file mode 100644 index c8df5797..00000000 --- a/nodes/deploy/parameters.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Collect secrets from AWS SSM Parameter Store and output as environment variable exports. -""" -import argparse -from dataclasses import dataclass -import sys -from typing import Any, Dict, Iterable, List, Optional - -import boto3 - - -@dataclass -class EnvironmentVariable: - name: str - value: str - - -def get_parameters(path: str) -> List[Dict[str, Any]]: - """ - Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters. - - Relies on the appropriate environment variables to authenticate against AWS: - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - """ - ssm = boto3.client("ssm") - next_token: Optional[bool] = True - parameters: List[Dict[str, Any]] = [] - while next_token is not None: - kwargs = {"Path": path, "Recursive": False, "WithDecryption": True} - if next_token is not True: - kwargs["NextToken"] = next_token - response = ssm.get_parameters_by_path(**kwargs) - new_parameters = response.get("Parameters", []) - parameters.extend(new_parameters) - next_token = response.get("NextToken") - - return parameters - - -def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable: - """ - Transforms parameters returned by the AWS SSM API into EnvironmentVariables. - """ - parameter_path = parameter_object.get("Name") - if parameter_path is None: - raise ValueError('Did not find "Name" in parameter object') - name = parameter_path.split("/")[-1].upper() - - value = parameter_object.get("Value") - if value is None: - raise ValueError('Did not find "Value" in parameter object') - - return EnvironmentVariable(name, value) - - -def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str: - """ - Produces a string which, when executed in a shell, exports the desired environment variables as - specified by env_vars. - """ - prefix = "export " if with_export else "" - return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars]) - - -def extract_handler(args: argparse.Namespace) -> None: - """ - Save environment variables to file. - """ - result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export) - with args.outfile as ofp: - print(result, file=ofp) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Materialize environment variables from AWS SSM Parameter Store" - ) - parser.set_defaults(func=lambda _: parser.print_help()) - subcommands = parser.add_subparsers(description="Parameters commands") - - parser_extract = subcommands.add_parser( - "extract", description="Parameters extract commands" - ) - parser_extract.set_defaults(func=lambda _: parser_extract.print_help()) - parser_extract.add_argument( - "-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout - ) - parser_extract.add_argument( - "--export", - action="store_true", - help="Set to output environment strings with export statements", - ) - parser_extract.add_argument( - "-p", - "--path", - default=None, - help="SSM path from which to pull environment variables (pull is NOT recursive)", - ) - parser_extract.set_defaults(func=extract_handler) - - args = parser.parse_args() - args.func(args)