Merge pull request #803 from moonstream-to/struct-nodebalancer

Updated nodebalancer placement

nodebalancer/.gitignore

@@ -0,0 +1,65 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,go
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,go
+
+### Go ###
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+### Go Patch ###
+/vendor/
+/Godeps/
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# Support for Project snippet scope
+.vscode/*.code-snippets
+
+# Ignore code-workspaces
+*.code-workspace
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,go
+
+# Custom
+.secrets/*
+dev.env
+prod.env
+test.env
+.venv
+fakenode

nodebalancer/deploy/deploy.bash

@@ -16,27 +16,15 @@ PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
 # Main
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
 APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}"
-APP_NODES_DIR="${APP_DIR}/nodes"
-PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
-PYTHON="${PYTHON_ENV_DIR}/bin/python"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
-AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
 SCRIPT_DIR="$(realpath $(dirname $0))"
-PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-NODE_BALANCER_CONFIG_PATH="${NODE_BALANCER_CONFIG_PATH:-/home/ubuntu/.nodebalancer/config.json}"
 
 # Service file
 NODE_BALANCER_SERVICE_FILE="nodebalancer.service"
 
 set -eu
 
-echo
-echo
-echo -e "${PREFIX_INFO} Retrieving deployment parameters"
-mkdir -p "${SECRETS_DIR}"
-AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
-
 echo
 echo
 echo -e "${PREFIX_INFO} Install checkenv"
@@ -56,8 +44,8 @@ echo
 echo
 echo -e "${PREFIX_INFO} Building executable load balancer for nodes script with Go"
 EXEC_DIR=$(pwd)
-cd "${APP_NODES_DIR}/node_balancer"
-HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_NODES_DIR}/node_balancer/nodebalancer" "${APP_NODES_DIR}/node_balancer/cmd/nodebalancer/"
+cd "${APP_DIR}/nodebalancer"
+HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/nodebalancer/nodebalancer" "${APP_DIR}/nodebalancer/cmd/nodebalancer/*.go"
 cd "${EXEC_DIR}"
 
 echo

nodebalancer/deploy/nodebalancer.service

@@ -5,11 +5,11 @@ StartLimitIntervalSec=300
 StartLimitBurst=3
 
 [Service]
-WorkingDirectory=/home/ubuntu/moonstream/nodes/node_balancer
+WorkingDirectory=/home/ubuntu/moonstream/nodebalancer
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream/nodes/node_balancer/nodebalancer server \
+ExecStart=/home/ubuntu/moonstream/nodebalancer/nodebalancer server \
     -host "${AWS_LOCAL_IPV4}" \
     -port 8544 \
     -healthcheck \

nodes/deploy/parameters.py

@@ -1,102 +0,0 @@
-"""
-Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
-"""
-import argparse
-from dataclasses import dataclass
-import sys
-from typing import Any, Dict, Iterable, List, Optional
-
-import boto3
-
-
-@dataclass
-class EnvironmentVariable:
-    name: str
-    value: str
-
-
-def get_parameters(path: str) -> List[Dict[str, Any]]:
-    """
-    Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
-
-    Relies on the appropriate environment variables to authenticate against AWS:
-    https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
-    """
-    ssm = boto3.client("ssm")
-    next_token: Optional[bool] = True
-    parameters: List[Dict[str, Any]] = []
-    while next_token is not None:
-        kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
-        if next_token is not True:
-            kwargs["NextToken"] = next_token
-        response = ssm.get_parameters_by_path(**kwargs)
-        new_parameters = response.get("Parameters", [])
-        parameters.extend(new_parameters)
-        next_token = response.get("NextToken")
-
-    return parameters
-
-
-def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
-    """
-    Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
-    """
-    parameter_path = parameter_object.get("Name")
-    if parameter_path is None:
-        raise ValueError('Did not find "Name" in parameter object')
-    name = parameter_path.split("/")[-1].upper()
-
-    value = parameter_object.get("Value")
-    if value is None:
-        raise ValueError('Did not find "Value" in parameter object')
-
-    return EnvironmentVariable(name, value)
-
-
-def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
-    """
-    Produces a string which, when executed in a shell, exports the desired environment variables as
-    specified by env_vars.
-    """
-    prefix = "export " if with_export else ""
-    return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
-
-
-def extract_handler(args: argparse.Namespace) -> None:
-    """
-    Save environment variables to file.
-    """
-    result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
-    with args.outfile as ofp:
-        print(result, file=ofp)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Materialize environment variables from AWS SSM Parameter Store"
-    )
-    parser.set_defaults(func=lambda _: parser.print_help())
-    subcommands = parser.add_subparsers(description="Parameters commands")
-
-    parser_extract = subcommands.add_parser(
-        "extract", description="Parameters extract commands"
-    )
-    parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
-    parser_extract.add_argument(
-        "-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
-    )
-    parser_extract.add_argument(
-        "--export",
-        action="store_true",
-        help="Set to output environment strings with export statements",
-    )
-    parser_extract.add_argument(
-        "-p",
-        "--path",
-        default=None,
-        help="SSM path from which to pull environment variables (pull is NOT recursive)",
-    )
-    parser_extract.set_defaults(func=extract_handler)
-
-    args = parser.parse_args()
-    args.func(args)