diff --git a/backend/moonstream/admin/cli.py b/backend/moonstream/admin/cli.py index 4047c46c..37e9a5e7 100644 --- a/backend/moonstream/admin/cli.py +++ b/backend/moonstream/admin/cli.py @@ -2,15 +2,20 @@ Moonstream CLI """ import argparse + +import logging import json from typing import Optional +from moonstreamdb.db import SessionLocal + +from ..settings import BUGOUT_BROOD_URL, BUGOUT_SPIRE_URL, MOONSTREAM_APPLICATION_ID +from ..web3_provider import yield_web3_provider from . import subscription_types, subscriptions -from ..settings import ( - BUGOUT_BROOD_URL, - BUGOUT_SPIRE_URL, - MOONSTREAM_APPLICATION_ID, -) +from .migrations import checksum_address + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) def parse_boolean_arg(raw_arg: Optional[str]) -> Optional[bool]: @@ -66,6 +71,29 @@ def cli_migrate_subscriptions(args: argparse.Namespace) -> None: return +def migrations_list(args: argparse.Namespace) -> None: + migrations_overview = f""" + + - id: 20211101 + name: {checksum_address.__name__} + description: {checksum_address.__doc__} + """ + logger.info(migrations_overview) + + +def migrations_run(args: argparse.Namespace) -> None: + web3_session = yield_web3_provider() + db_session = SessionLocal() + try: + if args.id == 20211101: + logger.info("Starting update of subscriptions in Brood resource...") + checksum_address.checksum_all_subscription_addresses(web3_session) + logger.info("Starting update of ethereum_labels in database...") + checksum_address.checksum_all_labels_addresses(db_session, web3_session) + finally: + db_session.close() + + def main() -> None: cli_description = f"""Moonstream Admin CLI @@ -279,6 +307,25 @@ This CLI is configured to work with the following API URLs: ) parser_subscription_migrate.set_defaults(func=cli_migrate_subscriptions) + parser_migrations = subcommands.add_parser( + "migrations", description="Manage database, resource and etc migrations" + ) + parser_migrations.set_defaults(func=lambda _: parser_migrations.print_help()) + subcommands_migrations = parser_migrations.add_subparsers( + description="Migration commands" + ) + parser_migrations_list = subcommands_migrations.add_parser( + "list", description="List migrations" + ) + parser_migrations_list.set_defaults(func=migrations_list) + parser_migrations_run = subcommands_migrations.add_parser( + "run", description="Run migration" + ) + parser_migrations_run.add_argument( + "-i", "--id", required=True, type=int, help="Provide migration ID" + ) + parser_migrations_run.set_defaults(func=migrations_run) + args = parser.parse_args() args.func(args) diff --git a/backend/moonstream/admin/migrations/__init__.py b/backend/moonstream/admin/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/moonstream/admin/migrations/checksum_address.py b/backend/moonstream/admin/migrations/checksum_address.py new file mode 100644 index 00000000..ddb5a8e0 --- /dev/null +++ b/backend/moonstream/admin/migrations/checksum_address.py @@ -0,0 +1,102 @@ +""" +Convert all addresses in user subscriptions +and ethereum_labels column to checksum address. +""" +import logging +from typing import List + +from bugout.data import BugoutResources +from bugout.exceptions import BugoutResponseException +from moonstreamdb.models import EthereumLabel +from sqlalchemy import func +from sqlalchemy.orm.session import Session +from web3 import Web3 + +from ...settings import BUGOUT_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_ADMIN_ACCESS_TOKEN +from ...settings import bugout_client as bc + +logger = logging.getLogger(__name__) + + +def checksum_all_subscription_addresses(web3: Web3) -> None: + """ + Parse all existing subscriptions at Brood resource + and replace address with checksum. + """ + resources: BugoutResources = bc.list_resources( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + params={"type": "subscription"}, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + for resource in resources.resources: + resource_data = resource.resource_data + try: + address = resource_data["address"] + resource_data["address"] = web3.toChecksumAddress(address) + updated_resource = bc.update_resource( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + resource_id=resource.id, + resource_data={"update": resource_data}, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + logger.info(f"Resource id: {updated_resource.id} updated") + except ValueError as e: + logger.info( + f"Not valid checksum address: {address}, probably " + "txpool or whalewatch subscription" + ) + continue + except BugoutResponseException as e: + logger.info(f"Bugout error: {e.status_code} with details: {e.detail}") + except Exception as e: + logger.info(f"Unexpected error: {repr(e)}") + continue + + +def checksum_all_labels_addresses(db_session: Session, web3: Web3) -> None: + """ + Convert all address to checksum in ethereum_labels column at database. + + Docs for SQLAlchemy mapping: + https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session.bulk_update_mappings + """ + query_limit = 500 + query_index = 0 + + malformed_addresses: List[str] = [] + + while True: + query = ( + db_session.query(EthereumLabel.id, EthereumLabel.address) + .filter(EthereumLabel.address == func.lower(EthereumLabel.address)) + .filter(EthereumLabel.address.not_in(malformed_addresses)) + .order_by(EthereumLabel.address) + .limit(query_limit) + .offset(query_index * query_limit) + ) + address_list = query.all() + address_list_len = len(address_list) + if address_list_len == 0: + break + + logger.info(f"Updating next {address_list_len} rows") + + # Build map of id and updated address checksum + mappings = [] + for address in address_list: + try: + checksum_address = web3.toChecksumAddress(address[1]) + mappings.append({"id": address[0], "address": checksum_address}) + except Exception as e: + logger.warn( + f"Unable checksum address: {address[1]}, added to malformed list" + ) + malformed_addresses.append(address[1]) + + db_session.bulk_update_mappings(EthereumLabel, mappings) + db_session.commit() + mappings[:] = [] + + query_index += 1 + + logger.warn(f"List of malformed addresses: {malformed_addresses}") diff --git a/backend/moonstream/admin/subscription_types.py b/backend/moonstream/admin/subscription_types.py index e666aebf..c4e0fbb4 100644 --- a/backend/moonstream/admin/subscription_types.py +++ b/backend/moonstream/admin/subscription_types.py @@ -5,16 +5,16 @@ import argparse import json from typing import Dict, List, Optional -from bugout.data import BugoutResources, BugoutResource +from bugout.data import BugoutResource, BugoutResources from sqlalchemy.sql.expression import update from ..data import SubscriptionTypeResourceData from ..settings import ( + BUGOUT_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_APPLICATION_ID, - bugout_client as bc, - BUGOUT_REQUEST_TIMEOUT_SECONDS, ) +from ..settings import bugout_client as bc CANONICAL_SUBSCRIPTION_TYPES = { "ethereum_blockchain": SubscriptionTypeResourceData( diff --git a/backend/moonstream/routes/subscriptions.py b/backend/moonstream/routes/subscriptions.py index 5b01bf23..7e6abe22 100644 --- a/backend/moonstream/routes/subscriptions.py +++ b/backend/moonstream/routes/subscriptions.py @@ -9,7 +9,8 @@ from typing import List, Optional, Dict, Any import boto3 # type: ignore from bugout.data import BugoutResource, BugoutResources from bugout.exceptions import BugoutResponseException -from fastapi import APIRouter, Request, Form +from fastapi import APIRouter, Depends, Request, Form +from web3 import Web3 from web3._utils.validation import validate_abi from ..admin import subscription_types @@ -21,6 +22,7 @@ from ..settings import ( bugout_client as bc, MOONSTREAM_SMARTCONTRACTS_ABI_BUCKET, ) +from ..web3_provider import yield_web3_provider logger = logging.getLogger(__name__) @@ -39,12 +41,30 @@ async def add_subscription_handler( label: str = Form(...), subscription_type_id: str = Form(...), abi: Optional[str] = Form(None), + web3: Web3 = Depends(yield_web3_provider), ) -> data.SubscriptionResourceData: """ Add subscription to blockchain stream data for user. """ token = request.state.token + if subscription_type_id != "ethereum_whalewatch": + try: + address = web3.toChecksumAddress(address) + except ValueError as e: + raise MoonstreamHTTPException( + status_code=400, + detail=str(e), + internal_error=e, + ) + except Exception as e: + logger.error(f"Failed to convert address to checksum address") + raise MoonstreamHTTPException( + status_code=500, + internal_error=e, + detail="Currently unable to convert address to checksum address", + ) + active_subscription_types_response = subscription_types.list_subscription_types( active_only=True ) diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 2aa1d355..d0467a87 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -2,6 +2,17 @@ # Deployment script - intended to run on Moonstream crawlers server +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + # Main AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" @@ -14,17 +25,20 @@ PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}" SCRIPT_DIR="$(realpath $(dirname $0))" PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py" +CHECKENV_REPO_URL="https://raw.githubusercontent.com/bugout-dev/checkenv/main/scripts" +CHECKENV_PARAMETERS_SCRIPT_URL="${CHECKENV_REPO_URL}/parameters.bash" +CHECKENV_NODES_CONNECTIONS_SCRIPT_URL="${CHECKENV_REPO_URL}/nodes-connections.bash" ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER="ethereum-trending.service" ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service" -SERVICE_FILE="moonstreamcrawlers.service" +ETHEREUM_CRAWLERS_SERVICE_FILE="moonstreamcrawlers.service" set -eu echo echo -echo "Building executable Ethereum transaction pool crawler script with Go" +echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go" EXEC_DIR=$(pwd) cd "${APP_CRAWLERS_DIR}/ethtxpool" HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go" @@ -32,7 +46,7 @@ cd "${EXEC_DIR}" echo echo -echo "Building executable server of moonstreamcrawlers with Go" +echo -e "${PREFIX_INFO} Building executable server of moonstreamcrawlers with Go" EXEC_DIR=$(pwd) cd "${APP_CRAWLERS_DIR}/server" HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go" @@ -40,19 +54,29 @@ cd "${EXEC_DIR}" echo echo -echo "Updating Python dependencies" +echo -e "${PREFIX_INFO} Updating Python dependencies" "${PIP}" install --upgrade pip "${PIP}" install -r "${APP_CRAWLERS_DIR}/mooncrawl/requirements.txt" echo echo -echo "Retrieving deployment parameters" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" mkdir -p "${SECRETS_DIR}" AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" extract -p "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}" echo echo -echo "Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}" +echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" +curl -s "${CHECKENV_PARAMETERS_SCRIPT_URL}" | bash /dev/stdin -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Updating nodes connection parameters" +curl -s "${CHECKENV_NODES_CONNECTIONS_SCRIPT_URL}" | bash /dev/stdin -v -f "${PARAMETERS_ENV_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Ethereum block with transactions syncronizer service definition with ${ETHEREUM_SYNCHRONIZE_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${ETHEREUM_SYNCHRONIZE_SERVICE}" systemctl daemon-reload @@ -60,7 +84,7 @@ systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE}" echo echo -echo "Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE}, ${ETHEREUM_TRENDING_TIMER}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER}" @@ -69,7 +93,7 @@ systemctl restart "${ETHEREUM_TRENDING_TIMER}" echo echo -echo "Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}" +echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}" systemctl daemon-reload @@ -77,9 +101,10 @@ systemctl restart "${ETHEREUM_TXPOOL_SERVICE}" echo echo -echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}" -chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}" -cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}" +echo -e "${PREFIX_INFO} Replacing existing moonstreamcrawlers service definition with ${ETHEREUM_CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ETHEREUM_CRAWLERS_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_CRAWLERS_SERVICE_FILE}" systemctl daemon-reload -systemctl restart "${SERVICE_FILE}" -systemctl status "${SERVICE_FILE}" +systemctl restart "${ETHEREUM_CRAWLERS_SERVICE_FILE}" +systemctl status "${ETHEREUM_CRAWLERS_SERVICE_FILE}" + diff --git a/crawlers/deploy/ethereum-txpool.service b/crawlers/deploy/ethereum-txpool.service index fc64c867..c8b45ea5 100644 --- a/crawlers/deploy/ethereum-txpool.service +++ b/crawlers/deploy/ethereum-txpool.service @@ -7,8 +7,8 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/crawlers/ethtxpool EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream/crawlers/ethtxpool/ethtxpool -geth "${MOONSTREAM_IPC_PATH}" +ExecStart=/home/ubuntu/moonstream/crawlers/ethtxpool/ethtxpool SyslogIdentifier=ethereum-txpool [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index e4aa4f55..6a4c9bc7 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -175,14 +175,16 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu } func main() { - var gethConnectionString string var intervalSeconds int - flag.StringVar(&gethConnectionString, "geth", "", "Geth IPC path/RPC url/Websockets URL") flag.IntVar(&intervalSeconds, "interval", 1, "Number of seconds to wait between RPC calls to query the transaction pool (default: 1)") flag.Parse() + var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR") + var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") + var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT) + sessionID := uuid.New().String() - + // Humbug crash client to collect errors crashReporter, err := humbugClient(sessionID, "moonstream-crawlers", os.Getenv("HUMBUG_REPORTER_CRAWLERS_TOKEN")) if err != nil { @@ -199,7 +201,7 @@ func main() { }() // Set connection with Ethereum blockchain via geth - gethClient, err := rpc.Dial(gethConnectionString) + gethClient, err := rpc.Dial(MOONSTREAM_IPC_PATH) if err != nil { panic(fmt.Sprintf("Could not connect to geth: %s", err.Error())) } diff --git a/crawlers/ethtxpool/sample.env b/crawlers/ethtxpool/sample.env index 809cc325..5b375379 100644 --- a/crawlers/ethtxpool/sample.env +++ b/crawlers/ethtxpool/sample.env @@ -1,3 +1,5 @@ +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export ETHTXPOOL_HUMBUG_CLIENT_ID="" export ETHTXPOOL_HUMBUG_TOKEN="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 00a8fbd9..fe85b631 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -4,8 +4,16 @@ from typing import cast # Bugout HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") -# Geth -MOONSTREAM_IPC_PATH = os.environ.get("MOONSTREAM_IPC_PATH", None) +# Geth connection address +MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1" +) +MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_IPC_PORT", 8545 +) +MOONSTREAM_IPC_PATH = ( + f"http://{MOONSTREAM_NODE_ETHEREUM_IPC_ADDR}:{MOONSTREAM_NODE_ETHEREUM_IPC_PORT}" +) MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a49b2bfb..01cdfb64 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -1,5 +1,6 @@ # Path to IPC socket to use for web3 connections -export MOONSTREAM_IPC_PATH=null +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_ETHERSCAN_TOKEN="" @@ -9,4 +10,4 @@ export COINMARKETCAP_API_KEY="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_ADMIN_ACCESS_TOKEN="" -export NFT_HUMBUG_TOKEN="" \ No newline at end of file +export NFT_HUMBUG_TOKEN="" diff --git a/crawlers/server/cmd/data.go b/crawlers/server/cmd/data.go new file mode 100644 index 00000000..9294c636 --- /dev/null +++ b/crawlers/server/cmd/data.go @@ -0,0 +1,13 @@ +package cmd + +type PingResponse struct { + Status string `json:"status"` +} + +type GethResponse struct { + Result string `json:"result"` +} + +type PingGethResponse struct { + CurrentBlock uint64 `json:"current_block"` +} diff --git a/crawlers/server/cmd/middlewares.go b/crawlers/server/cmd/middlewares.go new file mode 100644 index 00000000..444cfc7b --- /dev/null +++ b/crawlers/server/cmd/middlewares.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "log" + "net/http" + "strings" + "time" + + settings "github.com/bugout-dev/moonstream/crawlers/server/configs" +) + +// Handle panic errors to prevent server shutdown +func panicMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + log.Println("recovered", err) + http.Error(w, "Internal server error", 500) + } + }() + // There will be a defer with panic handler in each next function + next.ServeHTTP(w, r) + }) +} + +// Log requests in proper format +func logsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s %s\n", time.Since(start), r.Method, r.URL.Path, r.RemoteAddr) + }) +} + +// CORS middleware +func corsMiddleware(next http.Handler) http.Handler { + // Iterate over list of allowed origins + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, allowedOrigin := range strings.Split(settings.MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") { + if r.Header.Get("Origin") == allowedOrigin { + w.Header().Set("Access-Control-Allow-Origin", allowedOrigin) + } + } + if r.Method == "OPTIONS" { + w.Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS") + // Credentials are cookies, authorization headers, or TLS client certificates + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Allow-Headers", "Authorization") + } + next.ServeHTTP(w, r) + }) +} diff --git a/crawlers/server/cmd/routes.go b/crawlers/server/cmd/routes.go new file mode 100644 index 00000000..7b1dcad6 --- /dev/null +++ b/crawlers/server/cmd/routes.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "log" + "net/http" + "strconv" + "strings" + + settings "github.com/bugout-dev/moonstream/crawlers/server/configs" +) + +func pingRoute(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + response := PingResponse{Status: "ok"} + json.NewEncoder(w).Encode(response) +} + +// Fetch latest block from Geth +func pingGethRoute(w http.ResponseWriter, req *http.Request) { + postBody, err := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_blockNumber", + "params": []string{}, + "id": 1, + }) + if err != nil { + log.Printf("An error occurred due marshal postBody, error: %s", err) + http.Error(w, http.StatusText(500), http.StatusInternalServerError) + return + } + gethResponse, err := http.Post(settings.MOONSTREAM_IPC_PATH, "application/json", + bytes.NewBuffer(postBody)) + if err != nil { + log.Printf("Unable to request geth, error: %s", err) + http.Error(w, http.StatusText(500), http.StatusInternalServerError) + return + } + defer gethResponse.Body.Close() + + gethResponseBody, err := ioutil.ReadAll(gethResponse.Body) + if err != nil { + log.Printf("Unable to read geth response, error: %s", err) + http.Error(w, http.StatusText(500), http.StatusInternalServerError) + return + } + var obj GethResponse + _ = json.Unmarshal(gethResponseBody, &obj) + + blockNumberHex := strings.Replace(obj.Result, "0x", "", -1) + blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64) + if err != nil { + log.Printf("Unable to parse block number from hex to string, error: %s", err) + http.Error(w, http.StatusText(500), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + response := PingGethResponse{CurrentBlock: blockNumberStr} + json.NewEncoder(w).Encode(response) +} diff --git a/crawlers/server/cmd/server.go b/crawlers/server/cmd/server.go new file mode 100644 index 00000000..cef4e36b --- /dev/null +++ b/crawlers/server/cmd/server.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "flag" + "log" + "net/http" + "time" +) + +func InitServer() { + var listeningAddr string + var listeningPort string + flag.StringVar(&listeningAddr, "host", "127.0.0.1", "Server listening address") + flag.StringVar(&listeningPort, "port", "8080", "Server listening port") + flag.Parse() + + serverMux := http.NewServeMux() + serverMux.HandleFunc("/ping", pingRoute) + serverMux.HandleFunc("/status", pingGethRoute) + + // Set middlewares from bottom to top + serverHandler := corsMiddleware(serverMux) + serverHandler = logsMiddleware(serverHandler) + serverHandler = panicMiddleware(serverHandler) + + server := http.Server{ + Addr: listeningAddr + ":" + listeningPort, + Handler: serverHandler, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + log.Printf("Starting server at %s:%s\n", listeningAddr, listeningPort) + server.ListenAndServe() +} diff --git a/crawlers/server/configs/settings.go b/crawlers/server/configs/settings.go new file mode 100644 index 00000000..87e63d79 --- /dev/null +++ b/crawlers/server/configs/settings.go @@ -0,0 +1,14 @@ +package settings + +import ( + "fmt" + "os" +) + +// Geth configs +var MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_ADDR") +var MOONSTREAM_NODE_ETHEREUM_IPC_PORT = os.Getenv("MOONSTREAM_NODE_ETHEREUM_IPC_PORT") +var MOONSTREAM_IPC_PATH = fmt.Sprintf("http://%s:%s", MOONSTREAM_NODE_ETHEREUM_IPC_ADDR, MOONSTREAM_NODE_ETHEREUM_IPC_PORT) + +// CORS +var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") diff --git a/crawlers/server/dev.sh b/crawlers/server/dev.sh index 174a6682..2e464806 100755 --- a/crawlers/server/dev.sh +++ b/crawlers/server/dev.sh @@ -3,7 +3,7 @@ # Expects access to Python environment with the requirements for this project installed. set -e -MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}" -MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}" +MOONSTREAM_CLUSTER_SERVER_HOST="${MOONSTREAM_CLUSTER_SERVER_HOST:-0.0.0.0}" +MOONSTREAM_CLUSTER_SERVER_PORT="${MOONSTREAM_CLUSTER_SERVER_PORT:-8080}" -go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}" +go run main.go -host "${MOONSTREAM_CLUSTER_SERVER_HOST}" -port "${MOONSTREAM_CLUSTER_SERVER_PORT}" diff --git a/crawlers/server/go.mod b/crawlers/server/go.mod index 5a8fa99b..11b592d6 100644 --- a/crawlers/server/go.mod +++ b/crawlers/server/go.mod @@ -1,3 +1,3 @@ -module moonstreamdb +module github.com/bugout-dev/moonstream/crawlers/server go 1.17 diff --git a/crawlers/server/main.go b/crawlers/server/main.go index f0bd58db..014ea56a 100644 --- a/crawlers/server/main.go +++ b/crawlers/server/main.go @@ -1,118 +1,9 @@ package main import ( - "bytes" - "encoding/json" - "flag" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - "strings" + "github.com/bugout-dev/moonstream/crawlers/server/cmd" ) -var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH") -var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") - -type GethResponse struct { - Result string `json:"result"` -} - -type PingGethResponse struct { - CurrentBlock uint64 `json:"current_block"` -} - -type PingResponse struct { - Status string `json:"status"` -} - -// Extends handler with allowed CORS policies -func setupCorsResponse(w *http.ResponseWriter, req *http.Request) { - for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") { - for _, reqOrigin := range req.Header["Origin"] { - if reqOrigin == allowedOrigin { - (*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin) - } - } - - } - (*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS") -} - -func ping(w http.ResponseWriter, req *http.Request) { - setupCorsResponse(&w, req) - log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) - if (*req).Method == "OPTIONS" { - return - } - - w.Header().Set("Content-Type", "application/json") - response := PingResponse{Status: "ok"} - json.NewEncoder(w).Encode(response) -} - -func pingGeth(w http.ResponseWriter, req *http.Request) { - setupCorsResponse(&w, req) - log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) - if (*req).Method == "OPTIONS" { - return - } - - postBody, err := json.Marshal(map[string]interface{}{ - "jsonrpc": "2.0", - "method": "eth_blockNumber", - "params": []string{}, - "id": 1, - }) - if err != nil { - log.Println(err) - http.Error(w, http.StatusText(500), 500) - return - } - gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json", - bytes.NewBuffer(postBody)) - if err != nil { - log.Printf("Unable to request geth, error: %v", err) - http.Error(w, http.StatusText(500), 500) - return - } - defer gethResponse.Body.Close() - - gethResponseBody, err := ioutil.ReadAll(gethResponse.Body) - if err != nil { - log.Printf("Unable to read geth response, error: %v", err) - http.Error(w, http.StatusText(500), 500) - return - } - var obj GethResponse - _ = json.Unmarshal(gethResponseBody, &obj) - - blockNumberHex := strings.Replace(obj.Result, "0x", "", -1) - blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64) - if err != nil { - log.Printf("Unable to parse block number from hex to string, error: %v", err) - http.Error(w, http.StatusText(500), 500) - return - } - - w.Header().Set("Content-Type", "application/json") - response := PingGethResponse{CurrentBlock: blockNumberStr} - json.NewEncoder(w).Encode(response) -} - func main() { - var listenAddr string - var listenPort string - flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address") - flag.StringVar(&listenPort, "port", "8080", "Server listen port") - flag.Parse() - - address := listenAddr + ":" + listenPort - log.Printf("Starting server at %s\n", address) - - http.HandleFunc("/ping", ping) - http.HandleFunc("/status", pingGeth) - - http.ListenAndServe(address, nil) + cmd.InitServer() } diff --git a/crawlers/server/sample.env b/crawlers/server/sample.env index 053b87b4..7be46542 100644 --- a/crawlers/server/sample.env +++ b/crawlers/server/sample.env @@ -1,3 +1,3 @@ -export MOONSTREAM_CRAWLERS_SERVER_PORT="8080" -export MOONSTREAM_IPC_PATH=null +export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1" +export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to" diff --git a/db/server/cmd/data.go b/db/server/cmd/data.go new file mode 100644 index 00000000..097b8155 --- /dev/null +++ b/db/server/cmd/data.go @@ -0,0 +1,9 @@ +package cmd + +type PingResponse struct { + Status string `json:"status"` +} + +type BlockNumberResponse struct { + BlockNumber uint64 `json:"block_number"` +} diff --git a/db/server/cmd/db.go b/db/server/cmd/db.go new file mode 100644 index 00000000..6f97b0b2 --- /dev/null +++ b/db/server/cmd/db.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "database/sql" + "log" + + _ "github.com/lib/pq" + + settings "github.com/bugout-dev/moonstream/db/server/configs" +) + +func InitDB() *sql.DB { + db, err := sql.Open("postgres", settings.MOONSTREAM_DB_URI) + if err != nil { + // DSN parse error or another initialization error + log.Fatal(err) + } + + // Set the maximum number of concurrently idle connections, + // by default sql.DB allows a maximum of 2 idle connections. + db.SetMaxIdleConns(settings.MOONSTREAM_DB_MAX_IDLE_CONNS) + + // Set the maximum lifetime of a connection. + // Longer lifetime increase memory usage. + db.SetConnMaxLifetime(settings.MOONSTREAM_DB_CONN_MAX_LIFETIME) + + return db +} diff --git a/db/server/cmd/middlewares.go b/db/server/cmd/middlewares.go new file mode 100644 index 00000000..b0dec39f --- /dev/null +++ b/db/server/cmd/middlewares.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "log" + "net/http" + "strings" + "time" + + settings "github.com/bugout-dev/moonstream/db/server/configs" +) + +// Handle panic errors to prevent server shutdown +func panicMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + log.Println("recovered", err) + http.Error(w, "Internal server error", 500) + } + }() + // There will be a defer with panic handler in each next function + next.ServeHTTP(w, r) + }) +} + +// Log requests in proper format +func logsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s %s\n", time.Since(start), r.Method, r.URL.Path, r.RemoteAddr) + }) +} + +// CORS middleware +func corsMiddleware(next http.Handler) http.Handler { + // Iterate over list of allowed origins + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, allowedOrigin := range strings.Split(settings.MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") { + if r.Header.Get("Origin") == allowedOrigin { + w.Header().Set("Access-Control-Allow-Origin", allowedOrigin) + } + } + if r.Method == "OPTIONS" { + w.Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS") + // Credentials are cookies, authorization headers, or TLS client certificates + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Allow-Headers", "Authorization") + } + next.ServeHTTP(w, r) + }) +} diff --git a/db/server/cmd/routes.go b/db/server/cmd/routes.go new file mode 100644 index 00000000..dea7413c --- /dev/null +++ b/db/server/cmd/routes.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "database/sql" + "encoding/json" + "log" + "net/http" +) + +func pingRoute(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + response := PingResponse{Status: "ok"} + json.NewEncoder(w).Encode(response) +} + +// Fetch latest block record from database +func (es *extendedServer) blocksLatestRoute(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var latestBlock BlockNumberResponse + row := es.db.QueryRow("SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1") + err := row.Scan(&latestBlock.BlockNumber) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, "Row not found", http.StatusNotFound) + } else { + http.Error(w, "Internal server error", http.StatusInternalServerError) + } + log.Printf("An error occurred during sql operation: %s", err) + return + } + + json.NewEncoder(w).Encode(latestBlock) +} diff --git a/db/server/cmd/server.go b/db/server/cmd/server.go new file mode 100644 index 00000000..a4d5a71a --- /dev/null +++ b/db/server/cmd/server.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "database/sql" + "flag" + "log" + "net/http" + "time" +) + +type extendedServer struct { + db *sql.DB +} + +func InitServer() { + var listeningAddr string + var listeningPort string + flag.StringVar(&listeningAddr, "host", "127.0.0.1", "Server listening address") + flag.StringVar(&listeningPort, "port", "8080", "Server listening port") + flag.Parse() + + db := InitDB() + defer db.Close() + + es := extendedServer{db: db} + + serverMux := http.NewServeMux() + serverMux.HandleFunc("/ping", pingRoute) + serverMux.HandleFunc("/block/latest", es.blocksLatestRoute) + + // Set middlewares from bottom to top + serverHandler := corsMiddleware(serverMux) + serverHandler = logsMiddleware(serverHandler) + serverHandler = panicMiddleware(serverHandler) + + server := http.Server{ + Addr: listeningAddr + ":" + listeningPort, + Handler: serverHandler, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + log.Printf("Starting server at %s:%s\n", listeningAddr, listeningPort) + server.ListenAndServe() +} diff --git a/db/server/configs/settings.go b/db/server/configs/settings.go new file mode 100644 index 00000000..23b4d145 --- /dev/null +++ b/db/server/configs/settings.go @@ -0,0 +1,14 @@ +package settings + +import ( + "os" + "time" +) + +// Database configs +var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 +var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute +var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") + +// CORS +var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") diff --git a/db/server/go.mod b/db/server/go.mod index 570b3bc2..fcd34367 100644 --- a/db/server/go.mod +++ b/db/server/go.mod @@ -1,23 +1,5 @@ -module moonstreamdb +module github.com/bugout-dev/moonstream/db/server go 1.17 -require ( - gorm.io/driver/postgres v1.1.1 - gorm.io/gorm v1.21.15 -) - -require ( - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.10.0 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.1.1 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.8.1 // indirect - github.com/jackc/pgx/v4 v4.13.0 // indirect - github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.2 // indirect - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect - golang.org/x/text v0.3.7 // indirect -) +require github.com/lib/pq v1.10.2 diff --git a/db/server/go.sum b/db/server/go.sum index 33b809ff..3708eaf0 100644 --- a/db/server/go.sum +++ b/db/server/go.sum @@ -1,186 +1,2 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= -github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= -github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= -github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.1.1 h1:tWLmqYCyaoh89fi7DhM6QggujrOnmfo3H98AzgNAAu0= -gorm.io/driver/postgres v1.1.1/go.mod h1:tpe2xN7aCst1NUdYyWQyxPtnHC+Zfp6NEux9PXD1OU0= -gorm.io/gorm v1.21.15 h1:gAyaDoPw0lCyrSFWhBlahbUA1U4P5RViC1uIqoB+1Rk= -gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/db/server/main.go b/db/server/main.go index da9e452c..5a5449d7 100644 --- a/db/server/main.go +++ b/db/server/main.go @@ -1,99 +1,9 @@ package main import ( - "encoding/json" - "flag" - "log" - "net/http" - "os" - "strings" - - "gorm.io/driver/postgres" - "gorm.io/gorm" + "github.com/bugout-dev/moonstream/db/server/cmd" ) -var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") -var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") - -type PingResponse struct { - Status string `json:"status"` -} - -type BlockResponse struct { - BlockNumber uint64 `json:"block_number"` -} - -var dbConnection *gorm.DB - -func InitDB() *gorm.DB { - db, err := gorm.Open(postgres.Open(MOONSTREAM_DB_URI), &gorm.Config{}) - if err != nil { - log.Println("Database unavailable") - return nil - } - return db -} - -// Extends handler with allowed CORS policies -func setupCorsResponse(w *http.ResponseWriter, req *http.Request) { - for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") { - for _, reqOrigin := range req.Header["Origin"] { - if reqOrigin == allowedOrigin { - (*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin) - } - } - - } - (*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS") -} - -func ping(w http.ResponseWriter, req *http.Request) { - setupCorsResponse(&w, req) - log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) - if (*req).Method == "OPTIONS" { - return - } - - w.Header().Set("Content-Type", "application/json") - response := PingResponse{Status: "ok"} - json.NewEncoder(w).Encode(response) -} - -func blockLatest(w http.ResponseWriter, req *http.Request) { - setupCorsResponse(&w, req) - log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) - if (*req).Method == "OPTIONS" { - return - } - - w.Header().Set("Content-Type", "application/json") - - var latestBlock BlockResponse - - if dbConnection == nil { - http.Error(w, http.StatusText(500), 500) - return - } - query := "SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1" - dbConnection.Raw(query, 1).Scan(&latestBlock.BlockNumber) - - json.NewEncoder(w).Encode(latestBlock) -} - func main() { - var listenAddr string - var listenPort string - flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address") - flag.StringVar(&listenPort, "port", "8080", "Server listen port") - flag.Parse() - - dbConnection = InitDB() - - address := listenAddr + ":" + listenPort - log.Printf("Starting server at %s\n", address) - - http.HandleFunc("/ping", ping) - http.HandleFunc("/block/latest", blockLatest) - - http.ListenAndServe(address, nil) + cmd.InitServer() } diff --git a/db/server/sample.env b/db/server/sample.env index ef5a4be3..a31c731c 100644 --- a/db/server/sample.env +++ b/db/server/sample.env @@ -1,3 +1,2 @@ -export MOONSTREAM_DB_SERVER_PORT="8080" export MOONSTREAM_DB_URI="postgresql://:@:/" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"