From 86c59edd5cf6d848563640aa6f67504830ff15a6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 24 Mar 2022 19:28:10 +0000 Subject: [PATCH 1/4] DB status server also response with polygon block as for ethereum --- db/server/cmd/data.go | 5 +++-- db/server/cmd/routes.go | 17 +++++++++++++---- db/server/dev.sh | 9 +++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/db/server/cmd/data.go b/db/server/cmd/data.go index 097b8155..64bdd738 100644 --- a/db/server/cmd/data.go +++ b/db/server/cmd/data.go @@ -4,6 +4,7 @@ type PingResponse struct { Status string `json:"status"` } -type BlockNumberResponse struct { - BlockNumber uint64 `json:"block_number"` +type BlockLatestResponse struct { + EthereumBlockLatest uint64 `json:"ethereum_block_latest"` + PolygonBlockLatest uint64 `json:"polygon_block_latest"` } diff --git a/db/server/cmd/routes.go b/db/server/cmd/routes.go index dea7413c..0ba6db00 100644 --- a/db/server/cmd/routes.go +++ b/db/server/cmd/routes.go @@ -17,9 +17,18 @@ func pingRoute(w http.ResponseWriter, req *http.Request) { func (es *extendedServer) blocksLatestRoute(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") - var latestBlock BlockNumberResponse - row := es.db.QueryRow("SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1") - err := row.Scan(&latestBlock.BlockNumber) + var blockLatest BlockLatestResponse + row := es.db.QueryRow(`SELECT ethereum_blocks.block_number AS ethereum_block_latest, + polygon_blocks_joinquery.block_number AS polygon_block_latest +FROM ethereum_blocks + CROSS JOIN ( + SELECT block_number + FROM polygon_blocks + ORDER BY block_number DESC + ) AS polygon_blocks_joinquery +ORDER BY ethereum_blocks.block_number DESC +LIMIT 1`) + err := row.Scan(&blockLatest.EthereumBlockLatest, &blockLatest.PolygonBlockLatest) if err != nil { if err == sql.ErrNoRows { http.Error(w, "Row not found", http.StatusNotFound) @@ -30,5 +39,5 @@ func (es *extendedServer) blocksLatestRoute(w http.ResponseWriter, req *http.Req return } - json.NewEncoder(w).Encode(latestBlock) + json.NewEncoder(w).Encode(blockLatest) } diff --git a/db/server/dev.sh b/db/server/dev.sh index 8920f68e..4bd40f4b 100755 --- a/db/server/dev.sh +++ b/db/server/dev.sh @@ -1,9 +1,10 @@ #!/usr/bin/env sh -# Expects access to Python environment with the requirements for this project installed. +# Compile application and run with provided arguments set -e -MOONSTREAM_DB_SERVER_HOST="${MOONSTREAM_DB_SERVER_HOST:-0.0.0.0}" -MOONSTREAM_DB_SERVER_PORT="${MOONSTREAM_DB_SERVER_PORT:-8080}" +PROGRAM_NAME="moonstreamdb" -go run main.go -host "${MOONSTREAM_DB_SERVER_HOST}" -port "${MOONSTREAM_DB_SERVER_PORT}" +go build -o "$PROGRAM_NAME" . + +./"$PROGRAM_NAME" "$@" From deca9ea011c2f026e7693d981c74d3e02475b6b1 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 24 Mar 2022 19:46:07 +0000 Subject: [PATCH 2/4] Read only access to db for status server --- db/server/cmd/db.go | 2 +- db/server/configs/settings.go | 2 +- db/server/sample.env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/server/cmd/db.go b/db/server/cmd/db.go index 6f97b0b2..54760de5 100644 --- a/db/server/cmd/db.go +++ b/db/server/cmd/db.go @@ -10,7 +10,7 @@ import ( ) func InitDB() *sql.DB { - db, err := sql.Open("postgres", settings.MOONSTREAM_DB_URI) + db, err := sql.Open("postgres", settings.MOONSTREAM_DB_URI_READ_ONLY) if err != nil { // DSN parse error or another initialization error log.Fatal(err) diff --git a/db/server/configs/settings.go b/db/server/configs/settings.go index 23b4d145..8f3c9487 100644 --- a/db/server/configs/settings.go +++ b/db/server/configs/settings.go @@ -8,7 +8,7 @@ import ( // Database configs var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute -var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") +var MOONSTREAM_DB_URI_READ_ONLY = os.Getenv("MOONSTREAM_DB_URI_READ_ONLY") // CORS var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS") diff --git a/db/server/sample.env b/db/server/sample.env index a31c731c..56d78c4a 100644 --- a/db/server/sample.env +++ b/db/server/sample.env @@ -1,2 +1,2 @@ -export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to" From 98983a9f9e35e6fa912bcf48f60320bc384ca841 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 24 Mar 2022 20:43:21 +0000 Subject: [PATCH 3/4] Fetching both values with simple union --- db/server/cmd/routes.go | 42 +++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/db/server/cmd/routes.go b/db/server/cmd/routes.go index 0ba6db00..65d878e7 100644 --- a/db/server/cmd/routes.go +++ b/db/server/cmd/routes.go @@ -17,27 +17,37 @@ func pingRoute(w http.ResponseWriter, req *http.Request) { func (es *extendedServer) blocksLatestRoute(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") + var blockNumbers []uint64 var blockLatest BlockLatestResponse - row := es.db.QueryRow(`SELECT ethereum_blocks.block_number AS ethereum_block_latest, - polygon_blocks_joinquery.block_number AS polygon_block_latest -FROM ethereum_blocks - CROSS JOIN ( - SELECT block_number - FROM polygon_blocks - ORDER BY block_number DESC - ) AS polygon_blocks_joinquery -ORDER BY ethereum_blocks.block_number DESC -LIMIT 1`) - err := row.Scan(&blockLatest.EthereumBlockLatest, &blockLatest.PolygonBlockLatest) + rows, err := es.db.Query(`(SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1) + UNION + (SELECT block_number FROM polygon_blocks ORDER BY block_number DESC LIMIT 1)`) if err != nil { - if err == sql.ErrNoRows { - http.Error(w, "Row not found", http.StatusNotFound) - } else { - http.Error(w, "Internal server error", http.StatusInternalServerError) - } log.Printf("An error occurred during sql operation: %s", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) return } + defer rows.Close() + + for rows.Next() { + var bn uint64 + err := rows.Scan(&bn) + if err != nil { + if err == sql.ErrNoRows { + http.Error(w, "Row not found", http.StatusNotFound) + } else { + http.Error(w, "Internal server error", http.StatusInternalServerError) + } + log.Printf("An error occurred during scan sql response: %s", err) + return + } + blockNumbers = append(blockNumbers, bn) + } + + blockLatest = BlockLatestResponse{ + EthereumBlockLatest: blockNumbers[0], + PolygonBlockLatest: blockNumbers[1], + } json.NewEncoder(w).Encode(blockLatest) } From de1ea7eceb78a1c827ba5bedbee0dc607414c763 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 25 Mar 2022 14:28:22 +0000 Subject: [PATCH 4/4] Updated status page for monitoring server --- frontend/pages/status/index.js | 194 ++++++++++++------- frontend/sample.env | 1 - frontend/src/core/hooks/useStatus.js | 31 --- frontend/src/core/services/status.service.js | 25 +-- 4 files changed, 124 insertions(+), 127 deletions(-) diff --git a/frontend/pages/status/index.js b/frontend/pages/status/index.js index ece43f00..80eec26c 100644 --- a/frontend/pages/status/index.js +++ b/frontend/pages/status/index.js @@ -18,40 +18,39 @@ const Status = () => { const { serverListStatusCache, - crawlersStatusCache, - dbServerStatusCache, - latestBlockDBStatusCache, } = useStatus(); + console.log(serverListStatusCache?.data); + const moonstreamapiStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "moonstreamapi" + (i) => i.name === "moonstream_api" )[0]; const moonstreamCrawlersStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "moonstream_crawlers" + (i) => i.name === "moonstream_crawlers" + )[0]; + const nodeBalacerStatus = serverListStatusCache?.data?.filter( + (i) => i.name === "moonstream_node_balancer" )[0]; const nodeEthereumAStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_ethereum_a" - )[0]; - const nodeEthereumAGeth = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_ethereum_a_geth" + (i) => i.name === "node_ethereum_a" )[0]; const nodeEthereumBStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_ethereum_b" - )[0]; - const nodeEthereumBGeth = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_ethereum_b_geth" + (i) => i.name === "node_ethereum_b" )[0]; const nodePolygonAStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_polygon_a" - )[0]; - const nodePolygonAGeth = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_polygon_a_bor" + (i) => i.name === "node_polygon_a" )[0]; const nodePolygonBStatus = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_polygon_b" + (i) => i.name === "node_polygon_b" )[0]; - const nodePolygonBGeth = serverListStatusCache?.data?.filter( - (i) => i.status.name === "node_polygon_b_bor" + const dbServerStatus = serverListStatusCache?.data?.filter( + (i) => i.name === "moonstream_database" + )[0]; + const dbReplicaServerStatus = serverListStatusCache?.data?.filter( + (i) => i.name === "moonstream_database_replica" + )[0]; + const unimLeaderboardStatus = serverListStatusCache?.data?.filter( + (i) => i.name === "unim_leaderboard" )[0]; const StatusRow = (props) => { @@ -80,12 +79,12 @@ const Status = () => { - {moonstreamapiStatus?.status.body.status == "ok" + {moonstreamapiStatus?.status_code == 200 ? healthyStatusText : downStatusText} @@ -93,42 +92,33 @@ const Status = () => {
- + - {moonstreamCrawlersStatus?.status.body.status == "ok" + {moonstreamCrawlersStatus?.status_code == 200 ? healthyStatusText : downStatusText} - - - {!user - ? crawlersStatusCache?.data?.ethereum_txpool_timestamp - ? shortTimestamp( - crawlersStatusCache?.data?.ethereum_txpool_timestamp - ) - : downStatusText - : unauthorizedText} - - - - - {!user - ? crawlersStatusCache?.data?.ethereum_trending_timestamp - ? shortTimestamp( - crawlersStatusCache?.data?.ethereum_trending_timestamp - ) - : downStatusText - : unauthorizedText} + +
+ + + + {nodeBalacerStatus?.status_code == 200 + ? healthyStatusText + : downStatusText} @@ -137,20 +127,20 @@ const Status = () => { - {nodeEthereumAStatus?.status.body.status == "ok" + {nodeEthereumAStatus?.status_code == 200 ? healthyStatusText : downStatusText} - {nodeEthereumAGeth?.status.body.current_block - ? nodeEthereumAGeth.status.body.current_block + {nodeEthereumAStatus?.response?.current_block + ? nodeEthereumAStatus.response.current_block : 0} @@ -158,20 +148,20 @@ const Status = () => { - {nodeEthereumBStatus?.status.body.status == "ok" + {nodeEthereumBStatus?.status_code == 200 ? healthyStatusText : downStatusText} - {nodeEthereumBGeth?.status.body.current_block - ? nodeEthereumBGeth.status.body.current_block + {nodeEthereumBStatus?.response?.current_block + ? nodeEthereumBStatus.response.current_block : 0} @@ -179,20 +169,20 @@ const Status = () => { - {nodePolygonAStatus?.status.body.status == "ok" + {nodePolygonAStatus?.status_code == 200 ? healthyStatusText : downStatusText} - {nodePolygonAGeth?.status.body.current_block - ? nodePolygonAGeth.status.body.current_block + {nodePolygonAStatus?.response?.current_block + ? nodePolygonAStatus.response.current_block : 0} @@ -200,49 +190,111 @@ const Status = () => { - {nodePolygonBStatus?.status.body.status == "ok" + {nodePolygonBStatus?.status_code == 200 ? healthyStatusText : downStatusText} - {nodePolygonBGeth?.status.body.current_block - ? nodePolygonBGeth.status.body.current_block + {nodePolygonBStatus?.response?.current_block + ? nodePolygonBStatus.response.current_block : 0}
- + - {dbServerStatusCache?.data?.status == "ok" + {dbServerStatus?.status_code == 200 ? healthyStatusText : downStatusText} - {latestBlockDBStatusCache?.data?.block_number - ? latestBlockDBStatusCache.data.block_number + {dbServerStatus?.response?.ethereum_block_latest + ? dbServerStatus.response.ethereum_block_latest : 0} + + + {dbServerStatus?.response?.polygon_block_latest + ? dbServerStatus.response.polygon_block_latest + : 0} + + + +
+ + + + {dbReplicaServerStatus?.status_code == 200 + ? healthyStatusText + : downStatusText} + + + + + {dbReplicaServerStatus?.response?.ethereum_block_latest + ? dbReplicaServerStatus.response.ethereum_block_latest + : 0} + + + + + {dbReplicaServerStatus?.response?.polygon_block_latest + ? dbReplicaServerStatus.response.polygon_block_latest + : 0} + + + +
+ + + + {unimLeaderboardStatus?.status_code == 200 + ? healthyStatusText + : downStatusText} + + ); diff --git a/frontend/sample.env b/frontend/sample.env index de5e0769..0f905729 100644 --- a/frontend/sample.env +++ b/frontend/sample.env @@ -2,7 +2,6 @@ export NEXT_PUBLIC_MIXPANEL_TOKEN="" export NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY="" export NEXT_PUBLIC_BUGOUT_STATUS_URL=https://status.moonstream.to export NEXT_PUBLIC_MOONSTREAM_API_URL=https://api.moonstream.to -export NEXT_PUBLIC_MOONSTREAM_DB_URL=https://pg.moonstream.to export NEXT_PUBLIC_SIMIOTICS_AUTH_URL=https://auth.bugout.dev export NEXT_PUBLIC_SIMIOTICS_JOURNALS_URL=https://spire.bugout.dev export NEXT_PUBLIC_FRONTEND_VERSION="" diff --git a/frontend/src/core/hooks/useStatus.js b/frontend/src/core/hooks/useStatus.js index 0822a76b..c2389d35 100644 --- a/frontend/src/core/hooks/useStatus.js +++ b/frontend/src/core/hooks/useStatus.js @@ -7,18 +7,6 @@ const useStatus = () => { const response = await StatusService.serverListStatus(); return response.data; }; - const getCrawlersStatus = async () => { - const response = await StatusService.crawlersStatus(); - return response.data; - }; - const getDBServerStatus = async () => { - const response = await StatusService.dbServerStatus(); - return response.data; - }; - const getLatestBlockDBStatus = async () => { - const response = await StatusService.latestBlockDBStatus(); - return response.data; - }; const serverListStatusCache = useQuery( "serverListStatus", @@ -28,28 +16,9 @@ const useStatus = () => { retry: 0, } ); - const crawlersStatusCache = useQuery("crawlers", getCrawlersStatus, { - ...queryCacheProps, - retry: 0, - }); - const dbServerStatusCache = useQuery("dbServer", getDBServerStatus, { - ...queryCacheProps, - retry: 0, - }); - const latestBlockDBStatusCache = useQuery( - "latestBlockDB", - getLatestBlockDBStatus, - { - ...queryCacheProps, - retry: 0, - } - ); return { serverListStatusCache, - crawlersStatusCache, - dbServerStatusCache, - latestBlockDBStatusCache, }; }; diff --git a/frontend/src/core/services/status.service.js b/frontend/src/core/services/status.service.js index 9dff55ca..f72f5613 100644 --- a/frontend/src/core/services/status.service.js +++ b/frontend/src/core/services/status.service.js @@ -1,33 +1,10 @@ import { http } from "../utils"; const BUGOUT_STATUS_URL = process.env.NEXT_PUBLIC_BUGOUT_STATUS_URL; -const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL; -const DB_URL = process.env.NEXT_PUBLIC_MOONSTREAM_DB_URL; export const serverListStatus = () => { return http({ method: "GET", - url: `${BUGOUT_STATUS_URL}`, - }); -}; - -export const crawlersStatus = () => { - return http({ - method: "GET", - url: `${API_URL}/status`, - }); -}; - -export const dbServerStatus = () => { - return http({ - method: "GET", - url: `${DB_URL}/ping`, - }); -}; - -export const latestBlockDBStatus = () => { - return http({ - method: "GET", - url: `${DB_URL}/block/latest`, + url: `${BUGOUT_STATUS_URL}/status`, }); };