Merge branch 'main' into customized-dashboard

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -8,7 +8,7 @@ import logging
 import time
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Union
 from uuid import UUID
 
 import boto3  # type: ignore
@@ -417,6 +417,46 @@ def get_unique_address(
     )
 
 
+def get_blocks_state(
+    db_session: Session, blockchain_type: AvailableBlockchainType
+) -> Dict[str, int]:
+
+    """
+    Generate meta information about
+    """
+
+    blocks_state = {
+        "latest_stored_block": 0,
+        "latest_labelled_block": 0,
+        "earliest_labelled_block": 0,
+    }
+
+    label_model = get_label_model(blockchain_type)
+
+    transactions_model = get_transaction_model(blockchain_type)
+
+    max_transactions_number = db_session.query(
+        func.max(transactions_model.block_number).label("block_number")
+    ).scalar()
+
+    result = (
+        db_session.query(
+            func.min(label_model.block_number).label("earliest_labelled_block"),
+            func.max(label_model.block_number).label("latest_labelled_block"),
+            max_transactions_number,
+        ).filter(label_model.label == CRAWLER_LABEL)
+    ).one_or_none()
+
+    if result:
+        earliest_labelled_block, latest_labelled_block, latest_stored_block = result
+        blocks_state = {
+            "latest_stored_block": latest_stored_block,
+            "latest_labelled_block": latest_labelled_block,
+            "earliest_labelled_block": earliest_labelled_block,
+        }
+    return blocks_state
+
+
 def generate_list_of_names(
     type: str, subscription_filters: Dict[str, Any], read_abi: bool, abi_json: Any
 ):
@@ -583,7 +623,7 @@ def stats_generate_handler(args: argparse.Namespace):
                         continue
 
                     subscriptions_count += 1
-                    s3_data_object = {}
+                    s3_data_object: Dict[str, Any] = {}
 
                     extention_data = []
 
@@ -685,6 +725,10 @@ def stats_generate_handler(args: argparse.Namespace):
                             }
                         )
 
+                    current_blocks_state = get_blocks_state(
+                        db_session=db_session, blockchain_type=blockchain_type
+                    )
+
                     for timescale in [timescale.value for timescale in TimeScale]:
 
                         start_date = (
@@ -693,6 +737,8 @@ def stats_generate_handler(args: argparse.Namespace):
 
                         print(f"Timescale: {timescale}")
 
+                        s3_data_object["blocks_state"] = current_blocks_state
+
                         s3_data_object["web3_metric"] = extention_data
 
                         functions_calls_data = generate_data(

frontend/pages/status/index.js

@@ -1,11 +1,14 @@
-import React from "react";
+import React, { useContext } from "react";
 import { useStatus } from "../../src/core/hooks";
 import { Heading, Text, Flex, Spacer, chakra, Spinner } from "@chakra-ui/react";
 import { getLayout, getLayoutProps } from "../../src/layouts/InfoPageLayout";
+import UserContext from "../../src/core/providers/UserProvider/context";
 
 const Status = () => {
+  const user = useContext(UserContext);
   const healthyStatusText = "Available";
   const downStatusText = "Unavailable";
+  const unauthorizedText = "Please login";
   const healthyStatusColor = "green.900";
   const downStatusColor = "red.600";
 
@@ -14,14 +17,43 @@ const Status = () => {
   };
 
   const {
-    apiServerStatusCache,
+    serverListStatusCache,
-    ethereumClusterServerStatusCache,
-    gethStatusCache,
     crawlersStatusCache,
     dbServerStatusCache,
     latestBlockDBStatusCache,
   } = useStatus();
 
+  const moonstreamapiStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "moonstreamapi"
+  )[0];
+  const moonstreamCrawlersStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "moonstream_crawlers"
+  )[0];
+  const nodeEthereumAStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_ethereum_a"
+  )[0];
+  const nodeEthereumAGeth = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_ethereum_a_geth"
+  )[0];
+  const nodeEthereumBStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_ethereum_b"
+  )[0];
+  const nodeEthereumBGeth = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_ethereum_b_geth"
+  )[0];
+  const nodePolygonAStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_polygon_a"
+  )[0];
+  const nodePolygonAGeth = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_polygon_a_geth"
+  )[0];
+  const nodePolygonBStatus = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_polygon_b"
+  )[0];
+  const nodePolygonBGeth = serverListStatusCache?.data?.filter(
+    (i) => i.status.name === "node_polygon_b_geth"
+  )[0];
+
   const StatusRow = (props) => {
     return (
       <Flex mb={3}>
@@ -45,50 +77,44 @@ const Status = () => {
         {`Status page`}
       </Heading>
       <chakra.span pl={2} px={12} py={2} width="400px">
-        <StatusRow title="Backend server" cache={apiServerStatusCache}>
+        <StatusRow title="Backend server" cache={serverListStatusCache}>
           <Text
             color={
-              apiServerStatusCache?.data?.status == "ok"
+              moonstreamapiStatus?.status.body.status == "ok"
                 ? healthyStatusColor
                 : downStatusColor
             }
           >
-            {apiServerStatusCache?.data?.status == "ok"
+            {moonstreamapiStatus?.status.body.status == "ok"
               ? healthyStatusText
               : downStatusText}
           </Text>
         </StatusRow>
+
         <br />
-        <StatusRow
+
-          title="Crawlers server"
+        <StatusRow title="Crawlers server" cache={crawlersStatusCache}>
-          cache={ethereumClusterServerStatusCache}
-        >
           <Text
             color={
-              ethereumClusterServerStatusCache?.data?.status == "ok"
+              moonstreamCrawlersStatus?.status.body.status == "ok"
                 ? healthyStatusColor
                 : downStatusColor
             }
           >
-            {ethereumClusterServerStatusCache?.data
+            {moonstreamCrawlersStatus?.status.body.status == "ok"
               ? healthyStatusText
               : downStatusText}
           </Text>
         </StatusRow>
-        <StatusRow title="Latest block in Geth" cache={gethStatusCache}>
-          <Text>
-            {gethStatusCache?.data?.current_block
-              ? gethStatusCache.data.current_block
-              : 0}
-          </Text>
-        </StatusRow>
         <StatusRow title="Txpool latest record ts" cache={crawlersStatusCache}>
           <Text>
-            {crawlersStatusCache?.data?.ethereum_txpool_timestamp
+            {!user
-              ? shortTimestamp(
+              ? crawlersStatusCache?.data?.ethereum_txpool_timestamp
-                  crawlersStatusCache?.data?.ethereum_txpool_timestamp
+                ? shortTimestamp(
-                )
+                    crawlersStatusCache?.data?.ethereum_txpool_timestamp
-              : downStatusText}
+                  )
+                : downStatusText
+              : unauthorizedText}
           </Text>
         </StatusRow>
         <StatusRow
@@ -96,15 +122,104 @@ const Status = () => {
           cache={crawlersStatusCache}
         >
           <Text>
-            {crawlersStatusCache?.data?.ethereum_trending_timestamp
+            {!user
-              ? shortTimestamp(
+              ? crawlersStatusCache?.data?.ethereum_trending_timestamp
-                  crawlersStatusCache?.data?.ethereum_trending_timestamp
+                ? shortTimestamp(
-                )
+                    crawlersStatusCache?.data?.ethereum_trending_timestamp
-              : downStatusText}
+                  )
+                : downStatusText
+              : unauthorizedText}
           </Text>
         </StatusRow>
 
         <br />
+
+        <StatusRow title="Node Ethereum A" cache={serverListStatusCache}>
+          <Text
+            color={
+              nodeEthereumAStatus?.status.body.status == "ok"
+                ? healthyStatusColor
+                : downStatusColor
+            }
+          >
+            {nodeEthereumAStatus?.status.body.status == "ok"
+              ? healthyStatusText
+              : downStatusText}
+          </Text>
+        </StatusRow>
+        <StatusRow title="Current block" cache={serverListStatusCache}>
+          <Text>
+            {nodeEthereumAGeth?.status.body.current_block
+              ? nodeEthereumAGeth.status.body.current_block
+              : 0}
+          </Text>
+        </StatusRow>
+        <br />
+        <StatusRow title="Node Ethereum B" cache={serverListStatusCache}>
+          <Text
+            color={
+              nodeEthereumBStatus?.status.body.status == "ok"
+                ? healthyStatusColor
+                : downStatusColor
+            }
+          >
+            {nodeEthereumBStatus?.status.body.status == "ok"
+              ? healthyStatusText
+              : downStatusText}
+          </Text>
+        </StatusRow>
+        <StatusRow title="Current block" cache={serverListStatusCache}>
+          <Text>
+            {nodeEthereumBGeth?.status.body.current_block
+              ? nodeEthereumBGeth.status.body.current_block
+              : 0}
+          </Text>
+        </StatusRow>
+        <br />
+        <StatusRow title="Node Polygon A" cache={serverListStatusCache}>
+          <Text
+            color={
+              nodePolygonAStatus?.status.body.status == "ok"
+                ? healthyStatusColor
+                : downStatusColor
+            }
+          >
+            {nodePolygonAStatus?.status.body.status == "ok"
+              ? healthyStatusText
+              : downStatusText}
+          </Text>
+        </StatusRow>
+        <StatusRow title="Current block" cache={serverListStatusCache}>
+          <Text>
+            {nodePolygonAGeth?.status.body.current_block
+              ? nodePolygonAGeth.status.body.current_block
+              : 0}
+          </Text>
+        </StatusRow>
+        <br />
+        <StatusRow title="Node Polygon B" cache={serverListStatusCache}>
+          <Text
+            color={
+              nodePolygonBStatus?.status.body.status == "ok"
+                ? healthyStatusColor
+                : downStatusColor
+            }
+          >
+            {nodePolygonBStatus?.status.body.status == "ok"
+              ? healthyStatusText
+              : downStatusText}
+          </Text>
+        </StatusRow>
+        <StatusRow title="Current block" cache={serverListStatusCache}>
+          <Text>
+            {nodePolygonBGeth?.status.body.current_block
+              ? nodePolygonBGeth.status.body.current_block
+              : 0}
+          </Text>
+        </StatusRow>
+
+        <br />
+
         <StatusRow title="Database server" cache={dbServerStatusCache}>
           <Text
             color={

frontend/sample.env

@@ -1,10 +1,8 @@
 export NEXT_PUBLIC_MIXPANEL_TOKEN="<YOUR MIXPANEL TOKEN HERE>"
 export NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY="<stripe publishable key>"
-export NEXT_PUBLIC_MOONSTREAM_API_URL="<moonstream_api_url>"
+export NEXT_PUBLIC_BUGOUT_STATUS_URL=https://status.moonstream.to
-export NEXT_PUBLIC_MOONSTREAM_ETHEREUM_CLUSTER_URL="<moonstream_crawlers_url>"
+export NEXT_PUBLIC_MOONSTREAM_API_URL=https://api.moonstream.to
-export NEXT_PUBLIC_MOONSTREAM_DB_URL="<moonstream_db_url>"
+export NEXT_PUBLIC_MOONSTREAM_DB_URL=https://pg.moonstream.to
 export NEXT_PUBLIC_SIMIOTICS_AUTH_URL=https://auth.bugout.dev
 export NEXT_PUBLIC_SIMIOTICS_JOURNALS_URL=https://spire.bugout.dev
-export NEXT_PUBLIC_FRONTEND_VERSION="<frontend_version_number>"%
+export NEXT_PUBLIC_FRONTEND_VERSION="<frontend_version_number>"
-
-

frontend/src/core/hooks/useStatus.js

@@ -3,16 +3,8 @@ import { queryCacheProps } from "./hookCommon";
 import { StatusService } from "../../core/services";
 
 const useStatus = () => {
-  const getAPIServerStatus = async () => {
+  const getServerListStatus = async () => {
-    const response = await StatusService.apiServerStatus();
+    const response = await StatusService.serverListStatus();
-    return response.data;
-  };
-  const getEthereumClusterServerStatus = async () => {
-    const response = await StatusService.ethereumClusterServerStatus();
-    return response.data;
-  };
-  const getGethStatus = async () => {
-    const response = await StatusService.gethStatus();
     return response.data;
   };
   const getCrawlersStatus = async () => {
@@ -28,22 +20,14 @@ const useStatus = () => {
     return response.data;
   };
 
-  const apiServerStatusCache = useQuery("apiServer", getAPIServerStatus, {
+  const serverListStatusCache = useQuery(
-    ...queryCacheProps,
+    "serverListStatus",
-    retry: 0,
+    getServerListStatus,
-  });
-  const ethereumClusterServerStatusCache = useQuery(
-    "ethereumClusterServer",
-    getEthereumClusterServerStatus,
     {
       ...queryCacheProps,
       retry: 0,
     }
   );
-  const gethStatusCache = useQuery("geth", getGethStatus, {
-    ...queryCacheProps,
-    retry: 0,
-  });
   const crawlersStatusCache = useQuery("crawlers", getCrawlersStatus, {
     ...queryCacheProps,
     retry: 0,
@@ -62,9 +46,7 @@ const useStatus = () => {
   );
 
   return {
-    apiServerStatusCache,
+    serverListStatusCache,
-    ethereumClusterServerStatusCache,
-    gethStatusCache,
     crawlersStatusCache,
     dbServerStatusCache,
     latestBlockDBStatusCache,

frontend/src/core/services/status.service.js

@@ -1,28 +1,13 @@
 import { http } from "../utils";
 
+const BUGOUT_STATUS_URL = process.env.NEXT_PUBLIC_BUGOUT_STATUS_URL;
 const API_URL = process.env.NEXT_PUBLIC_MOONSTREAM_API_URL;
 const DB_URL = process.env.NEXT_PUBLIC_MOONSTREAM_DB_URL;
-const ETHEREUM_CLUSTER_URL =
-  process.env.NEXT_PUBLIC_MOONSTREAM_ETHEREUM_CLUSTER_URL;
 
-export const apiServerStatus = () => {
+export const serverListStatus = () => {
   return http({
     method: "GET",
-    url: `${API_URL}/ping`,
+    url: `${BUGOUT_STATUS_URL}`,
-  });
-};
-
-export const ethereumClusterServerStatus = () => {
-  return http({
-    method: "GET",
-    url: `${ETHEREUM_CLUSTER_URL}/ping`,
-  });
-};
-
-export const gethStatus = () => {
-  return http({
-    method: "GET",
-    url: `${ETHEREUM_CLUSTER_URL}/status`,
   });
 };