From b3412a3e36a9d6fec25551cdb73ec397485d7e0c Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Tue, 28 Sep 2021 13:44:24 +0000
Subject: [PATCH] Crawlers server for health checks

---
 crawlers/deploy/deploy.bash                | 18 +++++
 crawlers/deploy/moonstreamcrawlers.service | 14 ++++
 crawlers/server/dev.sh                     |  9 +++
 crawlers/server/go.mod                     |  3 +
 crawlers/server/main.go                    | 87 ++++++++++++++++++++++
 crawlers/server/sample.env                 |  2 +
 db/server/main.go                          |  5 --
 7 files changed, 133 insertions(+), 5 deletions(-)
 create mode 100644 crawlers/deploy/moonstreamcrawlers.service
 create mode 100755 crawlers/server/dev.sh
 create mode 100644 crawlers/server/go.mod
 create mode 100644 crawlers/server/main.go
 create mode 100644 crawlers/server/sample.env

diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash
index 3729d184..1f038ce5 100755
--- a/crawlers/deploy/deploy.bash
+++ b/crawlers/deploy/deploy.bash
@@ -19,6 +19,7 @@ ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
 ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
 ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
 ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
+SERVICE_FILE="moonstreamcrawlers.service"
 
 set -eu
 
@@ -30,6 +31,14 @@ cd "${APP_CRAWLERS_DIR}/ethtxpool"
 HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go"
 cd "${EXEC_DIR}"
 
+echo
+echo
+echo "Building executable server of moonstreamcrawlers with Go"
+EXEC_DIR=$(pwd)
+cd "${APP_CRAWLERS_DIR}/server"
+HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go"
+cd "${EXEC_DIR}"
+
 echo
 echo
 echo "Updating Python dependencies"
@@ -82,3 +91,12 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
 cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
 systemctl daemon-reload
 systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
+
+echo
+echo
+echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}"
+chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}"
+systemctl daemon-reload
+systemctl restart "${SERVICE_FILE}"
+systemctl status "${SERVICE_FILE}"
diff --git a/crawlers/deploy/moonstreamcrawlers.service b/crawlers/deploy/moonstreamcrawlers.service
new file mode 100644
index 00000000..3b6e7104
--- /dev/null
+++ b/crawlers/deploy/moonstreamcrawlers.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=moonstreamcrawlers-service
+After=network.target
+
+[Service]
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/server
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream/crawlers/server/moonstreamcrawlers -host 0.0.0.0 -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
+SyslogIdentifier=moonstreamcrawlers
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/crawlers/server/dev.sh b/crawlers/server/dev.sh
new file mode 100755
index 00000000..174a6682
--- /dev/null
+++ b/crawlers/server/dev.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Expects access to Python environment with the requirements for this project installed.
+set -e
+
+MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}"
+MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}"
+
+go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
diff --git a/crawlers/server/go.mod b/crawlers/server/go.mod
new file mode 100644
index 00000000..5a8fa99b
--- /dev/null
+++ b/crawlers/server/go.mod
@@ -0,0 +1,3 @@
+module moonstreamdb
+
+go 1.17
diff --git a/crawlers/server/main.go b/crawlers/server/main.go
new file mode 100644
index 00000000..205666f3
--- /dev/null
+++ b/crawlers/server/main.go
@@ -0,0 +1,87 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+)
+
+var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_DB_URI")
+
+type GethEthSyncingResponse struct {
+	CurrentBlock string `json:"currentBlock"`
+}
+
+type GethResponse struct {
+	Result GethEthSyncingResponse `json:"result"`
+}
+
+type PingGethResponse struct {
+	Status       string `json:"status"`
+	CurrentBlock string `json:"current_block"`
+}
+
+type PingResponse struct {
+	Status string `json:"status"`
+}
+
+func ping(w http.ResponseWriter, req *http.Request) {
+	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+
+	w.Header().Set("Content-Type", "application/json")
+	response := PingResponse{Status: "ok"}
+	json.NewEncoder(w).Encode(response)
+}
+
+func pingGeth(w http.ResponseWriter, req *http.Request) {
+	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+
+	postBody, err := json.Marshal(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"method":  "eth_syncing",
+		"id":      1,
+	})
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json",
+		bytes.NewBuffer(postBody))
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	defer gethResponse.Body.Close()
+
+	gethResponseBody, err := ioutil.ReadAll(gethResponse.Body)
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	var obj GethResponse
+	_ = json.Unmarshal(gethResponseBody, &obj)
+
+	w.Header().Set("Content-Type", "application/json")
+	response := PingGethResponse{Status: "ok", CurrentBlock: obj.Result.CurrentBlock}
+	json.NewEncoder(w).Encode(response)
+}
+
+func main() {
+	var listenAddr string
+	var listenPort string
+	flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address")
+	flag.StringVar(&listenPort, "port", "8080", "Server listen port")
+	flag.Parse()
+
+	address := listenAddr + ":" + listenPort
+	log.Printf("Starting server at %s\n", address)
+
+	http.HandleFunc("/ping", ping)
+	http.HandleFunc("/ping/geth", pingGeth)
+
+	http.ListenAndServe(address, nil)
+}
diff --git a/crawlers/server/sample.env b/crawlers/server/sample.env
new file mode 100644
index 00000000..fc9b5a4a
--- /dev/null
+++ b/crawlers/server/sample.env
@@ -0,0 +1,2 @@
+export MOONSTREAM_CRAWLERS_SERVER_PORT="8080"
+export MOONSTREAM_IPC_PATH=null
diff --git a/db/server/main.go b/db/server/main.go
index 2da96b3b..fd5f4899 100644
--- a/db/server/main.go
+++ b/db/server/main.go
@@ -13,11 +13,6 @@ import (
 
 var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
 
-type Error interface {
-	error
-	Status() int
-}
-
 type PingResponse struct {
 	Status string `json:"status"`
 }