Working go status servers

crawlers/server/cmd/data.go

@@ -0,0 +1,13 @@
+package cmd
+
+type PingResponse struct {
+	Status string `json:"status"`
+}
+
+type GethResponse struct {
+	Result string `json:"result"`
+}
+
+type PingGethResponse struct {
+	CurrentBlock uint64 `json:"current_block"`
+}

crawlers/server/cmd/middlewares.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	settings "github.com/bugout-dev/moonstream/crawlers/server/configs"
+)
+
+// Handle panic errors to prevent server shutdown
+func panicMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Println("recovered", err)
+				http.Error(w, "Internal server error", 500)
+			}
+		}()
+		// There will be a defer with panic handler in each next function
+		next.ServeHTTP(w, r)
+	})
+}
+
+// Log requests in proper format
+func logsMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		next.ServeHTTP(w, r)
+		log.Printf("%s %s %s %s\n", time.Since(start), r.Method, r.URL.Path, r.RemoteAddr)
+	})
+}
+
+// CORS middleware
+func corsMiddleware(next http.Handler) http.Handler {
+	// Iterate over list of allowed origins
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for _, allowedOrigin := range strings.Split(settings.MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
+			if r.Header.Get("Origin") == allowedOrigin {
+				w.Header().Set("Access-Control-Allow-Origin", allowedOrigin)
+			}
+		}
+		if r.Method == "OPTIONS" {
+			w.Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
+			// Credentials are cookies, authorization headers, or TLS client certificates
+			w.Header().Set("Access-Control-Allow-Credentials", "true")
+			w.Header().Set("Access-Control-Allow-Headers", "Authorization")
+		}
+		next.ServeHTTP(w, r)
+	})
+}

crawlers/server/cmd/routes.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	settings "github.com/bugout-dev/moonstream/crawlers/server/configs"
+)
+
+func pingRoute(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	response := PingResponse{Status: "ok"}
+	json.NewEncoder(w).Encode(response)
+}
+
+// Fetch latest block from Geth
+func pingGethRoute(w http.ResponseWriter, req *http.Request) {
+	postBody, err := json.Marshal(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"method":  "eth_blockNumber",
+		"params":  []string{},
+		"id":      1,
+	})
+	if err != nil {
+		log.Printf("An error occurred due marshal postBody, error: %s", err)
+		http.Error(w, http.StatusText(500), http.StatusInternalServerError)
+		return
+	}
+	gethResponse, err := http.Post(settings.MOONSTREAM_IPC_PATH, "application/json",
+		bytes.NewBuffer(postBody))
+	if err != nil {
+		log.Printf("Unable to request geth, error: %s", err)
+		http.Error(w, http.StatusText(500), http.StatusInternalServerError)
+		return
+	}
+	defer gethResponse.Body.Close()
+
+	gethResponseBody, err := ioutil.ReadAll(gethResponse.Body)
+	if err != nil {
+		log.Printf("Unable to read geth response, error: %s", err)
+		http.Error(w, http.StatusText(500), http.StatusInternalServerError)
+		return
+	}
+	var obj GethResponse
+	_ = json.Unmarshal(gethResponseBody, &obj)
+
+	blockNumberHex := strings.Replace(obj.Result, "0x", "", -1)
+	blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64)
+	if err != nil {
+		log.Printf("Unable to parse block number from hex to string, error: %s", err)
+		http.Error(w, http.StatusText(500), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	response := PingGethResponse{CurrentBlock: blockNumberStr}
+	json.NewEncoder(w).Encode(response)
+}

crawlers/server/cmd/server.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"time"
+)
+
+func InitServer() {
+	var listeningAddr string
+	var listeningPort string
+	flag.StringVar(&listeningAddr, "host", "127.0.0.1", "Server listening address")
+	flag.StringVar(&listeningPort, "port", "8080", "Server listening port")
+	flag.Parse()
+
+	serverMux := http.NewServeMux()
+	serverMux.HandleFunc("/ping", pingRoute)
+	serverMux.HandleFunc("/status", pingGethRoute)
+
+	// Set middlewares from bottom to top
+	serverHandler := corsMiddleware(serverMux)
+	serverHandler = logsMiddleware(serverHandler)
+	serverHandler = panicMiddleware(serverHandler)
+
+	server := http.Server{
+		Addr:         listeningAddr + ":" + listeningPort,
+		Handler:      serverHandler,
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+
+	log.Printf("Starting server at %s:%s\n", listeningAddr, listeningPort)
+	server.ListenAndServe()
+}

crawlers/server/configs/settings.go

@@ -0,0 +1,11 @@
+package settings
+
+import (
+	"os"
+)
+
+// Geth configs
+var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH")
+
+// CORS
+var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")

crawlers/server/dev.sh

@@ -3,7 +3,7 @@
 # Expects access to Python environment with the requirements for this project installed.
 set -e
 
-MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}"
+MOONSTREAM_CLUSTER_SERVER_HOST="${MOONSTREAM_CLUSTER_SERVER_HOST:-0.0.0.0}"
-MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}"
+MOONSTREAM_CLUSTER_SERVER_PORT="${MOONSTREAM_CLUSTER_SERVER_PORT:-8080}"
 
-go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
+go run main.go -host "${MOONSTREAM_CLUSTER_SERVER_HOST}" -port "${MOONSTREAM_CLUSTER_SERVER_PORT}"

crawlers/server/go.mod

@@ -1,3 +1,3 @@
-module moonstreamdb
+module github.com/bugout-dev/moonstream/crawlers/server
 
 go 1.17

crawlers/server/main.go

@@ -1,118 +1,9 @@
 package main
 
 import (
-	"bytes"
+	"github.com/bugout-dev/moonstream/crawlers/server/cmd"
-	"encoding/json"
-	"flag"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strconv"
-	"strings"
 )
 
-var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH")
-var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")
-
-type GethResponse struct {
-	Result string `json:"result"`
-}
-
-type PingGethResponse struct {
-	CurrentBlock uint64 `json:"current_block"`
-}
-
-type PingResponse struct {
-	Status string `json:"status"`
-}
-
-// Extends handler with allowed CORS policies
-func setupCorsResponse(w *http.ResponseWriter, req *http.Request) {
-	for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
-		for _, reqOrigin := range req.Header["Origin"] {
-			if reqOrigin == allowedOrigin {
-				(*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin)
-			}
-		}
-
-	}
-	(*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
-}
-
-func ping(w http.ResponseWriter, req *http.Request) {
-	setupCorsResponse(&w, req)
-	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
-	if (*req).Method == "OPTIONS" {
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	response := PingResponse{Status: "ok"}
-	json.NewEncoder(w).Encode(response)
-}
-
-func pingGeth(w http.ResponseWriter, req *http.Request) {
-	setupCorsResponse(&w, req)
-	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
-	if (*req).Method == "OPTIONS" {
-		return
-	}
-
-	postBody, err := json.Marshal(map[string]interface{}{
-		"jsonrpc": "2.0",
-		"method":  "eth_blockNumber",
-		"params":  []string{},
-		"id":      1,
-	})
-	if err != nil {
-		log.Println(err)
-		http.Error(w, http.StatusText(500), 500)
-		return
-	}
-	gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json",
-		bytes.NewBuffer(postBody))
-	if err != nil {
-		log.Printf("Unable to request geth, error: %v", err)
-		http.Error(w, http.StatusText(500), 500)
-		return
-	}
-	defer gethResponse.Body.Close()
-
-	gethResponseBody, err := ioutil.ReadAll(gethResponse.Body)
-	if err != nil {
-		log.Printf("Unable to read geth response, error: %v", err)
-		http.Error(w, http.StatusText(500), 500)
-		return
-	}
-	var obj GethResponse
-	_ = json.Unmarshal(gethResponseBody, &obj)
-
-	blockNumberHex := strings.Replace(obj.Result, "0x", "", -1)
-	blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64)
-	if err != nil {
-		log.Printf("Unable to parse block number from hex to string, error: %v", err)
-		http.Error(w, http.StatusText(500), 500)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	response := PingGethResponse{CurrentBlock: blockNumberStr}
-	json.NewEncoder(w).Encode(response)
-}
-
 func main() {
-	var listenAddr string
+	cmd.InitServer()
-	var listenPort string
-	flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address")
-	flag.StringVar(&listenPort, "port", "8080", "Server listen port")
-	flag.Parse()
-
-	address := listenAddr + ":" + listenPort
-	log.Printf("Starting server at %s\n", address)
-
-	http.HandleFunc("/ping", ping)
-	http.HandleFunc("/status", pingGeth)
-
-	http.ListenAndServe(address, nil)
 }

crawlers/server/sample.env

@@ -1,3 +1,2 @@
-export MOONSTREAM_CRAWLERS_SERVER_PORT="8080"
 export MOONSTREAM_IPC_PATH=null
 export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to,https://alpha.moonstream.to"

db/server/cmd/middlewares.go

@@ -45,6 +45,7 @@ func corsMiddleware(next http.Handler) http.Handler {
 			w.Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
 			// Credentials are cookies, authorization headers, or TLS client certificates
 			w.Header().Set("Access-Control-Allow-Credentials", "true")
+			w.Header().Set("Access-Control-Allow-Headers", "Authorization")
 		}
 		next.ServeHTTP(w, r)
 	})

db/server/configs/settings.go

@@ -12,4 +12,3 @@ var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
 
 // CORS
 var MOONSTREAM_CORS_ALLOWED_ORIGINS = os.Getenv("MOONSTREAM_CORS_ALLOWED_ORIGINS")
-