From 66a6b0eb635a95946104a7ffebae72c28b7988c3 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 27 Sep 2021 13:00:45 +0000 Subject: [PATCH 1/8] Select last block from db with server --- db/server/go.mod | 17 +++++++++++++++++ db/server/main.go | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/db/server/go.mod b/db/server/go.mod index 5a8fa99b..2f789bca 100644 --- a/db/server/go.mod +++ b/db/server/go.mod @@ -1,3 +1,20 @@ module moonstreamdb go 1.17 + +require ( + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.10.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.8.1 // indirect + github.com/jackc/pgx/v4 v4.13.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.2 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/text v0.3.7 // indirect + gorm.io/driver/postgres v1.1.1 // indirect + gorm.io/gorm v1.21.15 // indirect +) diff --git a/db/server/main.go b/db/server/main.go index e9eb2811..47aeb11b 100644 --- a/db/server/main.go +++ b/db/server/main.go @@ -3,24 +3,51 @@ package main import ( "encoding/json" "fmt" + "log" "net/http" + "os" + + "gorm.io/driver/postgres" + "gorm.io/gorm" ) type PingResponse struct { Status string `json:"status"` } +type BlockResponse struct { + BlockNumber uint64 `json:"block_number"` +} + func ping(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") response := PingResponse{Status: "ok"} json.NewEncoder(w).Encode(response) } +func blockLatest(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var latestBlock BlockResponse + + MOONSTREAM_DB_URI := os.Getenv("MOONSTREAM_DB_URI") + db, err := gorm.Open(postgres.Open(MOONSTREAM_DB_URI), &gorm.Config{}) + if err != nil { + log.Print(err) + } + + query := "SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1" + db.Raw(query, 1).Scan(&latestBlock.BlockNumber) + + json.NewEncoder(w).Encode(latestBlock) +} + func main() { address := "0.0.0.0:8931" fmt.Printf("Starting server at %s\n", address) http.HandleFunc("/ping", ping) + http.HandleFunc("/block/latest", blockLatest) http.ListenAndServe(address, nil) } From f7a10b740247dae28f072da4ab4cff77255d0ff6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 28 Sep 2021 12:00:37 +0000 Subject: [PATCH 2/8] DB server to handle request for latest block --- db/deploy/deploy.bash | 17 +++ db/deploy/moonstreamdb.service | 3 +- db/server/dev.sh | 9 ++ db/server/go.mod | 7 +- db/server/go.sum | 186 +++++++++++++++++++++++++++++++++ db/server/main.go | 28 +++-- db/server/sample.env | 2 + 7 files changed, 243 insertions(+), 9 deletions(-) create mode 100755 db/server/dev.sh create mode 100644 db/server/go.sum create mode 100644 db/server/sample.env diff --git a/db/deploy/deploy.bash b/db/deploy/deploy.bash index b2c30312..a35e87b4 100755 --- a/db/deploy/deploy.bash +++ b/db/deploy/deploy.bash @@ -5,15 +5,32 @@ # Main APP_DIR="${APP_DIR:-/home/ubuntu/moonstream}" APP_DB_SERVER_DIR="${APP_DIR}/db/server" +SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" SCRIPT_DIR="$(realpath $(dirname $0))" SERVICE_FILE="${SCRIPT_DIR}/moonstreamdb.service" set -eu +echo +echo +echo "Retrieving deployment parameters for GCL Secret Manager" +mkdir -p "${SECRETS_DIR}" +echo "" > "${SECRETS_DIR}/app.env" +SECRET_NAMES=$(gcloud beta secrets list --filter="labels.product=moonstream" --format="get(name)") +for secret in $SECRET_NAMES +do + secret_key=$(echo "${secret}" | awk -F'/' '{print $NF}') + secret_val=$(gcloud secrets versions access latest --secret="${secret_key}") + echo "${secret_key}=\"${secret_val}\"" >> "${SECRETS_DIR}/app.env" +done + echo echo echo "Building executable database server script with Go" +EXEC_DIR=$(pwd) +cd "${APP_DB_SERVER_DIR}" HOME=/root /usr/local/go/bin/go build -o "${APP_DB_SERVER_DIR}/moonstreamdb" "${APP_DB_SERVER_DIR}/main.go" +cd "${EXEC_DIR}" echo echo diff --git a/db/deploy/moonstreamdb.service b/db/deploy/moonstreamdb.service index 70e7dfea..5f0b2b6f 100644 --- a/db/deploy/moonstreamdb.service +++ b/db/deploy/moonstreamdb.service @@ -6,7 +6,8 @@ After=network.target User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/db/server -ExecStart=/home/ubuntu/moonstream/db/server/moonstreamdb +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream/db/server/moonstreamdb -host 0.0.0.0 -port "${MOONSTREAM_DB_SERVER_PORT}" SyslogIdentifier=moonstreamdb [Install] diff --git a/db/server/dev.sh b/db/server/dev.sh new file mode 100755 index 00000000..8920f68e --- /dev/null +++ b/db/server/dev.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh + +# Expects access to Python environment with the requirements for this project installed. +set -e + +MOONSTREAM_DB_SERVER_HOST="${MOONSTREAM_DB_SERVER_HOST:-0.0.0.0}" +MOONSTREAM_DB_SERVER_PORT="${MOONSTREAM_DB_SERVER_PORT:-8080}" + +go run main.go -host "${MOONSTREAM_DB_SERVER_HOST}" -port "${MOONSTREAM_DB_SERVER_PORT}" diff --git a/db/server/go.mod b/db/server/go.mod index 2f789bca..570b3bc2 100644 --- a/db/server/go.mod +++ b/db/server/go.mod @@ -2,6 +2,11 @@ module moonstreamdb go 1.17 +require ( + gorm.io/driver/postgres v1.1.1 + gorm.io/gorm v1.21.15 +) + require ( github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.10.0 // indirect @@ -15,6 +20,4 @@ require ( github.com/jinzhu/now v1.1.2 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/text v0.3.7 // indirect - gorm.io/driver/postgres v1.1.1 // indirect - gorm.io/gorm v1.21.15 // indirect ) diff --git a/db/server/go.sum b/db/server/go.sum new file mode 100644 index 00000000..33b809ff --- /dev/null +++ b/db/server/go.sum @@ -0,0 +1,186 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.1.1 h1:tWLmqYCyaoh89fi7DhM6QggujrOnmfo3H98AzgNAAu0= +gorm.io/driver/postgres v1.1.1/go.mod h1:tpe2xN7aCst1NUdYyWQyxPtnHC+Zfp6NEux9PXD1OU0= +gorm.io/gorm v1.21.15 h1:gAyaDoPw0lCyrSFWhBlahbUA1U4P5RViC1uIqoB+1Rk= +gorm.io/gorm v1.21.15/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/db/server/main.go b/db/server/main.go index 47aeb11b..2da96b3b 100644 --- a/db/server/main.go +++ b/db/server/main.go @@ -2,7 +2,7 @@ package main import ( "encoding/json" - "fmt" + "flag" "log" "net/http" "os" @@ -11,6 +11,13 @@ import ( "gorm.io/gorm" ) +var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") + +type Error interface { + error + Status() int +} + type PingResponse struct { Status string `json:"status"` } @@ -20,20 +27,23 @@ type BlockResponse struct { } func ping(w http.ResponseWriter, req *http.Request) { + log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) + w.Header().Set("Content-Type", "application/json") response := PingResponse{Status: "ok"} json.NewEncoder(w).Encode(response) } func blockLatest(w http.ResponseWriter, req *http.Request) { + log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) + w.Header().Set("Content-Type", "application/json") var latestBlock BlockResponse - - MOONSTREAM_DB_URI := os.Getenv("MOONSTREAM_DB_URI") db, err := gorm.Open(postgres.Open(MOONSTREAM_DB_URI), &gorm.Config{}) if err != nil { - log.Print(err) + http.Error(w, http.StatusText(500), 500) + return } query := "SELECT block_number FROM ethereum_blocks ORDER BY block_number DESC LIMIT 1" @@ -43,8 +53,14 @@ func blockLatest(w http.ResponseWriter, req *http.Request) { } func main() { - address := "0.0.0.0:8931" - fmt.Printf("Starting server at %s\n", address) + var listenAddr string + var listenPort string + flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address") + flag.StringVar(&listenPort, "port", "8080", "Server listen port") + flag.Parse() + + address := listenAddr + ":" + listenPort + log.Printf("Starting server at %s\n", address) http.HandleFunc("/ping", ping) http.HandleFunc("/block/latest", blockLatest) diff --git a/db/server/sample.env b/db/server/sample.env new file mode 100644 index 00000000..412c015d --- /dev/null +++ b/db/server/sample.env @@ -0,0 +1,2 @@ +export MOONSTREAM_DB_SERVER_PORT="8080" +export MOONSTREAM_DB_URI="postgresql://:@:/" From b3412a3e36a9d6fec25551cdb73ec397485d7e0c Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 28 Sep 2021 13:44:24 +0000 Subject: [PATCH 3/8] Crawlers server for health checks --- crawlers/deploy/deploy.bash | 18 +++++ crawlers/deploy/moonstreamcrawlers.service | 14 ++++ crawlers/server/dev.sh | 9 +++ crawlers/server/go.mod | 3 + crawlers/server/main.go | 87 ++++++++++++++++++++++ crawlers/server/sample.env | 2 + db/server/main.go | 5 -- 7 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 crawlers/deploy/moonstreamcrawlers.service create mode 100755 crawlers/server/dev.sh create mode 100644 crawlers/server/go.mod create mode 100644 crawlers/server/main.go create mode 100644 crawlers/server/sample.env diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 3729d184..1f038ce5 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -19,6 +19,7 @@ ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER="ethereum-trending.service" ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service" +SERVICE_FILE="moonstreamcrawlers.service" set -eu @@ -30,6 +31,14 @@ cd "${APP_CRAWLERS_DIR}/ethtxpool" HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go" cd "${EXEC_DIR}" +echo +echo +echo "Building executable server of moonstreamcrawlers with Go" +EXEC_DIR=$(pwd) +cd "${APP_CRAWLERS_DIR}/server" +HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go" +cd "${EXEC_DIR}" + echo echo echo "Updating Python dependencies" @@ -82,3 +91,12 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}" systemctl daemon-reload systemctl restart "${ETHEREUM_TXPOOL_SERVICE}" + +echo +echo +echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}" +systemctl daemon-reload +systemctl restart "${SERVICE_FILE}" +systemctl status "${SERVICE_FILE}" diff --git a/crawlers/deploy/moonstreamcrawlers.service b/crawlers/deploy/moonstreamcrawlers.service new file mode 100644 index 00000000..3b6e7104 --- /dev/null +++ b/crawlers/deploy/moonstreamcrawlers.service @@ -0,0 +1,14 @@ +[Unit] +Description=moonstreamcrawlers-service +After=network.target + +[Service] +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/server +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream/crawlers/server/moonstreamcrawlers -host 0.0.0.0 -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}" +SyslogIdentifier=moonstreamcrawlers + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/server/dev.sh b/crawlers/server/dev.sh new file mode 100755 index 00000000..174a6682 --- /dev/null +++ b/crawlers/server/dev.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh + +# Expects access to Python environment with the requirements for this project installed. +set -e + +MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}" +MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}" + +go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}" diff --git a/crawlers/server/go.mod b/crawlers/server/go.mod new file mode 100644 index 00000000..5a8fa99b --- /dev/null +++ b/crawlers/server/go.mod @@ -0,0 +1,3 @@ +module moonstreamdb + +go 1.17 diff --git a/crawlers/server/main.go b/crawlers/server/main.go new file mode 100644 index 00000000..205666f3 --- /dev/null +++ b/crawlers/server/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "io/ioutil" + "log" + "net/http" + "os" +) + +var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_DB_URI") + +type GethEthSyncingResponse struct { + CurrentBlock string `json:"currentBlock"` +} + +type GethResponse struct { + Result GethEthSyncingResponse `json:"result"` +} + +type PingGethResponse struct { + Status string `json:"status"` + CurrentBlock string `json:"current_block"` +} + +type PingResponse struct { + Status string `json:"status"` +} + +func ping(w http.ResponseWriter, req *http.Request) { + log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) + + w.Header().Set("Content-Type", "application/json") + response := PingResponse{Status: "ok"} + json.NewEncoder(w).Encode(response) +} + +func pingGeth(w http.ResponseWriter, req *http.Request) { + log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String()) + + postBody, err := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_syncing", + "id": 1, + }) + if err != nil { + http.Error(w, http.StatusText(500), 500) + return + } + gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json", + bytes.NewBuffer(postBody)) + if err != nil { + http.Error(w, http.StatusText(500), 500) + return + } + defer gethResponse.Body.Close() + + gethResponseBody, err := ioutil.ReadAll(gethResponse.Body) + if err != nil { + http.Error(w, http.StatusText(500), 500) + return + } + var obj GethResponse + _ = json.Unmarshal(gethResponseBody, &obj) + + w.Header().Set("Content-Type", "application/json") + response := PingGethResponse{Status: "ok", CurrentBlock: obj.Result.CurrentBlock} + json.NewEncoder(w).Encode(response) +} + +func main() { + var listenAddr string + var listenPort string + flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address") + flag.StringVar(&listenPort, "port", "8080", "Server listen port") + flag.Parse() + + address := listenAddr + ":" + listenPort + log.Printf("Starting server at %s\n", address) + + http.HandleFunc("/ping", ping) + http.HandleFunc("/ping/geth", pingGeth) + + http.ListenAndServe(address, nil) +} diff --git a/crawlers/server/sample.env b/crawlers/server/sample.env new file mode 100644 index 00000000..fc9b5a4a --- /dev/null +++ b/crawlers/server/sample.env @@ -0,0 +1,2 @@ +export MOONSTREAM_CRAWLERS_SERVER_PORT="8080" +export MOONSTREAM_IPC_PATH=null diff --git a/db/server/main.go b/db/server/main.go index 2da96b3b..fd5f4899 100644 --- a/db/server/main.go +++ b/db/server/main.go @@ -13,11 +13,6 @@ import ( var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") -type Error interface { - error - Status() int -} - type PingResponse struct { Status string `json:"status"` } From 286f735f30f4b6c33938954cfeadfc2a4f8facb8 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 29 Sep 2021 14:08:58 +0000 Subject: [PATCH 4/8] Status API endpoint for backend --- backend/moonstream/api.py | 44 +++++++++++++++++++++++++++++++++++++- backend/moonstream/data.py | 5 +++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 6e687eb9..340cb2b9 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -4,17 +4,26 @@ The Moonstream HTTP API import logging import time +from bugout.data import BugoutSearchResults +from bugout.exceptions import BugoutResponseException +from bugout.journal import SearchOrder from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from . import data +from .middleware import MoonstreamHTTPException from .routes.address_info import app as addressinfo_api from .routes.nft import app as nft_api from .routes.subscriptions import app as subscriptions_api from .routes.streams import app as streams_api from .routes.txinfo import app as txinfo_api from .routes.users import app as users_api -from .settings import ORIGINS +from .settings import ( + bugout_client as bc, + ORIGINS, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, +) from .version import MOONSTREAM_VERSION logging.basicConfig(level=logging.INFO) @@ -46,6 +55,39 @@ async def now_handler() -> data.NowResponse: return data.NowResponse(epoch_time=time.time()) +@app.get("/status", tags=["streams"], response_model=data.StatusResponse) +async def status_handler() -> data.StatusResponse: + """ + Get latest records and their creation timestamp for crawlers: + - ethereum_txpool + - ethereum_trending + """ + crawl_types_timestamp = {"ethereum_txpool": None, "ethereum_trending": None} + for crawl_type in crawl_types_timestamp.keys(): + try: + search_results: BugoutSearchResults = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_DATA_JOURNAL_ID, + query=f"tag:crawl_type:{crawl_type}", + limit=1, + content=False, + timeout=10.0, + order=SearchOrder.DESCENDING, + ) + if len(search_results.results) == 1: + crawl_types_timestamp[crawl_type] = search_results.results[0].created_at + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Unable to get status for crawler with type: {crawl_type}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return data.StatusResponse( + ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"], + ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"], + ) + + app.mount("/subscriptions", subscriptions_api) app.mount("/users", users_api) app.mount("/streams", streams_api) diff --git a/backend/moonstream/data.py b/backend/moonstream/data.py index 05e49ea2..7a7089a2 100644 --- a/backend/moonstream/data.py +++ b/backend/moonstream/data.py @@ -67,6 +67,11 @@ class NowResponse(BaseModel): epoch_time: float +class StatusResponse(BaseModel): + ethereum_txpool_timestamp: Optional[datetime] = None + ethereum_trending_timestamp: Optional[datetime] = None + + class SubscriptionUpdate(BaseModel): update: Dict[str, Any] drop_keys: List[str] = Field(default_factory=list) From 10206840d2068bcdeadb5c9c7981e72058c13e31 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 29 Sep 2021 14:25:41 +0000 Subject: [PATCH 5/8] Geth status returns block number --- crawlers/server/main.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/crawlers/server/main.go b/crawlers/server/main.go index 205666f3..923d1574 100644 --- a/crawlers/server/main.go +++ b/crawlers/server/main.go @@ -8,21 +8,18 @@ import ( "log" "net/http" "os" + "strconv" + "strings" ) -var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_DB_URI") - -type GethEthSyncingResponse struct { - CurrentBlock string `json:"currentBlock"` -} +var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_IPC_PATH") type GethResponse struct { - Result GethEthSyncingResponse `json:"result"` + Result string `json:"result"` } type PingGethResponse struct { - Status string `json:"status"` - CurrentBlock string `json:"current_block"` + CurrentBlock uint64 `json:"current_block"` } type PingResponse struct { @@ -42,16 +39,19 @@ func pingGeth(w http.ResponseWriter, req *http.Request) { postBody, err := json.Marshal(map[string]interface{}{ "jsonrpc": "2.0", - "method": "eth_syncing", + "method": "eth_blockNumber", + "params": []string{}, "id": 1, }) if err != nil { + log.Println(err) http.Error(w, http.StatusText(500), 500) return } gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json", bytes.NewBuffer(postBody)) if err != nil { + log.Printf("Unable to request geth, error: %v", err) http.Error(w, http.StatusText(500), 500) return } @@ -59,14 +59,23 @@ func pingGeth(w http.ResponseWriter, req *http.Request) { gethResponseBody, err := ioutil.ReadAll(gethResponse.Body) if err != nil { + log.Printf("Unable to read geth response, error: %v", err) http.Error(w, http.StatusText(500), 500) return } var obj GethResponse _ = json.Unmarshal(gethResponseBody, &obj) + blockNumberHex := strings.Replace(obj.Result, "0x", "", -1) + blockNumberStr, err := strconv.ParseUint(blockNumberHex, 16, 64) + if err != nil { + log.Printf("Unable to parse block number from hex to string, error: %v", err) + http.Error(w, http.StatusText(500), 500) + return + } + w.Header().Set("Content-Type", "application/json") - response := PingGethResponse{Status: "ok", CurrentBlock: obj.Result.CurrentBlock} + response := PingGethResponse{CurrentBlock: blockNumberStr} json.NewEncoder(w).Encode(response) } @@ -81,7 +90,7 @@ func main() { log.Printf("Starting server at %s\n", address) http.HandleFunc("/ping", ping) - http.HandleFunc("/ping/geth", pingGeth) + http.HandleFunc("/status", pingGeth) http.ListenAndServe(address, nil) } From c473a899c5a58e077762afe237650d66ce5354d3 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 29 Sep 2021 14:34:29 +0000 Subject: [PATCH 6/8] Fixed mypy --- backend/moonstream/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 340cb2b9..8b5e0bc5 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -3,6 +3,7 @@ The Moonstream HTTP API """ import logging import time +from typing import Any, Dict from bugout.data import BugoutSearchResults from bugout.exceptions import BugoutResponseException @@ -62,7 +63,10 @@ async def status_handler() -> data.StatusResponse: - ethereum_txpool - ethereum_trending """ - crawl_types_timestamp = {"ethereum_txpool": None, "ethereum_trending": None} + crawl_types_timestamp: Dict[str, Any] = { + "ethereum_txpool": None, + "ethereum_trending": None, + } for crawl_type in crawl_types_timestamp.keys(): try: search_results: BugoutSearchResults = bc.search( From b01f96fa8061017f85908b6d62ba73cf76679351 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 29 Sep 2021 16:02:32 +0000 Subject: [PATCH 7/8] Files structure update of status --- backend/moonstream/api.py | 50 ++---------------- backend/moonstream/routes/status.py | 78 +++++++++++++++++++++++++++++ backend/moonstream/routes/users.py | 1 - 3 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 backend/moonstream/routes/status.py diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 8b5e0bc5..cfa34320 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -3,28 +3,19 @@ The Moonstream HTTP API """ import logging import time -from typing import Any, Dict -from bugout.data import BugoutSearchResults -from bugout.exceptions import BugoutResponseException -from bugout.journal import SearchOrder from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from . import data -from .middleware import MoonstreamHTTPException from .routes.address_info import app as addressinfo_api from .routes.nft import app as nft_api +from .routes.status import app as status_api from .routes.subscriptions import app as subscriptions_api from .routes.streams import app as streams_api from .routes.txinfo import app as txinfo_api from .routes.users import app as users_api -from .settings import ( - bugout_client as bc, - ORIGINS, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_DATA_JOURNAL_ID, -) +from .settings import ORIGINS from .version import MOONSTREAM_VERSION logging.basicConfig(level=logging.INFO) @@ -56,44 +47,9 @@ async def now_handler() -> data.NowResponse: return data.NowResponse(epoch_time=time.time()) -@app.get("/status", tags=["streams"], response_model=data.StatusResponse) -async def status_handler() -> data.StatusResponse: - """ - Get latest records and their creation timestamp for crawlers: - - ethereum_txpool - - ethereum_trending - """ - crawl_types_timestamp: Dict[str, Any] = { - "ethereum_txpool": None, - "ethereum_trending": None, - } - for crawl_type in crawl_types_timestamp.keys(): - try: - search_results: BugoutSearchResults = bc.search( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=MOONSTREAM_DATA_JOURNAL_ID, - query=f"tag:crawl_type:{crawl_type}", - limit=1, - content=False, - timeout=10.0, - order=SearchOrder.DESCENDING, - ) - if len(search_results.results) == 1: - crawl_types_timestamp[crawl_type] = search_results.results[0].created_at - except BugoutResponseException as e: - raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: - logger.error(f"Unable to get status for crawler with type: {crawl_type}") - raise MoonstreamHTTPException(status_code=500, internal_error=e) - - return data.StatusResponse( - ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"], - ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"], - ) - - app.mount("/subscriptions", subscriptions_api) app.mount("/users", users_api) +app.mount("/status", status_api) app.mount("/streams", streams_api) app.mount("/txinfo", txinfo_api) app.mount("/address_info", addressinfo_api) diff --git a/backend/moonstream/routes/status.py b/backend/moonstream/routes/status.py new file mode 100644 index 00000000..ccf41ef1 --- /dev/null +++ b/backend/moonstream/routes/status.py @@ -0,0 +1,78 @@ +import logging +from typing import Any, Dict + +from bugout.data import BugoutSearchResults +from bugout.exceptions import BugoutResponseException +from bugout.journal import SearchOrder +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from .. import data +from ..middleware import MoonstreamHTTPException +from ..settings import ( + bugout_client as bc, + DOCS_TARGET_PATH, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, + ORIGINS, +) +from ..version import MOONSTREAM_VERSION + +logger = logging.getLogger(__name__) + +tags_metadata = [ + {"name": "status", "description": "Status of Moonstream API services."} +] +app = FastAPI( + title=f"Moonstream users API.", + description="User, token and password handlers.", + version=MOONSTREAM_VERSION, + openapi_tags=tags_metadata, + openapi_url="/openapi.json", + docs_url=None, + redoc_url=f"/{DOCS_TARGET_PATH}", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/", tags=["status"], response_model=data.StatusResponse) +async def status_handler() -> data.StatusResponse: + """ + Get latest records and their creation timestamp for crawlers: + - ethereum_txpool + - ethereum_trending + """ + crawl_types_timestamp: Dict[str, Any] = { + "ethereum_txpool": None, + "ethereum_trending": None, + } + for crawl_type in crawl_types_timestamp.keys(): + try: + search_results: BugoutSearchResults = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_DATA_JOURNAL_ID, + query=f"tag:crawl_type:{crawl_type}", + limit=1, + content=False, + timeout=10.0, + order=SearchOrder.DESCENDING, + ) + if len(search_results.results) == 1: + crawl_types_timestamp[crawl_type] = search_results.results[0].created_at + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Unable to get status for crawler with type: {crawl_type}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return data.StatusResponse( + ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"], + ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"], + ) diff --git a/backend/moonstream/routes/users.py b/backend/moonstream/routes/users.py index c414fabf..237f031e 100644 --- a/backend/moonstream/routes/users.py +++ b/backend/moonstream/routes/users.py @@ -7,7 +7,6 @@ import uuid from bugout.data import BugoutToken, BugoutUser, BugoutResource, BugoutUserTokens from bugout.exceptions import BugoutResponseException - from fastapi import ( Body, FastAPI, From e17c6473778a758e71960f7f104cefbecff46ea4 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 29 Sep 2021 16:19:31 +0000 Subject: [PATCH 8/8] Moved status logic to actions py --- backend/moonstream/actions.py | 36 +++++++++++++ backend/moonstream/api.py | 25 ++++++++- backend/moonstream/routes/status.py | 78 ----------------------------- 3 files changed, 59 insertions(+), 80 deletions(-) delete mode 100644 backend/moonstream/routes/status.py diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 112fc36e..72e894f9 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -5,6 +5,8 @@ from enum import Enum import uuid import boto3 # type: ignore +from bugout.data import BugoutSearchResults +from bugout.journal import SearchOrder from moonstreamdb.models import ( EthereumAddress, EthereumLabel, @@ -20,12 +22,20 @@ from .settings import ( MOONSTREAM_APPLICATION_ID, bugout_client as bc, BUGOUT_REQUEST_TIMEOUT_SECONDS, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, ) logger = logging.getLogger(__name__) ETHERSCAN_SMARTCONTRACT_LABEL_NAME = "etherscan_smartcontract" +class StatusAPIException(Exception): + """ + Raised during checking Moonstream API statuses. + """ + + def get_contract_source_info( db_session: Session, contract_address: str ) -> Optional[data.EthereumSmartContractSourceInfo]: @@ -192,3 +202,29 @@ def create_onboarding_resource( timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) return resource + + +def check_api_status(): + crawl_types_timestamp: Dict[str, Any] = { + "ethereum_txpool": None, + "ethereum_trending": None, + } + for crawl_type in crawl_types_timestamp.keys(): + try: + search_results: BugoutSearchResults = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_DATA_JOURNAL_ID, + query=f"tag:crawl_type:{crawl_type}", + limit=1, + content=False, + timeout=10.0, + order=SearchOrder.DESCENDING, + ) + if len(search_results.results) == 1: + crawl_types_timestamp[crawl_type] = search_results.results[0].created_at + except Exception: + raise StatusAPIException( + f"Unable to get status for crawler with type: {crawl_type}" + ) + + return crawl_types_timestamp diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index cfa34320..0d8d0ed5 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -7,10 +7,11 @@ import time from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from . import actions from . import data +from .middleware import MoonstreamHTTPException from .routes.address_info import app as addressinfo_api from .routes.nft import app as nft_api -from .routes.status import app as status_api from .routes.subscriptions import app as subscriptions_api from .routes.streams import app as streams_api from .routes.txinfo import app as txinfo_api @@ -47,9 +48,29 @@ async def now_handler() -> data.NowResponse: return data.NowResponse(epoch_time=time.time()) +@app.get("/status", response_model=data.StatusResponse) +async def status_handler() -> data.StatusResponse: + """ + Get latest records and their creation timestamp for crawlers: + - ethereum_txpool + - ethereum_trending + """ + try: + crawl_types_timestamp = actions.check_api_status() + except actions.StatusAPIException: + raise MoonstreamHTTPException(status_code=500) + except Exception as e: + logger.error(f"Unhandled status exception, error: {e}") + raise MoonstreamHTTPException(status_code=500) + + return data.StatusResponse( + ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"], + ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"], + ) + + app.mount("/subscriptions", subscriptions_api) app.mount("/users", users_api) -app.mount("/status", status_api) app.mount("/streams", streams_api) app.mount("/txinfo", txinfo_api) app.mount("/address_info", addressinfo_api) diff --git a/backend/moonstream/routes/status.py b/backend/moonstream/routes/status.py deleted file mode 100644 index ccf41ef1..00000000 --- a/backend/moonstream/routes/status.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging -from typing import Any, Dict - -from bugout.data import BugoutSearchResults -from bugout.exceptions import BugoutResponseException -from bugout.journal import SearchOrder -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware - -from .. import data -from ..middleware import MoonstreamHTTPException -from ..settings import ( - bugout_client as bc, - DOCS_TARGET_PATH, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_DATA_JOURNAL_ID, - ORIGINS, -) -from ..version import MOONSTREAM_VERSION - -logger = logging.getLogger(__name__) - -tags_metadata = [ - {"name": "status", "description": "Status of Moonstream API services."} -] -app = FastAPI( - title=f"Moonstream users API.", - description="User, token and password handlers.", - version=MOONSTREAM_VERSION, - openapi_tags=tags_metadata, - openapi_url="/openapi.json", - docs_url=None, - redoc_url=f"/{DOCS_TARGET_PATH}", -) - -app.add_middleware( - CORSMiddleware, - allow_origins=ORIGINS, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -@app.get("/", tags=["status"], response_model=data.StatusResponse) -async def status_handler() -> data.StatusResponse: - """ - Get latest records and their creation timestamp for crawlers: - - ethereum_txpool - - ethereum_trending - """ - crawl_types_timestamp: Dict[str, Any] = { - "ethereum_txpool": None, - "ethereum_trending": None, - } - for crawl_type in crawl_types_timestamp.keys(): - try: - search_results: BugoutSearchResults = bc.search( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=MOONSTREAM_DATA_JOURNAL_ID, - query=f"tag:crawl_type:{crawl_type}", - limit=1, - content=False, - timeout=10.0, - order=SearchOrder.DESCENDING, - ) - if len(search_results.results) == 1: - crawl_types_timestamp[crawl_type] = search_results.results[0].created_at - except BugoutResponseException as e: - raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: - logger.error(f"Unable to get status for crawler with type: {crawl_type}") - raise MoonstreamHTTPException(status_code=500, internal_error=e) - - return data.StatusResponse( - ethereum_txpool_timestamp=crawl_types_timestamp["ethereum_txpool"], - ethereum_trending_timestamp=crawl_types_timestamp["ethereum_trending"], - )