mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Functionality to report at humbug journal with corrupt blocks

pull/557/head
kompotkot 2022-03-03 21:05:33 +00:00
rodzic 45cc528582
commit 1642ed9e0a
6 zmienionych plików z 116 dodań i 16 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

74
crawlers/ldb/cmd/chain.go
Wyświetl plik

@ -2,11 +2,11 @@ package cmd
import (
"database/sql"
"encoding/json"
"fmt"
"time"
humbug "github.com/bugout-dev/humbug/go/pkg"
"github.com/bugout-dev/moonstream/crawlers/ldb/configs"
_ "github.com/lib/pq"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
@ -16,6 +16,7 @@ import (
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/node"
_ "github.com/lib/pq"
"gopkg.in/urfave/cli.v1"
)
@ -37,6 +38,10 @@ type LocalConnections struct {
Database *sql.DB
}
type HumbugReporter struct {
Reporter *humbug.HumbugReporter
}
func setLocalChain(ctx *cli.Context) error {
cfg := gethConfig{
Eth: ethconfig.Defaults,
@ -164,31 +169,63 @@ func (lc *LocalConnections) getTxsFromDB(blockNumber uint64) (Transactions, erro
return transactions, nil
}
type CorruptBlocks struct {
Numbers []uint64
}
var corruptBlocks CorruptBlocks
// Write down inconsistent state between database and blockchain
// **source** (string): Source of nonconformity [blockchain, database]
func recordNonconformity(number uint64, source string) {
fmt.Println(number, source)
/*
- number (uint64): Block number
- source (string): Source of nonconformity [blockchain, database]
*/
func (cb *CorruptBlocks) recordNonconformity(number uint64, source string) {
cb.Numbers = append(cb.Numbers, number)
fmt.Println(cb.Numbers)
}
func (r *HumbugReporter) submitReport(start, end uint64) error {
content, err := json.Marshal(corruptBlocks)
if err != nil {
return fmt.Errorf("Unable to marshal to json: %v", err)
}
report := humbug.Report{
Title: fmt.Sprintf("LDB verifier %d-%d", start, end),
Content: string(content),
Tags: []string{
fmt.Sprintf("start:%d", start),
fmt.Sprintf("end:%d", end),
},
}
r.Reporter.Publish(report)
fmt.Println("Error published")
return nil
}
func verify(start, end uint64) error {
for i := start; i <= end; i++ {
var cnt uint64 // Counter until report formed and sent to Humbug
for i := start; i < end; i++ {
header, err := localConnections.getBlockFromChain(i)
if err != nil {
fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err)
recordNonconformity(i, "blockchain")
corruptBlocks.recordNonconformity(i, "blockchain")
continue
}
block, err := localConnections.getBlockFromDB(header)
if err != nil {
fmt.Printf("Unable to get block: %d, err: %v\n", block.BlockNumber, err)
recordNonconformity(block.BlockNumber, "database")
corruptBlocks.recordNonconformity(i, "database")
continue
}
if header.Number.Uint64() != block.BlockNumber {
fmt.Printf("Incorrect %d block retrieved from database\n", block.BlockNumber)
recordNonconformity(block.BlockNumber, "database")
corruptBlocks.recordNonconformity(i, "database")
continue
}
@ -198,18 +235,31 @@ func verify(start, end uint64) error {
dbTxs, err := localConnections.getTxsFromDB(header.Number.Uint64())
if err != nil {
fmt.Printf("Unable to get transactions: %d, err: %v\n", block.BlockNumber, err)
recordNonconformity(block.BlockNumber, "database")
corruptBlocks.recordNonconformity(i, "database")
continue
}
if chainTxs.Quantity != dbTxs.Quantity {
fmt.Printf("Different number of transactions in block %d, err %v\n", block.BlockNumber, err)
recordNonconformity(block.BlockNumber, "database")
corruptBlocks.recordNonconformity(i, "database")
continue
}
fmt.Printf("Processed block number: %d\r", i)
time.Sleep(1 * time.Second)
cnt++
if cnt >= configs.BLOCK_RANGE_REPORT {
err := humbugReporter.submitReport(start, end)
if err != nil {
fmt.Printf("Unable to send humbug report: %v", err)
}
cnt = 0
}
}
err := humbugReporter.submitReport(start, end)
if err != nil {
fmt.Printf("Unable to send humbug report: %v", err)
}
fmt.Println("")

31
crawlers/ldb/cmd/cli.go
Wyświetl plik

@ -8,11 +8,18 @@ import (
"sort"
"strconv"
"github.com/bugout-dev/moonstream/crawlers/ldb/configs"
"github.com/bugout-dev/humbug/go/pkg"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/google/uuid"
"gopkg.in/urfave/cli.v1"
)
var (
localConnections *LocalConnections
humbugReporter *HumbugReporter
DataDirFlag = cli.StringFlag{
Name: "datadir",
Usage: "Data directory for the databases and keystore",
@ -25,6 +32,18 @@ var (
}
)
// Generate humbug client
func setHumbugClient(sessionID string) error {
consent := humbug.CreateHumbugConsent(humbug.True)
reporter, err := humbug.CreateHumbugReporter(consent, configs.HUMBUG_LDB_CLIENT_ID, sessionID, configs.HUMBUG_LDB_TOKEN)
if err != nil {
return fmt.Errorf("Unable to generate humbug reporter: %v", err)
}
humbugReporter.Reporter = reporter
return nil
}
// Parse start and end blocks from command line input
func startEndBlock(ctx *cli.Context) (uint64, uint64, error) {
start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32)
@ -39,8 +58,6 @@ func startEndBlock(ctx *cli.Context) (uint64, uint64, error) {
return start, end, nil
}
var localConnections *LocalConnections
func processAddCommand(ctx *cli.Context) error {
if ctx.NArg() != 2 {
return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage)
@ -168,8 +185,16 @@ func LDBCLI() {
sort.Sort(cli.CommandsByName(app.Commands))
localConnections = &LocalConnections{}
humbugReporter = &HumbugReporter{}
err := app.Run(os.Args)
// Humbug client to be able write data in Bugout journal
sessionID := uuid.New().String()
err := setHumbugClient(sessionID)
if err != nil {
log.Fatal(err)
}
err = app.Run(os.Args)
if err != nil {
log.Fatal(err)
}

19
crawlers/ldb/configs/settings.go
Wyświetl plik

@ -1,11 +1,30 @@
package configs
import (
"log"
"os"
"strconv"
"time"
)
var BLOCK_RANGE_REPORT uint64 = 100000
// Database configs
var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30
var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute
var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
// Humber configs
var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID")
var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN")
var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT")
func init() {
if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" {
ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 0, 16)
if err != nil {
log.Fatal(err)
}
BLOCK_RANGE_REPORT = ldbRangeReport
}
}

3
crawlers/ldb/go.mod
Wyświetl plik

@ -3,7 +3,9 @@ module github.com/bugout-dev/moonstream/crawlers/ldb
go 1.17
require (
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205
github.com/ethereum/go-ethereum v1.10.16
github.com/google/uuid v1.1.5
github.com/lib/pq v1.0.0
gopkg.in/urfave/cli.v1 v1.20.0
)
@ -22,7 +24,6 @@ require (
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.1.5 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/graph-gophers/graphql-go v1.3.0 // indirect
github.com/hashicorp/go-bexpr v0.1.10 // indirect

2
crawlers/ldb/go.sum
Wyświetl plik

@ -69,6 +69,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 h1:UQ7XGjvoOVKGRIuTFXgqGtU/UgMOk8+ikpoHWrWefjQ=
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205/go.mod h1:U/NXHfc3tzGeQz+xVfpifXdPZi7p6VV8xdP/4ZKeWJU=
github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=

3
crawlers/ldb/sample.env
Wyświetl plik

@ -1 +1,4 @@
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export HUMBUG_LDB_CLIENT_ID="<client_id_of_crawl_service>"
export HUMBUG_LDB_TOKEN="<generate_humbug_integration_with_token_at_https://bugout.dev/account/teams>"
export HUMBUG_LDB_BLOCK_RANGE_REPORT=100000