kopia lustrzana https://github.com/bugout-dev/moonstream
Verification command in goroutines
rodzic
39c391dfa0
commit
838c75ae29
|
@ -73,50 +73,117 @@ func show(blockNumbers []uint64) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO(kompotkot): Find way to remove Number
|
||||||
|
type Result struct {
|
||||||
|
ErrorOutput string
|
||||||
|
ErrorSource string
|
||||||
|
Number uint64
|
||||||
|
Output string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Job struct {
|
||||||
|
BlockNumber uint64
|
||||||
|
Results chan<- Result
|
||||||
|
}
|
||||||
|
|
||||||
// Run verification flow of blockchain with database data
|
// Run verification flow of blockchain with database data
|
||||||
func verify(blockchain string, blockNumbers []uint64) error {
|
func verify(blockchain string, blockNumbers []uint64, workers int) error {
|
||||||
for _, bn := range blockNumbers {
|
jobsCh := make(chan Job, workers)
|
||||||
chainBlock, err := localConnections.getChainBlock(bn)
|
resultCh := make(chan Result, len(blockNumbers))
|
||||||
if err != nil {
|
doneCh := make(chan struct{}, workers)
|
||||||
description := fmt.Sprintf("Unable to get block: %d from chain, err %v", bn, err)
|
|
||||||
fmt.Println(description)
|
// Add jobs
|
||||||
corruptBlocks.registerCorruptBlock(bn, "blockchain", description)
|
go func() {
|
||||||
continue
|
for _, bn := range blockNumbers {
|
||||||
|
jobsCh <- Job{
|
||||||
|
BlockNumber: bn,
|
||||||
|
Results: resultCh,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
close(jobsCh)
|
||||||
|
}()
|
||||||
|
|
||||||
dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String())
|
for i := 0; i < workers; i++ {
|
||||||
|
// Do jobs
|
||||||
|
go func() {
|
||||||
|
for job := range jobsCh {
|
||||||
|
chainBlock, err := localConnections.getChainBlock(job.BlockNumber)
|
||||||
|
if err != nil {
|
||||||
|
job.Results <- Result{
|
||||||
|
ErrorOutput: fmt.Sprintf("Unable to get block: %d from chain, err %v", job.BlockNumber, err),
|
||||||
|
ErrorSource: "blockchain",
|
||||||
|
Number: job.BlockNumber,
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String())
|
||||||
description := fmt.Sprintf("Unable to get block: %d, err: %v", bn, err)
|
|
||||||
fmt.Println(description)
|
if err != nil {
|
||||||
corruptBlocks.registerCorruptBlock(bn, "database", description)
|
job.Results <- Result{
|
||||||
continue
|
ErrorOutput: fmt.Sprintf("Unable to get block: %d, err: %v", job.BlockNumber, err),
|
||||||
|
ErrorSource: "database",
|
||||||
|
Number: job.BlockNumber,
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if dbBlock.Number == nil {
|
||||||
|
job.Results <- Result{
|
||||||
|
ErrorOutput: fmt.Sprintf("Block %d not presented in database", job.BlockNumber),
|
||||||
|
ErrorSource: "database",
|
||||||
|
Number: job.BlockNumber,
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if chainBlock.NumberU64() != dbBlock.Number.Uint64() {
|
||||||
|
job.Results <- Result{
|
||||||
|
ErrorOutput: fmt.Sprintf("Incorrect %d block retrieved from database", job.BlockNumber),
|
||||||
|
ErrorSource: "database",
|
||||||
|
Number: job.BlockNumber,
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
chainTxs := localConnections.getChainTxs(chainBlock.Hash(), job.BlockNumber)
|
||||||
|
|
||||||
|
if len(chainTxs) != len(dbBlock.Transactions) {
|
||||||
|
job.Results <- Result{
|
||||||
|
ErrorOutput: fmt.Sprintf("Different number of transactions in block %d, err %v", job.BlockNumber, err),
|
||||||
|
ErrorSource: "database",
|
||||||
|
Number: job.BlockNumber,
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
job.Results <- Result{
|
||||||
|
Output: fmt.Sprintf("Processed block number: %d", job.BlockNumber),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
doneCh <- struct{}{}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Await completion
|
||||||
|
go func() {
|
||||||
|
for i := 0; i < workers; i++ {
|
||||||
|
<-doneCh
|
||||||
}
|
}
|
||||||
|
close(resultCh)
|
||||||
|
}()
|
||||||
|
|
||||||
if dbBlock.Number == nil {
|
for result := range resultCh {
|
||||||
description := fmt.Sprintf("Block %d not presented in database", bn)
|
if result.ErrorOutput != "" {
|
||||||
fmt.Println(description)
|
fmt.Println(result.ErrorOutput)
|
||||||
corruptBlocks.registerCorruptBlock(bn, "database", description)
|
corruptBlocks.registerCorruptBlock(result.Number, result.ErrorSource, result.ErrorOutput)
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
if result.Output != "" {
|
||||||
if chainBlock.NumberU64() != dbBlock.Number.Uint64() {
|
fmt.Println(result.Output)
|
||||||
description := fmt.Sprintf("Incorrect %d block retrieved from database", bn)
|
|
||||||
fmt.Println(description)
|
|
||||||
corruptBlocks.registerCorruptBlock(bn, "database", description)
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
if result.Output != "" && result.ErrorOutput != "" {
|
||||||
chainTxs := localConnections.getChainTxs(chainBlock.Hash(), bn)
|
fmt.Printf("Unprocessable result with error: %s and output: %s", result.ErrorOutput, result.Output)
|
||||||
|
|
||||||
if len(chainTxs) != len(dbBlock.Transactions) {
|
|
||||||
description := fmt.Sprintf("Different number of transactions in block %d, err %v", bn, err)
|
|
||||||
fmt.Println(description)
|
|
||||||
corruptBlocks.registerCorruptBlock(bn, "database", description)
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Processed block number: %d\r", bn)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/bugout-dev/moonstream/crawlers/ldb/configs"
|
// "github.com/bugout-dev/moonstream/crawlers/ldb/configs"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
@ -33,9 +33,15 @@ var (
|
||||||
Usage: `Blockchain garbage collection mode ("full", "archive")`,
|
Usage: `Blockchain garbage collection mode ("full", "archive")`,
|
||||||
Value: "full",
|
Value: "full",
|
||||||
}
|
}
|
||||||
|
ThreadsFlag = cli.IntFlag{
|
||||||
|
Name: "threads",
|
||||||
|
Usage: "Number of threads to use",
|
||||||
|
Value: 2,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// Block step generator, yield list of blocks with length equal blockStep
|
// Block step generator, yield list of blocks with length equal blockStep
|
||||||
|
// TODO(kompotkot): Not-safe method with slices in channel, re-write this function
|
||||||
func BlockYield(start, end, blockStep uint64) chan []uint64 {
|
func BlockYield(start, end, blockStep uint64) chan []uint64 {
|
||||||
ch := make(chan []uint64)
|
ch := make(chan []uint64)
|
||||||
|
|
||||||
|
@ -173,6 +179,10 @@ func processVerifyCommand(ctx *cli.Context) error {
|
||||||
if blockchain != "ethereum" && blockchain != "polygon" {
|
if blockchain != "ethereum" && blockchain != "polygon" {
|
||||||
return fmt.Errorf("Unsupported blockchain provided")
|
return fmt.Errorf("Unsupported blockchain provided")
|
||||||
}
|
}
|
||||||
|
threads := ctx.GlobalInt(ThreadsFlag.Name)
|
||||||
|
if threads <= 0 {
|
||||||
|
threads = 1
|
||||||
|
}
|
||||||
|
|
||||||
start, end, err := startEndBlock(ctx)
|
start, end, err := startEndBlock(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -191,23 +201,11 @@ func processVerifyCommand(ctx *cli.Context) error {
|
||||||
return fmt.Errorf("Unable to set database connection: %v", err)
|
return fmt.Errorf("Unable to set database connection: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cnt := uint64(0)
|
|
||||||
reportStart := uint64(start)
|
|
||||||
for blocks := range BlockYield(start, end, BlockNumberStep) {
|
for blocks := range BlockYield(start, end, BlockNumberStep) {
|
||||||
err = verify(blockchain, blocks)
|
err = verify(blockchain, blocks, threads)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error occurred due verify acction: %v", err)
|
return fmt.Errorf("Error occurred due verify acction: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cnt += BlockNumberStep
|
|
||||||
if cnt >= configs.BLOCK_RANGE_REPORT {
|
|
||||||
err := humbugReporter.submitReport(reportStart, blocks[len(blocks)-1]+1, "")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Unable to send humbug report: %v", err)
|
|
||||||
}
|
|
||||||
reportStart = blocks[len(blocks)-1] + 1
|
|
||||||
cnt = 0
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = humbugReporter.submitReport(start, end, "Total ")
|
err = humbugReporter.submitReport(start, end, "Total ")
|
||||||
|
@ -230,6 +228,7 @@ func LDBCLI() {
|
||||||
BlockchainFlag,
|
BlockchainFlag,
|
||||||
DataDirFlag,
|
DataDirFlag,
|
||||||
GCModeFlag,
|
GCModeFlag,
|
||||||
|
ThreadsFlag,
|
||||||
}
|
}
|
||||||
|
|
||||||
app.Commands = []cli.Command{
|
app.Commands = []cli.Command{
|
||||||
|
@ -267,6 +266,7 @@ func LDBCLI() {
|
||||||
BlockchainFlag,
|
BlockchainFlag,
|
||||||
DataDirFlag,
|
DataDirFlag,
|
||||||
GCModeFlag,
|
GCModeFlag,
|
||||||
|
ThreadsFlag,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,8 +5,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var BLOCK_RANGE_REPORT uint64 = 100000
|
|
||||||
|
|
||||||
// Database configs
|
// Database configs
|
||||||
var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30
|
var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30
|
||||||
var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute
|
var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute
|
||||||
|
|
Ładowanie…
Reference in New Issue