2022-03-03 20:11:40 +00:00
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"log"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"sort"
|
|
|
|
"strconv"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/cmd/utils"
|
2022-03-03 21:05:33 +00:00
|
|
|
"github.com/google/uuid"
|
2022-03-03 20:11:40 +00:00
|
|
|
"gopkg.in/urfave/cli.v1"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2022-03-06 18:51:36 +00:00
|
|
|
// Block steps used to prevent long executor tasks and data loss possibility
|
2022-03-06 18:10:55 +00:00
|
|
|
BlockNumberStep = uint64(1000)
|
|
|
|
|
2022-03-05 19:49:53 +00:00
|
|
|
BlockchainFlag = cli.StringFlag{
|
|
|
|
Name: "blockchain",
|
|
|
|
Usage: `Which blockchain to crawl ("ethereum", "polygon")`,
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
DataDirFlag = cli.StringFlag{
|
|
|
|
Name: "datadir",
|
|
|
|
Usage: "Data directory for the databases and keystore",
|
|
|
|
Value: "/home/ubuntu/nodes/ethereum",
|
|
|
|
}
|
|
|
|
GCModeFlag = cli.StringFlag{
|
|
|
|
Name: "gcmode",
|
|
|
|
Usage: `Blockchain garbage collection mode ("full", "archive")`,
|
|
|
|
Value: "full",
|
|
|
|
}
|
2022-03-06 22:59:47 +00:00
|
|
|
ThreadsFlag = cli.IntFlag{
|
|
|
|
Name: "threads",
|
|
|
|
Usage: "Number of threads to use",
|
|
|
|
Value: 2,
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
)
|
|
|
|
|
2022-03-06 18:10:55 +00:00
|
|
|
// Block step generator, yield list of blocks with length equal blockStep
|
2022-03-06 22:59:47 +00:00
|
|
|
// TODO(kompotkot): Not-safe method with slices in channel, re-write this function
|
2022-03-06 18:10:55 +00:00
|
|
|
func BlockYield(start, end, blockStep uint64) chan []uint64 {
|
|
|
|
ch := make(chan []uint64)
|
|
|
|
|
|
|
|
go func(ch chan []uint64) {
|
|
|
|
currentBlock := start
|
|
|
|
var tempEnd uint64
|
|
|
|
|
|
|
|
for {
|
|
|
|
if currentBlock >= end {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
tempEnd = currentBlock + blockStep
|
|
|
|
if tempEnd >= end {
|
|
|
|
tempEnd = end
|
|
|
|
}
|
|
|
|
|
|
|
|
var blocks []uint64
|
|
|
|
for i := currentBlock; i < tempEnd; i++ {
|
2022-03-06 23:33:28 +00:00
|
|
|
blocks = append(blocks, i) // Blocking operation
|
2022-03-06 18:10:55 +00:00
|
|
|
}
|
|
|
|
ch <- blocks
|
|
|
|
|
|
|
|
currentBlock += blockStep
|
|
|
|
}
|
|
|
|
|
|
|
|
close(ch)
|
|
|
|
}(ch)
|
|
|
|
|
|
|
|
return ch
|
|
|
|
}
|
|
|
|
|
2022-03-03 20:11:40 +00:00
|
|
|
// Parse start and end blocks from command line input
|
|
|
|
func startEndBlock(ctx *cli.Context) (uint64, uint64, error) {
|
2022-03-06 18:10:55 +00:00
|
|
|
inputStart, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32)
|
2022-03-03 20:11:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return 0, 0, err
|
|
|
|
}
|
2022-03-06 18:10:55 +00:00
|
|
|
inputEnd, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32)
|
2022-03-03 20:11:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return 0, 0, err
|
|
|
|
}
|
|
|
|
|
2022-03-06 18:10:55 +00:00
|
|
|
var start, end uint64
|
|
|
|
if inputStart < inputEnd {
|
|
|
|
start = inputStart
|
|
|
|
end = inputEnd
|
|
|
|
} else if inputStart > inputEnd {
|
|
|
|
start = inputEnd
|
|
|
|
end = inputStart
|
|
|
|
} else {
|
|
|
|
return 0, 0, fmt.Errorf("Start and end block can't be equal")
|
|
|
|
}
|
|
|
|
|
2022-03-03 20:11:40 +00:00
|
|
|
return start, end, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func processAddCommand(ctx *cli.Context) error {
|
|
|
|
if ctx.NArg() != 2 {
|
|
|
|
return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage)
|
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
blockchain := ctx.GlobalString(BlockchainFlag.Name)
|
|
|
|
if blockchain != "ethereum" && blockchain != "polygon" {
|
|
|
|
return fmt.Errorf("Unsupported blockchain provided")
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
start, end, err := startEndBlock(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to parse block range: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = setLocalChain(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to set blockchain: %v", err)
|
|
|
|
}
|
|
|
|
defer localConnections.Stack.Close()
|
|
|
|
defer localConnections.ChainDB.Close()
|
|
|
|
|
2022-03-05 19:49:53 +00:00
|
|
|
err = setDatabase()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to set database connection: %v", err)
|
|
|
|
}
|
|
|
|
|
2022-03-06 18:10:55 +00:00
|
|
|
for blocks := range BlockYield(start, end, BlockNumberStep) {
|
|
|
|
err = add(blockchain, blocks)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Error occurred due add acction: %v", err)
|
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
localConnections.Chain.Stop()
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func processShowCommand(ctx *cli.Context) error {
|
|
|
|
if ctx.NArg() != 2 {
|
|
|
|
return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage)
|
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
blockchain := ctx.GlobalString(BlockchainFlag.Name)
|
|
|
|
if blockchain != "ethereum" && blockchain != "polygon" {
|
|
|
|
return fmt.Errorf("Unsupported blockchain provided")
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
start, end, err := startEndBlock(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to parse block range: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = setLocalChain(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to set blockchain: %v", err)
|
|
|
|
}
|
|
|
|
defer localConnections.Stack.Close()
|
|
|
|
defer localConnections.ChainDB.Close()
|
|
|
|
|
2022-03-06 18:10:55 +00:00
|
|
|
for blocks := range BlockYield(start, end, BlockNumberStep) {
|
|
|
|
err = show(blocks)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Error occurred due show acction: %v", err)
|
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
localConnections.Chain.Stop()
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func processVerifyCommand(ctx *cli.Context) error {
|
|
|
|
if ctx.NArg() != 2 {
|
|
|
|
return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage)
|
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
blockchain := ctx.GlobalString(BlockchainFlag.Name)
|
|
|
|
if blockchain != "ethereum" && blockchain != "polygon" {
|
|
|
|
return fmt.Errorf("Unsupported blockchain provided")
|
|
|
|
}
|
2022-03-06 22:59:47 +00:00
|
|
|
threads := ctx.GlobalInt(ThreadsFlag.Name)
|
|
|
|
if threads <= 0 {
|
|
|
|
threads = 1
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
start, end, err := startEndBlock(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to parse block range: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = setLocalChain(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to set blockchain: %v", err)
|
|
|
|
}
|
|
|
|
defer localConnections.Stack.Close()
|
|
|
|
defer localConnections.ChainDB.Close()
|
|
|
|
|
|
|
|
err = setDatabase()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to set database connection: %v", err)
|
|
|
|
}
|
|
|
|
|
2022-03-06 18:10:55 +00:00
|
|
|
for blocks := range BlockYield(start, end, BlockNumberStep) {
|
2022-03-06 22:59:47 +00:00
|
|
|
err = verify(blockchain, blocks, threads)
|
2022-03-06 18:10:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Error occurred due verify acction: %v", err)
|
|
|
|
}
|
2022-03-06 18:51:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
err = humbugReporter.submitReport(start, end, "Total ")
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to send humbug report: %v", err)
|
2022-03-05 19:49:53 +00:00
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
|
|
|
localConnections.Chain.Stop()
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func LDBCLI() {
|
|
|
|
app := cli.NewApp()
|
|
|
|
app.Name = filepath.Base(os.Args[0])
|
|
|
|
app.Author = "Bugout.dev"
|
|
|
|
app.Email = "engineering@bugout.dev"
|
|
|
|
app.Usage = "blockchain ldb extractor command line interface"
|
|
|
|
app.Flags = []cli.Flag{
|
2022-03-05 19:49:53 +00:00
|
|
|
BlockchainFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
DataDirFlag,
|
|
|
|
GCModeFlag,
|
2022-03-06 22:59:47 +00:00
|
|
|
ThreadsFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
}
|
2022-03-05 19:49:53 +00:00
|
|
|
|
2022-03-03 20:11:40 +00:00
|
|
|
app.Commands = []cli.Command{
|
|
|
|
{
|
|
|
|
Name: "add",
|
|
|
|
Action: utils.MigrateFlags(processAddCommand),
|
|
|
|
ArgsUsage: "<start_block> <end_block>",
|
|
|
|
Usage: "Add new blocks with transactions to database",
|
|
|
|
Description: "This command request blocks from blockchain and adds to database.",
|
|
|
|
Flags: []cli.Flag{
|
2022-03-05 19:49:53 +00:00
|
|
|
BlockchainFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
DataDirFlag,
|
|
|
|
GCModeFlag,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "show",
|
|
|
|
Action: utils.MigrateFlags(processShowCommand),
|
|
|
|
ArgsUsage: "<start_block> <end_block>",
|
|
|
|
Usage: "Show block with transactions",
|
|
|
|
Description: "This command print out requested blocks.",
|
|
|
|
Flags: []cli.Flag{
|
2022-03-05 19:49:53 +00:00
|
|
|
BlockchainFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
DataDirFlag,
|
|
|
|
GCModeFlag,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Name: "verify",
|
|
|
|
Action: utils.MigrateFlags(processVerifyCommand),
|
|
|
|
ArgsUsage: "<start_block> <end_block>",
|
|
|
|
Usage: "Verify blocks with transactions at database",
|
|
|
|
Description: "This command compare blocks in database and in blockchain for difference.",
|
|
|
|
Flags: []cli.Flag{
|
2022-03-05 19:49:53 +00:00
|
|
|
BlockchainFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
DataDirFlag,
|
|
|
|
GCModeFlag,
|
2022-03-06 22:59:47 +00:00
|
|
|
ThreadsFlag,
|
2022-03-03 20:11:40 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
sort.Sort(cli.FlagsByName(app.Flags))
|
|
|
|
sort.Sort(cli.CommandsByName(app.Commands))
|
|
|
|
|
2022-03-05 19:13:42 +00:00
|
|
|
// Initialize local connections
|
2022-03-03 20:11:40 +00:00
|
|
|
localConnections = &LocalConnections{}
|
2022-03-03 21:05:33 +00:00
|
|
|
|
2022-03-05 19:13:42 +00:00
|
|
|
// Initialize humbug client to be able write data in Bugout journal
|
|
|
|
humbugReporter = &HumbugReporter{}
|
2022-03-03 21:05:33 +00:00
|
|
|
sessionID := uuid.New().String()
|
|
|
|
err := setHumbugClient(sessionID)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
2022-03-03 20:11:40 +00:00
|
|
|
|
2022-03-03 21:05:33 +00:00
|
|
|
err = app.Run(os.Args)
|
2022-03-03 20:11:40 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|