mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Update access CLI and new client keys to support call limits

pull/786/head
kompotkot 2023-05-18 14:16:10 +00:00
rodzic 759a51db46
commit 40c9c21777
8 zmienionych plików z 203 dodań i 32 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

202
nodes/node_balancer/cmd/nodebalancer/cli.go
Wyświetl plik

@ -6,6 +6,7 @@ import (
"fmt"
"log"
"os"
"time"
bugout "github.com/bugout-dev/bugout-go/pkg"
"github.com/google/uuid"
@ -16,11 +17,19 @@ var (
stateCLI StateCLI
bugoutClient bugout.BugoutClient
DEFAULT_ACCESS_NAME = ""
DEFAULT_ACCESS_DESCRIPTION = ""
DEFAULT_BLOCKCHAIN_ACCESS = true
DEFAULT_EXTENDED_METHODS = true
DEFAULT_PERIOD_DURATION = int64(86400) // 1 day
DEFAULT_MAX_CALLS_PER_PERIOD = int64(10000)
)
// Command Line Interface state
type StateCLI struct {
addAccessCmd *flag.FlagSet
updateAccessCmd *flag.FlagSet
generateConfigCmd *flag.FlagSet
deleteAccessCmd *flag.FlagSet
serverCmd *flag.FlagSet
@ -31,13 +40,21 @@ type StateCLI struct {
configPathFlag string
helpFlag bool
// Add user access flags
// Add/update user access flags
userIDFlag string
accessIDFlag string
accessNameFlag string
accessDescriptionFlag string
blockchainAccessFlag bool
extendedMethodsFlag bool
blockchainAccessFlag bool
extendedMethodsFlag bool
PeriodDurationFlag int64
MaxCallsPerPeriodFlag int64
// Update user access flags
PeriodStartTsFlag int64
CallsPerPeriodFlag int64
// Server flags
listeningAddrFlag string
@ -58,8 +75,8 @@ optional arguments:
-h, --help show this help message and exit
subcommands:
{%[1]s,%[2]s,%[3]s,%[4]s,%[5]s,%[6]s}
`, s.addAccessCmd.Name(), s.generateConfigCmd.Name(), s.deleteAccessCmd.Name(), s.serverCmd.Name(), s.usersCmd.Name(), s.versionCmd.Name())
{%[1]s,%[2]s,%[3]s,%[4]s,%[5]s,%[6]s,%[7]s}
`, s.addAccessCmd.Name(), s.updateAccessCmd.Name(), s.generateConfigCmd.Name(), s.deleteAccessCmd.Name(), s.serverCmd.Name(), s.usersCmd.Name(), s.versionCmd.Name())
}
// Check if required flags are set
@ -67,15 +84,19 @@ func (s *StateCLI) checkRequirements() {
if s.helpFlag {
switch {
case s.addAccessCmd.Parsed():
fmt.Printf("Add new user access token\n\n")
fmt.Printf("Add new user access resource\n\n")
s.addAccessCmd.PrintDefaults()
os.Exit(0)
case s.updateAccessCmd.Parsed():
fmt.Printf("Update user access resource\n\n")
s.updateAccessCmd.PrintDefaults()
os.Exit(0)
case s.generateConfigCmd.Parsed():
fmt.Printf("Generate new configuration\n\n")
s.generateConfigCmd.PrintDefaults()
os.Exit(0)
case s.deleteAccessCmd.Parsed():
fmt.Printf("Delete user access token\n\n")
fmt.Printf("Delete user access resource\n\n")
s.deleteAccessCmd.PrintDefaults()
os.Exit(0)
case s.serverCmd.Parsed():
@ -111,6 +132,12 @@ func (s *StateCLI) checkRequirements() {
s.addAccessCmd.PrintDefaults()
os.Exit(1)
}
case s.updateAccessCmd.Parsed():
if s.userIDFlag == "" && s.accessIDFlag == "" {
fmt.Printf("User ID or access ID should be specified\n\n")
s.updateAccessCmd.PrintDefaults()
os.Exit(1)
}
case s.deleteAccessCmd.Parsed():
if s.userIDFlag == "" && s.accessIDFlag == "" {
fmt.Printf("User or access ID flag should be specified\n\n")
@ -146,6 +173,7 @@ func (s *StateCLI) checkRequirements() {
func (s *StateCLI) populateCLI() {
// Subcommands setup
s.addAccessCmd = flag.NewFlagSet("add-access", flag.ExitOnError)
s.updateAccessCmd = flag.NewFlagSet("update-access", flag.ExitOnError)
s.generateConfigCmd = flag.NewFlagSet("generate-config", flag.ExitOnError)
s.deleteAccessCmd = flag.NewFlagSet("delete-access", flag.ExitOnError)
s.serverCmd = flag.NewFlagSet("server", flag.ExitOnError)
@ -153,22 +181,29 @@ func (s *StateCLI) populateCLI() {
s.versionCmd = flag.NewFlagSet("version", flag.ExitOnError)
// Common flag pointers
for _, fs := range []*flag.FlagSet{s.addAccessCmd, s.generateConfigCmd, s.deleteAccessCmd, s.serverCmd, s.usersCmd, s.versionCmd} {
for _, fs := range []*flag.FlagSet{s.addAccessCmd, s.updateAccessCmd, s.generateConfigCmd, s.deleteAccessCmd, s.serverCmd, s.usersCmd, s.versionCmd} {
fs.BoolVar(&s.helpFlag, "help", false, "Show help message")
fs.StringVar(&s.configPathFlag, "config", "", "Path to configuration file (default: ~/.nodebalancer/config.txt)")
fs.StringVar(&s.configPathFlag, "config", "", "Path to configuration file (default: ~/.nodebalancer/config.json)")
}
// Add, delete and list user access subcommand flag pointers
for _, fs := range []*flag.FlagSet{s.addAccessCmd, s.deleteAccessCmd, s.usersCmd} {
for _, fs := range []*flag.FlagSet{s.addAccessCmd, s.updateAccessCmd, s.deleteAccessCmd, s.usersCmd} {
fs.StringVar(&s.userIDFlag, "user-id", "", "Bugout user ID")
fs.StringVar(&s.accessIDFlag, "access-id", "", "UUID for access identification")
}
// Add user access subcommand flag pointers
s.addAccessCmd.StringVar(&s.accessNameFlag, "name", "", "Name of access")
s.addAccessCmd.StringVar(&s.accessDescriptionFlag, "description", "", "Description of access")
s.addAccessCmd.BoolVar(&s.blockchainAccessFlag, "blockchain-access", false, "Provide if allow direct access to blockchain nodes")
s.addAccessCmd.BoolVar(&s.extendedMethodsFlag, "extended-methods", false, "Provide to be able to execute not whitelisted methods")
// Add/update user access subcommand flag pointers
for _, fs := range []*flag.FlagSet{s.addAccessCmd, s.updateAccessCmd} {
fs.StringVar(&s.accessNameFlag, "name", DEFAULT_ACCESS_NAME, fmt.Sprintf("Name of access (default: %s)", DEFAULT_ACCESS_NAME))
fs.StringVar(&s.accessDescriptionFlag, "description", DEFAULT_ACCESS_DESCRIPTION, fmt.Sprintf("Description of access (default: %s)", DEFAULT_ACCESS_DESCRIPTION))
fs.BoolVar(&s.blockchainAccessFlag, "blockchain-access", DEFAULT_BLOCKCHAIN_ACCESS, fmt.Sprintf("Specify this flag to grant direct access to blockchain nodes (default: %t)", DEFAULT_BLOCKCHAIN_ACCESS))
fs.BoolVar(&s.extendedMethodsFlag, "extended-methods", DEFAULT_EXTENDED_METHODS, fmt.Sprintf("Specify this flag to grant execution availability to not whitelisted methods (default: %t)", DEFAULT_EXTENDED_METHODS))
fs.Int64Var(&s.PeriodDurationFlag, "period-duration", DEFAULT_PERIOD_DURATION, fmt.Sprintf("Access period duration in seconds (default: %d)", DEFAULT_PERIOD_DURATION))
fs.Int64Var(&s.MaxCallsPerPeriodFlag, "max-calls-per-period", DEFAULT_MAX_CALLS_PER_PERIOD, fmt.Sprintf("Max available calls to node during the period (default: %d)", DEFAULT_MAX_CALLS_PER_PERIOD))
}
s.updateAccessCmd.Int64Var(&s.PeriodStartTsFlag, "period-start-ts", 0, "When period starts in unix timestamp format (default: now)")
s.updateAccessCmd.Int64Var(&s.CallsPerPeriodFlag, "calls-per-period", 0, "Current number of calls to node during the period (default: 0)")
// Server subcommand flag pointers
s.serverCmd.StringVar(&s.listeningAddrFlag, "host", "127.0.0.1", "Server listening address")
@ -198,6 +233,10 @@ func cli() {
// Parse subcommands and appropriate FlagSet
switch os.Args[1] {
case "generate-config":
stateCLI.generateConfigCmd.Parse(os.Args[2:])
stateCLI.checkRequirements()
case "add-access":
stateCLI.addAccessCmd.Parse(os.Args[2:])
stateCLI.checkRequirements()
@ -209,6 +248,11 @@ func cli() {
Description: stateCLI.accessDescriptionFlag,
BlockchainAccess: stateCLI.blockchainAccessFlag,
ExtendedMethods: stateCLI.extendedMethodsFlag,
PeriodDuration: stateCLI.PeriodDurationFlag,
PeriodStartTs: time.Now().Unix(),
MaxCallsPerPeriod: stateCLI.MaxCallsPerPeriodFlag,
CallsPerPeriod: 0,
}
_, err := bugoutClient.Brood.FindUser(
NB_CONTROLLER_TOKEN,
@ -226,17 +270,128 @@ func cli() {
fmt.Printf("Unable to create user access, err: %v\n", err)
os.Exit(1)
}
resource_data, err := json.Marshal(resource.ResourceData)
resourceData, err := json.Marshal(resource.ResourceData)
if err != nil {
fmt.Printf("Unable to encode resource %s data interface to json, err: %v\n", resource.Id, err)
os.Exit(1)
}
var newUserAccess ClientResourceData
err = json.Unmarshal(resourceData, &newUserAccess)
if err != nil {
fmt.Printf("Unable to decode resource %s data json to structure, err: %v\n", resource.Id, err)
os.Exit(1)
}
newUserAccess.ResourceID = resource.Id
userAccessJson, err := json.Marshal(newUserAccess)
if err != nil {
fmt.Printf("Unable to encode resource %s data interface to json, err: %v", resource.Id, err)
os.Exit(1)
}
fmt.Println(string(resource_data))
fmt.Println(string(userAccessJson))
case "generate-config":
stateCLI.generateConfigCmd.Parse(os.Args[2:])
case "update-access":
stateCLI.updateAccessCmd.Parse(os.Args[2:])
stateCLI.checkRequirements()
queryParameters := make(map[string]string)
if stateCLI.userIDFlag != "" {
queryParameters["user_id"] = stateCLI.userIDFlag
}
if stateCLI.accessIDFlag != "" {
queryParameters["access_id"] = stateCLI.accessIDFlag
}
resources, err := bugoutClient.Brood.GetResources(
NB_CONTROLLER_TOKEN,
NB_APPLICATION_ID,
queryParameters,
)
if err != nil {
fmt.Printf("Unable to get Bugout resources, err: %v\n", err)
os.Exit(1)
}
resourcesLen := len(resources.Resources)
if resourcesLen == 0 {
fmt.Printf("There are no access resource with provided user-id %s or access-id %s\n", stateCLI.userIDFlag, stateCLI.accessIDFlag)
os.Exit(1)
}
if resourcesLen > 1 {
fmt.Printf("There are several %d access resources with provided user-id %s or access-id %s\n", resourcesLen, stateCLI.userIDFlag, stateCLI.accessIDFlag)
os.Exit(1)
}
resource := resources.Resources[0]
resource_data, err := json.Marshal(resource.ResourceData)
if err != nil {
fmt.Printf("Unable to encode resource %s data interface to json, err: %v\n", resource.Id, err)
os.Exit(1)
}
var currentUserAccess ClientResourceData
err = json.Unmarshal(resource_data, &currentUserAccess)
if err != nil {
fmt.Printf("Unable to decode resource %s data json to structure, err: %v\n", resource.Id, err)
os.Exit(1)
}
currentUserAccess.ResourceID = resource.Id
// TODO(kompotkot): Since we are using bool flags I moved with ugly solution.
// Let's find better one when have free time or will re-write flag Set.
update := make(map[string]interface{})
if stateCLI.accessNameFlag != currentUserAccess.Name && stateCLI.accessNameFlag != DEFAULT_ACCESS_NAME {
update["name"] = stateCLI.accessNameFlag
}
if stateCLI.accessDescriptionFlag != currentUserAccess.Description && stateCLI.accessDescriptionFlag != DEFAULT_ACCESS_DESCRIPTION {
update["description"] = stateCLI.accessDescriptionFlag
}
if stateCLI.blockchainAccessFlag != currentUserAccess.BlockchainAccess && stateCLI.blockchainAccessFlag != DEFAULT_BLOCKCHAIN_ACCESS {
update["blockchain_access"] = stateCLI.blockchainAccessFlag
}
if stateCLI.extendedMethodsFlag != currentUserAccess.ExtendedMethods && stateCLI.extendedMethodsFlag != DEFAULT_EXTENDED_METHODS {
update["extended_methods"] = stateCLI.extendedMethodsFlag
}
if stateCLI.PeriodDurationFlag != currentUserAccess.PeriodDuration && stateCLI.PeriodDurationFlag != DEFAULT_PERIOD_DURATION {
update["period_duration"] = stateCLI.PeriodDurationFlag
}
if stateCLI.MaxCallsPerPeriodFlag != currentUserAccess.MaxCallsPerPeriod && stateCLI.MaxCallsPerPeriodFlag != DEFAULT_MAX_CALLS_PER_PERIOD {
update["max_calls_per_period"] = stateCLI.MaxCallsPerPeriodFlag
}
if stateCLI.PeriodStartTsFlag != currentUserAccess.PeriodStartTs && stateCLI.PeriodStartTsFlag != 0 {
update["period_start_ts"] = stateCLI.PeriodStartTsFlag
}
if stateCLI.CallsPerPeriodFlag != currentUserAccess.CallsPerPeriod && stateCLI.CallsPerPeriodFlag != 0 {
update["calls_per_period"] = stateCLI.CallsPerPeriodFlag
}
updatedResource, err := bugoutClient.Brood.UpdateResource(
NB_CONTROLLER_TOKEN,
resource.Id,
update,
[]string{},
)
if err != nil {
fmt.Printf("Unable to update Bugout resource, err: %v\n", err)
os.Exit(1)
}
updatedResourceData, err := json.Marshal(updatedResource.ResourceData)
if err != nil {
fmt.Printf("Unable to encode resource %s data interface to json, err: %v\n", resource.Id, err)
os.Exit(1)
}
var updatedUserAccess ClientResourceData
err = json.Unmarshal(updatedResourceData, &updatedUserAccess)
if err != nil {
fmt.Printf("Unable to decode resource %s data json to structure, err: %v\n", resource.Id, err)
os.Exit(1)
}
updatedUserAccess.ResourceID = updatedResource.Id
userAccessJson, err := json.Marshal(updatedUserAccess)
if err != nil {
fmt.Printf("Unable to marshal user access struct, err: %v\n", err)
os.Exit(1)
}
fmt.Println(string(userAccessJson))
case "delete-access":
stateCLI.deleteAccessCmd.Parse(os.Args[2:])
stateCLI.checkRequirements()
@ -265,18 +420,19 @@ func cli() {
fmt.Printf("Unable to delete resource %s, err: %v\n", resource.Id, err)
continue
}
resource_data, err := json.Marshal(deletedResource.ResourceData)
deletedResourceData, err := json.Marshal(deletedResource.ResourceData)
if err != nil {
fmt.Printf("Unable to encode resource %s data interface to json, err: %v\n", resource.Id, err)
continue
}
var userAccess ClientResourceData
err = json.Unmarshal(resource_data, &userAccess)
var deletedUserAccess ClientResourceData
err = json.Unmarshal(deletedResourceData, &deletedUserAccess)
if err != nil {
fmt.Printf("Unable to decode resource %s data json to structure, err: %v\n", resource.Id, err)
continue
}
userAccesses = append(userAccesses, userAccess)
deletedUserAccess.ResourceID = deletedResource.Id
userAccesses = append(userAccesses, deletedUserAccess)
}
userAccessesJson, err := json.Marshal(userAccesses)

11
nodes/node_balancer/cmd/nodebalancer/clients.go
Wyświetl plik

@ -12,6 +12,8 @@ var (
// Structure to define user access according with Brood resources
type ClientResourceData struct {
ResourceID string `json:"resource_id"`
UserID string `json:"user_id"`
AccessID string `json:"access_id"`
Name string `json:"name"`
@ -19,7 +21,14 @@ type ClientResourceData struct {
BlockchainAccess bool `json:"blockchain_access"`
ExtendedMethods bool `json:"extended_methods"`
LastAccessTs int64 `json:"last_access_ts"`
PeriodDuration int64 `json:"period_duration"`
PeriodStartTs int64 `json:"period_start_ts"`
MaxCallsPerPeriod int64 `json:"max_calls_per_period"`
CallsPerPeriod int64 `json:"calls_per_period"`
LastAccessTs int64 `json:"last_access_ts"`
LastSessionAccessTs int64 `json:"last_session_access_ts"` // When last session with nodebalancer where started
LastSessionCallsCounter int64 `json:"last_session_calls_counter"`
dataSource string
}

1
nodes/node_balancer/cmd/nodebalancer/configs.go
Wyświetl plik

@ -32,6 +32,7 @@ var (
NB_CACHE_CLEANING_INTERVAL = time.Second * 10
NB_CACHE_ACCESS_ID_LIFETIME = int64(120)
NB_CACHE_ACCESS_ID_SESSION_LIFETIME = int64(600)
NB_MAX_COUNTER_NUMBER = uint64(10000000)

9
nodes/node_balancer/cmd/nodebalancer/middleware.go
Wyświetl plik

@ -29,6 +29,7 @@ type AccessCache struct {
mux sync.RWMutex
}
// CreateAccessCache generates empty cache of client access
func CreateAccessCache() {
accessIdCache = AccessCache{
accessIds: make(map[string]ClientResourceData),
@ -65,6 +66,8 @@ func (ac *AccessCache) UpdateAccessIdAtCache(accessId, dataSource string) {
// Add new access id with data to cache
func (ac *AccessCache) AddAccessIdToCache(clientResourceData ClientResourceData, dataSource string) {
tsNow := time.Now().Unix()
ac.mux.Lock()
ac.accessIds[clientResourceData.AccessID] = ClientResourceData{
UserID: clientResourceData.UserID,
@ -74,7 +77,8 @@ func (ac *AccessCache) AddAccessIdToCache(clientResourceData ClientResourceData,
BlockchainAccess: clientResourceData.BlockchainAccess,
ExtendedMethods: clientResourceData.ExtendedMethods,
LastAccessTs: time.Now().Unix(),
LastAccessTs: tsNow,
LastSessionAccessTs: tsNow,
dataSource: dataSource,
}
@ -90,6 +94,9 @@ func (ac *AccessCache) Cleanup() (int64, int64) {
if tsNow-aData.LastAccessTs > NB_CACHE_ACCESS_ID_LIFETIME {
delete(ac.accessIds, aId)
removedAccessIds++
} else if tsNow-aData.LastSessionAccessTs > NB_CACHE_ACCESS_ID_SESSION_LIFETIME {
delete(ac.accessIds, aId)
removedAccessIds++
} else {
totalAccessIds++
}

1
nodes/node_balancer/cmd/nodebalancer/server.go
Wyświetl plik

@ -118,6 +118,7 @@ func Server() {
// Record system information
reporter.Publish(humbug.SystemReport())
// Fetch access id for internal usage (crawlers, infrastructure, etc)
resources, err := bugoutClient.Brood.GetResources(
NB_CONTROLLER_TOKEN,
NB_APPLICATION_ID,

2
nodes/node_balancer/cmd/nodebalancer/version.go
Wyświetl plik

@ -1,3 +1,3 @@
package main
var NB_VERSION = "0.2.1"
var NB_VERSION = "0.2.2"

3
nodes/node_balancer/go.mod
Wyświetl plik

@ -3,8 +3,7 @@ module github.com/bugout-dev/moonstream/nodes/node_balancer
go 1.17
require (
github.com/bugout-dev/bugout-go v0.3.4
github.com/bugout-dev/bugout-go v0.4.1
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205
github.com/google/uuid v1.3.0
github.com/lib/pq v1.10.4
)

6
nodes/node_balancer/go.sum
Wyświetl plik

@ -23,8 +23,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bugout-dev/bugout-go v0.3.4 h1:UJVaXv7ACcChoYIl0Zx38axV65s2vLH2kWZ76H/YK2s=
github.com/bugout-dev/bugout-go v0.3.4/go.mod h1:P4+788iHtt/32u2wIaRTaiXTWpvSVBYxZ01qQ8N7eB8=
github.com/bugout-dev/bugout-go v0.4.1 h1:idZ4k+/skHj217/q8OmHBoYdzwJrqCY5Vd7S8FM6zlo=
github.com/bugout-dev/bugout-go v0.4.1/go.mod h1:P4+788iHtt/32u2wIaRTaiXTWpvSVBYxZ01qQ8N7eB8=
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 h1:UQ7XGjvoOVKGRIuTFXgqGtU/UgMOk8+ikpoHWrWefjQ=
github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205/go.mod h1:U/NXHfc3tzGeQz+xVfpifXdPZi7p6VV8xdP/4ZKeWJU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@ -107,8 +107,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=