From fd643d15f0f44081a6469be7613b1162b8a29258 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 02:04:57 +0000 Subject: [PATCH] Drop: pogreb support (#175) followup of #173 close #95 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/175 --- .woodpecker.yml | 13 --- Justfile | 2 +- cmd/certs.go | 61 +------------ cmd/flags.go | 9 +- cmd/setup.go | 26 +----- go.mod | 1 - go.sum | 2 - server/certificates/certificates.go | 8 -- server/database/interface.go | 2 - server/database/mock.go | 5 -- server/database/pogreb.go | 134 ---------------------------- server/database/xorm.go | 5 -- 12 files changed, 6 insertions(+), 262 deletions(-) delete mode 100644 server/database/pogreb.go diff --git a/.woodpecker.yml b/.woodpecker.yml index ba44f82..5b07053 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -65,19 +65,6 @@ pipeline: - RAW_DOMAIN=raw.localhost.mock.directory - PORT=4430 - # TODO: remove in next version - integration-tests-legacy: - group: test - image: codeberg.org/6543/docker-images/golang_just - commands: - - just integration - environment: - - ACME_API=https://acme.mock.directory - - PAGES_DOMAIN=localhost.mock.directory - - RAW_DOMAIN=raw.localhost.mock.directory - - PORT=4430 - - DB_TYPE= - release: image: plugins/gitea-release settings: diff --git a/Justfile b/Justfile index 683222d..0db7845 100644 --- a/Justfile +++ b/Justfile @@ -27,7 +27,7 @@ fmt: tool-gofumpt clean: go clean ./... - rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json + rm -rf build/ integration/certs.sqlite integration/acme-account.json tool-golangci: @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ diff --git a/cmd/certs.go b/cmd/certs.go index 96244b7..6012b6e 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -4,11 +4,7 @@ import ( "fmt" "time" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/database" ) var Certs = &cli.Command{ @@ -25,63 +21,8 @@ var Certs = &cli.Command{ Usage: "remove a certificate from the database", Action: removeCert, }, - { - Name: "migrate", - Usage: "migrate from \"pogreb\" driver to dbms driver", - Action: migrateCerts, - }, }, - Flags: append(CertStorageFlags, []cli.Flag{ - &cli.BoolFlag{ - Name: "verbose", - Usage: "print trace info", - EnvVars: []string{"VERBOSE"}, - Value: false, - }, - }...), -} - -func migrateCerts(ctx *cli.Context) error { - dbType := ctx.String("db-type") - if dbType == "" { - dbType = "sqlite3" - } - dbConn := ctx.String("db-conn") - dbPogrebConn := ctx.String("db-pogreb") - verbose := ctx.Bool("verbose") - - log.Level(zerolog.InfoLevel) - if verbose { - log.Level(zerolog.TraceLevel) - } - - xormDB, err := database.NewXormDB(dbType, dbConn) - if err != nil { - return fmt.Errorf("could not connect to database: %w", err) - } - defer xormDB.Close() - - pogrebDB, err := database.NewPogreb(dbPogrebConn) - if err != nil { - return fmt.Errorf("could not open database: %w", err) - } - defer pogrebDB.Close() - - fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn) - - certs, err := pogrebDB.Items(0, 0) - if err != nil { - return err - } - - for _, cert := range certs { - if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil { - return err - } - } - - fmt.Println("... done") - return nil + Flags: CertStorageFlags, } func listCerts(ctx *cli.Context) error { diff --git a/cmd/flags.go b/cmd/flags.go index da3febc..a37b179 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -6,16 +6,9 @@ import ( var ( CertStorageFlags = []cli.Flag{ - &cli.StringFlag{ - // TODO: remove in next version - // DEPRICATED - Name: "db-pogreb", - Value: "key-database.pogreb", - EnvVars: []string{"DB_POGREB"}, - }, &cli.StringFlag{ Name: "db-type", - Value: "", // TODO: "sqlite3" in next version + Value: "sqlite3", EnvVars: []string{"DB_TYPE"}, }, &cli.StringFlag{ diff --git a/cmd/setup.go b/cmd/setup.go index 3d1d6ee..bccba03 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -10,29 +10,9 @@ import ( ) func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { - if ctx.String("db-type") != "" { - log.Trace().Msg("use xorm mode") - certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to database: %w", err) - } - } else { - // TODO: remove in next version - fmt.Println(` -###################### -## W A R N I N G !!! # -###################### - -You use "pogreb" witch is deprecated and will be removed in the next version. -Please switch to sqlite, mysql or postgres !!! - -The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`) - log.Error().Msg("depricated \"pogreb\" used\n") - - certDB, err = database.NewPogreb(ctx.String("db-pogreb")) - if err != nil { - return nil, nil, fmt.Errorf("could not create database: %w", err) - } + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) } closeFn = func() { diff --git a/go.mod b/go.mod index 2e75c8e..944e2ad 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a - github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 github.com/go-sql-driver/mysql v1.6.0 github.com/joho/godotenv v1.4.0 diff --git a/go.sum b/go.sum index c1042c9..b5a7568 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,6 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= -github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= -github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 555539e..23fad17 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -510,14 +510,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } } log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) - - // compact the database - msg, err := certDB.Compact() - if err != nil { - log.Error().Err(err).Msg("Compacting key database failed") - } else { - log.Debug().Msgf("Compacted key database: %s", msg) - } } // update main cert diff --git a/server/database/interface.go b/server/database/interface.go index a0edc91..92068c5 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -14,8 +14,6 @@ type CertDB interface { Get(name string) (*certificate.Resource, error) Delete(key string) error Items(page, pageSize int) ([]*Cert, error) - // Compact deprecated // TODO: remove in next version - Compact() (string, error) } type Cert struct { diff --git a/server/database/mock.go b/server/database/mock.go index 7c3735e..6148287 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -37,11 +37,6 @@ func (p tmpDB) Delete(key string) error { return nil } -func (p tmpDB) Compact() (string, error) { - p.intern.Truncate() - return "Truncate done", nil -} - func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { return nil, fmt.Errorf("items not implemented for tmpDB") } diff --git a/server/database/pogreb.go b/server/database/pogreb.go deleted file mode 100644 index 9a53faf..0000000 --- a/server/database/pogreb.go +++ /dev/null @@ -1,134 +0,0 @@ -package database - -import ( - "bytes" - "context" - "encoding/gob" - "errors" - "fmt" - "time" - - "github.com/akrylysov/pogreb" - "github.com/akrylysov/pogreb/fs" - "github.com/go-acme/lego/v4/certificate" - "github.com/rs/zerolog/log" -) - -var _ CertDB = aDB{} - -type aDB struct { - ctx context.Context - cancel context.CancelFunc - intern *pogreb.DB - syncInterval time.Duration -} - -func (p aDB) Close() error { - p.cancel() - return p.intern.Sync() -} - -func (p aDB) Put(name string, cert *certificate.Resource) error { - var resGob bytes.Buffer - if err := gob.NewEncoder(&resGob).Encode(cert); err != nil { - return err - } - return p.intern.Put([]byte(name), resGob.Bytes()) -} - -func (p aDB) Get(name string) (*certificate.Resource, error) { - cert := &certificate.Resource{} - resBytes, err := p.intern.Get([]byte(name)) - if err != nil { - return nil, err - } - if resBytes == nil { - return nil, nil - } - if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil { - return nil, err - } - return cert, nil -} - -func (p aDB) Delete(key string) error { - return p.intern.Delete([]byte(key)) -} - -func (p aDB) Compact() (string, error) { - result, err := p.intern.Compact() - if err != nil { - return "", err - } - return fmt.Sprintf("%+v", result), nil -} - -func (p aDB) Items(_, _ int) ([]*Cert, error) { - items := make([]*Cert, 0, p.intern.Count()) - iterator := p.intern.Items() - for { - key, resBytes, err := iterator.Next() - if err != nil { - if errors.Is(err, pogreb.ErrIterationDone) { - break - } - return nil, err - } - - res := &certificate.Resource{} - if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil { - return nil, err - } - - cert, err := toCert(string(key), res) - if err != nil { - return nil, err - } - - items = append(items, cert) - } - - return items, nil -} - -var _ CertDB = &aDB{} - -func (p aDB) sync() { - for { - err := p.intern.Sync() - if err != nil { - log.Error().Err(err).Msg("Syncing cert database failed") - } - select { - case <-p.ctx.Done(): - return - case <-time.After(p.syncInterval): - } - } -} - -func NewPogreb(path string) (CertDB, error) { - if path == "" { - return nil, fmt.Errorf("path not set") - } - db, err := pogreb.Open(path, &pogreb.Options{ - BackgroundSyncInterval: 30 * time.Second, - BackgroundCompactionInterval: 6 * time.Hour, - FileSystem: fs.OSMMap, - }) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithCancel(context.Background()) - result := &aDB{ - ctx: ctx, - cancel: cancel, - intern: db, - syncInterval: 5 * time.Minute, - } - - go result.sync() - - return result, nil -} diff --git a/server/database/xorm.go b/server/database/xorm.go index a14f887..4b43cbb 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -106,11 +106,6 @@ func (x xDB) Delete(domain string) error { return err } -func (x xDB) Compact() (string, error) { - // not needed - return "", nil -} - // Items return al certs from db, if pageSize is 0 it does not use limit func (x xDB) Items(page, pageSize int) ([]*Cert, error) { // paginated return