From 2c7ded7f71ab700d1b53e5eedd5a336e557bf3dd Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 29 Mar 2024 23:15:08 +0100 Subject: [PATCH] Move redis config to CacheConfig struct, add cache prefixes & trace logging --- README.md | 1 + config/config.go | 6 +++++- config/setup.go | 10 +++++++--- server/cache/redis.go | 26 ++++++++++++++++---------- server/gitea/client.go | 14 +++++++------- server/startup.go | 16 ++++++++-------- server/upstream/redirects.go | 2 +- 7 files changed, 45 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f47a196..79f447f 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg See for available values & additional environment variables. - `NO_DNS_01` (default: `false`): Disable the use of ACME DNS. This means that the wildcard certificate is self-signed and all domains and subdomains will have a distinct certificate. Because this may lead to a rate limit from the ACME provider, this option is not recommended for Gitea/Forgejo instances with open registrations or a great number of users/orgs. - `LOG_LEVEL` (default: warn): Set this to specify the level of logging. +- `REDIS_URL` (default: use in-memory cache): Set this to use Redis as a cache server. ## Contributing to the development diff --git a/config/config.go b/config/config.go index 51a54cb..0e0979b 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,7 @@ type Config struct { Gitea GiteaConfig Database DatabaseConfig ACME ACMEConfig - RedisURL string `default:""` + Cache CacheConfig } type ServerConfig struct { @@ -46,3 +46,7 @@ type ACMEConfig struct { NoDNS01 bool `default:"false"` AccountConfigFile string `default:"acme-account.json"` } + +type CacheConfig struct { + RedisURL string `default:""` +} diff --git a/config/setup.go b/config/setup.go index 9ebb9bc..60f9222 100644 --- a/config/setup.go +++ b/config/setup.go @@ -54,9 +54,7 @@ func MergeConfig(ctx *cli.Context, config *Config) { mergeGiteaConfig(ctx, &config.Gitea) mergeDatabaseConfig(ctx, &config.Database) mergeACMEConfig(ctx, &config.ACME) - if ctx.IsSet("redis-url") { - config.RedisURL = ctx.String("redis-url") - } + mergeCacheConfig(ctx, &config.Cache) } func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { @@ -151,3 +149,9 @@ func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { config.AccountConfigFile = ctx.String("acme-account-config") } } + +func mergeCacheConfig(ctx *cli.Context, config *CacheConfig) { + if ctx.IsSet("redis-url") { + config.RedisURL = ctx.String("redis-url") + } +} diff --git a/server/cache/redis.go b/server/cache/redis.go index 1e5c7c9..b56d118 100644 --- a/server/cache/redis.go +++ b/server/cache/redis.go @@ -9,35 +9,41 @@ import ( ) type RedisCache struct { - ctx context.Context - rdb *redis.Client + name string + ctx context.Context + rdb *redis.Client } func (r *RedisCache) Set(key string, value []byte, ttl time.Duration) error { - return r.rdb.Set(r.ctx, key, value, ttl).Err() + log.Trace().Str("key", r.name+"::"+key).Int("len(value)", len(value)).Bytes("value", value).Msg("Set in Redis.") + return r.rdb.Set(r.ctx, r.name+"::"+key, value, ttl).Err() } func (r *RedisCache) Get(key string) ([]byte, bool) { - val, err := r.rdb.Get(r.ctx, key).Bytes() + val, err := r.rdb.Get(r.ctx, r.name+"::"+key).Bytes() if err != nil { - if errors.Is(err, redis.Nil) { - log.Error().Err(err).Str("key", key).Msg("Couldn't request key from cache.") + if !errors.Is(err, redis.Nil) { + log.Error().Err(err).Str("key", r.name+"::"+key).Msg("Couldn't request key from cache.") + } else { + log.Trace().Str("key", r.name+"::"+key).Msg("Get from Redis, doesn't exist.") } return nil, false } else { + log.Trace().Str("key", r.name+"::"+key).Int("len(value)", len(val)).Msg("Get from Redis.") return val, true } } func (r *RedisCache) Remove(key string) { - err := r.rdb.Del(r.ctx, key).Err() - if err == nil { - log.Error().Err(err).Str("key", key).Msg("Couldn't delete key from cache.") + err := r.rdb.Del(r.ctx, r.name+"::"+key).Err() + if err != nil { + log.Error().Err(err).Str("key", r.name+"::"+key).Msg("Couldn't delete key from cache.") } } -func NewRedisCache(opts *redis.Options) ICache { +func NewRedisCache(name string, opts *redis.Options) ICache { return &RedisCache{ + name, context.Background(), redis.NewClient(opts), } diff --git a/server/gitea/client.go b/server/gitea/client.go index 8287d31..8f98dd9 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -117,8 +117,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str // handle if cache entry exist if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok { cache := FileResponseFromMetadataString(string(cacheMetadata)) - cacheBodyString, _ := client.responseCache.Get(cacheKey + "|Body") - cache.Body = []byte(cacheBodyString) + cache.Body, _ = client.responseCache.Get(cacheKey + "|Body") // TODO: don't grab the content from the cache if the ETag matches?! cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) @@ -136,8 +135,9 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str log.Debug().Msg("[cache] is empty") // TODO: empty files aren't cached anyways; but when closing the issue please make sure that a missing body cache key is also handled correctly. } - } + } // TODO: handle missing pages if they redirect to a index.html } + // TODO: metadata not written, is close ever called? log.Trace().Msg("file not in cache") // not in cache, open reader via gitea api reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) @@ -284,12 +284,12 @@ func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) { cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner) if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil { - return exist.(bool), nil + return string(exist) == "true", nil } _, resp, err := client.sdkClient.GetUserInfo(owner) if resp.StatusCode == http.StatusOK && err == nil { - if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return true, nil @@ -299,14 +299,14 @@ func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) { _, resp, err = client.sdkClient.GetOrg(owner) if resp.StatusCode == http.StatusOK && err == nil { - if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte("true"), ownerExistenceCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return true, nil } else if resp.StatusCode != http.StatusNotFound { return false, err } - if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte("false"), ownerExistenceCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return false, nil diff --git a/server/startup.go b/server/startup.go index 98c8c62..d93ff5d 100644 --- a/server/startup.go +++ b/server/startup.go @@ -78,24 +78,24 @@ func Serve(ctx *cli.Context) error { dnsLookupCache := mcache.New() var redisErr error = nil - createCache := func() cache.ICache { - if cfg.RedisURL != "" { - opts, err := redis.ParseURL(cfg.RedisURL) + createCache := func(name string) cache.ICache { + if cfg.Cache.RedisURL != "" { + opts, err := redis.ParseURL(cfg.Cache.RedisURL) if err != nil { redisErr = err } - return cache.NewRedisCache(opts) + return cache.NewRedisCache(name, opts) } return cache.NewInMemoryCache() } // challengeCache stores the certificate challenges - challengeCache := createCache() + challengeCache := createCache("challenge") // canonicalDomainCache stores canonical domains - canonicalDomainCache := createCache() + canonicalDomainCache := createCache("canonicalDomain") // redirectsCache stores redirects in _redirects files - redirectsCache := createCache() + redirectsCache := createCache("redirects") // clientResponseCache stores responses from the Gitea server - clientResponseCache := createCache() + clientResponseCache := createCache("clientResponse") if redisErr != nil { return redisErr } diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index d09214b..c8613b8 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -31,7 +31,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I // Check for cached redirects if cachedValue, ok := redirectsCache.Get(cacheKey); ok { redirects := []Redirect{} - err := json.Unmarshal(cachedValue, redirects) + err := json.Unmarshal(cachedValue, &redirects) if err != nil { log.Error().Err(err).Msgf("could not parse redirects for key %s", cacheKey) // It's okay to continue, the array stays empty.