From 42d5802b9ba77556dfa529e20caf0442ac6b1fa4 Mon Sep 17 00:00:00 2001 From: deblan Date: Tue, 14 Feb 2023 03:03:00 +0000 Subject: [PATCH] Allow to define default branches (#125) This try to address #115 Co-authored-by: Simon Vieille Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/125 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: deblan Co-committed-by: deblan --- cmd/flags.go | 7 ++++ cmd/main.go | 7 ++++ go.mod | 3 +- go.sum | 4 +++ integration/main_test.go | 1 + server/certificates/certificates.go | 3 +- server/dns/dns.go | 10 +++--- server/handler/handler.go | 4 ++- server/handler/handler_custom_domain.go | 5 +-- server/handler/handler_sub_domain.go | 46 ++++++++++++++++++++----- server/handler/handler_test.go | 1 + 11 files changed, 73 insertions(+), 18 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 5bc638b..a71dd35 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -112,6 +112,13 @@ var ( Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", EnvVars: []string{"LOG_LEVEL"}, }, + // Default branches to fetch assets from + &cli.StringSliceFlag{ + Name: "pages-branch", + Usage: "define a branch to fetch assets from", + EnvVars: []string{"PAGES_BRANCHES"}, + Value: cli.NewStringSlice("pages"), + }, // ############################ // ### ACME Client Settings ### diff --git a/cmd/main.go b/cmd/main.go index aa00f54..45e151d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := ctx.String("gitea-root") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") + defaultBranches := ctx.StringSlice("pages-branch") mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningHost := ctx.String("host") @@ -63,6 +64,10 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix = "." + mainDomainSuffix } + if len(defaultBranches) == 0 { + return fmt.Errorf("no default branches set (PAGES_BRANCHES)") + } + // Init ssl cert database certDB, closeFn, err := openCertDB(ctx) if err != nil { @@ -104,6 +109,7 @@ func Serve(ctx *cli.Context) error { listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaClient, acmeClient, + defaultBranches[0], keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB)) @@ -131,6 +137,7 @@ func Serve(ctx *cli.Context) error { giteaClient, rawInfoPage, BlacklistedPaths, allowedCorsDomains, + defaultBranches, dnsLookupCache, canonicalDomainCache) // Start the ssl listener diff --git a/go.mod b/go.mod index 944e2ad..bfde7f7 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 + golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb xorm.io/xorm v1.3.2 ) @@ -117,7 +118,7 @@ require ( golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect diff --git a/go.sum b/go.sum index b5a7568..b10305c 100644 --- a/go.sum +++ b/go.sum @@ -768,6 +768,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -899,6 +901,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/integration/main_test.go b/integration/main_test.go index a6579fd..a397110 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -39,6 +39,7 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("ACME_API", "https://acme.mock.directory") setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") + setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master") setEnvIfNotSet("PORT", "4430") setEnvIfNotSet("HTTP_PORT", "8880") setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index ce1420d..3ae891a 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -30,6 +30,7 @@ var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, + firstDefaultBranch string, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, certDB database.CertDB, ) *tls.Config { @@ -68,7 +69,7 @@ func TLSConfig(mainDomainSuffix string, domain = mainDomainSuffix } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs domain = mainDomainSuffix diff --git a/server/dns/dns.go b/server/dns/dns.go index 2719d4d..c11b278 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -11,9 +11,11 @@ import ( // lookupCacheTimeout specifies the timeout for the DNS lookup cache. var lookupCacheTimeout = 15 * time.Minute +var defaultPagesRepo = "pages" + // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { +func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error @@ -50,10 +52,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG targetBranch = cnameParts[len(cnameParts)-3] } if targetRepo == "" { - targetRepo = "pages" + targetRepo = defaultPagesRepo } - if targetBranch == "" && targetRepo != "pages" { - targetBranch = "pages" + if targetBranch == "" && targetRepo != defaultPagesRepo { + targetBranch = firstDefaultBranch } // if targetBranch is still empty, the caller must find the default branch return diff --git a/server/handler/handler.go b/server/handler/handler.go index 78301e9..a944c7e 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -17,7 +17,6 @@ const ( headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" headerAccessControlAllowMethods = "Access-Control-Allow-Methods" defaultPagesRepo = "pages" - defaultPagesBranch = "pages" ) // Handler handles a single HTTP request to the web server. @@ -25,6 +24,7 @@ func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, rawInfoPage string, blacklistedPaths, allowedCorsDomains []string, + defaultPagesBranches []string, dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { @@ -98,6 +98,7 @@ func Handler(mainDomainSuffix, rawDomain string, log.Debug().Msg("subdomain request detecded") handleSubDomain(log, ctx, giteaClient, mainDomainSuffix, + defaultPagesBranches, trimmedHost, pathElements, canonicalDomainCache) @@ -107,6 +108,7 @@ func Handler(mainDomainSuffix, rawDomain string, mainDomainSuffix, trimmedHost, pathElements, + defaultPagesBranches[0], dnsLookupCache, canonicalDomainCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index a541b74..1b85f62 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -18,10 +18,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g mainDomainSuffix string, trimmedHost string, pathElements []string, + firstDefaultBranch string, dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) { // Serve pages from custom domains - targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache) + targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner == "" { html.ReturnErrorPage(ctx, "could not obtain repo owner from custom domain", @@ -52,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! - targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache) + targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect) return diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 2a75e9f..68f4822 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/rs/zerolog" + "golang.org/x/exp/slices" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" @@ -17,6 +18,7 @@ import ( func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix string, + defaultPagesBranches []string, trimmedHost string, pathElements []string, canonicalDomainCache cache.SetGetKey, @@ -63,12 +65,21 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Check if the first directory is a branch for the defaultPagesRepo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { + targetBranch := pathElements[0][1:] + + // if the default pages branch can be determined exactly, it does not need to be set + if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) { + // example.codeberg.org/@pages/... redirects to example.codeberg.org/... + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) + return + } + log.Debug().Msg("main domain preparations, now trying with specified branch") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, TargetRepo: defaultPagesRepo, - TargetBranch: pathElements[0][1:], + TargetBranch: targetBranch, TargetPath: path.Join(pathElements[1:]...), }, true); works { log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") @@ -81,19 +92,36 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite return } - // Check if the first directory is a repo with a defaultPagesRepo branch - // example.codeberg.page/myrepo/index.html - // example.codeberg.page/pages/... is not allowed here. - log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != defaultPagesRepo { + for _, defaultPagesBranch := range defaultPagesBranches { + // Check if the first directory is a repo with a default pages branch + // example.codeberg.page/myrepo/index.html + // example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here. + log.Debug().Msg("main domain preparations, now trying with specified repo") + if pathElements[0] != defaultPagesBranch { + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: pathElements[0], + TargetBranch: defaultPagesBranch, + TargetPath: path.Join(pathElements[1:]...), + }, false); works { + log.Debug().Msg("tryBranch, now trying upstream 5") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + } + + // Try to use the defaultPagesRepo on an default pages branch + // example.codeberg.page/index.html + log.Debug().Msg("main domain preparations, now trying with default repo") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, - TargetRepo: pathElements[0], + TargetRepo: defaultPagesRepo, TargetBranch: defaultPagesBranch, - TargetPath: path.Join(pathElements[1:]...), + TargetPath: path.Join(pathElements...), }, false); works { - log.Debug().Msg("tryBranch, now trying upstream 5") + log.Debug().Msg("tryBranch, now trying upstream 6") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) return } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index 626564a..ed063b2 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -18,6 +18,7 @@ func TestHandlerPerformance(t *testing.T) { "https://docs.codeberg.org/pages/raw-content/", []string{"/.well-known/acme-challenge/"}, []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, + []string{"pages"}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), )