kopia lustrzana https://codeberg.org/Codeberg/pages-server
Merge pull request 'Refactor: restructure in packages and dont use golbal vars' (#18) from 6543/codeberg-pages:refactoring into main
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/18pull/58/head v2.5
Moritz Marquardt
commit
e73c79da77
2
Justfile
2
Justfile
|
|
@ -6,7 +6,7 @@ dev:
|
|||
export PAGES_DOMAIN=localhost.mock.directory
|
||||
export RAW_DOMAIN=raw.localhost.mock.directory
|
||||
export PORT=4430
|
||||
go run .
|
||||
go run . --verbose
|
||||
|
||||
build:
|
||||
CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
|
||||
|
|
|
|||
26
README.md
26
README.md
|
|
@ -2,10 +2,10 @@
|
|||
|
||||
- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
|
||||
- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
|
||||
- `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources.
|
||||
- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
|
||||
- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
|
||||
- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
|
||||
- `REDIRECT_RAW_INFO` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
|
||||
- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
|
||||
- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).
|
||||
ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
|
||||
- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
|
||||
|
|
@ -15,3 +15,25 @@
|
|||
- `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
|
||||
- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.
|
||||
See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
|
||||
- `DEBUG` (default: false): Set this to true to enable debug logging.
|
||||
|
||||
```
|
||||
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
|
||||
//
|
||||
// Mapping custom domains is not static anymore, but can be done with DNS:
|
||||
//
|
||||
// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The
|
||||
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
|
||||
//
|
||||
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
|
||||
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
|
||||
// www.example.org. IN CNAME main.pages.example.codeberg.page.
|
||||
//
|
||||
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
|
||||
// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
|
||||
// record that points to your repo (just like the CNAME record):
|
||||
// example.org IN ALIAS codeberg.page.
|
||||
// example.org IN TXT main.pages.example.codeberg.page.
|
||||
//
|
||||
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
|
||||
```
|
||||
|
|
|
|||
599
certificates.go
599
certificates.go
|
|
@ -1,599 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
"github.com/go-acme/lego/v4/certificate"
|
||||
"github.com/go-acme/lego/v4/challenge"
|
||||
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
|
||||
"github.com/go-acme/lego/v4/providers/dns"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/big"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/akrylysov/pogreb"
|
||||
"github.com/reugn/equalizer"
|
||||
|
||||
"github.com/go-acme/lego/v4/certcrypto"
|
||||
"github.com/go-acme/lego/v4/lego"
|
||||
"github.com/go-acme/lego/v4/registration"
|
||||
)
|
||||
|
||||
// tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates.
|
||||
var tlsConfig = &tls.Config{
|
||||
// check DNS name & get certificate from Let's Encrypt
|
||||
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
|
||||
sniBytes := []byte(sni)
|
||||
if len(sni) < 1 {
|
||||
return nil, errors.New("missing sni")
|
||||
}
|
||||
|
||||
if info.SupportedProtos != nil {
|
||||
for _, proto := range info.SupportedProtos {
|
||||
if proto == tlsalpn01.ACMETLS1Protocol {
|
||||
challenge, ok := challengeCache.Get(sni)
|
||||
if !ok {
|
||||
return nil, errors.New("no challenge for this domain")
|
||||
}
|
||||
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetOwner := ""
|
||||
if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) {
|
||||
// deliver default certificate for the main domain (*.codeberg.page)
|
||||
sniBytes = MainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
} else {
|
||||
var targetRepo, targetBranch string
|
||||
targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni)
|
||||
if targetOwner == "" {
|
||||
// DNS not set up, return main certificate to redirect to the docs
|
||||
sniBytes = MainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
} else {
|
||||
_, _ = targetRepo, targetBranch
|
||||
_, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni)
|
||||
if !valid {
|
||||
sniBytes = MainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if tlsCertificate, ok := keyCache.Get(sni); ok {
|
||||
// we can use an existing certificate object
|
||||
return tlsCertificate.(*tls.Certificate), nil
|
||||
}
|
||||
|
||||
var tlsCertificate tls.Certificate
|
||||
var err error
|
||||
var ok bool
|
||||
if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok {
|
||||
// request a new certificate
|
||||
if bytes.Equal(sniBytes, MainDomainSuffix) {
|
||||
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
|
||||
}
|
||||
|
||||
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &tlsCertificate, nil
|
||||
},
|
||||
PreferServerCipherSuites: true,
|
||||
NextProtos: []string{
|
||||
"http/1.1",
|
||||
tlsalpn01.ACMETLS1Protocol,
|
||||
},
|
||||
|
||||
// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
|
||||
// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: []uint16{
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
|
||||
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
|
||||
},
|
||||
}
|
||||
|
||||
var keyCache = mcache.New()
|
||||
var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{
|
||||
BackgroundSyncInterval: 30 * time.Second,
|
||||
BackgroundCompactionInterval: 6 * time.Hour,
|
||||
FileSystem: fs.OSMMap,
|
||||
})
|
||||
|
||||
func CheckUserLimit(user string) error {
|
||||
userLimit, ok := acmeClientCertificateLimitPerUser[user]
|
||||
if !ok {
|
||||
// Each Codeberg user can only add 10 new domains per day.
|
||||
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
|
||||
acmeClientCertificateLimitPerUser[user] = userLimit
|
||||
}
|
||||
if !userLimit.Ask() {
|
||||
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var myAcmeAccount AcmeAccount
|
||||
var myAcmeConfig *lego.Config
|
||||
|
||||
type AcmeAccount struct {
|
||||
Email string
|
||||
Registration *registration.Resource
|
||||
Key crypto.PrivateKey `json:"-"`
|
||||
KeyPEM string `json:"Key"`
|
||||
}
|
||||
|
||||
func (u *AcmeAccount) GetEmail() string {
|
||||
return u.Email
|
||||
}
|
||||
func (u AcmeAccount) GetRegistration() *registration.Resource {
|
||||
return u.Registration
|
||||
}
|
||||
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
|
||||
return u.Key
|
||||
}
|
||||
|
||||
var acmeClient, mainDomainAcmeClient *lego.Client
|
||||
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
|
||||
|
||||
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
|
||||
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
|
||||
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
|
||||
|
||||
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
|
||||
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
|
||||
|
||||
var challengeCache = mcache.New()
|
||||
|
||||
type AcmeTLSChallengeProvider struct{}
|
||||
|
||||
var _ challenge.Provider = AcmeTLSChallengeProvider{}
|
||||
|
||||
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
|
||||
return challengeCache.Set(domain, keyAuth, 1*time.Hour)
|
||||
}
|
||||
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
|
||||
challengeCache.Remove(domain)
|
||||
return nil
|
||||
}
|
||||
|
||||
type AcmeHTTPChallengeProvider struct{}
|
||||
|
||||
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
|
||||
|
||||
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
|
||||
return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
|
||||
}
|
||||
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
|
||||
challengeCache.Remove(domain + "/" + token)
|
||||
return nil
|
||||
}
|
||||
|
||||
func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) {
|
||||
// parse certificate from database
|
||||
res := &certificate.Resource{}
|
||||
if !PogrebGet(keyDatabase, sni, res) {
|
||||
return tls.Certificate{}, false
|
||||
}
|
||||
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(sni, MainDomainSuffix) {
|
||||
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// renew certificates 7 days before they expire
|
||||
if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
|
||||
if res.CSR != nil && len(res.CSR) > 0 {
|
||||
// CSR stores the time when the renewal shall be tried again
|
||||
nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
|
||||
if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
|
||||
return tlsCertificate, true
|
||||
}
|
||||
}
|
||||
go (func() {
|
||||
res.CSR = nil // acme client doesn't like CSR to be set
|
||||
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "")
|
||||
if err != nil {
|
||||
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
|
||||
}
|
||||
})()
|
||||
}
|
||||
}
|
||||
|
||||
return tlsCertificate, true
|
||||
}
|
||||
|
||||
var obtainLocks = sync.Map{}
|
||||
|
||||
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) {
|
||||
name := strings.TrimPrefix(domains[0], "*")
|
||||
if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
|
||||
domains = domains[1:]
|
||||
}
|
||||
|
||||
// lock to avoid simultaneous requests
|
||||
_, working := obtainLocks.LoadOrStore(name, struct{}{})
|
||||
if working {
|
||||
for working {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, working = obtainLocks.Load(name)
|
||||
}
|
||||
cert, ok := retrieveCertFromDB([]byte(name))
|
||||
if !ok {
|
||||
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
defer obtainLocks.Delete(name)
|
||||
|
||||
if acmeClient == nil {
|
||||
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil
|
||||
}
|
||||
|
||||
// request actual cert
|
||||
var res *certificate.Resource
|
||||
var err error
|
||||
if renew != nil && renew.CertURL != "" {
|
||||
if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
|
||||
acmeClientRequestLimit.Take()
|
||||
}
|
||||
log.Printf("Renewing certificate for %v", domains)
|
||||
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
|
||||
if err != nil {
|
||||
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
|
||||
res = nil
|
||||
}
|
||||
}
|
||||
if res == nil {
|
||||
if user != "" {
|
||||
if err := CheckUserLimit(user); err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
|
||||
acmeClientOrderLimit.Take()
|
||||
acmeClientRequestLimit.Take()
|
||||
}
|
||||
log.Printf("Requesting new certificate for %v", domains)
|
||||
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
|
||||
Domains: domains,
|
||||
Bundle: true,
|
||||
MustStaple: false,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
|
||||
if renew != nil && renew.CertURL != "" {
|
||||
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
|
||||
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
|
||||
// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
|
||||
renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
|
||||
PogrebPut(keyDatabase, []byte(name), renew)
|
||||
return tlsCertificate, nil
|
||||
}
|
||||
} else {
|
||||
return mockCert(domains[0], err.Error()), err
|
||||
}
|
||||
}
|
||||
log.Printf("Obtained certificate for %v", domains)
|
||||
|
||||
PogrebPut(keyDatabase, []byte(name), res)
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
return tlsCertificate, nil
|
||||
}
|
||||
|
||||
func mockCert(domain string, msg string) tls.Certificate {
|
||||
key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
template := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{
|
||||
CommonName: domain,
|
||||
Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
|
||||
OrganizationalUnit: []string{
|
||||
"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
|
||||
"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
|
||||
"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
|
||||
"Error message: " + msg,
|
||||
},
|
||||
},
|
||||
|
||||
// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
|
||||
NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
|
||||
NotBefore: time.Now(),
|
||||
|
||||
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
certBytes, err := x509.CreateCertificate(
|
||||
rand.Reader,
|
||||
&template,
|
||||
&template,
|
||||
&key.(*rsa.PrivateKey).PublicKey,
|
||||
key,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
err = pem.Encode(out, &pem.Block{
|
||||
Bytes: certBytes,
|
||||
Type: "CERTIFICATE",
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
outBytes := out.Bytes()
|
||||
res := &certificate.Resource{
|
||||
PrivateKey: certcrypto.PEMEncode(key),
|
||||
Certificate: outBytes,
|
||||
IssuerCertificate: outBytes,
|
||||
Domain: domain,
|
||||
}
|
||||
databaseName := domain
|
||||
if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) {
|
||||
databaseName = string(MainDomainSuffix)
|
||||
}
|
||||
PogrebPut(keyDatabase, []byte(databaseName), res)
|
||||
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return tlsCertificate
|
||||
}
|
||||
|
||||
func setupCertificates() {
|
||||
if keyDatabaseErr != nil {
|
||||
panic(keyDatabaseErr)
|
||||
}
|
||||
|
||||
if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") {
|
||||
panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory"))
|
||||
}
|
||||
|
||||
// getting main cert before ACME account so that we can panic here on database failure without hitting rate limits
|
||||
mainCertBytes, err := keyDatabase.Get(MainDomainSuffix)
|
||||
if err != nil {
|
||||
// key database is not working
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if account, err := ioutil.ReadFile("acme-account.json"); err == nil {
|
||||
err = json.Unmarshal(account, &myAcmeAccount)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
|
||||
myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
|
||||
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
|
||||
_, err := lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
}
|
||||
} else if os.IsNotExist(err) {
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
myAcmeAccount = AcmeAccount{
|
||||
Email: envOr("ACME_EMAIL", "noreply@example.email"),
|
||||
Key: privateKey,
|
||||
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
|
||||
}
|
||||
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
|
||||
myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
|
||||
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
|
||||
tempClient, err := lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
// accept terms & log in to EAB
|
||||
if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" {
|
||||
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
myAcmeAccount.Registration = reg
|
||||
}
|
||||
} else {
|
||||
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
|
||||
TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true",
|
||||
Kid: os.Getenv("ACME_EAB_KID"),
|
||||
HmacEncoded: os.Getenv("ACME_EAB_HMAC"),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
myAcmeAccount.Registration = reg
|
||||
}
|
||||
}
|
||||
|
||||
if myAcmeAccount.Registration != nil {
|
||||
acmeAccountJson, err := json.Marshal(myAcmeAccount)
|
||||
if err != nil {
|
||||
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
|
||||
select {}
|
||||
}
|
||||
err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600)
|
||||
if err != nil {
|
||||
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
|
||||
select {}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
acmeClient, err = lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
|
||||
}
|
||||
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
|
||||
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
if os.Getenv("DNS_PROVIDER") == "" {
|
||||
// using mock server, don't use wildcard certs
|
||||
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
|
||||
}
|
||||
} else {
|
||||
provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER"))
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
|
||||
}
|
||||
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if mainCertBytes == nil {
|
||||
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "")
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
go (func() {
|
||||
for {
|
||||
err := keyDatabase.Sync()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Syncinc key database failed: %s", err)
|
||||
}
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
})()
|
||||
go (func() {
|
||||
for {
|
||||
// clean up expired certs
|
||||
now := time.Now()
|
||||
expiredCertCount := 0
|
||||
keyDatabaseIterator := keyDatabase.Items()
|
||||
key, resBytes, err := keyDatabaseIterator.Next()
|
||||
for err == nil {
|
||||
if !bytes.Equal(key, MainDomainSuffix) {
|
||||
resGob := bytes.NewBuffer(resBytes)
|
||||
resDec := gob.NewDecoder(resGob)
|
||||
res := &certificate.Resource{}
|
||||
err = resDec.Decode(res)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
|
||||
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
|
||||
err := keyDatabase.Delete(key)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
|
||||
} else {
|
||||
expiredCertCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
key, resBytes, err = keyDatabaseIterator.Next()
|
||||
}
|
||||
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
|
||||
|
||||
// compact the database
|
||||
result, err := keyDatabase.Compact()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Compacting key database failed: %s", err)
|
||||
} else {
|
||||
log.Printf("[INFO] Compacted key database (%+v)", result)
|
||||
}
|
||||
|
||||
// update main cert
|
||||
res := &certificate.Resource{}
|
||||
if !PogrebGet(keyDatabase, MainDomainSuffix, res) {
|
||||
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
|
||||
} else {
|
||||
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
|
||||
|
||||
// renew main certificate 30 days before it expires
|
||||
if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
|
||||
go (func() {
|
||||
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "")
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
|
||||
}
|
||||
})()
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(12 * time.Hour)
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/database"
|
||||
)
|
||||
|
||||
var Certs = &cli.Command{
|
||||
Name: "certs",
|
||||
Usage: "manage certs manually",
|
||||
Action: certs,
|
||||
}
|
||||
|
||||
func certs(ctx *cli.Context) error {
|
||||
if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
|
||||
if ctx.Args().Len() == 1 {
|
||||
println("--remove-certificate requires at least one domain as an argument")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
domains := ctx.Args().Slice()[2:]
|
||||
|
||||
// TODO: make "key-database.pogreb" set via flag
|
||||
keyDatabase, err := database.New("key-database.pogreb")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create database: %v", err)
|
||||
}
|
||||
|
||||
for _, domain := range domains {
|
||||
if err := keyDatabase.Delete([]byte(domain)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if err := keyDatabase.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var ServeFlags = []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"DEBUG"},
|
||||
},
|
||||
|
||||
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
|
||||
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
|
||||
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
|
||||
&cli.StringFlag{
|
||||
Name: "pages-domain",
|
||||
Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
|
||||
EnvVars: []string{"PAGES_DOMAIN"},
|
||||
Value: "codeberg.page",
|
||||
},
|
||||
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
|
||||
&cli.StringFlag{
|
||||
Name: "gitea-root",
|
||||
Usage: "specifies the root URL of the Gitea instance, without a trailing slash.",
|
||||
EnvVars: []string{"GITEA_ROOT"},
|
||||
Value: "https://codeberg.org",
|
||||
},
|
||||
// GiteaApiToken specifies an api token for the Gitea instance
|
||||
&cli.StringFlag{
|
||||
Name: "gitea-api-token",
|
||||
Usage: "specifies an api token for the Gitea instance",
|
||||
EnvVars: []string{"GITEA_API_TOKEN"},
|
||||
Value: "",
|
||||
},
|
||||
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
|
||||
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
|
||||
// (set to []byte(nil) to disable raw content hosting)
|
||||
&cli.StringFlag{
|
||||
Name: "raw-domain",
|
||||
Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
|
||||
EnvVars: []string{"RAW_DOMAIN"},
|
||||
Value: "raw.codeberg.page",
|
||||
},
|
||||
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
|
||||
&cli.StringFlag{
|
||||
Name: "raw-info-page",
|
||||
Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
|
||||
EnvVars: []string{"RAW_INFO_PAGE"},
|
||||
Value: "https://docs.codeberg.org/codeberg-pages/raw-content/",
|
||||
},
|
||||
|
||||
// Server
|
||||
&cli.StringFlag{
|
||||
Name: "host",
|
||||
Usage: "specifies host of listening address",
|
||||
EnvVars: []string{"HOST"},
|
||||
Value: "[::]",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "port",
|
||||
Usage: "specifies port of listening address",
|
||||
EnvVars: []string{"PORT"},
|
||||
Value: "443",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "enable-http-server",
|
||||
// TODO: desc
|
||||
EnvVars: []string{"ENABLE_HTTP_SERVER"},
|
||||
},
|
||||
|
||||
// ACME
|
||||
&cli.StringFlag{
|
||||
Name: "acme-api-endpoint",
|
||||
EnvVars: []string{"ACME_API"},
|
||||
Value: "https://acme-v02.api.letsencrypt.org/directory",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "acme-email",
|
||||
EnvVars: []string{"ACME_EMAIL"},
|
||||
Value: "noreply@example.email",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "acme-use-rate-limits",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"ACME_USE_RATE_LIMITS"},
|
||||
Value: true,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "acme-accept-terms",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"ACME_ACCEPT_TERMS"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "acme-eab-kid",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"ACME_EAB_KID"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "acme-eab-hmac",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"ACME_EAB_HMAC"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "dns-provider",
|
||||
// TODO: Usage
|
||||
EnvVars: []string{"DNS_PROVIDER"},
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"codeberg.org/codeberg/pages/server"
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/certificates"
|
||||
"codeberg.org/codeberg/pages/server/database"
|
||||
)
|
||||
|
||||
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
|
||||
// TODO: make it a flag
|
||||
var AllowedCorsDomains = [][]byte{
|
||||
[]byte("fonts.codeberg.org"),
|
||||
[]byte("design.codeberg.org"),
|
||||
}
|
||||
|
||||
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
|
||||
// TODO: Make it a flag too
|
||||
var BlacklistedPaths = [][]byte{
|
||||
[]byte("/.well-known/acme-challenge/"),
|
||||
}
|
||||
|
||||
// Serve sets up and starts the web server.
|
||||
func Serve(ctx *cli.Context) error {
|
||||
verbose := ctx.Bool("verbose")
|
||||
if !verbose {
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
}
|
||||
|
||||
giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
|
||||
giteaAPIToken := ctx.String("gitea-api-token")
|
||||
rawDomain := ctx.String("raw-domain")
|
||||
mainDomainSuffix := []byte(ctx.String("pages-domain"))
|
||||
rawInfoPage := ctx.String("raw-info-page")
|
||||
listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
|
||||
enableHTTPServer := ctx.Bool("enable-http-server")
|
||||
|
||||
acmeAPI := ctx.String("acme-api-endpoint")
|
||||
acmeMail := ctx.String("acme-email")
|
||||
acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
|
||||
acmeAcceptTerms := ctx.Bool("acme-accept-terms")
|
||||
acmeEabKID := ctx.String("acme-eab-kid")
|
||||
acmeEabHmac := ctx.String("acme-eab-hmac")
|
||||
dnsProvider := ctx.String("dns-provider")
|
||||
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
|
||||
return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
|
||||
}
|
||||
|
||||
allowedCorsDomains := AllowedCorsDomains
|
||||
if len(rawDomain) != 0 {
|
||||
allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain))
|
||||
}
|
||||
|
||||
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
|
||||
if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
|
||||
mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
|
||||
}
|
||||
|
||||
keyCache := cache.NewKeyValueCache()
|
||||
challengeCache := cache.NewKeyValueCache()
|
||||
// canonicalDomainCache stores canonical domains
|
||||
var canonicalDomainCache = cache.NewKeyValueCache()
|
||||
// dnsLookupCache stores DNS lookups for custom domains
|
||||
var dnsLookupCache = cache.NewKeyValueCache()
|
||||
// branchTimestampCache stores branch timestamps for faster cache checking
|
||||
var branchTimestampCache = cache.NewKeyValueCache()
|
||||
// fileResponseCache stores responses from the Gitea server
|
||||
// TODO: make this an MRU cache with a size limit
|
||||
var fileResponseCache = cache.NewKeyValueCache()
|
||||
|
||||
// Create handler based on settings
|
||||
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
|
||||
giteaRoot, rawInfoPage, giteaAPIToken,
|
||||
BlacklistedPaths, allowedCorsDomains,
|
||||
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
|
||||
fastServer := server.SetupServer(handler)
|
||||
httpServer := server.SetupHTTPACMEChallengeServer(challengeCache)
|
||||
|
||||
// Setup listener and TLS
|
||||
log.Info().Msgf("Listening on https://%s", listeningAddress)
|
||||
listener, err := net.Listen("tcp", listeningAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't create listener: %s", err)
|
||||
}
|
||||
|
||||
// TODO: make "key-database.pogreb" set via flag
|
||||
certDB, err := database.New("key-database.pogreb")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create database: %v", err)
|
||||
}
|
||||
defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
|
||||
|
||||
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
|
||||
giteaRoot, giteaAPIToken, dnsProvider,
|
||||
acmeUseRateLimits,
|
||||
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
|
||||
certDB))
|
||||
|
||||
acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
interval := 12 * time.Hour
|
||||
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
|
||||
defer cancelCertMaintain()
|
||||
go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
|
||||
|
||||
if enableHTTPServer {
|
||||
go func() {
|
||||
err := httpServer.ListenAndServe("[::]:80")
|
||||
if err != nil {
|
||||
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Start the web fastServer
|
||||
err = fastServer.Serve(listener)
|
||||
if err != nil {
|
||||
log.Panic().Err(err).Msg("Couldn't start fastServer")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
package debug_stepper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var Enabled = strings.HasSuffix(os.Args[0], ".test") || os.Getenv("DEBUG") == "1"
|
||||
|
||||
var Logger = func(s string, i ...interface{}) {
|
||||
fmt.Printf(s, i...)
|
||||
}
|
||||
|
||||
type Stepper struct {
|
||||
Name string
|
||||
Start time.Time
|
||||
LastStep time.Time
|
||||
Completion time.Time
|
||||
}
|
||||
|
||||
func Start(name string) *Stepper {
|
||||
if !Enabled {
|
||||
return nil
|
||||
}
|
||||
t := time.Now()
|
||||
Logger("%s: started at %s\n", name, t.Format(time.RFC3339))
|
||||
return &Stepper{
|
||||
Name: name,
|
||||
Start: t,
|
||||
LastStep: t,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stepper) Debug(text string) {
|
||||
if !Enabled {
|
||||
return
|
||||
}
|
||||
t := time.Now()
|
||||
Logger("%s: %s (at %s, %s since last step, %s since start)\n", s.Name, text, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
|
||||
}
|
||||
|
||||
func (s *Stepper) Step(description string) {
|
||||
if !Enabled {
|
||||
return
|
||||
}
|
||||
if s.Completion != (time.Time{}) {
|
||||
Logger("%s: already completed all tasks.\n")
|
||||
return
|
||||
}
|
||||
t := time.Now()
|
||||
Logger("%s: completed %s at %s (%s)\n", s.Name, description, t.Format(time.RFC3339), t.Sub(s.LastStep).String())
|
||||
s.LastStep = t
|
||||
}
|
||||
|
||||
func (s *Stepper) Complete() {
|
||||
if !Enabled {
|
||||
return
|
||||
}
|
||||
if s.Completion != (time.Time{}) {
|
||||
Logger("%s: already completed all tasks.\n")
|
||||
return
|
||||
}
|
||||
t := time.Now()
|
||||
Logger("%s: completed all tasks at %s (%s since last step; total time: %s)\n", s.Name, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
|
||||
s.Completion = t
|
||||
}
|
||||
113
domains.go
113
domains.go
|
|
@ -1,113 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/valyala/fasthttp"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
|
||||
var DnsLookupCacheTimeout = 15 * time.Minute
|
||||
|
||||
// dnsLookupCache stores DNS lookups for custom domains
|
||||
var dnsLookupCache = mcache.New()
|
||||
|
||||
// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
|
||||
// If everything is fine, it returns the target data.
|
||||
func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch string) {
|
||||
// Get CNAME or TXT
|
||||
var cname string
|
||||
var err error
|
||||
if cachedName, ok := dnsLookupCache.Get(domain); ok {
|
||||
cname = cachedName.(string)
|
||||
} else {
|
||||
cname, err = net.LookupCNAME(domain)
|
||||
cname = strings.TrimSuffix(cname, ".")
|
||||
if err != nil || !strings.HasSuffix(cname, string(MainDomainSuffix)) {
|
||||
cname = ""
|
||||
// TODO: check if the A record matches!
|
||||
names, err := net.LookupTXT(domain)
|
||||
if err == nil {
|
||||
for _, name := range names {
|
||||
name = strings.TrimSuffix(name, ".")
|
||||
if strings.HasSuffix(name, string(MainDomainSuffix)) {
|
||||
cname = name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout)
|
||||
}
|
||||
if cname == "" {
|
||||
return
|
||||
}
|
||||
cnameParts := strings.Split(strings.TrimSuffix(cname, string(MainDomainSuffix)), ".")
|
||||
targetOwner = cnameParts[len(cnameParts)-1]
|
||||
if len(cnameParts) > 1 {
|
||||
targetRepo = cnameParts[len(cnameParts)-2]
|
||||
}
|
||||
if len(cnameParts) > 2 {
|
||||
targetBranch = cnameParts[len(cnameParts)-3]
|
||||
}
|
||||
if targetRepo == "" {
|
||||
targetRepo = "pages"
|
||||
}
|
||||
if targetBranch == "" && targetRepo != "pages" {
|
||||
targetBranch = "pages"
|
||||
}
|
||||
// if targetBranch is still empty, the caller must find the default branch
|
||||
return
|
||||
}
|
||||
|
||||
// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
|
||||
var CanonicalDomainCacheTimeout = 15 * time.Minute
|
||||
|
||||
// canonicalDomainCache stores canonical domains
|
||||
var canonicalDomainCache = mcache.New()
|
||||
|
||||
// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`).
|
||||
func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain string) (canonicalDomain string, valid bool) {
|
||||
domains := []string{}
|
||||
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
|
||||
domains = cachedValue.([]string)
|
||||
for _, domain := range domains {
|
||||
if domain == actualDomain {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
req := fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + GiteaApiToken)
|
||||
res := fasthttp.AcquireResponse()
|
||||
|
||||
err := upstreamClient.Do(req, res)
|
||||
if err == nil && res.StatusCode() == fasthttp.StatusOK {
|
||||
for _, domain := range strings.Split(string(res.Body()), "\n") {
|
||||
domain = strings.ToLower(domain)
|
||||
domain = strings.TrimSpace(domain)
|
||||
domain = strings.TrimPrefix(domain, "http://")
|
||||
domain = strings.TrimPrefix(domain, "https://")
|
||||
if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
|
||||
domains = append(domains, domain)
|
||||
}
|
||||
if domain == actualDomain {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
domains = append(domains, targetOwner+string(MainDomainSuffix))
|
||||
if domains[len(domains)-1] == actualDomain {
|
||||
valid = true
|
||||
}
|
||||
if targetRepo != "" && targetRepo != "pages" {
|
||||
domains[len(domains)-1] += "/" + targetRepo
|
||||
}
|
||||
_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout)
|
||||
}
|
||||
canonicalDomain = domains[0]
|
||||
return
|
||||
}
|
||||
3
go.mod
3
go.mod
|
|
@ -7,6 +7,9 @@ require (
|
|||
github.com/akrylysov/pogreb v0.10.1
|
||||
github.com/go-acme/lego/v4 v4.5.3
|
||||
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
|
||||
github.com/rs/zerolog v1.26.0
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/urfave/cli/v2 v2.3.0
|
||||
github.com/valyala/fasthttp v1.31.0
|
||||
github.com/valyala/fastjson v1.6.3
|
||||
)
|
||||
|
|
|
|||
19
go.sum
19
go.sum
|
|
@ -95,10 +95,12 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
|
|||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
|
||||
github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -148,6 +150,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
|
|||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA=
|
||||
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
|
|
@ -422,6 +425,10 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjI
|
|||
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
|
||||
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
|
||||
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI=
|
||||
|
|
@ -429,6 +436,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
|
|||
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE=
|
||||
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
|
|
@ -478,7 +486,9 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
|
|||
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
|
||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
|
||||
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
|
|
@ -499,6 +509,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
|
|||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
|
|
@ -557,6 +568,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -589,8 +601,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
|
|||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -653,8 +666,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -708,6 +722,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
|
|||
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
|
|||
542
handler.go
542
handler.go
|
|
@ -1,542 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
debug_stepper "codeberg.org/codeberg/pages/debug-stepper"
|
||||
"fmt"
|
||||
"github.com/OrlovEvgeny/go-mcache"
|
||||
"github.com/valyala/fasthttp"
|
||||
"github.com/valyala/fastjson"
|
||||
"io"
|
||||
"mime"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// handler handles a single HTTP request to the web server.
|
||||
func handler(ctx *fasthttp.RequestCtx) {
|
||||
s := debug_stepper.Start("handler")
|
||||
defer s.Complete()
|
||||
|
||||
ctx.Response.Header.Set("Server", "Codeberg Pages")
|
||||
|
||||
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
|
||||
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
|
||||
// Enable browser caching for up to 10 minutes
|
||||
ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
|
||||
|
||||
trimmedHost := TrimHostPort(ctx.Request.Host())
|
||||
|
||||
// Add HSTS for RawDomain and MainDomainSuffix
|
||||
if hsts := GetHSTSHeader(trimmedHost); hsts != "" {
|
||||
ctx.Response.Header.Set("Strict-Transport-Security", hsts)
|
||||
}
|
||||
|
||||
// Block all methods not required for static pages
|
||||
if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
|
||||
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
|
||||
ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
// Block blacklisted paths (like ACME challenges)
|
||||
for _, blacklistedPath := range BlacklistedPaths {
|
||||
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
|
||||
returnErrorPage(ctx, fasthttp.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Allow CORS for specified domains
|
||||
if ctx.IsOptions() {
|
||||
allowCors := false
|
||||
for _, allowedCorsDomain := range AllowedCorsDomains {
|
||||
if bytes.Equal(trimmedHost, allowedCorsDomain) {
|
||||
allowCors = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowCors {
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
|
||||
}
|
||||
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
|
||||
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare request information to Gitea
|
||||
var targetOwner, targetRepo, targetBranch, targetPath string
|
||||
var targetOptions = &upstreamOptions{
|
||||
ForbiddenMimeTypes: map[string]struct{}{},
|
||||
TryIndexPages: true,
|
||||
}
|
||||
|
||||
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
|
||||
// also disallow search indexing and add a Link header to the canonical URL.
|
||||
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
|
||||
if repo == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if the branch exists, otherwise treat it as a file path
|
||||
branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch)
|
||||
if branchTimestampResult == nil {
|
||||
// branch doesn't exist
|
||||
return false
|
||||
}
|
||||
|
||||
// Branch exists, use it
|
||||
targetRepo = repo
|
||||
targetPath = strings.Trim(strings.Join(path, "/"), "/")
|
||||
targetBranch = branchTimestampResult.branch
|
||||
|
||||
targetOptions.BranchTimestamp = branchTimestampResult.timestamp
|
||||
|
||||
if canonicalLink != "" {
|
||||
// Hide from search machines & add canonical link
|
||||
ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
|
||||
ctx.Response.Header.Set("Link",
|
||||
strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
|
||||
"; rel=\"canonical\"",
|
||||
)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
|
||||
var tryUpstream = func() {
|
||||
// check if a canonical domain exists on a request on MainDomain
|
||||
if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
|
||||
canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "")
|
||||
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(MainDomainSuffix)) {
|
||||
canonicalPath := string(ctx.RequestURI())
|
||||
if targetRepo != "pages" {
|
||||
canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2]
|
||||
}
|
||||
ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Try to request the file from the Gitea API
|
||||
if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, targetOptions) {
|
||||
returnErrorPage(ctx, ctx.Response.StatusCode())
|
||||
}
|
||||
}
|
||||
|
||||
s.Step("preparations")
|
||||
|
||||
if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) {
|
||||
// Serve raw content from RawDomain
|
||||
s.Debug("raw domain")
|
||||
|
||||
targetOptions.TryIndexPages = false
|
||||
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
|
||||
targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
if len(pathElements) < 2 {
|
||||
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
|
||||
ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
targetOwner = pathElements[0]
|
||||
targetRepo = pathElements[1]
|
||||
|
||||
// raw.codeberg.org/example/myrepo/@main/index.html
|
||||
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
|
||||
s.Step("raw domain preparations, now trying with specified branch")
|
||||
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
|
||||
string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
|
||||
) {
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
return
|
||||
}
|
||||
s.Debug("missing branch")
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
} else {
|
||||
s.Step("raw domain preparations, now trying with default branch")
|
||||
tryBranch(targetRepo, "", pathElements[2:],
|
||||
string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
|
||||
)
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
return
|
||||
}
|
||||
|
||||
} else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
|
||||
// Serve pages from subdomains of MainDomainSuffix
|
||||
s.Debug("main domain suffix")
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix))
|
||||
targetRepo = pathElements[0]
|
||||
targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
|
||||
|
||||
if targetOwner == "www" {
|
||||
// www.codeberg.page redirects to codeberg.page
|
||||
ctx.Redirect("https://" + string(MainDomainSuffix[1:]) + string(ctx.Path()), fasthttp.StatusPermanentRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a repo with the second directory as a branch
|
||||
// example.codeberg.page/myrepo/@main/index.html
|
||||
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
|
||||
if targetRepo == "pages" {
|
||||
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
|
||||
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
s.Step("main domain preparations, now trying with specified repo & branch")
|
||||
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
|
||||
"/"+pathElements[0]+"/%p",
|
||||
) {
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a branch for the "pages" repo
|
||||
// example.codeberg.page/@main/index.html
|
||||
if strings.HasPrefix(pathElements[0], "@") {
|
||||
s.Step("main domain preparations, now trying with specified branch")
|
||||
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a repo with a "pages" branch
|
||||
// example.codeberg.page/myrepo/index.html
|
||||
// example.codeberg.page/pages/... is not allowed here.
|
||||
s.Step("main domain preparations, now trying with specified repo")
|
||||
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
return
|
||||
}
|
||||
|
||||
// Try to use the "pages" repo on its default branch
|
||||
// example.codeberg.page/index.html
|
||||
s.Step("main domain preparations, now trying with default repo/branch")
|
||||
if tryBranch("pages", "", pathElements, "") {
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
return
|
||||
}
|
||||
|
||||
// Couldn't find a valid repo/branch
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
} else {
|
||||
trimmedHostStr := string(trimmedHost)
|
||||
|
||||
// Serve pages from external domains
|
||||
targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr)
|
||||
if targetOwner == "" {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
canonicalLink := ""
|
||||
if strings.HasPrefix(pathElements[0], "@") {
|
||||
targetBranch = pathElements[0][1:]
|
||||
pathElements = pathElements[1:]
|
||||
canonicalLink = "/%p"
|
||||
}
|
||||
|
||||
// Try to use the given repo on the given branch or the default branch
|
||||
s.Step("custom domain preparations, now trying with details from DNS")
|
||||
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
|
||||
canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr)
|
||||
if !valid {
|
||||
returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
|
||||
return
|
||||
} else if canonicalDomain != trimmedHostStr {
|
||||
// only redirect if the target is also a codeberg page!
|
||||
targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0])
|
||||
if targetOwner != "" {
|
||||
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
s.Step("tryBranch, now trying upstream")
|
||||
tryUpstream()
|
||||
return
|
||||
} else {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
|
||||
// with the provided status code.
|
||||
func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
|
||||
ctx.Response.SetStatusCode(code)
|
||||
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
|
||||
message := fasthttp.StatusMessage(code)
|
||||
if code == fasthttp.StatusMisdirectedRequest {
|
||||
message += " - domain not specified in <code>.domains</code> file"
|
||||
}
|
||||
if code == fasthttp.StatusFailedDependency {
|
||||
message += " - target repo/branch doesn't exist or is private"
|
||||
}
|
||||
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
|
||||
}
|
||||
|
||||
// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
|
||||
var DefaultBranchCacheTimeout = 15 * time.Minute
|
||||
|
||||
// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
|
||||
// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
|
||||
// picked up faster, while still allowing the content to be cached longer if nothing changes.
|
||||
var BranchExistanceCacheTimeout = 5 * time.Minute
|
||||
|
||||
// branchTimestampCache stores branch timestamps for faster cache checking
|
||||
var branchTimestampCache = mcache.New()
|
||||
|
||||
type branchTimestamp struct {
|
||||
branch string
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
|
||||
// on your available memory.
|
||||
var FileCacheTimeout = 5 * time.Minute
|
||||
|
||||
// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
|
||||
var FileCacheSizeLimit = 1024 * 1024
|
||||
|
||||
// fileResponseCache stores responses from the Gitea server
|
||||
// TODO: make this an MRU cache with a size limit
|
||||
var fileResponseCache = mcache.New()
|
||||
|
||||
type fileResponse struct {
|
||||
exists bool
|
||||
mimeType string
|
||||
body []byte
|
||||
}
|
||||
|
||||
// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
|
||||
// (or nil if the branch doesn't exist)
|
||||
func getBranchTimestamp(owner, repo, branch string) *branchTimestamp {
|
||||
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return result.(*branchTimestamp)
|
||||
}
|
||||
result := &branchTimestamp{}
|
||||
result.branch = branch
|
||||
if branch == "" {
|
||||
// Get default branch
|
||||
var body = make([]byte, 0)
|
||||
status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
|
||||
return nil
|
||||
}
|
||||
result.branch = fastjson.GetString(body, "default_branch")
|
||||
}
|
||||
|
||||
var body = make([]byte, 0)
|
||||
status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+GiteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
|
||||
return result
|
||||
}
|
||||
|
||||
var upstreamClient = fasthttp.Client{
|
||||
ReadTimeout: 10 * time.Second,
|
||||
MaxConnDuration: 60 * time.Second,
|
||||
MaxConnWaitTimeout: 1000 * time.Millisecond,
|
||||
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
}
|
||||
|
||||
// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
|
||||
func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, targetBranch string, targetPath string, options *upstreamOptions) (final bool) {
|
||||
s := debug_stepper.Start("upstream")
|
||||
defer s.Complete()
|
||||
|
||||
if options.ForbiddenMimeTypes == nil {
|
||||
options.ForbiddenMimeTypes = map[string]struct{}{}
|
||||
}
|
||||
|
||||
// Check if the branch exists and when it was modified
|
||||
if options.BranchTimestamp == (time.Time{}) {
|
||||
branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch)
|
||||
|
||||
if branch == nil {
|
||||
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return true
|
||||
}
|
||||
targetBranch = branch.branch
|
||||
options.BranchTimestamp = branch.timestamp
|
||||
}
|
||||
|
||||
if targetOwner == "" || targetRepo == "" || targetBranch == "" {
|
||||
returnErrorPage(ctx, fasthttp.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the browser has a cached version
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
|
||||
if !ifModifiedSince.Before(options.BranchTimestamp) {
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
|
||||
return true
|
||||
}
|
||||
}
|
||||
s.Step("preparations")
|
||||
|
||||
// Make a GET request to the upstream URL
|
||||
uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
|
||||
var req *fasthttp.Request
|
||||
var res *fasthttp.Response
|
||||
var cachedResponse fileResponse
|
||||
var err error
|
||||
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
|
||||
cachedResponse = cachedValue.(fileResponse)
|
||||
} else {
|
||||
req = fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + uri + "?access_token=" + GiteaApiToken)
|
||||
res = fasthttp.AcquireResponse()
|
||||
res.SetBodyStream(&strings.Reader{}, -1)
|
||||
err = upstreamClient.Do(req, res)
|
||||
}
|
||||
s.Step("acquisition")
|
||||
|
||||
// Handle errors
|
||||
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
|
||||
if options.TryIndexPages {
|
||||
// copy the options struct & try if an index page exists
|
||||
optionsForIndexPages := *options
|
||||
optionsForIndexPages.TryIndexPages = false
|
||||
optionsForIndexPages.AppendTrailingSlash = true
|
||||
for _, indexPage := range IndexPages {
|
||||
if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
// compatibility fix for GitHub Pages (/example → /example.html)
|
||||
optionsForIndexPages.AppendTrailingSlash = false
|
||||
optionsForIndexPages.RedirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
|
||||
if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath + ".html", &optionsForIndexPages) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
|
||||
if res != nil {
|
||||
// Update cache if the request is fresh
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, FileCacheTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
|
||||
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
|
||||
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
|
||||
// options.AppendTrailingSlash is only true when looking for index pages
|
||||
if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
|
||||
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
|
||||
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if options.RedirectIfExists != "" {
|
||||
ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
s.Step("error handling")
|
||||
|
||||
// Set the MIME type
|
||||
mimeType := mime.TypeByExtension(path.Ext(targetPath))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
|
||||
if options.DefaultMimeType != "" {
|
||||
mimeType = options.DefaultMimeType
|
||||
} else {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
ctx.Response.Header.SetContentType(mimeType)
|
||||
|
||||
// Everything's okay so far
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusOK)
|
||||
ctx.Response.Header.SetLastModified(options.BranchTimestamp)
|
||||
|
||||
s.Step("response preparations")
|
||||
|
||||
// Write the response body to the original request
|
||||
var cacheBodyWriter bytes.Buffer
|
||||
if res != nil {
|
||||
if res.Header.ContentLength() > FileCacheSizeLimit {
|
||||
err = res.BodyWriteTo(ctx.Response.BodyWriter())
|
||||
} else {
|
||||
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
|
||||
}
|
||||
} else {
|
||||
_, err = ctx.Write(cachedResponse.body)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
|
||||
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
s.Step("response")
|
||||
|
||||
if res != nil && ctx.Err() == nil {
|
||||
cachedResponse.exists = true
|
||||
cachedResponse.mimeType = mimeType
|
||||
cachedResponse.body = cacheBodyWriter.Bytes()
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// upstreamOptions provides various options for the upstream request.
|
||||
type upstreamOptions struct {
|
||||
DefaultMimeType string
|
||||
ForbiddenMimeTypes map[string]struct{}
|
||||
TryIndexPages bool
|
||||
AppendTrailingSlash bool
|
||||
RedirectIfExists string
|
||||
BranchTimestamp time.Time
|
||||
}
|
||||
|
|
@ -51,6 +51,7 @@ frontend https_sni_frontend
|
|||
###################################################
|
||||
acl use_http_backend req.ssl_sni -i "codeberg.org"
|
||||
acl use_http_backend req.ssl_sni -i "join.codeberg.org"
|
||||
# TODO: use this if no SNI exists
|
||||
use_backend https_termination_backend if use_http_backend
|
||||
|
||||
############################
|
||||
|
|
|
|||
56
helpers.go
56
helpers.go
|
|
@ -1,56 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"github.com/akrylysov/pogreb"
|
||||
)
|
||||
|
||||
// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
|
||||
// string for custom domains.
|
||||
func GetHSTSHeader(host []byte) string {
|
||||
if bytes.HasSuffix(host, MainDomainSuffix) || bytes.Equal(host, RawDomain) {
|
||||
return "max-age=63072000; includeSubdomains; preload"
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func TrimHostPort(host []byte) []byte {
|
||||
i := bytes.IndexByte(host, ':')
|
||||
if i >= 0 {
|
||||
return host[:i]
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) {
|
||||
var resGob bytes.Buffer
|
||||
resEnc := gob.NewEncoder(&resGob)
|
||||
err := resEnc.Encode(obj)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = db.Put(name, resGob.Bytes())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool {
|
||||
resBytes, err := db.Get(name)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if resBytes == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
resGob := bytes.NewBuffer(resBytes)
|
||||
resDec := gob.NewDecoder(resGob)
|
||||
err = resDec.Decode(obj)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package html
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
|
||||
// with the provided status code.
|
||||
func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
|
||||
ctx.Response.SetStatusCode(code)
|
||||
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
|
||||
message := fasthttp.StatusMessage(code)
|
||||
if code == fasthttp.StatusMisdirectedRequest {
|
||||
message += " - domain not specified in <code>.domains</code> file"
|
||||
}
|
||||
if code == fasthttp.StatusFailedDependency {
|
||||
message += " - target repo/branch doesn't exist or is private"
|
||||
}
|
||||
// TODO: use template engine?
|
||||
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package html
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed 404.html
|
||||
var NotFoundPage []byte
|
||||
161
main.go
161
main.go
|
|
@ -1,159 +1,32 @@
|
|||
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
|
||||
//
|
||||
// Mapping custom domains is not static anymore, but can be done with DNS:
|
||||
//
|
||||
// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
|
||||
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
|
||||
//
|
||||
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
|
||||
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
|
||||
// www.example.org. IN CNAME main.pages.example.codeberg.page.
|
||||
//
|
||||
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
|
||||
// for "example.org" (if your provider allows ALIAS or similar records):
|
||||
// example.org IN ALIAS codeberg.page.
|
||||
//
|
||||
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
_ "embed"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"codeberg.org/codeberg/pages/cmd"
|
||||
)
|
||||
|
||||
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
|
||||
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
|
||||
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
|
||||
var MainDomainSuffix = []byte("." + envOr("PAGES_DOMAIN", "codeberg.page"))
|
||||
var (
|
||||
// can be changed with -X on compile
|
||||
version = "dev"
|
||||
)
|
||||
|
||||
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
|
||||
var GiteaRoot = []byte(envOr("GITEA_ROOT", "https://codeberg.org"))
|
||||
|
||||
var GiteaApiToken = envOr("GITEA_API_TOKEN", "")
|
||||
|
||||
//go:embed 404.html
|
||||
var NotFoundPage []byte
|
||||
|
||||
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
|
||||
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
|
||||
// (set to []byte(nil) to disable raw content hosting)
|
||||
var RawDomain = []byte(envOr("RAW_DOMAIN", "raw.codeberg.org"))
|
||||
|
||||
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
|
||||
var RawInfoPage = envOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
|
||||
|
||||
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
|
||||
var AllowedCorsDomains = [][]byte{
|
||||
RawDomain,
|
||||
[]byte("fonts.codeberg.org"),
|
||||
[]byte("design.codeberg.org"),
|
||||
}
|
||||
|
||||
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
|
||||
var BlacklistedPaths = [][]byte{
|
||||
[]byte("/.well-known/acme-challenge/"),
|
||||
}
|
||||
|
||||
// IndexPages lists pages that may be considered as index pages for directories.
|
||||
var IndexPages = []string{
|
||||
"index.html",
|
||||
}
|
||||
|
||||
// main sets up and starts the web server.
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" {
|
||||
if len(os.Args) < 2 {
|
||||
println("--remove-certificate requires at least one domain as an argument")
|
||||
os.Exit(1)
|
||||
}
|
||||
if keyDatabaseErr != nil {
|
||||
panic(keyDatabaseErr)
|
||||
}
|
||||
for _, domain := range os.Args[2:] {
|
||||
if err := keyDatabase.Delete([]byte(domain)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if err := keyDatabase.Sync(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(0)
|
||||
app := cli.NewApp()
|
||||
app.Name = "pages-server"
|
||||
app.Version = version
|
||||
app.Usage = "pages server"
|
||||
app.Action = cmd.Serve
|
||||
app.Flags = cmd.ServeFlags
|
||||
app.Commands = []*cli.Command{
|
||||
cmd.Certs,
|
||||
}
|
||||
|
||||
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
|
||||
if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
|
||||
MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
|
||||
}
|
||||
GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
|
||||
|
||||
// Use HOST and PORT environment variables to determine listening address
|
||||
address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "443"))
|
||||
log.Printf("Listening on https://%s", address)
|
||||
|
||||
// Enable compression by wrapping the handler() method with the compression function provided by FastHTTP
|
||||
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
|
||||
|
||||
server := &fasthttp.Server{
|
||||
Handler: compressedHandler,
|
||||
DisablePreParseMultipartForm: false,
|
||||
MaxRequestBodySize: 0,
|
||||
NoDefaultServerHeader: true,
|
||||
NoDefaultDate: true,
|
||||
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
|
||||
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
MaxConnsPerIP: 100,
|
||||
}
|
||||
|
||||
// Setup listener and TLS
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't create listener: %s", err)
|
||||
}
|
||||
listener = tls.NewListener(listener, tlsConfig)
|
||||
|
||||
setupCertificates()
|
||||
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
|
||||
go (func() {
|
||||
challengePath := []byte("/.well-known/acme-challenge/")
|
||||
err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
|
||||
if bytes.HasPrefix(ctx.Path(), challengePath) {
|
||||
challenge, ok := challengeCache.Get(string(TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
|
||||
if !ok || challenge == nil {
|
||||
ctx.SetStatusCode(http.StatusNotFound)
|
||||
ctx.SetBodyString("no challenge for this token")
|
||||
}
|
||||
ctx.SetBodyString(challenge.(string))
|
||||
} else {
|
||||
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't start HTTP server: %s", err)
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
||||
// Start the web server
|
||||
err = server.Serve(listener)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't start server: %s", err)
|
||||
if err := app.Run(os.Args); err != nil {
|
||||
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// envOr reads an environment variable and returns a default value if it's empty.
|
||||
func envOr(env string, or string) string {
|
||||
if v := os.Getenv(env); v != "" {
|
||||
return v
|
||||
}
|
||||
return or
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
package cache
|
||||
|
||||
import "time"
|
||||
|
||||
type SetGetKey interface {
|
||||
Set(key string, value interface{}, ttl time.Duration) error
|
||||
Get(key string) (interface{}, bool)
|
||||
Remove(key string)
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package cache
|
||||
|
||||
import "github.com/OrlovEvgeny/go-mcache"
|
||||
|
||||
func NewKeyValueCache() SetGetKey {
|
||||
return mcache.New()
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package certificates
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
|
||||
"github.com/go-acme/lego/v4/registration"
|
||||
)
|
||||
|
||||
type AcmeAccount struct {
|
||||
Email string
|
||||
Registration *registration.Resource
|
||||
Key crypto.PrivateKey `json:"-"`
|
||||
KeyPEM string `json:"Key"`
|
||||
}
|
||||
|
||||
// make sure AcmeAccount match User interface
|
||||
var _ registration.User = &AcmeAccount{}
|
||||
|
||||
func (u *AcmeAccount) GetEmail() string {
|
||||
return u.Email
|
||||
}
|
||||
func (u AcmeAccount) GetRegistration() *registration.Resource {
|
||||
return u.Registration
|
||||
}
|
||||
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
|
||||
return u.Key
|
||||
}
|
||||
|
|
@ -0,0 +1,522 @@
|
|||
package certificates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-acme/lego/v4/certcrypto"
|
||||
"github.com/go-acme/lego/v4/certificate"
|
||||
"github.com/go-acme/lego/v4/challenge"
|
||||
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
|
||||
"github.com/go-acme/lego/v4/lego"
|
||||
"github.com/go-acme/lego/v4/providers/dns"
|
||||
"github.com/go-acme/lego/v4/registration"
|
||||
"github.com/reugn/equalizer"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/database"
|
||||
dnsutils "codeberg.org/codeberg/pages/server/dns"
|
||||
"codeberg.org/codeberg/pages/server/upstream"
|
||||
)
|
||||
|
||||
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
|
||||
func TLSConfig(mainDomainSuffix []byte,
|
||||
giteaRoot, giteaAPIToken, dnsProvider string,
|
||||
acmeUseRateLimits bool,
|
||||
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
|
||||
certDB database.CertDB) *tls.Config {
|
||||
return &tls.Config{
|
||||
// check DNS name & get certificate from Let's Encrypt
|
||||
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
|
||||
sniBytes := []byte(sni)
|
||||
if len(sni) < 1 {
|
||||
return nil, errors.New("missing sni")
|
||||
}
|
||||
|
||||
if info.SupportedProtos != nil {
|
||||
for _, proto := range info.SupportedProtos {
|
||||
if proto == tlsalpn01.ACMETLS1Protocol {
|
||||
challenge, ok := challengeCache.Get(sni)
|
||||
if !ok {
|
||||
return nil, errors.New("no challenge for this domain")
|
||||
}
|
||||
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetOwner := ""
|
||||
if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
|
||||
// deliver default certificate for the main domain (*.codeberg.page)
|
||||
sniBytes = mainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
} else {
|
||||
var targetRepo, targetBranch string
|
||||
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
|
||||
if targetOwner == "" {
|
||||
// DNS not set up, return main certificate to redirect to the docs
|
||||
sniBytes = mainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
} else {
|
||||
_, _ = targetRepo, targetBranch
|
||||
_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
|
||||
if !valid {
|
||||
sniBytes = mainDomainSuffix
|
||||
sni = string(sniBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if tlsCertificate, ok := keyCache.Get(sni); ok {
|
||||
// we can use an existing certificate object
|
||||
return tlsCertificate.(*tls.Certificate), nil
|
||||
}
|
||||
|
||||
var tlsCertificate tls.Certificate
|
||||
var err error
|
||||
var ok bool
|
||||
if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
|
||||
// request a new certificate
|
||||
if bytes.Equal(sniBytes, mainDomainSuffix) {
|
||||
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
|
||||
}
|
||||
|
||||
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tlsCertificate, nil
|
||||
},
|
||||
PreferServerCipherSuites: true,
|
||||
NextProtos: []string{
|
||||
"http/1.1",
|
||||
tlsalpn01.ACMETLS1Protocol,
|
||||
},
|
||||
|
||||
// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
|
||||
// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: []uint16{
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
|
||||
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func checkUserLimit(user string) error {
|
||||
userLimit, ok := acmeClientCertificateLimitPerUser[user]
|
||||
if !ok {
|
||||
// Each Codeberg user can only add 10 new domains per day.
|
||||
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
|
||||
acmeClientCertificateLimitPerUser[user] = userLimit
|
||||
}
|
||||
if !userLimit.Ask() {
|
||||
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var acmeClient, mainDomainAcmeClient *lego.Client
|
||||
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
|
||||
|
||||
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
|
||||
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
|
||||
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
|
||||
|
||||
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
|
||||
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
|
||||
|
||||
type AcmeTLSChallengeProvider struct {
|
||||
challengeCache cache.SetGetKey
|
||||
}
|
||||
|
||||
// make sure AcmeTLSChallengeProvider match Provider interface
|
||||
var _ challenge.Provider = AcmeTLSChallengeProvider{}
|
||||
|
||||
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
|
||||
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
|
||||
}
|
||||
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
|
||||
a.challengeCache.Remove(domain)
|
||||
return nil
|
||||
}
|
||||
|
||||
type AcmeHTTPChallengeProvider struct {
|
||||
challengeCache cache.SetGetKey
|
||||
}
|
||||
|
||||
// make sure AcmeHTTPChallengeProvider match Provider interface
|
||||
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
|
||||
|
||||
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
|
||||
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
|
||||
}
|
||||
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
|
||||
a.challengeCache.Remove(domain + "/" + token)
|
||||
return nil
|
||||
}
|
||||
|
||||
func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
|
||||
// parse certificate from database
|
||||
res, err := certDB.Get(sni)
|
||||
if err != nil {
|
||||
panic(err) // TODO: no panic
|
||||
}
|
||||
if res == nil {
|
||||
return tls.Certificate{}, false
|
||||
}
|
||||
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// TODO: document & put into own function
|
||||
if !bytes.Equal(sni, mainDomainSuffix) {
|
||||
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// renew certificates 7 days before they expire
|
||||
if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
|
||||
// TODO: add ValidUntil to custom res struct
|
||||
if res.CSR != nil && len(res.CSR) > 0 {
|
||||
// CSR stores the time when the renewal shall be tried again
|
||||
nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
|
||||
if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
|
||||
return tlsCertificate, true
|
||||
}
|
||||
}
|
||||
go (func() {
|
||||
res.CSR = nil // acme client doesn't like CSR to be set
|
||||
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
|
||||
if err != nil {
|
||||
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
|
||||
}
|
||||
})()
|
||||
}
|
||||
}
|
||||
|
||||
return tlsCertificate, true
|
||||
}
|
||||
|
||||
var obtainLocks = sync.Map{}
|
||||
|
||||
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
|
||||
name := strings.TrimPrefix(domains[0], "*")
|
||||
if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
|
||||
domains = domains[1:]
|
||||
}
|
||||
|
||||
// lock to avoid simultaneous requests
|
||||
_, working := obtainLocks.LoadOrStore(name, struct{}{})
|
||||
if working {
|
||||
for working {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, working = obtainLocks.Load(name)
|
||||
}
|
||||
cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
|
||||
if !ok {
|
||||
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
defer obtainLocks.Delete(name)
|
||||
|
||||
if acmeClient == nil {
|
||||
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil
|
||||
}
|
||||
|
||||
// request actual cert
|
||||
var res *certificate.Resource
|
||||
var err error
|
||||
if renew != nil && renew.CertURL != "" {
|
||||
if acmeUseRateLimits {
|
||||
acmeClientRequestLimit.Take()
|
||||
}
|
||||
log.Printf("Renewing certificate for %v", domains)
|
||||
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
|
||||
if err != nil {
|
||||
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
|
||||
res = nil
|
||||
}
|
||||
}
|
||||
if res == nil {
|
||||
if user != "" {
|
||||
if err := checkUserLimit(user); err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if acmeUseRateLimits {
|
||||
acmeClientOrderLimit.Take()
|
||||
acmeClientRequestLimit.Take()
|
||||
}
|
||||
log.Printf("Requesting new certificate for %v", domains)
|
||||
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
|
||||
Domains: domains,
|
||||
Bundle: true,
|
||||
MustStaple: false,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
|
||||
if renew != nil && renew.CertURL != "" {
|
||||
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
|
||||
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
|
||||
// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
|
||||
renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
|
||||
if err := keyDatabase.Put(name, renew); err != nil {
|
||||
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
|
||||
}
|
||||
return tlsCertificate, nil
|
||||
}
|
||||
}
|
||||
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
|
||||
}
|
||||
log.Printf("Obtained certificate for %v", domains)
|
||||
|
||||
if err := keyDatabase.Put(name, res); err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
return tlsCertificate, nil
|
||||
}
|
||||
|
||||
func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
|
||||
const configFile = "acme-account.json"
|
||||
var myAcmeAccount AcmeAccount
|
||||
var myAcmeConfig *lego.Config
|
||||
|
||||
if account, err := ioutil.ReadFile(configFile); err == nil {
|
||||
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
|
||||
myAcmeConfig.CADirURL = acmeAPI
|
||||
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
|
||||
|
||||
// Validate Config
|
||||
_, err := lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
// TODO: should we fail hard instead?
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
}
|
||||
return myAcmeConfig, nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
myAcmeAccount = AcmeAccount{
|
||||
Email: acmeMail,
|
||||
Key: privateKey,
|
||||
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
|
||||
}
|
||||
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
|
||||
myAcmeConfig.CADirURL = acmeAPI
|
||||
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
|
||||
tempClient, err := lego.NewClient(myAcmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
// accept terms & log in to EAB
|
||||
if acmeEabKID == "" || acmeEabHmac == "" {
|
||||
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
myAcmeAccount.Registration = reg
|
||||
}
|
||||
} else {
|
||||
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
|
||||
TermsOfServiceAgreed: acmeAcceptTerms,
|
||||
Kid: acmeEabKID,
|
||||
HmacEncoded: acmeEabHmac,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
myAcmeAccount.Registration = reg
|
||||
}
|
||||
}
|
||||
|
||||
if myAcmeAccount.Registration != nil {
|
||||
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
|
||||
if err != nil {
|
||||
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
|
||||
select {}
|
||||
}
|
||||
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
|
||||
if err != nil {
|
||||
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
|
||||
select {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return myAcmeConfig, nil
|
||||
}
|
||||
|
||||
func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
|
||||
// getting main cert before ACME account so that we can fail here without hitting rate limits
|
||||
mainCertBytes, err := certDB.Get(mainDomainSuffix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cert database is not working")
|
||||
}
|
||||
|
||||
acmeClient, err = lego.NewClient(acmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
|
||||
}
|
||||
if enableHTTPServer {
|
||||
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
|
||||
} else {
|
||||
if dnsProvider == "" {
|
||||
// using mock server, don't use wildcard certs
|
||||
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
|
||||
}
|
||||
} else {
|
||||
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
|
||||
}
|
||||
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if mainCertBytes == nil {
|
||||
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
|
||||
for {
|
||||
// clean up expired certs
|
||||
now := time.Now()
|
||||
expiredCertCount := 0
|
||||
keyDatabaseIterator := certDB.Items()
|
||||
key, resBytes, err := keyDatabaseIterator.Next()
|
||||
for err == nil {
|
||||
if !bytes.Equal(key, mainDomainSuffix) {
|
||||
resGob := bytes.NewBuffer(resBytes)
|
||||
resDec := gob.NewDecoder(resGob)
|
||||
res := &certificate.Resource{}
|
||||
err = resDec.Decode(res)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
|
||||
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
|
||||
err := certDB.Delete(key)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
|
||||
} else {
|
||||
expiredCertCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
key, resBytes, err = keyDatabaseIterator.Next()
|
||||
}
|
||||
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
|
||||
|
||||
// compact the database
|
||||
result, err := certDB.Compact()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Compacting key database failed: %s", err)
|
||||
} else {
|
||||
log.Printf("[INFO] Compacted key database (%+v)", result)
|
||||
}
|
||||
|
||||
// update main cert
|
||||
res, err := certDB.Get(mainDomainSuffix)
|
||||
if err != nil {
|
||||
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
|
||||
} else if res == nil {
|
||||
log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
|
||||
} else {
|
||||
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
|
||||
|
||||
// renew main certificate 30 days before it expires
|
||||
if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
|
||||
go (func() {
|
||||
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
|
||||
}
|
||||
})()
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package certificates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/go-acme/lego/v4/certcrypto"
|
||||
"github.com/go-acme/lego/v4/certificate"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/database"
|
||||
)
|
||||
|
||||
func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate {
|
||||
key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
template := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{
|
||||
CommonName: domain,
|
||||
Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
|
||||
OrganizationalUnit: []string{
|
||||
"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
|
||||
"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
|
||||
"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
|
||||
"Error message: " + msg,
|
||||
},
|
||||
},
|
||||
|
||||
// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
|
||||
NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
|
||||
NotBefore: time.Now(),
|
||||
|
||||
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
certBytes, err := x509.CreateCertificate(
|
||||
rand.Reader,
|
||||
&template,
|
||||
&template,
|
||||
&key.(*rsa.PrivateKey).PublicKey,
|
||||
key,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
err = pem.Encode(out, &pem.Block{
|
||||
Bytes: certBytes,
|
||||
Type: "CERTIFICATE",
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
outBytes := out.Bytes()
|
||||
res := &certificate.Resource{
|
||||
PrivateKey: certcrypto.PEMEncode(key),
|
||||
Certificate: outBytes,
|
||||
IssuerCertificate: outBytes,
|
||||
Domain: domain,
|
||||
}
|
||||
databaseName := domain
|
||||
if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] {
|
||||
databaseName = mainDomainSuffix
|
||||
}
|
||||
if err := keyDatabase.Put(databaseName, res); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return tlsCertificate
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"github.com/akrylysov/pogreb"
|
||||
"github.com/go-acme/lego/v4/certificate"
|
||||
)
|
||||
|
||||
type CertDB interface {
|
||||
Close() error
|
||||
Put(name string, cert *certificate.Resource) error
|
||||
Get(name []byte) (*certificate.Resource, error)
|
||||
Delete(key []byte) error
|
||||
Compact() (pogreb.CompactionResult, error)
|
||||
Items() *pogreb.ItemIterator
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/akrylysov/pogreb"
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
"github.com/go-acme/lego/v4/certificate"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type aDB struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
intern *pogreb.DB
|
||||
syncInterval time.Duration
|
||||
}
|
||||
|
||||
func (p aDB) Close() error {
|
||||
p.cancel()
|
||||
return p.intern.Sync()
|
||||
}
|
||||
|
||||
func (p aDB) Put(name string, cert *certificate.Resource) error {
|
||||
var resGob bytes.Buffer
|
||||
if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.intern.Put([]byte(name), resGob.Bytes())
|
||||
}
|
||||
|
||||
func (p aDB) Get(name []byte) (*certificate.Resource, error) {
|
||||
cert := &certificate.Resource{}
|
||||
resBytes, err := p.intern.Get(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resBytes == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
|
||||
func (p aDB) Delete(key []byte) error {
|
||||
return p.intern.Delete(key)
|
||||
}
|
||||
|
||||
func (p aDB) Compact() (pogreb.CompactionResult, error) {
|
||||
return p.intern.Compact()
|
||||
}
|
||||
|
||||
func (p aDB) Items() *pogreb.ItemIterator {
|
||||
return p.intern.Items()
|
||||
}
|
||||
|
||||
var _ CertDB = &aDB{}
|
||||
|
||||
func (p aDB) sync() {
|
||||
for {
|
||||
err := p.intern.Sync()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Syncing cert database failed")
|
||||
}
|
||||
select {
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
case <-time.After(p.syncInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p aDB) compact() {
|
||||
for {
|
||||
err := p.intern.Sync()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Syncing cert database failed")
|
||||
}
|
||||
select {
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
case <-time.After(p.syncInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func New(path string) (CertDB, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("path not set")
|
||||
}
|
||||
db, err := pogreb.Open(path, &pogreb.Options{
|
||||
BackgroundSyncInterval: 30 * time.Second,
|
||||
BackgroundCompactionInterval: 6 * time.Hour,
|
||||
FileSystem: fs.OSMMap,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := &aDB{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
intern: db,
|
||||
syncInterval: 5 * time.Minute,
|
||||
}
|
||||
|
||||
go result.sync()
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package dns
|
||||
|
||||
import "time"
|
||||
|
||||
// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
|
||||
var lookupCacheTimeout = 15 * time.Minute
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
package dns
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
)
|
||||
|
||||
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
|
||||
// If everything is fine, it returns the target data.
|
||||
func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
|
||||
// Get CNAME or TXT
|
||||
var cname string
|
||||
var err error
|
||||
if cachedName, ok := dnsLookupCache.Get(domain); ok {
|
||||
cname = cachedName.(string)
|
||||
} else {
|
||||
cname, err = net.LookupCNAME(domain)
|
||||
cname = strings.TrimSuffix(cname, ".")
|
||||
if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) {
|
||||
cname = ""
|
||||
// TODO: check if the A record matches!
|
||||
names, err := net.LookupTXT(domain)
|
||||
if err == nil {
|
||||
for _, name := range names {
|
||||
name = strings.TrimSuffix(name, ".")
|
||||
if strings.HasSuffix(name, mainDomainSuffix) {
|
||||
cname = name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
|
||||
}
|
||||
if cname == "" {
|
||||
return
|
||||
}
|
||||
cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".")
|
||||
targetOwner = cnameParts[len(cnameParts)-1]
|
||||
if len(cnameParts) > 1 {
|
||||
targetRepo = cnameParts[len(cnameParts)-2]
|
||||
}
|
||||
if len(cnameParts) > 2 {
|
||||
targetBranch = cnameParts[len(cnameParts)-3]
|
||||
}
|
||||
if targetRepo == "" {
|
||||
targetRepo = "pages"
|
||||
}
|
||||
if targetBranch == "" && targetRepo != "pages" {
|
||||
targetBranch = "pages"
|
||||
}
|
||||
// if targetBranch is still empty, the caller must find the default branch
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/dns"
|
||||
"codeberg.org/codeberg/pages/server/upstream"
|
||||
"codeberg.org/codeberg/pages/server/utils"
|
||||
)
|
||||
|
||||
// Handler handles a single HTTP request to the web server.
|
||||
func Handler(mainDomainSuffix, rawDomain []byte,
|
||||
giteaRoot, rawInfoPage, giteaAPIToken string,
|
||||
blacklistedPaths, allowedCorsDomains [][]byte,
|
||||
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
|
||||
return func(ctx *fasthttp.RequestCtx) {
|
||||
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
|
||||
|
||||
ctx.Response.Header.Set("Server", "Codeberg Pages")
|
||||
|
||||
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
|
||||
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
|
||||
// Enable browser caching for up to 10 minutes
|
||||
ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
|
||||
|
||||
trimmedHost := utils.TrimHostPort(ctx.Request.Host())
|
||||
|
||||
// Add HSTS for RawDomain and MainDomainSuffix
|
||||
if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
|
||||
ctx.Response.Header.Set("Strict-Transport-Security", hsts)
|
||||
}
|
||||
|
||||
// Block all methods not required for static pages
|
||||
if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
|
||||
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
|
||||
ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
// Block blacklisted paths (like ACME challenges)
|
||||
for _, blacklistedPath := range blacklistedPaths {
|
||||
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Allow CORS for specified domains
|
||||
if ctx.IsOptions() {
|
||||
allowCors := false
|
||||
for _, allowedCorsDomain := range allowedCorsDomains {
|
||||
if bytes.Equal(trimmedHost, allowedCorsDomain) {
|
||||
allowCors = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowCors {
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
|
||||
}
|
||||
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
|
||||
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare request information to Gitea
|
||||
var targetOwner, targetRepo, targetBranch, targetPath string
|
||||
var targetOptions = &upstream.Options{
|
||||
ForbiddenMimeTypes: map[string]struct{}{},
|
||||
TryIndexPages: true,
|
||||
}
|
||||
|
||||
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
|
||||
// also disallow search indexing and add a Link header to the canonical URL.
|
||||
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
|
||||
if repo == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if the branch exists, otherwise treat it as a file path
|
||||
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
|
||||
if branchTimestampResult == nil {
|
||||
// branch doesn't exist
|
||||
return false
|
||||
}
|
||||
|
||||
// Branch exists, use it
|
||||
targetRepo = repo
|
||||
targetPath = strings.Trim(strings.Join(path, "/"), "/")
|
||||
targetBranch = branchTimestampResult.Branch
|
||||
|
||||
targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
|
||||
|
||||
if canonicalLink != "" {
|
||||
// Hide from search machines & add canonical link
|
||||
ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
|
||||
ctx.Response.Header.Set("Link",
|
||||
strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
|
||||
"; rel=\"canonical\"",
|
||||
)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
log.Debug().Msg("preparations")
|
||||
if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
|
||||
// Serve raw content from RawDomain
|
||||
log.Debug().Msg("raw domain")
|
||||
|
||||
targetOptions.TryIndexPages = false
|
||||
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
|
||||
targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
if len(pathElements) < 2 {
|
||||
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
|
||||
ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
targetOwner = pathElements[0]
|
||||
targetRepo = pathElements[1]
|
||||
|
||||
// raw.codeberg.org/example/myrepo/@main/index.html
|
||||
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
|
||||
log.Debug().Msg("raw domain preparations, now trying with specified branch")
|
||||
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
|
||||
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
|
||||
) {
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
return
|
||||
}
|
||||
log.Debug().Msg("missing branch")
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug().Msg("raw domain preparations, now trying with default branch")
|
||||
tryBranch(targetRepo, "", pathElements[2:],
|
||||
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
|
||||
)
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
return
|
||||
|
||||
} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
|
||||
// Serve pages from subdomains of MainDomainSuffix
|
||||
log.Debug().Msg("main domain suffix")
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
|
||||
targetRepo = pathElements[0]
|
||||
targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
|
||||
|
||||
if targetOwner == "www" {
|
||||
// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
|
||||
ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a repo with the second directory as a branch
|
||||
// example.codeberg.page/myrepo/@main/index.html
|
||||
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
|
||||
if targetRepo == "pages" {
|
||||
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
|
||||
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
|
||||
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
|
||||
"/"+pathElements[0]+"/%p",
|
||||
) {
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
} else {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a branch for the "pages" repo
|
||||
// example.codeberg.page/@main/index.html
|
||||
if strings.HasPrefix(pathElements[0], "@") {
|
||||
log.Debug().Msg("main domain preparations, now trying with specified branch")
|
||||
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
} else {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the first directory is a repo with a "pages" branch
|
||||
// example.codeberg.page/myrepo/index.html
|
||||
// example.codeberg.page/pages/... is not allowed here.
|
||||
log.Debug().Msg("main domain preparations, now trying with specified repo")
|
||||
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
return
|
||||
}
|
||||
|
||||
// Try to use the "pages" repo on its default branch
|
||||
// example.codeberg.page/index.html
|
||||
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
|
||||
if tryBranch("pages", "", pathElements, "") {
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
return
|
||||
}
|
||||
|
||||
// Couldn't find a valid repo/branch
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
} else {
|
||||
trimmedHostStr := string(trimmedHost)
|
||||
|
||||
// Serve pages from external domains
|
||||
targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
|
||||
if targetOwner == "" {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
|
||||
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
|
||||
canonicalLink := ""
|
||||
if strings.HasPrefix(pathElements[0], "@") {
|
||||
targetBranch = pathElements[0][1:]
|
||||
pathElements = pathElements[1:]
|
||||
canonicalLink = "/%p"
|
||||
}
|
||||
|
||||
// Try to use the given repo on the given branch or the default branch
|
||||
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
|
||||
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
|
||||
canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
|
||||
if !valid {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
|
||||
return
|
||||
} else if canonicalDomain != trimmedHostStr {
|
||||
// only redirect if the target is also a codeberg page!
|
||||
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
|
||||
if targetOwner != "" {
|
||||
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug().Msg("tryBranch, now trying upstream")
|
||||
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
|
||||
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
|
||||
giteaRoot, giteaAPIToken,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache)
|
||||
return
|
||||
}
|
||||
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,29 @@
|
|||
package main
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/valyala/fasthttp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
)
|
||||
|
||||
func TestHandlerPerformance(t *testing.T) {
|
||||
testHandler := Handler(
|
||||
[]byte("codeberg.page"),
|
||||
[]byte("raw.codeberg.org"),
|
||||
"https://codeberg.org",
|
||||
"https://docs.codeberg.org/pages/raw-content/",
|
||||
"",
|
||||
[][]byte{[]byte("/.well-known/acme-challenge/")},
|
||||
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
|
||||
cache.NewKeyValueCache(),
|
||||
cache.NewKeyValueCache(),
|
||||
cache.NewKeyValueCache(),
|
||||
cache.NewKeyValueCache(),
|
||||
)
|
||||
|
||||
ctx := &fasthttp.RequestCtx{
|
||||
Request: *fasthttp.AcquireRequest(),
|
||||
Response: *fasthttp.AcquireResponse(),
|
||||
|
|
@ -15,7 +31,7 @@ func TestHandlerPerformance(t *testing.T) {
|
|||
ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
|
||||
fmt.Printf("Start: %v\n", time.Now())
|
||||
start := time.Now()
|
||||
handler(ctx)
|
||||
testHandler(ctx)
|
||||
end := time.Now()
|
||||
fmt.Printf("Done: %v\n", time.Now())
|
||||
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
|
||||
|
|
@ -28,7 +44,7 @@ func TestHandlerPerformance(t *testing.T) {
|
|||
ctx.Response.ResetBody()
|
||||
fmt.Printf("Start: %v\n", time.Now())
|
||||
start = time.Now()
|
||||
handler(ctx)
|
||||
testHandler(ctx)
|
||||
end = time.Now()
|
||||
fmt.Printf("Done: %v\n", time.Now())
|
||||
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
|
||||
|
|
@ -42,7 +58,7 @@ func TestHandlerPerformance(t *testing.T) {
|
|||
ctx.Request.SetRequestURI("http://example.momar.xyz/")
|
||||
fmt.Printf("Start: %v\n", time.Now())
|
||||
start = time.Now()
|
||||
handler(ctx)
|
||||
testHandler(ctx)
|
||||
end = time.Now()
|
||||
fmt.Printf("Done: %v\n", time.Now())
|
||||
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
)
|
||||
|
||||
// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
|
||||
// string for custom domains.
|
||||
func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
|
||||
if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
|
||||
return "max-age=63072000; includeSubdomains; preload"
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/utils"
|
||||
)
|
||||
|
||||
func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
|
||||
// Enable compression by wrapping the handler with the compression function provided by FastHTTP
|
||||
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
|
||||
|
||||
return &fasthttp.Server{
|
||||
Handler: compressedHandler,
|
||||
DisablePreParseMultipartForm: true,
|
||||
MaxRequestBodySize: 0,
|
||||
NoDefaultServerHeader: true,
|
||||
NoDefaultDate: true,
|
||||
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
|
||||
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
MaxConnsPerIP: 100,
|
||||
}
|
||||
}
|
||||
|
||||
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
|
||||
challengePath := []byte("/.well-known/acme-challenge/")
|
||||
|
||||
return &fasthttp.Server{
|
||||
Handler: func(ctx *fasthttp.RequestCtx) {
|
||||
if bytes.HasPrefix(ctx.Path(), challengePath) {
|
||||
challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
|
||||
if !ok || challenge == nil {
|
||||
ctx.SetStatusCode(http.StatusNotFound)
|
||||
ctx.SetBodyString("no challenge for this token")
|
||||
}
|
||||
ctx.SetBodyString(challenge.(string))
|
||||
} else {
|
||||
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
"codeberg.org/codeberg/pages/server/upstream"
|
||||
)
|
||||
|
||||
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
|
||||
func tryUpstream(ctx *fasthttp.RequestCtx,
|
||||
mainDomainSuffix, trimmedHost []byte,
|
||||
|
||||
targetOptions *upstream.Options,
|
||||
targetOwner, targetRepo, targetBranch, targetPath,
|
||||
|
||||
giteaRoot, giteaAPIToken string,
|
||||
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
|
||||
|
||||
// check if a canonical domain exists on a request on MainDomain
|
||||
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
|
||||
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
|
||||
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
|
||||
canonicalPath := string(ctx.RequestURI())
|
||||
if targetRepo != "pages" {
|
||||
path := strings.SplitN(canonicalPath, "/", 3)
|
||||
if len(path) >= 3 {
|
||||
canonicalPath = "/" + path[2]
|
||||
}
|
||||
}
|
||||
ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
targetOptions.TargetOwner = targetOwner
|
||||
targetOptions.TargetRepo = targetRepo
|
||||
targetOptions.TargetBranch = targetBranch
|
||||
targetOptions.TargetPath = targetPath
|
||||
|
||||
// Try to request the file from the Gitea API
|
||||
if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
|
||||
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package upstream
|
||||
|
||||
import "time"
|
||||
|
||||
// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
|
||||
var defaultBranchCacheTimeout = 15 * time.Minute
|
||||
|
||||
// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
|
||||
// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
|
||||
// picked up faster, while still allowing the content to be cached longer if nothing changes.
|
||||
var branchExistenceCacheTimeout = 5 * time.Minute
|
||||
|
||||
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
|
||||
// on your available memory.
|
||||
var fileCacheTimeout = 5 * time.Minute
|
||||
|
||||
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
|
||||
var fileCacheSizeLimit = 1024 * 1024
|
||||
|
||||
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
|
||||
var canonicalDomainCacheTimeout = 15 * time.Minute
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
)
|
||||
|
||||
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
|
||||
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
|
||||
domains := []string{}
|
||||
valid := false
|
||||
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
|
||||
domains = cachedValue.([]string)
|
||||
for _, domain := range domains {
|
||||
if domain == actualDomain {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
req := fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
|
||||
res := fasthttp.AcquireResponse()
|
||||
|
||||
err := client.Do(req, res)
|
||||
if err == nil && res.StatusCode() == fasthttp.StatusOK {
|
||||
for _, domain := range strings.Split(string(res.Body()), "\n") {
|
||||
domain = strings.ToLower(domain)
|
||||
domain = strings.TrimSpace(domain)
|
||||
domain = strings.TrimPrefix(domain, "http://")
|
||||
domain = strings.TrimPrefix(domain, "https://")
|
||||
if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
|
||||
domains = append(domains, domain)
|
||||
}
|
||||
if domain == actualDomain {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
domains = append(domains, targetOwner+mainDomainSuffix)
|
||||
if domains[len(domains)-1] == actualDomain {
|
||||
valid = true
|
||||
}
|
||||
if targetRepo != "" && targetRepo != "pages" {
|
||||
domains[len(domains)-1] += "/" + targetRepo
|
||||
}
|
||||
_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
|
||||
}
|
||||
return domains[0], valid
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
)
|
||||
|
||||
type branchTimestamp struct {
|
||||
Branch string
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
|
||||
// (or nil if the branch doesn't exist)
|
||||
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
|
||||
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return result.(*branchTimestamp)
|
||||
}
|
||||
result := &branchTimestamp{}
|
||||
result.Branch = branch
|
||||
if branch == "" {
|
||||
// Get default branch
|
||||
var body = make([]byte, 0)
|
||||
// TODO: use header for API key?
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
|
||||
return nil
|
||||
}
|
||||
result.Branch = fastjson.GetString(body, "default_branch")
|
||||
}
|
||||
|
||||
var body = make([]byte, 0)
|
||||
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
|
||||
if err != nil || status != 200 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
|
||||
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
|
||||
return result
|
||||
}
|
||||
|
||||
type fileResponse struct {
|
||||
exists bool
|
||||
mimeType string
|
||||
body []byte
|
||||
}
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
package upstream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
||||
"codeberg.org/codeberg/pages/html"
|
||||
"codeberg.org/codeberg/pages/server/cache"
|
||||
)
|
||||
|
||||
// upstreamIndexPages lists pages that may be considered as index pages for directories.
|
||||
var upstreamIndexPages = []string{
|
||||
"index.html",
|
||||
}
|
||||
|
||||
// Options provides various options for the upstream request.
|
||||
type Options struct {
|
||||
TargetOwner,
|
||||
TargetRepo,
|
||||
TargetBranch,
|
||||
TargetPath,
|
||||
|
||||
DefaultMimeType string
|
||||
ForbiddenMimeTypes map[string]struct{}
|
||||
TryIndexPages bool
|
||||
BranchTimestamp time.Time
|
||||
// internal
|
||||
appendTrailingSlash bool
|
||||
redirectIfExists string
|
||||
}
|
||||
|
||||
var client = fasthttp.Client{
|
||||
ReadTimeout: 10 * time.Second,
|
||||
MaxConnDuration: 60 * time.Second,
|
||||
MaxConnWaitTimeout: 1000 * time.Millisecond,
|
||||
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
|
||||
}
|
||||
|
||||
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
|
||||
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
|
||||
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
|
||||
|
||||
if o.ForbiddenMimeTypes == nil {
|
||||
o.ForbiddenMimeTypes = map[string]struct{}{}
|
||||
}
|
||||
|
||||
// Check if the branch exists and when it was modified
|
||||
if o.BranchTimestamp.IsZero() {
|
||||
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
|
||||
|
||||
if branch == nil {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
|
||||
return true
|
||||
}
|
||||
o.TargetBranch = branch.Branch
|
||||
o.BranchTimestamp = branch.Timestamp
|
||||
}
|
||||
|
||||
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if the browser has a cached version
|
||||
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
|
||||
if !ifModifiedSince.Before(o.BranchTimestamp) {
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
|
||||
return true
|
||||
}
|
||||
}
|
||||
log.Debug().Msg("preparations")
|
||||
|
||||
// Make a GET request to the upstream URL
|
||||
uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
|
||||
var req *fasthttp.Request
|
||||
var res *fasthttp.Response
|
||||
var cachedResponse fileResponse
|
||||
var err error
|
||||
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
|
||||
cachedResponse = cachedValue.(fileResponse)
|
||||
} else {
|
||||
req = fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
|
||||
res = fasthttp.AcquireResponse()
|
||||
res.SetBodyStream(&strings.Reader{}, -1)
|
||||
err = client.Do(req, res)
|
||||
}
|
||||
log.Debug().Msg("acquisition")
|
||||
|
||||
// Handle errors
|
||||
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
|
||||
if o.TryIndexPages {
|
||||
// copy the o struct & try if an index page exists
|
||||
optionsForIndexPages := *o
|
||||
optionsForIndexPages.TryIndexPages = false
|
||||
optionsForIndexPages.appendTrailingSlash = true
|
||||
for _, indexPage := range upstreamIndexPages {
|
||||
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
|
||||
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, fileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
// compatibility fix for GitHub Pages (/example → /example.html)
|
||||
optionsForIndexPages.appendTrailingSlash = false
|
||||
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
|
||||
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
|
||||
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, fileCacheTimeout)
|
||||
return true
|
||||
}
|
||||
}
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
|
||||
if res != nil {
|
||||
// Update cache if the request is fresh
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
|
||||
exists: false,
|
||||
}, fileCacheTimeout)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
|
||||
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
|
||||
// o.appendTrailingSlash is only true when looking for index pages
|
||||
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
|
||||
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
|
||||
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
if o.redirectIfExists != "" {
|
||||
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("error handling")
|
||||
|
||||
// Set the MIME type
|
||||
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
|
||||
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
|
||||
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
|
||||
if o.DefaultMimeType != "" {
|
||||
mimeType = o.DefaultMimeType
|
||||
} else {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
}
|
||||
ctx.Response.Header.SetContentType(mimeType)
|
||||
|
||||
// Everything's okay so far
|
||||
ctx.Response.SetStatusCode(fasthttp.StatusOK)
|
||||
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
|
||||
|
||||
log.Debug().Msg("response preparations")
|
||||
|
||||
// Write the response body to the original request
|
||||
var cacheBodyWriter bytes.Buffer
|
||||
if res != nil {
|
||||
if res.Header.ContentLength() > fileCacheSizeLimit {
|
||||
err = res.BodyWriteTo(ctx.Response.BodyWriter())
|
||||
} else {
|
||||
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
|
||||
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
|
||||
}
|
||||
} else {
|
||||
_, err = ctx.Write(cachedResponse.body)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
|
||||
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
log.Debug().Msg("response")
|
||||
|
||||
if res != nil && ctx.Err() == nil {
|
||||
cachedResponse.exists = true
|
||||
cachedResponse.mimeType = mimeType
|
||||
cachedResponse.body = cacheBodyWriter.Bytes()
|
||||
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package utils
|
||||
|
||||
import "bytes"
|
||||
|
||||
func TrimHostPort(host []byte) []byte {
|
||||
i := bytes.IndexByte(host, ':')
|
||||
if i >= 0 {
|
||||
return host[:i]
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTrimHostPort(t *testing.T) {
|
||||
assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
|
||||
assert.EqualValues(t, "", TrimHostPort([]byte(":")))
|
||||
assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
|
||||
}
|
||||
Ładowanie…
Reference in New Issue