2021-02-03 08:12:30 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
2021-02-04 16:44:58 +00:00
|
|
|
"database/sql"
|
2021-11-01 09:51:55 +00:00
|
|
|
"errors"
|
2021-02-03 08:12:30 +00:00
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2021-05-12 11:52:51 +00:00
|
|
|
"os"
|
2021-10-30 15:30:05 +00:00
|
|
|
"regexp"
|
2021-02-03 08:12:30 +00:00
|
|
|
"strings"
|
2021-11-01 09:51:55 +00:00
|
|
|
"syscall"
|
2021-02-03 08:12:30 +00:00
|
|
|
|
2021-02-04 16:44:58 +00:00
|
|
|
"html/template"
|
2021-02-03 08:12:30 +00:00
|
|
|
"lieu/database"
|
|
|
|
"lieu/types"
|
|
|
|
"lieu/util"
|
|
|
|
)
|
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
type RequestHandler struct {
|
2021-04-25 10:20:39 +00:00
|
|
|
config types.Config
|
|
|
|
db *sql.DB
|
2021-04-25 09:12:43 +00:00
|
|
|
}
|
|
|
|
|
2021-05-11 18:13:50 +00:00
|
|
|
type TemplateView struct {
|
2021-04-13 22:32:53 +00:00
|
|
|
SiteName string
|
|
|
|
Data interface{}
|
|
|
|
}
|
|
|
|
|
2021-02-03 08:12:30 +00:00
|
|
|
type SearchData struct {
|
2021-10-20 16:08:20 +00:00
|
|
|
Query string
|
|
|
|
Title string
|
2021-10-20 16:55:37 +00:00
|
|
|
Site string
|
2021-10-20 16:08:20 +00:00
|
|
|
Pages []types.PageData
|
|
|
|
IsInternal bool
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
|
|
|
|
2021-05-11 18:55:07 +00:00
|
|
|
type IndexData struct {
|
|
|
|
Tagline string
|
|
|
|
Placeholder string
|
|
|
|
}
|
|
|
|
|
2021-04-13 22:32:53 +00:00
|
|
|
type ListData struct {
|
|
|
|
Title string
|
|
|
|
URLs []types.PageData
|
|
|
|
}
|
|
|
|
|
2021-02-03 08:12:30 +00:00
|
|
|
type AboutData struct {
|
|
|
|
DomainCount int
|
2021-05-11 18:39:14 +00:00
|
|
|
WebringName string
|
2021-11-10 15:52:59 +00:00
|
|
|
LastCrawl string
|
2021-02-03 08:12:30 +00:00
|
|
|
PageCount string
|
|
|
|
TermCount string
|
|
|
|
FilteredLink string
|
|
|
|
RingLink string
|
|
|
|
}
|
|
|
|
|
2021-04-13 22:32:53 +00:00
|
|
|
var templates = template.Must(template.ParseFiles(
|
|
|
|
"html/head.html", "html/nav.html", "html/footer.html",
|
|
|
|
"html/about.html", "html/index.html", "html/list.html", "html/search.html", "html/webring.html"))
|
2021-04-25 08:53:15 +00:00
|
|
|
|
2021-05-12 11:52:51 +00:00
|
|
|
const useURLTitles = true
|
|
|
|
|
2021-10-30 15:30:05 +00:00
|
|
|
var sitePattern = regexp.MustCompile(`site:\S+`)
|
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request) {
|
2021-02-03 08:12:30 +00:00
|
|
|
var query string
|
2021-05-11 18:13:50 +00:00
|
|
|
view := &TemplateView{}
|
2021-02-03 08:12:30 +00:00
|
|
|
|
2021-10-20 16:55:37 +00:00
|
|
|
var domain string
|
2021-02-03 08:12:30 +00:00
|
|
|
if req.Method == http.MethodGet {
|
|
|
|
params := req.URL.Query()
|
2021-04-25 16:48:34 +00:00
|
|
|
if words, exists := params["q"]; exists && words[0] != "" {
|
2021-05-11 18:39:14 +00:00
|
|
|
query = words[0]
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
2021-10-20 16:55:37 +00:00
|
|
|
|
2021-10-20 17:06:45 +00:00
|
|
|
// how to use: https://gist.github.com/cblgh/29991ba0a9e65cccbe14f4afd7c975f1
|
2021-10-20 16:55:37 +00:00
|
|
|
if parts, exists := params["site"]; exists && parts[0] != "" {
|
|
|
|
// make sure we only have the domain, and no protocol prefix
|
|
|
|
domain = strings.TrimPrefix(parts[0], "https://")
|
|
|
|
domain = strings.TrimPrefix(domain, "http://")
|
|
|
|
domain = strings.TrimSuffix(domain, "/")
|
2021-10-30 15:30:05 +00:00
|
|
|
} else if sitePattern.MatchString(query) {
|
|
|
|
// if user searched with "site:<domain>" in text box, behave the same way as if a query param was used
|
|
|
|
domain = sitePattern.FindString(query)[5:]
|
|
|
|
}
|
|
|
|
// if clear button was used -> clear site param / search text
|
|
|
|
if parts, exists := params["clear"]; exists && parts[0] != "" {
|
|
|
|
domain = ""
|
|
|
|
query = sitePattern.ReplaceAllString(query, "")
|
2021-10-20 16:55:37 +00:00
|
|
|
}
|
2021-04-25 16:48:34 +00:00
|
|
|
}
|
|
|
|
|
2021-05-11 18:39:14 +00:00
|
|
|
if len(query) == 0 {
|
2021-05-11 18:55:07 +00:00
|
|
|
view.Data = IndexData{Tagline: h.config.General.Tagline, Placeholder: h.config.General.Placeholder}
|
2021-05-11 18:39:14 +00:00
|
|
|
h.renderView(res, "index", view)
|
2021-02-03 08:12:30 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2021-10-20 16:55:37 +00:00
|
|
|
var pages []types.PageData
|
|
|
|
if domain != "" {
|
|
|
|
pages = database.SearchWordsBySite(h.db, util.Inflect(strings.Fields(query)), domain)
|
|
|
|
} else {
|
|
|
|
pages = database.SearchWordsByScore(h.db, util.Inflect(strings.Fields(query)))
|
|
|
|
}
|
2021-02-03 08:12:30 +00:00
|
|
|
|
|
|
|
if useURLTitles {
|
|
|
|
for i, pageData := range pages {
|
|
|
|
prettyURL, err := url.QueryUnescape(strings.TrimPrefix(strings.TrimPrefix(pageData.URL, "http://"), "https://"))
|
|
|
|
util.Check(err)
|
|
|
|
pageData.Title = prettyURL
|
|
|
|
pages[i] = pageData
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-11 18:13:50 +00:00
|
|
|
view.Data = SearchData{
|
2021-10-20 16:08:20 +00:00
|
|
|
Title: "Results",
|
|
|
|
Query: query,
|
2021-10-20 16:55:37 +00:00
|
|
|
Site: domain,
|
2021-10-20 16:08:20 +00:00
|
|
|
Pages: pages,
|
|
|
|
IsInternal: true,
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
2021-05-11 18:39:14 +00:00
|
|
|
h.renderView(res, "search", view)
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
|
|
|
|
2021-10-16 20:04:18 +00:00
|
|
|
func (h RequestHandler) externalSearchRoute(res http.ResponseWriter, req *http.Request) {
|
|
|
|
var query string
|
|
|
|
view := &TemplateView{}
|
|
|
|
|
|
|
|
if req.Method == http.MethodGet {
|
|
|
|
params := req.URL.Query()
|
|
|
|
if words, exists := params["q"]; exists && words[0] != "" {
|
|
|
|
query = words[0]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pages := database.FulltextSearchWords(h.db, query)
|
|
|
|
|
|
|
|
if useURLTitles {
|
|
|
|
for i, pageData := range pages {
|
|
|
|
prettyURL, err := url.QueryUnescape(strings.TrimPrefix(strings.TrimPrefix(pageData.URL, "http://"), "https://"))
|
|
|
|
util.Check(err)
|
|
|
|
pageData.Title = prettyURL
|
|
|
|
pages[i] = pageData
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
view.Data = SearchData{
|
2021-10-20 16:08:20 +00:00
|
|
|
Title: "External Results",
|
|
|
|
Query: query,
|
|
|
|
Pages: pages,
|
|
|
|
IsInternal: false,
|
2021-10-16 20:04:18 +00:00
|
|
|
}
|
|
|
|
h.renderView(res, "search", view)
|
|
|
|
}
|
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
func (h RequestHandler) aboutRoute(res http.ResponseWriter, req *http.Request) {
|
2021-05-11 18:13:50 +00:00
|
|
|
view := &TemplateView{}
|
2021-04-13 22:32:53 +00:00
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
pageCount := util.Humanize(database.GetPageCount(h.db))
|
|
|
|
wordCount := util.Humanize(database.GetWordCount(h.db))
|
|
|
|
domainCount := database.GetDomainCount(h.db)
|
2021-11-10 15:52:59 +00:00
|
|
|
lastCrawl := database.GetLastCrawl(h.db)
|
2021-02-03 08:12:30 +00:00
|
|
|
|
2021-05-11 18:13:50 +00:00
|
|
|
view.Data = AboutData{
|
2021-05-11 18:39:14 +00:00
|
|
|
WebringName: h.config.General.Name,
|
2021-02-03 08:12:30 +00:00
|
|
|
DomainCount: domainCount,
|
|
|
|
PageCount: pageCount,
|
|
|
|
TermCount: wordCount,
|
2021-11-10 15:52:59 +00:00
|
|
|
LastCrawl: lastCrawl,
|
2021-02-03 08:12:30 +00:00
|
|
|
FilteredLink: "/filtered",
|
2021-04-25 09:12:43 +00:00
|
|
|
RingLink: h.config.General.URL,
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
2021-05-11 18:39:14 +00:00
|
|
|
h.renderView(res, "about", view)
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request) {
|
2021-05-11 18:13:50 +00:00
|
|
|
view := &TemplateView{}
|
2021-05-11 18:39:14 +00:00
|
|
|
|
2021-02-03 08:12:30 +00:00
|
|
|
var URLs []types.PageData
|
2021-04-25 09:12:43 +00:00
|
|
|
for _, domain := range util.ReadList(h.config.Crawler.BannedDomains, "\n") {
|
2021-02-03 08:12:30 +00:00
|
|
|
u, err := url.Parse(domain)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
u.Scheme = "https"
|
|
|
|
p := types.PageData{Title: domain, URL: u.String()}
|
|
|
|
URLs = append(URLs, p)
|
|
|
|
}
|
2021-05-11 18:39:14 +00:00
|
|
|
|
2021-05-11 18:13:50 +00:00
|
|
|
view.Data = ListData{
|
2021-02-03 08:12:30 +00:00
|
|
|
Title: "Filtered Domains",
|
|
|
|
URLs: URLs,
|
|
|
|
}
|
2021-05-11 18:39:14 +00:00
|
|
|
h.renderView(res, "list", view)
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request) {
|
|
|
|
link := database.GetRandomPage(h.db)
|
2021-02-04 16:44:58 +00:00
|
|
|
http.Redirect(res, req, link, http.StatusSeeOther)
|
2021-02-03 08:12:30 +00:00
|
|
|
}
|
|
|
|
|
2021-10-16 20:04:18 +00:00
|
|
|
func (h RequestHandler) randomExternalRoute(res http.ResponseWriter, req *http.Request) {
|
|
|
|
link := database.GetRandomExternalLink(h.db)
|
|
|
|
http.Redirect(res, req, link, http.StatusSeeOther)
|
|
|
|
}
|
|
|
|
|
2021-05-11 18:46:42 +00:00
|
|
|
func (h RequestHandler) webringRoute(res http.ResponseWriter, req *http.Request) {
|
|
|
|
http.Redirect(res, req, h.config.General.URL, http.StatusSeeOther)
|
|
|
|
}
|
|
|
|
|
2021-05-11 18:39:14 +00:00
|
|
|
func (h RequestHandler) renderView(res http.ResponseWriter, tmpl string, view *TemplateView) {
|
|
|
|
view.SiteName = h.config.General.Name
|
2021-05-12 11:52:51 +00:00
|
|
|
var errTemp error
|
|
|
|
if _, exists := os.LookupEnv("LIEU_DEV"); exists {
|
|
|
|
var templates = template.Must(template.ParseFiles(
|
|
|
|
"html/head.html", "html/nav.html", "html/footer.html",
|
|
|
|
"html/about.html", "html/index.html", "html/list.html", "html/search.html", "html/webring.html"))
|
|
|
|
errTemp = templates.ExecuteTemplate(res, tmpl+".html", view)
|
|
|
|
} else {
|
|
|
|
errTemp = templates.ExecuteTemplate(res, tmpl+".html", view)
|
|
|
|
}
|
2021-11-01 09:51:55 +00:00
|
|
|
if errors.Is(errTemp, syscall.EPIPE) {
|
|
|
|
fmt.Println("had a broken pipe, continuing")
|
|
|
|
} else {
|
|
|
|
util.Check(errTemp)
|
|
|
|
}
|
2021-04-13 22:32:53 +00:00
|
|
|
}
|
|
|
|
|
2022-02-20 14:36:07 +00:00
|
|
|
func WriteTheme(config types.Config) {
|
|
|
|
theme := config.Theme
|
2022-03-07 10:21:23 +00:00
|
|
|
// no theme is set, use the default
|
|
|
|
if theme.Foreground == "" {
|
|
|
|
return
|
|
|
|
}
|
2022-02-20 14:36:07 +00:00
|
|
|
colors := fmt.Sprintf(`:root {
|
|
|
|
--primary: %s;
|
|
|
|
--secondary: %s;
|
|
|
|
--link: %s;
|
|
|
|
}\n`, theme.Foreground, theme.Background, theme.Links)
|
|
|
|
err := os.WriteFile("html/assets/theme.css", []byte(colors), 0644)
|
|
|
|
util.Check(err)
|
|
|
|
}
|
|
|
|
|
2021-02-03 08:12:30 +00:00
|
|
|
func Serve(config types.Config) {
|
2022-02-20 14:36:07 +00:00
|
|
|
WriteTheme(config)
|
2021-02-04 14:53:03 +00:00
|
|
|
db := database.InitDB(config.Data.Database)
|
2021-04-25 10:20:39 +00:00
|
|
|
handler := RequestHandler{config: config, db: db}
|
2021-02-04 14:53:03 +00:00
|
|
|
|
2021-04-25 09:12:43 +00:00
|
|
|
http.HandleFunc("/about", handler.aboutRoute)
|
|
|
|
http.HandleFunc("/", handler.searchRoute)
|
2021-10-20 16:08:20 +00:00
|
|
|
http.HandleFunc("/outgoing", handler.externalSearchRoute)
|
|
|
|
http.HandleFunc("/random/outgoing", handler.randomExternalRoute)
|
2021-04-25 09:12:43 +00:00
|
|
|
http.HandleFunc("/random", handler.randomRoute)
|
2021-05-11 18:46:42 +00:00
|
|
|
http.HandleFunc("/webring", handler.webringRoute)
|
2021-04-25 09:12:43 +00:00
|
|
|
http.HandleFunc("/filtered", handler.filteredRoute)
|
2021-02-03 08:12:30 +00:00
|
|
|
|
|
|
|
fileserver := http.FileServer(http.Dir("html/assets/"))
|
2021-04-13 22:32:53 +00:00
|
|
|
http.Handle("/assets/", http.StripPrefix("/assets/", fileserver))
|
2021-02-03 08:12:30 +00:00
|
|
|
|
|
|
|
portstr := fmt.Sprintf(":%d", config.General.Port)
|
2021-04-13 22:32:53 +00:00
|
|
|
fmt.Println("Listening on port: ", portstr)
|
2021-02-03 08:12:30 +00:00
|
|
|
|
|
|
|
http.ListenAndServe(portstr, nil)
|
|
|
|
}
|