Merge branch 'main' into improve-web-ui

README.md

@@ -18,6 +18,12 @@ engine, a way for personal webrings to increase serendipitous connexions.
 
 ## Usage
 
+### How to search
+
+For the full search syntax (including how to use `site:` and `-site:`), see the [search syntax and API documentation](docs/querying.md).
+
+### Getting Lieu running
+
 ```
 $ lieu help
 Lieu: neighbourhood search engine
@@ -77,6 +83,8 @@ port = 10001
 
 [theme]
 # colors specified in hex (or valid css names) which determine the theme of the lieu instance
+# NOTE: If (and only if) all three values are set lieu uses those to generate the file html/assets/theme.css at startup.
+# You can also write directly to that file istead of adding this section to your configuration file
 foreground = "#ffffff"
 background = "#000000"
 links = "#ffffff"

database/database.go

@@ -19,10 +19,13 @@ import (
 	"log"
 	"net/url"
 	"strings"
+	"regexp"
 
 	_ "github.com/mattn/go-sqlite3"
 )
 
+var languageCodeSanityRegex = regexp.MustCompile("^[a-zA-Z\\-0-9]+$")
+
 func InitDB(filepath string) *sql.DB {
 	db, err := sql.Open("sqlite3", filepath)
 	if err != nil {
@@ -95,17 +98,19 @@ query params:
 &order=score, &order=count
 */
 
+var emptyStringArray = []string{}
+
 func SearchWordsByScore(db *sql.DB, words []string) []types.PageData {
-	return searchWords(db, words, true)
+	return SearchWords(db, words, true, emptyStringArray, emptyStringArray, emptyStringArray)
 }
 
 func SearchWordsBySite(db *sql.DB, words []string, domain string) []types.PageData {
 	// search words by site is same as search words by score, but adds a domain condition
-	return searchWords(db, words, true, domain)
+	return SearchWords(db, words, true, []string{domain}, emptyStringArray, emptyStringArray)
 }
 
 func SearchWordsByCount(db *sql.DB, words []string) []types.PageData {
-	return searchWords(db, words, false)
+	return SearchWords(db, words, false, emptyStringArray, emptyStringArray, emptyStringArray)
 }
 
 func FulltextSearchWords(db *sql.DB, phrase string) []types.PageData {
@@ -222,12 +227,16 @@ func countQuery(db *sql.DB, table string) int {
 	return count
 }
 
-func searchWords(db *sql.DB, words []string, searchByScore bool, domain ...string) []types.PageData {
-	var wordlist []string
+func SearchWords(db *sql.DB, words []string, searchByScore bool, domain []string, nodomain []string, language []string) []types.PageData {
 	var args []interface{}
-	for _, word := range words {
-		wordlist = append(wordlist, "word = ?")
-		args = append(args, strings.ToLower(word))
+
+	wordlist := []string{"1"}
+	if len(words) > 0 && words[0] != "" {
+		wordlist = make([]string, 0)
+		for _, word := range words {
+			wordlist = append(wordlist, "word = ?")
+			args = append(args, strings.ToLower(word))
+		}
 	}
 
 	// the domains conditional defaults to just 'true' i.e. no domain condition
@@ -240,6 +249,28 @@ func searchWords(db *sql.DB, words []string, searchByScore bool, domain ...strin
 		}
 	}
 
+	nodomains := []string{"1"}
+	if len(nodomain) > 0 && nodomain[0] != "" {
+		nodomains = make([]string, 0)
+		for _, d := range nodomain {
+			nodomains = append(nodomains, "domain != ?")
+			args = append(args, d)
+		}
+	}
+
+	//This needs some wildcard support …
+	languages := []string{"1"}
+	if len(language) > 0 && language[0] != "" {
+		languages = make([]string, 0)
+		for _, d := range language {
+			// Do a little check to avoid the database being DOSed
+			if languageCodeSanityRegex.MatchString(d) {
+				languages = append(languages, "lang LIKE ?")
+				args = append(args, d+"%")
+			}
+		}
+	}
+
 	orderType := "SUM(score)"
 	if !searchByScore {
 		orderType = "COUNT(*)"
@@ -250,11 +281,13 @@ func searchWords(db *sql.DB, words []string, searchByScore bool, domain ...strin
     FROM inv_index inv INNER JOIN pages p ON inv.url = p.url 
     WHERE (%s)
     AND (%s)
+    AND (%s)
+    AND (%s)
     GROUP BY inv.url 
     ORDER BY %s
     DESC
     LIMIT 15
-    `, strings.Join(wordlist, " OR "), strings.Join(domains, " OR "), orderType)
+    `, strings.Join(wordlist, " OR "), strings.Join(domains, " OR "), strings.Join(nodomains, " AND "), strings.Join(languages, " OR "), orderType)
 
 	stmt, err := db.Prepare(query)
 	util.Check(err)

docs/querying.md

@@ -0,0 +1,41 @@
+# Querying Lieu
+
+## Search Syntax
+
+* `cat dog` - search for pages about cats or dogs, most probably both
+* `fox site:example.org` - search example.org (if indexed) for term "fox"
+* `fox -site:example.org` - search all indexed sites except `example.org` for term "fox"
+* `emoji lang:de` - search pages that claim to mainly contain German content for the term "emoji"
+
+When searching, capitalisation and inflection do not matter, as search terms are:
+
+* Converted to lowercase using the go standard library
+* Passed through [jinzhu's inflection library](https://github.com/jinzhu/inflection) for
+  converting to a possible singular form (intended to work with English nouns)
+
+## Search API
+
+Lieu currently only renders its results to HTML. A query can be passed to the `/` endpoint using a `GET` request.
+
+It supports two URL parameters:
+* `q` - used for the search query
+* `site` - accepts one domain name and will have the same effect as the `site:<domain>` syntax.
+  You can use this to make your webrings search engine double as a searchbox on your website.
+
+### Examples
+To search `example.org` for the term "ssh" using `https://search.webring.example`:
+
+```
+https://search.webring.example/?q=ssh&site=example.org
+```
+
+Adding a form element, to use Lieu as a search engine, to the HTML at example.org:
+
+```
+<form method="GET" action="https://search.webring.example">
+	<label for="search">Search example.org</label>
+	<input type="search" minlength="1" required="" name="q" placeholder="Your search query here" id="search">
+	<input type="hidden" name="site" value="example.org"> <!-- This hidden field tells lieu to only search example.org -->
+	<button type="submit">Let's go!</button>
+</form>
+```

html/assets/favicon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="420" height="420" fill="none" version="1.1" xmlns="http://www.w3.org/2000/svg"><g stroke-linecap="round"><rect width="420" height="420" fill="#000" stroke-width="12.8"/></g><path d="m210 87c-53.5 0-104 27.1-149 71.9l-28.6 28.6 50.7 50.7 28.3-27.9 4.76 13.4-38.8 38.8 57.3 57.3 34.5-34.5v2.72 4.5 40.5h81v-40.5-4.5-2.72l34.5 34.5 57.3-57.3-38.8-38.8 4.76-13.4 28.3 27.9 50.7-50.7-28.6-28.6c-44.8-44.8-95.1-71.9-149-71.9zm0 81c11.2 0 19.5 8.25 19.5 19.5s-8.25 19.5-19.5 19.5-19.5-8.25-19.5-19.5 8.25-19.5 19.5-19.5z" color="#000000" fill="#fff" stroke-linecap="square" stroke-linejoin="round" style="-inkscape-stroke:none"/></svg>

html/head.html

@@ -15,7 +15,7 @@
         <link href="/assets/theme.css" rel="stylesheet">
 
         <link rel="icon" href="/assets/favicon.ico">
-        <link rel="icon" href="/assets/logo.svg" type="image/svg+xml">
+        <link rel="icon" href="/assets/favicon.svg" type="image/svg+xml">
         <link rel="shortcut icon" href="/assets/favicon.png">
         <link rel="apple-touch-icon" href="/assets/favicon.png">
         <meta name="theme-color" content="#000000">

html/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /*?

server/server.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"regexp"
 	"strings"
 	"syscall"
 
@@ -62,17 +61,21 @@ var templates = template.Must(template.ParseFiles(
 
 const useURLTitles = true
 
-var sitePattern = regexp.MustCompile(`site:\S+`)
-
 func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request) {
 	var query string
+	var domain string
 	view := &TemplateView{}
 
-	var domain string
-	if req.Method == http.MethodGet {
+	var domains = []string{}
+	var nodomains = []string{}
+	var langs = []string{}
+	var queryFields = []string{}
+		
+	if req.Method == http.MethodGet{
 		params := req.URL.Query()
 		if words, exists := params["q"]; exists && words[0] != "" {
 			query = words[0]
+			queryFields = strings.Fields(query)
 		}
 
 		// how to use: https://gist.github.com/cblgh/29991ba0a9e65cccbe14f4afd7c975f1
@@ -81,29 +84,36 @@ func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request)
 			domain = strings.TrimPrefix(parts[0], "https://")
 			domain = strings.TrimPrefix(domain, "http://")
 			domain = strings.TrimSuffix(domain, "/")
-		} else if sitePattern.MatchString(query) {
-			// if user searched with "site:<domain>" in text box, behave the same way as if a query param was used
-			domain = sitePattern.FindString(query)[5:]
+			domains = append(domains, domain)
 		}
-		// if clear button was used -> clear site param / search text
-		if parts, exists := params["clear"]; exists && parts[0] != "" {
-			domain = ""
-			query = sitePattern.ReplaceAllString(query, "")
+
+		// don't process if there are too many fields
+		if len(queryFields) <= 100 {
+			var newQueryFields []string;
+			for _, word := range queryFields {
+				// This could be more efficient by splitting arrays, but I'm going with the more readable version for now
+				if strings.HasPrefix(word, "site:") {
+					domains = append(domains, strings.TrimPrefix(word, "site:"))
+				} else if strings.HasPrefix(word, "-site:") {
+					nodomains = append(nodomains, strings.TrimPrefix(word, "-site:"))
+				} else if strings.HasPrefix(word, "lang:") {
+					langs = append(langs, strings.TrimPrefix(word, "lang:"))
+				} else {
+					newQueryFields = append(newQueryFields, word)
+				}
+			}
+			queryFields = newQueryFields;
 		}
+		
 	}
 
-	if len(query) == 0 {
+	if len(queryFields) == 0 || len(queryFields) > 100 || len(query) >= 8192 {
 		view.Data = IndexData{Tagline: h.config.General.Tagline, Placeholder: h.config.General.Placeholder}
 		h.renderView(res, "index", view)
 		return
 	}
 
-	var pages []types.PageData
-	if domain != "" {
-		pages = database.SearchWordsBySite(h.db, util.Inflect(strings.Fields(query)), domain)
-	} else {
-		pages = database.SearchWordsByScore(h.db, util.Inflect(strings.Fields(query)))
-	}
+	var pages = database.SearchWords(h.db, util.Inflect(queryFields), true, domains, nodomains, langs)
 
 	if useURLTitles {
 		for i, pageData := range pages {
@@ -232,14 +242,15 @@ func (h RequestHandler) renderView(res http.ResponseWriter, tmpl string, view *T
 func WriteTheme(config types.Config) {
 	theme := config.Theme
 	// no theme is set, use the default
-	if theme.Foreground == "" {
+	if theme.Foreground == "" || theme.Background == "" || theme.Links =="" {
 		return
 	}
-	colors := fmt.Sprintf(`:root {
+	colors := fmt.Sprintf(`/*This file will be automatically regenerated by lieu on startup if the theme colors are set in the configuration file*/
+:root {
   --primary: %s;
   --secondary: %s;
   --link: %s;
-}\n`, theme.Foreground, theme.Background, theme.Links)
+}`, theme.Foreground, theme.Background, theme.Links)
 	err := os.WriteFile("html/assets/theme.css", []byte(colors), 0644)
 	util.Check(err)
 }
@@ -257,8 +268,9 @@ func Serve(config types.Config) {
 	http.HandleFunc("/webring", handler.webringRoute)
 	http.HandleFunc("/filtered", handler.filteredRoute)
 
-	fileserver := http.FileServer(http.Dir("html/assets/"))
-	http.Handle("/assets/", http.StripPrefix("/assets/", fileserver))
+	fileserver := http.FileServer(http.Dir("html/"))
+	http.Handle("/assets/", fileserver)
+	http.Handle("/robots.txt", fileserver)
 
 	portstr := fmt.Sprintf(":%d", config.General.Port)
 	fmt.Println("Listening on port: ", portstr)