kopia lustrzana https://github.com/cblgh/lieu
merge eotl changes :)
cblgh
commit
44914e0f21
|
|
@ -1,3 +1,7 @@
|
|||
# Lieu
|
||||
data/
|
||||
searchengine.db
|
||||
|
||||
#~top ignores~
|
||||
node_modules/
|
||||
*.vim
|
||||
|
|
|
|||
11
README.md
11
README.md
|
|
@ -1,4 +1,5 @@
|
|||
# Lieu
|
||||
|
||||
_an alternative search engine_
|
||||
|
||||
Created in response to the environs of apathy concerning the use of hypertext
|
||||
|
|
@ -10,11 +11,13 @@ engine, a way for personal webrings to increase serendipitous connexions.
|
|||
|
||||
|
||||
## Goals
|
||||
|
||||
* Enable serendipitous discovery
|
||||
* Support personal communities
|
||||
* Be reusable, easily
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
$ lieu help
|
||||
Lieu: neighbourhood search engine
|
||||
|
|
@ -28,6 +31,7 @@ Commands
|
|||
|
||||
Example:
|
||||
lieu precrawl > data/webring.txt
|
||||
lieu crawl > data/crawled.txt
|
||||
lieu ingest
|
||||
lieu host
|
||||
```
|
||||
|
|
@ -39,12 +43,13 @@ the files Lieu reads from, as defined in the config file. See below for a
|
|||
typical workflow.
|
||||
|
||||
### Workflow
|
||||
|
||||
* Edit the config
|
||||
* Add domains to crawl in `config.crawler.webring`
|
||||
* **If you have a webpage with links you want to crawl:**
|
||||
* Set the config's `url` field to that page
|
||||
* Populate the list of domains to crawl with `precrawl`: `lieu precrawl > data/webring.txt`
|
||||
* Crawl: `lieu crawl > data/source.txt`
|
||||
* Crawl: `lieu crawl > data/crawled.txt`
|
||||
* Create database: `lieu ingest`
|
||||
* Host engine: `lieu host`
|
||||
|
||||
|
|
@ -52,6 +57,7 @@ After ingesting the data with `lieu ingest`, you can also use lieu to search the
|
|||
corpus in the terminal with `lieu search`.
|
||||
|
||||
## Config
|
||||
|
||||
The config file is written in [TOML](https://toml.io/en/).
|
||||
|
||||
```toml
|
||||
|
|
@ -85,6 +91,7 @@ boringDomains = "data/boring-domains.txt"
|
|||
```
|
||||
|
||||
For your own use, the following config fields should be customized:
|
||||
|
||||
* `name`
|
||||
* `url `
|
||||
* `port`
|
||||
|
|
@ -93,6 +100,7 @@ For your own use, the following config fields should be customized:
|
|||
* `bannedDomains`
|
||||
|
||||
The following config-defined files can stay as-is unless you have specific requirements:
|
||||
|
||||
* `database`
|
||||
* `heuristics`
|
||||
* `wordlist`
|
||||
|
|
@ -102,5 +110,6 @@ For a full rundown of the files and their various jobs, see the [files
|
|||
description](docs/files.md).
|
||||
|
||||
### License
|
||||
|
||||
Source code `AGPL-3.0-or-later`, Inter is available under `SIL OPEN FONT
|
||||
LICENSE Version 1.1`, Noto Serif is licensed as `Apache License, Version 2.0`.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gocolly/colly/v2"
|
||||
|
|
@ -85,7 +85,6 @@ func getDomains(links []string) []string {
|
|||
return domains
|
||||
}
|
||||
|
||||
|
||||
func findSuffix(suffixes []string, query string) bool {
|
||||
for _, suffix := range suffixes {
|
||||
if strings.HasSuffix(strings.ToLower(query), suffix) {
|
||||
|
|
@ -208,8 +207,8 @@ func Crawl(config types.Config) {
|
|||
c.AllowURLRevisit = false
|
||||
c.DisallowedDomains = getBannedDomains(config.Crawler.BannedDomains)
|
||||
|
||||
delay, _ := time.ParseDuration("200ms")
|
||||
c.Limit(&colly.LimitRule{DomainGlob: "*", Delay: delay, Parallelism: 3})
|
||||
delay, _ := time.ParseDuration("200ms")
|
||||
c.Limit(&colly.LimitRule{DomainGlob: "*", Delay: delay, Parallelism: 3})
|
||||
|
||||
boringDomains := getBoringDomains(config.Crawler.BoringDomains)
|
||||
boringWords := getBoringWords(config.Crawler.BoringWords)
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -66,6 +66,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
|||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
|
|
@ -75,6 +76,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
|
|||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -101,6 +103,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
|
|
@ -114,9 +118,14 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
|
|||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0 h1:u0bliLHgSO64Pb0xbhtwNIHspZc11X8M1bJqBkYl4Co=
|
||||
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0/go.mod h1:bWzMdWN2SiLomDzvESYfljDnNu60fUM2ATO8j09tZ5Y=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<link href="/links/about.css" rel="stylesheet">
|
||||
<title>Lieu—webring search engine</title>
|
||||
<link rel="shortcut icon" href="/links/favicon.png">
|
||||
<link rel="apple touch icon" href="/links/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
<a class="about-link" href="/about">About</a>
|
||||
<main>
|
||||
<h1><a href="/">Lieu</a></h1>
|
||||
<h2>the search for the new—endless</h2>
|
||||
<p>
|
||||
<span class="lieu">Lieu</span>—an alternative search engine. Created in response to the environs of
|
||||
apathy concerning the use of hypertext search and discovery. In <span class="lieu">Lieu</span>, the
|
||||
internet is not what is made searchable, but instead one's own neighbourhood. Put differently,
|
||||
<span class="lieu">Lieu</span> is a neighbourhood search engine, a way for personal webrings to increase
|
||||
serendipitous connexions.
|
||||
</p>
|
||||
<p>
|
||||
This instance indexes the <a href="{{.RingLink}}">{{.InstanceName}}</a>—{{ .DomainCount }} domains,
|
||||
{{ .PageCount }} pages, {{ .TermCount }} search terms.
|
||||
Some domains of the webring have been filtered out for a better search experience,
|
||||
see <a href="{{.FilteredLink}}">the filtered list</a>. Visit a <a href="/random">random page</a>.
|
||||
</p>
|
||||
<p><span class="lieu">Lieu</span> was created by <a href="https://cblgh.org/support.html">cblgh</a> at the onset of 2021.</p>
|
||||
<p>For Lieu's AGPL licensed source code, <a href="https://github.com/cblgh/lieu">the repository</a>.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{{ template "head" . }}
|
||||
{{ template "nav" . }}
|
||||
<main id="about">
|
||||
<p>
|
||||
<span class="lieu">Lieu</span>—an alternative search engine. Created in response to the environs of
|
||||
apathy concerning the use of hypertext search and discovery. In <span class="lieu">Lieu</span>, the
|
||||
internet is not what is made searchable, but instead one's own neighbourhood. Put differently,
|
||||
<span class="lieu">Lieu</span> is a neighbourhood search engine, a way for personal webrings to increase
|
||||
serendipitous connexions.
|
||||
</p>
|
||||
<p>
|
||||
This instance indexes the <a href="{{ .Data.RingLink }}">{{ .Data.WebringName }}</a> - {{ .Data.DomainCount }} domains,
|
||||
{{ .Data.PageCount }} pages, {{ .Data.TermCount }} search terms.
|
||||
Some domains of the webring have been filtered out for a better search experience,
|
||||
see <a href="{{ .Data.FilteredLink }}">the filtered list</a>.
|
||||
Visit a <a href="/random">random page</a>.
|
||||
</p>
|
||||
<p><span class="lieu">Lieu</span> was created by <a href="https://cblgh.org/support.html">cblgh</a> at the onset of 2021.</p>
|
||||
<p>For Lieu's AGPL licensed source code, <a href="https://github.com/cblgh/lieu">the repository</a>.</p>
|
||||
</main>
|
||||
{{ template "footer" . }}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
@import url("base.css");
|
||||
|
||||
html {
|
||||
max-width: 31rem;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 3rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-family: "Noto Serif";
|
||||
font-style: italic;
|
||||
font-weight: 400;
|
||||
font-size: 1.5rem;
|
||||
margin-top: 0;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.lieu {
|
||||
font-family: "Noto Serif";
|
||||
font-weight: 400;
|
||||
}
|
||||
|
|
@ -42,9 +42,9 @@ html {
|
|||
font-family: "Inter UI", sans-serif;
|
||||
background: var(--secondary);
|
||||
color: var(--primary);
|
||||
max-width: 650px;
|
||||
padding-bottom: 2rem;
|
||||
padding-left: 2rem;
|
||||
padding-right: 2rem;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
|
|
@ -141,7 +141,10 @@ and (max-device-width : 720px)
|
|||
padding-left: 0.75rem;
|
||||
padding-right: 0.75rem;
|
||||
font-size: 30pt;
|
||||
max-width: 100vw;
|
||||
max-width: 100vw !important;
|
||||
}
|
||||
#results {
|
||||
display: grid;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
@import url('base.css');
|
||||
|
||||
main {
|
||||
columns: 2;
|
||||
}
|
||||
|
||||
.entry {
|
||||
-webkit-column-break-inside: avoid;
|
||||
-moz-column-break-inside:avoid;
|
||||
-moz-page-break-inside:avoid;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid-column;
|
||||
}
|
||||
|
||||
.link {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
@media
|
||||
only screen
|
||||
and (min-device-width : 320px)
|
||||
and (max-device-width : 720px)
|
||||
{
|
||||
main {
|
||||
columns: 1 !important;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
@import url("about.css");
|
||||
|
||||
html {
|
||||
max-width: 100vw;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
main {
|
||||
display: grid;
|
||||
justify-items: center;
|
||||
align-items: center;
|
||||
margin-top: 10rem;
|
||||
}
|
||||
|
||||
.search-container {
|
||||
grid-template-columns: 19rem 3rem;
|
||||
}
|
||||
|
||||
.lieu-container {
|
||||
justify-items: start;
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
@import url("base.css");
|
||||
|
||||
h1 {
|
||||
font-size: 3rem;
|
||||
margin-bottom: 0rem;
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-family: "Noto Serif";
|
||||
font-weight: 400;
|
||||
font-size: 1.5rem;
|
||||
margin-top: 0;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.lieu-container h2 {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
header {
|
||||
clear: both;
|
||||
display: grid;
|
||||
grid-auto-flow: column;
|
||||
grid-template-columns: max-content max-content 1fr;
|
||||
grid-column-gap: 1rem;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
header h2 a, header h2 a:hover {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
header ul {
|
||||
justify-self: end;
|
||||
margin-top: 0.5rem;
|
||||
grid-column-start: 3;
|
||||
}
|
||||
|
||||
header ul li {
|
||||
margin-left: 1.5rem;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
header ul li:first-of-type {
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
main {
|
||||
display: grid;
|
||||
justify-items: left;
|
||||
align-items: left;
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
main#results {
|
||||
display: block;
|
||||
margin-top: 4rem;
|
||||
columns: 2;
|
||||
max-width: 1200px;
|
||||
}
|
||||
|
||||
main#about {
|
||||
max-width: 600px;
|
||||
}
|
||||
|
||||
.lieu {
|
||||
font-family: "Noto Serif";
|
||||
font-weight: 400;
|
||||
}
|
||||
|
||||
.search-container {
|
||||
grid-template-columns: 19rem 3rem;
|
||||
}
|
||||
|
||||
.lieu-container {
|
||||
display: grid;
|
||||
justify-items: center;
|
||||
align-items: center;
|
||||
margin-top: 5rem;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.entry {
|
||||
-webkit-column-break-inside: avoid;
|
||||
-moz-column-break-inside:avoid;
|
||||
-moz-page-break-inside:avoid;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid-column;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.entry p {
|
||||
color: var(--primary);
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
.link {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
@media only screen and (min-device-width : 320px) and (max-device-width : 720px) {
|
||||
main {
|
||||
columns: 1 !important;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{{ define "footer" }}
|
||||
</body>
|
||||
</html>
|
||||
{{ end }}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{{ define "head" }}
|
||||
<html lang="en">
|
||||
<head>
|
||||
<link href="/assets/style.css" rel="stylesheet">
|
||||
<title>Lieu—webring search engine</title>
|
||||
<link rel="shortcut icon" href="/assets/favicon.png">
|
||||
<link rel="apple touch icon" href="/assets/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
{{ end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<link href="/links/startpage.css" rel="stylesheet">
|
||||
<title>Lieu—webring search engine</title>
|
||||
<link rel="shortcut icon" href="/links/favicon.png">
|
||||
<link rel="apple touch icon" href="/links/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
<a class="about-link" href="/about">About</a>
|
||||
<main>
|
||||
<div class="lieu-container">
|
||||
<h1><a href="/">Lieu</a></h1>
|
||||
<h2>the search for the new—endless</h2>
|
||||
<form class="search-container">
|
||||
<input type="search" required minlength="1" name="q" placeholder="tracking" value="{{ .Query }}" class="search-box">
|
||||
<button type="submit" class="search-button"></button>
|
||||
</form>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{{ template "head" . }}
|
||||
<header>
|
||||
<h2><a href="https://github.com/cblgh/lieu">Lieu</a></h2>
|
||||
<ul>
|
||||
<li><a href="/webring">Webring</a></li>
|
||||
<li><a href="/about">About</a></li>
|
||||
</ul>
|
||||
</header>
|
||||
<div class="clear"></div>
|
||||
<main>
|
||||
<div class="lieu-container">
|
||||
<h1>
|
||||
{{ .SiteName }}
|
||||
</h1>
|
||||
<h2>
|
||||
the search for the new—endless
|
||||
</h2>
|
||||
<form class="search-container">
|
||||
<input type="search" required minlength="1" name="q" placeholder="tracking" value="{{ .Data.Query }}" class="search-box">
|
||||
<button type="submit" class="search-button"></button>
|
||||
</form>
|
||||
</div>
|
||||
</main>
|
||||
{{ template "footer" . }}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<link href="/links/search.css" rel="stylesheet">
|
||||
<title>{{ .Title }}—Lieu</title>
|
||||
<link rel="shortcut icon" href="/links/favicon.png">
|
||||
<link rel="apple touch icon" href="/links/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
<a class="about-link" href="/about">About</a>
|
||||
<h1><a href="/">Lieu</a></h1>
|
||||
<h2>{{ .Title }}</h2>
|
||||
<main>
|
||||
<ul>
|
||||
{{ range .URLs }}
|
||||
<li>
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
</li>
|
||||
{{ end }}
|
||||
</ul>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{{ template "head" . }}
|
||||
{{ template "nav" . }}
|
||||
<main>
|
||||
<ul>
|
||||
{{ range .Data.URLs }}
|
||||
<li>
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
</li>
|
||||
{{ end }}
|
||||
</ul>
|
||||
</main>
|
||||
{{ template "footer" . }}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{{ define "nav" }}
|
||||
<header>
|
||||
<h2>
|
||||
<a href="/">{{ .SiteName }}</a>
|
||||
</h2>
|
||||
<ul>
|
||||
<li><a href="/">Search</a></li>
|
||||
<li><a href="/webring">Webring</a></li>
|
||||
<li><a href="/about">About</a></li>
|
||||
</ul>
|
||||
</header>
|
||||
{{ end }}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<link href="/links/search.css" rel="stylesheet">
|
||||
<title>{{ .Query }}—Lieu</title>
|
||||
<link rel="shortcut icon" href="/links/favicon.png">
|
||||
<link rel="apple touch icon" href="/links/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
<a class="about-link" href="/about">About</a>
|
||||
<h1><a href="/">Lieu</a></h1>
|
||||
<form method="GET" class="search-container">
|
||||
<input type="search" minlength="1" required name="q" value="{{ .Query }}" class="search-box">
|
||||
<button type="submit" class="search-button"></button>
|
||||
</form>
|
||||
<main>
|
||||
{{ range .Pages }}
|
||||
<div class="entry">
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
<p>{{ .About }}</p>
|
||||
</div>
|
||||
{{ end }}
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{{ template "head" . }}
|
||||
{{ template "nav" . }}
|
||||
<form method="GET" class="search-container">
|
||||
<input type="search" minlength="1" required name="q" value="{{ .Data.Query }}" class="search-box">
|
||||
<button type="submit" class="search-button"></button>
|
||||
</form>
|
||||
<main id="results">
|
||||
{{ range .Data.Pages }}
|
||||
<div class="entry">
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
<p>{{ .About }}</p>
|
||||
</div>
|
||||
{{ end }}
|
||||
{{ template "footer" . }}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<link href="/links/style.css" rel="stylesheet">
|
||||
<title>indexed domains—Lieu</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Lieu</h1>
|
||||
<a class="about-link" href="/about">About</a>
|
||||
<main>
|
||||
{{ range .Domains }}
|
||||
<div class="entry">
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
<p>{{ .About }}</p>
|
||||
</div>
|
||||
{{ end }}
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{{ template "head" . }}
|
||||
{{ template "nav" . }}
|
||||
{{ range .Data.Domains }}
|
||||
<div class="entry">
|
||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||
<p>{{ .About }}</p>
|
||||
</div>
|
||||
{{ end }}
|
||||
{{ template "footer" . }}
|
||||
|
|
@ -18,46 +18,50 @@ type RequestHandler struct {
|
|||
db *sql.DB
|
||||
}
|
||||
|
||||
type TemplateView struct {
|
||||
SiteName string
|
||||
Data interface{}
|
||||
}
|
||||
|
||||
type SearchData struct {
|
||||
Query string
|
||||
Pages []types.PageData
|
||||
}
|
||||
|
||||
type AboutData struct {
|
||||
DomainCount int
|
||||
InstanceName string
|
||||
PageCount string
|
||||
TermCount string
|
||||
FilteredLink string
|
||||
RingLink string
|
||||
}
|
||||
|
||||
type ListData struct {
|
||||
Title string
|
||||
URLs []types.PageData
|
||||
}
|
||||
|
||||
type AboutData struct {
|
||||
DomainCount int
|
||||
WebringName string
|
||||
PageCount string
|
||||
TermCount string
|
||||
FilteredLink string
|
||||
RingLink string
|
||||
}
|
||||
|
||||
const useURLTitles = true
|
||||
|
||||
var indexView = template.Must(template.ParseFiles("html/index-template.html"))
|
||||
var aboutView = template.Must(template.ParseFiles("html/about-template.html"))
|
||||
var listView = template.Must(template.ParseFiles("html/list-template.html"))
|
||||
var searchResultsView = template.Must(template.ParseFiles("html/search-template.html"))
|
||||
var templates = template.Must(template.ParseFiles(
|
||||
"html/head.html", "html/nav.html", "html/footer.html",
|
||||
"html/about.html", "html/index.html", "html/list.html", "html/search.html", "html/webring.html"))
|
||||
|
||||
func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request) {
|
||||
var query string
|
||||
view := &TemplateView{}
|
||||
|
||||
if req.Method == http.MethodGet {
|
||||
params := req.URL.Query()
|
||||
if words, exists := params["q"]; exists && words[0] != "" {
|
||||
query = words[0]
|
||||
}
|
||||
query = words[0]
|
||||
}
|
||||
}
|
||||
|
||||
if len(query) == 0 {
|
||||
var empty interface{}
|
||||
err := indexView.Execute(res, empty)
|
||||
util.Check(err)
|
||||
if len(query) == 0 {
|
||||
view.Data = SearchData{}
|
||||
h.renderView(res, "index", view)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -72,32 +76,34 @@ func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request)
|
|||
}
|
||||
}
|
||||
|
||||
data := SearchData{
|
||||
view.Data = SearchData{
|
||||
Query: query,
|
||||
Pages: pages,
|
||||
}
|
||||
err := searchResultsView.Execute(res, data)
|
||||
util.Check(err)
|
||||
h.renderView(res, "search", view)
|
||||
}
|
||||
|
||||
func (h RequestHandler) aboutRoute(res http.ResponseWriter, req *http.Request) {
|
||||
view := &TemplateView{}
|
||||
|
||||
pageCount := util.Humanize(database.GetPageCount(h.db))
|
||||
wordCount := util.Humanize(database.GetWordCount(h.db))
|
||||
domainCount := database.GetDomainCount(h.db)
|
||||
|
||||
data := AboutData{
|
||||
InstanceName: h.config.General.Name,
|
||||
view.Data = AboutData{
|
||||
WebringName: h.config.General.Name,
|
||||
DomainCount: domainCount,
|
||||
PageCount: pageCount,
|
||||
TermCount: wordCount,
|
||||
FilteredLink: "/filtered",
|
||||
RingLink: h.config.General.URL,
|
||||
}
|
||||
err := aboutView.Execute(res, data)
|
||||
util.Check(err)
|
||||
h.renderView(res, "about", view)
|
||||
}
|
||||
|
||||
func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request) {
|
||||
view := &TemplateView{}
|
||||
|
||||
var URLs []types.PageData
|
||||
for _, domain := range util.ReadList(h.config.Crawler.BannedDomains, "\n") {
|
||||
u, err := url.Parse(domain)
|
||||
|
|
@ -108,12 +114,12 @@ func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request
|
|||
p := types.PageData{Title: domain, URL: u.String()}
|
||||
URLs = append(URLs, p)
|
||||
}
|
||||
data := ListData{
|
||||
|
||||
view.Data = ListData{
|
||||
Title: "Filtered Domains",
|
||||
URLs: URLs,
|
||||
}
|
||||
err := listView.Execute(res, data)
|
||||
util.Check(err)
|
||||
h.renderView(res, "list", view)
|
||||
}
|
||||
|
||||
func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request) {
|
||||
|
|
@ -121,6 +127,12 @@ func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request)
|
|||
http.Redirect(res, req, link, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (h RequestHandler) renderView(res http.ResponseWriter, tmpl string, view *TemplateView) {
|
||||
view.SiteName = h.config.General.Name
|
||||
errTemp := templates.ExecuteTemplate(res, tmpl+".html", view)
|
||||
util.Check(errTemp)
|
||||
}
|
||||
|
||||
func Serve(config types.Config) {
|
||||
db := database.InitDB(config.Data.Database)
|
||||
handler := RequestHandler{config: config, db: db}
|
||||
|
|
@ -131,10 +143,10 @@ func Serve(config types.Config) {
|
|||
http.HandleFunc("/filtered", handler.filteredRoute)
|
||||
|
||||
fileserver := http.FileServer(http.Dir("html/assets/"))
|
||||
http.Handle("/links/", http.StripPrefix("/links/", fileserver))
|
||||
http.Handle("/assets/", http.StripPrefix("/assets/", fileserver))
|
||||
|
||||
portstr := fmt.Sprintf(":%d", config.General.Port)
|
||||
fmt.Println("listening on", portstr)
|
||||
fmt.Println("Listening on port: ", portstr)
|
||||
|
||||
http.ListenAndServe(portstr, nil)
|
||||
}
|
||||
|
|
|
|||
58
util/util.go
58
util/util.go
|
|
@ -1,18 +1,18 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"os"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"lieu/types"
|
||||
"github.com/jinzhu/inflection"
|
||||
"github.com/komkom/toml"
|
||||
"lieu/types"
|
||||
)
|
||||
|
||||
func Inflect(words []string) []string {
|
||||
|
|
@ -30,17 +30,17 @@ func Check(err error) {
|
|||
}
|
||||
|
||||
func DatabaseDoesNotExist(filepath string) {
|
||||
fmt.Printf("lieu: database %s does not exist\n", filepath)
|
||||
fmt.Println("lieu: try running `lieu ingest` if you have already crawled source data")
|
||||
Exit()
|
||||
fmt.Printf("lieu: database %s does not exist\n", filepath)
|
||||
fmt.Println("lieu: try running `lieu ingest` if you have already crawled source data")
|
||||
Exit()
|
||||
}
|
||||
|
||||
func CheckFileExists (path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
return os.IsExist(err)
|
||||
func CheckFileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
return os.IsExist(err)
|
||||
}
|
||||
|
||||
func Humanize(n int) string {
|
||||
|
|
@ -64,30 +64,30 @@ func Contains(arr []string, query string) bool {
|
|||
|
||||
func ReadList(filepath, sep string) []string {
|
||||
data, err := ioutil.ReadFile(filepath)
|
||||
if err != nil || len(data) == 0{
|
||||
if err != nil || len(data) == 0 {
|
||||
return []string{}
|
||||
}
|
||||
return strings.Split(strings.TrimSuffix(string(data), sep), sep)
|
||||
}
|
||||
|
||||
func CheckPortOpen(port int) bool {
|
||||
tcpaddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
tcpaddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
l, err := net.ListenTCP("tcp", tcpaddr)
|
||||
defer l.Close()
|
||||
l, err := net.ListenTCP("tcp", tcpaddr)
|
||||
defer l.Close()
|
||||
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func ReadConfig() types.Config {
|
||||
data, err := ioutil.ReadFile("lieu.toml")
|
||||
Check(err)
|
||||
Check(err)
|
||||
|
||||
var conf types.Config
|
||||
decoder := json.NewDecoder(toml.New(bytes.NewBuffer(data)))
|
||||
|
|
@ -98,8 +98,8 @@ func ReadConfig() types.Config {
|
|||
return conf
|
||||
}
|
||||
|
||||
func WriteMockConfig () {
|
||||
conf := []byte(`[general]
|
||||
func WriteMockConfig() {
|
||||
conf := []byte(`[general]
|
||||
name = "Sweet Webring"
|
||||
# used by the precrawl command and linked to in /about route
|
||||
url = "https://example.com/"
|
||||
|
|
@ -127,10 +127,10 @@ boringWords = "data/boring-words.txt"
|
|||
# domains that won't be output as outgoing links
|
||||
boringDomains = "data/boring-domains.txt"
|
||||
`)
|
||||
err := ioutil.WriteFile("lieu.toml", conf, 0644)
|
||||
err := ioutil.WriteFile("lieu.toml", conf, 0644)
|
||||
Check(err)
|
||||
}
|
||||
|
||||
func Exit () {
|
||||
os.Exit(0)
|
||||
func Exit() {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue