kopia lustrzana https://github.com/cblgh/lieu
merge eotl changes :)
commit
44914e0f21
|
@ -1,3 +1,7 @@
|
||||||
|
# Lieu
|
||||||
|
data/
|
||||||
|
searchengine.db
|
||||||
|
|
||||||
#~top ignores~
|
#~top ignores~
|
||||||
node_modules/
|
node_modules/
|
||||||
*.vim
|
*.vim
|
||||||
|
|
11
README.md
11
README.md
|
@ -1,4 +1,5 @@
|
||||||
# Lieu
|
# Lieu
|
||||||
|
|
||||||
_an alternative search engine_
|
_an alternative search engine_
|
||||||
|
|
||||||
Created in response to the environs of apathy concerning the use of hypertext
|
Created in response to the environs of apathy concerning the use of hypertext
|
||||||
|
@ -10,11 +11,13 @@ engine, a way for personal webrings to increase serendipitous connexions.
|
||||||
|
|
||||||
|
|
||||||
## Goals
|
## Goals
|
||||||
|
|
||||||
* Enable serendipitous discovery
|
* Enable serendipitous discovery
|
||||||
* Support personal communities
|
* Support personal communities
|
||||||
* Be reusable, easily
|
* Be reusable, easily
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```
|
```
|
||||||
$ lieu help
|
$ lieu help
|
||||||
Lieu: neighbourhood search engine
|
Lieu: neighbourhood search engine
|
||||||
|
@ -28,6 +31,7 @@ Commands
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
lieu precrawl > data/webring.txt
|
lieu precrawl > data/webring.txt
|
||||||
|
lieu crawl > data/crawled.txt
|
||||||
lieu ingest
|
lieu ingest
|
||||||
lieu host
|
lieu host
|
||||||
```
|
```
|
||||||
|
@ -39,12 +43,13 @@ the files Lieu reads from, as defined in the config file. See below for a
|
||||||
typical workflow.
|
typical workflow.
|
||||||
|
|
||||||
### Workflow
|
### Workflow
|
||||||
|
|
||||||
* Edit the config
|
* Edit the config
|
||||||
* Add domains to crawl in `config.crawler.webring`
|
* Add domains to crawl in `config.crawler.webring`
|
||||||
* **If you have a webpage with links you want to crawl:**
|
* **If you have a webpage with links you want to crawl:**
|
||||||
* Set the config's `url` field to that page
|
* Set the config's `url` field to that page
|
||||||
* Populate the list of domains to crawl with `precrawl`: `lieu precrawl > data/webring.txt`
|
* Populate the list of domains to crawl with `precrawl`: `lieu precrawl > data/webring.txt`
|
||||||
* Crawl: `lieu crawl > data/source.txt`
|
* Crawl: `lieu crawl > data/crawled.txt`
|
||||||
* Create database: `lieu ingest`
|
* Create database: `lieu ingest`
|
||||||
* Host engine: `lieu host`
|
* Host engine: `lieu host`
|
||||||
|
|
||||||
|
@ -52,6 +57,7 @@ After ingesting the data with `lieu ingest`, you can also use lieu to search the
|
||||||
corpus in the terminal with `lieu search`.
|
corpus in the terminal with `lieu search`.
|
||||||
|
|
||||||
## Config
|
## Config
|
||||||
|
|
||||||
The config file is written in [TOML](https://toml.io/en/).
|
The config file is written in [TOML](https://toml.io/en/).
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
|
@ -85,6 +91,7 @@ boringDomains = "data/boring-domains.txt"
|
||||||
```
|
```
|
||||||
|
|
||||||
For your own use, the following config fields should be customized:
|
For your own use, the following config fields should be customized:
|
||||||
|
|
||||||
* `name`
|
* `name`
|
||||||
* `url `
|
* `url `
|
||||||
* `port`
|
* `port`
|
||||||
|
@ -93,6 +100,7 @@ For your own use, the following config fields should be customized:
|
||||||
* `bannedDomains`
|
* `bannedDomains`
|
||||||
|
|
||||||
The following config-defined files can stay as-is unless you have specific requirements:
|
The following config-defined files can stay as-is unless you have specific requirements:
|
||||||
|
|
||||||
* `database`
|
* `database`
|
||||||
* `heuristics`
|
* `heuristics`
|
||||||
* `wordlist`
|
* `wordlist`
|
||||||
|
@ -102,5 +110,6 @@ For a full rundown of the files and their various jobs, see the [files
|
||||||
description](docs/files.md).
|
description](docs/files.md).
|
||||||
|
|
||||||
### License
|
### License
|
||||||
|
|
||||||
Source code `AGPL-3.0-or-later`, Inter is available under `SIL OPEN FONT
|
Source code `AGPL-3.0-or-later`, Inter is available under `SIL OPEN FONT
|
||||||
LICENSE Version 1.1`, Noto Serif is licensed as `Apache License, Version 2.0`.
|
LICENSE Version 1.1`, Noto Serif is licensed as `Apache License, Version 2.0`.
|
||||||
|
|
|
@ -9,7 +9,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/gocolly/colly/v2"
|
"github.com/gocolly/colly/v2"
|
||||||
|
@ -85,7 +85,6 @@ func getDomains(links []string) []string {
|
||||||
return domains
|
return domains
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func findSuffix(suffixes []string, query string) bool {
|
func findSuffix(suffixes []string, query string) bool {
|
||||||
for _, suffix := range suffixes {
|
for _, suffix := range suffixes {
|
||||||
if strings.HasSuffix(strings.ToLower(query), suffix) {
|
if strings.HasSuffix(strings.ToLower(query), suffix) {
|
||||||
|
@ -208,8 +207,8 @@ func Crawl(config types.Config) {
|
||||||
c.AllowURLRevisit = false
|
c.AllowURLRevisit = false
|
||||||
c.DisallowedDomains = getBannedDomains(config.Crawler.BannedDomains)
|
c.DisallowedDomains = getBannedDomains(config.Crawler.BannedDomains)
|
||||||
|
|
||||||
delay, _ := time.ParseDuration("200ms")
|
delay, _ := time.ParseDuration("200ms")
|
||||||
c.Limit(&colly.LimitRule{DomainGlob: "*", Delay: delay, Parallelism: 3})
|
c.Limit(&colly.LimitRule{DomainGlob: "*", Delay: delay, Parallelism: 3})
|
||||||
|
|
||||||
boringDomains := getBoringDomains(config.Crawler.BoringDomains)
|
boringDomains := getBoringDomains(config.Crawler.BoringDomains)
|
||||||
boringWords := getBoringWords(config.Crawler.BoringWords)
|
boringWords := getBoringWords(config.Crawler.BoringWords)
|
||||||
|
|
9
go.sum
9
go.sum
|
@ -66,6 +66,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
||||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||||
|
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
|
||||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
@ -75,6 +76,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
|
||||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
@ -101,6 +103,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
|
||||||
|
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
@ -114,9 +118,14 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
|
||||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||||
|
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||||
|
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0 h1:u0bliLHgSO64Pb0xbhtwNIHspZc11X8M1bJqBkYl4Co=
|
||||||
|
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0/go.mod h1:bWzMdWN2SiLomDzvESYfljDnNu60fUM2ATO8j09tZ5Y=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
|
|
@ -1,30 +0,0 @@
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<link href="/links/about.css" rel="stylesheet">
|
|
||||||
<title>Lieu—webring search engine</title>
|
|
||||||
<link rel="shortcut icon" href="/links/favicon.png">
|
|
||||||
<link rel="apple touch icon" href="/links/favicon.png">
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<a class="about-link" href="/about">About</a>
|
|
||||||
<main>
|
|
||||||
<h1><a href="/">Lieu</a></h1>
|
|
||||||
<h2>the search for the new—endless</h2>
|
|
||||||
<p>
|
|
||||||
<span class="lieu">Lieu</span>—an alternative search engine. Created in response to the environs of
|
|
||||||
apathy concerning the use of hypertext search and discovery. In <span class="lieu">Lieu</span>, the
|
|
||||||
internet is not what is made searchable, but instead one's own neighbourhood. Put differently,
|
|
||||||
<span class="lieu">Lieu</span> is a neighbourhood search engine, a way for personal webrings to increase
|
|
||||||
serendipitous connexions.
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
This instance indexes the <a href="{{.RingLink}}">{{.InstanceName}}</a>—{{ .DomainCount }} domains,
|
|
||||||
{{ .PageCount }} pages, {{ .TermCount }} search terms.
|
|
||||||
Some domains of the webring have been filtered out for a better search experience,
|
|
||||||
see <a href="{{.FilteredLink}}">the filtered list</a>. Visit a <a href="/random">random page</a>.
|
|
||||||
</p>
|
|
||||||
<p><span class="lieu">Lieu</span> was created by <a href="https://cblgh.org/support.html">cblgh</a> at the onset of 2021.</p>
|
|
||||||
<p>For Lieu's AGPL licensed source code, <a href="https://github.com/cblgh/lieu">the repository</a>.</p>
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
{{ template "head" . }}
|
||||||
|
{{ template "nav" . }}
|
||||||
|
<main id="about">
|
||||||
|
<p>
|
||||||
|
<span class="lieu">Lieu</span>—an alternative search engine. Created in response to the environs of
|
||||||
|
apathy concerning the use of hypertext search and discovery. In <span class="lieu">Lieu</span>, the
|
||||||
|
internet is not what is made searchable, but instead one's own neighbourhood. Put differently,
|
||||||
|
<span class="lieu">Lieu</span> is a neighbourhood search engine, a way for personal webrings to increase
|
||||||
|
serendipitous connexions.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
This instance indexes the <a href="{{ .Data.RingLink }}">{{ .Data.WebringName }}</a> - {{ .Data.DomainCount }} domains,
|
||||||
|
{{ .Data.PageCount }} pages, {{ .Data.TermCount }} search terms.
|
||||||
|
Some domains of the webring have been filtered out for a better search experience,
|
||||||
|
see <a href="{{ .Data.FilteredLink }}">the filtered list</a>.
|
||||||
|
Visit a <a href="/random">random page</a>.
|
||||||
|
</p>
|
||||||
|
<p><span class="lieu">Lieu</span> was created by <a href="https://cblgh.org/support.html">cblgh</a> at the onset of 2021.</p>
|
||||||
|
<p>For Lieu's AGPL licensed source code, <a href="https://github.com/cblgh/lieu">the repository</a>.</p>
|
||||||
|
</main>
|
||||||
|
{{ template "footer" . }}
|
|
@ -1,24 +0,0 @@
|
||||||
@import url("base.css");
|
|
||||||
|
|
||||||
html {
|
|
||||||
max-width: 31rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 {
|
|
||||||
font-size: 3rem;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
h2 {
|
|
||||||
font-family: "Noto Serif";
|
|
||||||
font-style: italic;
|
|
||||||
font-weight: 400;
|
|
||||||
font-size: 1.5rem;
|
|
||||||
margin-top: 0;
|
|
||||||
margin-bottom: 2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.lieu {
|
|
||||||
font-family: "Noto Serif";
|
|
||||||
font-weight: 400;
|
|
||||||
}
|
|
|
@ -42,9 +42,9 @@ html {
|
||||||
font-family: "Inter UI", sans-serif;
|
font-family: "Inter UI", sans-serif;
|
||||||
background: var(--secondary);
|
background: var(--secondary);
|
||||||
color: var(--primary);
|
color: var(--primary);
|
||||||
max-width: 650px;
|
|
||||||
padding-bottom: 2rem;
|
padding-bottom: 2rem;
|
||||||
padding-left: 2rem;
|
padding-left: 2rem;
|
||||||
|
padding-right: 2rem;
|
||||||
margin-top: 2rem;
|
margin-top: 2rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -141,7 +141,10 @@ and (max-device-width : 720px)
|
||||||
padding-left: 0.75rem;
|
padding-left: 0.75rem;
|
||||||
padding-right: 0.75rem;
|
padding-right: 0.75rem;
|
||||||
font-size: 30pt;
|
font-size: 30pt;
|
||||||
max-width: 100vw;
|
max-width: 100vw !important;
|
||||||
|
}
|
||||||
|
#results {
|
||||||
|
display: grid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,27 +0,0 @@
|
||||||
@import url('base.css');
|
|
||||||
|
|
||||||
main {
|
|
||||||
columns: 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.entry {
|
|
||||||
-webkit-column-break-inside: avoid;
|
|
||||||
-moz-column-break-inside:avoid;
|
|
||||||
-moz-page-break-inside:avoid;
|
|
||||||
page-break-inside: avoid;
|
|
||||||
break-inside: avoid-column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.link {
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media
|
|
||||||
only screen
|
|
||||||
and (min-device-width : 320px)
|
|
||||||
and (max-device-width : 720px)
|
|
||||||
{
|
|
||||||
main {
|
|
||||||
columns: 1 !important;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
@import url("about.css");
|
|
||||||
|
|
||||||
html {
|
|
||||||
max-width: 100vw;
|
|
||||||
}
|
|
||||||
|
|
||||||
h2 {
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
main {
|
|
||||||
display: grid;
|
|
||||||
justify-items: center;
|
|
||||||
align-items: center;
|
|
||||||
margin-top: 10rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-container {
|
|
||||||
grid-template-columns: 19rem 3rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.lieu-container {
|
|
||||||
justify-items: start;
|
|
||||||
}
|
|
|
@ -0,0 +1,106 @@
|
||||||
|
@import url("base.css");
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
font-size: 3rem;
|
||||||
|
margin-bottom: 0rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
font-family: "Noto Serif";
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 1.5rem;
|
||||||
|
margin-top: 0;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lieu-container h2 {
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
header {
|
||||||
|
clear: both;
|
||||||
|
display: grid;
|
||||||
|
grid-auto-flow: column;
|
||||||
|
grid-template-columns: max-content max-content 1fr;
|
||||||
|
grid-column-gap: 1rem;
|
||||||
|
align-items: start;
|
||||||
|
}
|
||||||
|
|
||||||
|
header h2 a, header h2 a:hover {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
header ul {
|
||||||
|
justify-self: end;
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
grid-column-start: 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
header ul li {
|
||||||
|
margin-left: 1.5rem;
|
||||||
|
display: inline-block;
|
||||||
|
}
|
||||||
|
|
||||||
|
header ul li:first-of-type {
|
||||||
|
margin-left: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
display: grid;
|
||||||
|
justify-items: left;
|
||||||
|
align-items: left;
|
||||||
|
margin-top: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
main#results {
|
||||||
|
display: block;
|
||||||
|
margin-top: 4rem;
|
||||||
|
columns: 2;
|
||||||
|
max-width: 1200px;
|
||||||
|
}
|
||||||
|
|
||||||
|
main#about {
|
||||||
|
max-width: 600px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lieu {
|
||||||
|
font-family: "Noto Serif";
|
||||||
|
font-weight: 400;
|
||||||
|
}
|
||||||
|
|
||||||
|
.search-container {
|
||||||
|
grid-template-columns: 19rem 3rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lieu-container {
|
||||||
|
display: grid;
|
||||||
|
justify-items: center;
|
||||||
|
align-items: center;
|
||||||
|
margin-top: 5rem;
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.entry {
|
||||||
|
-webkit-column-break-inside: avoid;
|
||||||
|
-moz-column-break-inside:avoid;
|
||||||
|
-moz-page-break-inside:avoid;
|
||||||
|
page-break-inside: avoid;
|
||||||
|
break-inside: avoid-column;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.entry p {
|
||||||
|
color: var(--primary);
|
||||||
|
opacity: 0.45;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link {
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media only screen and (min-device-width : 320px) and (max-device-width : 720px) {
|
||||||
|
main {
|
||||||
|
columns: 1 !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
{{ define "footer" }}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
{{ end }}
|
|
@ -0,0 +1,10 @@
|
||||||
|
{{ define "head" }}
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<link href="/assets/style.css" rel="stylesheet">
|
||||||
|
<title>Lieu—webring search engine</title>
|
||||||
|
<link rel="shortcut icon" href="/assets/favicon.png">
|
||||||
|
<link rel="apple touch icon" href="/assets/favicon.png">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
{{ end }}
|
|
@ -1,21 +0,0 @@
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<link href="/links/startpage.css" rel="stylesheet">
|
|
||||||
<title>Lieu—webring search engine</title>
|
|
||||||
<link rel="shortcut icon" href="/links/favicon.png">
|
|
||||||
<link rel="apple touch icon" href="/links/favicon.png">
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<a class="about-link" href="/about">About</a>
|
|
||||||
<main>
|
|
||||||
<div class="lieu-container">
|
|
||||||
<h1><a href="/">Lieu</a></h1>
|
|
||||||
<h2>the search for the new—endless</h2>
|
|
||||||
<form class="search-container">
|
|
||||||
<input type="search" required minlength="1" name="q" placeholder="tracking" value="{{ .Query }}" class="search-box">
|
|
||||||
<button type="submit" class="search-button"></button>
|
|
||||||
</form>
|
|
||||||
</div>
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
{{ template "head" . }}
|
||||||
|
<header>
|
||||||
|
<h2><a href="https://github.com/cblgh/lieu">Lieu</a></h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="/webring">Webring</a></li>
|
||||||
|
<li><a href="/about">About</a></li>
|
||||||
|
</ul>
|
||||||
|
</header>
|
||||||
|
<div class="clear"></div>
|
||||||
|
<main>
|
||||||
|
<div class="lieu-container">
|
||||||
|
<h1>
|
||||||
|
{{ .SiteName }}
|
||||||
|
</h1>
|
||||||
|
<h2>
|
||||||
|
the search for the new—endless
|
||||||
|
</h2>
|
||||||
|
<form class="search-container">
|
||||||
|
<input type="search" required minlength="1" name="q" placeholder="tracking" value="{{ .Data.Query }}" class="search-box">
|
||||||
|
<button type="submit" class="search-button"></button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
{{ template "footer" . }}
|
|
@ -1,22 +0,0 @@
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<link href="/links/search.css" rel="stylesheet">
|
|
||||||
<title>{{ .Title }}—Lieu</title>
|
|
||||||
<link rel="shortcut icon" href="/links/favicon.png">
|
|
||||||
<link rel="apple touch icon" href="/links/favicon.png">
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<a class="about-link" href="/about">About</a>
|
|
||||||
<h1><a href="/">Lieu</a></h1>
|
|
||||||
<h2>{{ .Title }}</h2>
|
|
||||||
<main>
|
|
||||||
<ul>
|
|
||||||
{{ range .URLs }}
|
|
||||||
<li>
|
|
||||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
|
||||||
</li>
|
|
||||||
{{ end }}
|
|
||||||
</ul>
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
{{ template "head" . }}
|
||||||
|
{{ template "nav" . }}
|
||||||
|
<main>
|
||||||
|
<ul>
|
||||||
|
{{ range .Data.URLs }}
|
||||||
|
<li>
|
||||||
|
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||||
|
</li>
|
||||||
|
{{ end }}
|
||||||
|
</ul>
|
||||||
|
</main>
|
||||||
|
{{ template "footer" . }}
|
|
@ -0,0 +1,12 @@
|
||||||
|
{{ define "nav" }}
|
||||||
|
<header>
|
||||||
|
<h2>
|
||||||
|
<a href="/">{{ .SiteName }}</a>
|
||||||
|
</h2>
|
||||||
|
<ul>
|
||||||
|
<li><a href="/">Search</a></li>
|
||||||
|
<li><a href="/webring">Webring</a></li>
|
||||||
|
<li><a href="/about">About</a></li>
|
||||||
|
</ul>
|
||||||
|
</header>
|
||||||
|
{{ end }}
|
|
@ -1,24 +0,0 @@
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<link href="/links/search.css" rel="stylesheet">
|
|
||||||
<title>{{ .Query }}—Lieu</title>
|
|
||||||
<link rel="shortcut icon" href="/links/favicon.png">
|
|
||||||
<link rel="apple touch icon" href="/links/favicon.png">
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<a class="about-link" href="/about">About</a>
|
|
||||||
<h1><a href="/">Lieu</a></h1>
|
|
||||||
<form method="GET" class="search-container">
|
|
||||||
<input type="search" minlength="1" required name="q" value="{{ .Query }}" class="search-box">
|
|
||||||
<button type="submit" class="search-button"></button>
|
|
||||||
</form>
|
|
||||||
<main>
|
|
||||||
{{ range .Pages }}
|
|
||||||
<div class="entry">
|
|
||||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
|
||||||
<p>{{ .About }}</p>
|
|
||||||
</div>
|
|
||||||
{{ end }}
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
{{ template "head" . }}
|
||||||
|
{{ template "nav" . }}
|
||||||
|
<form method="GET" class="search-container">
|
||||||
|
<input type="search" minlength="1" required name="q" value="{{ .Data.Query }}" class="search-box">
|
||||||
|
<button type="submit" class="search-button"></button>
|
||||||
|
</form>
|
||||||
|
<main id="results">
|
||||||
|
{{ range .Data.Pages }}
|
||||||
|
<div class="entry">
|
||||||
|
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||||
|
<p>{{ .About }}</p>
|
||||||
|
</div>
|
||||||
|
{{ end }}
|
||||||
|
{{ template "footer" . }}
|
|
@ -1,19 +0,0 @@
|
||||||
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<link href="/links/style.css" rel="stylesheet">
|
|
||||||
<title>indexed domains—Lieu</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Lieu</h1>
|
|
||||||
<a class="about-link" href="/about">About</a>
|
|
||||||
<main>
|
|
||||||
{{ range .Domains }}
|
|
||||||
<div class="entry">
|
|
||||||
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
|
||||||
<p>{{ .About }}</p>
|
|
||||||
</div>
|
|
||||||
{{ end }}
|
|
||||||
</main>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
{{ template "head" . }}
|
||||||
|
{{ template "nav" . }}
|
||||||
|
{{ range .Data.Domains }}
|
||||||
|
<div class="entry">
|
||||||
|
<a class="link" href="{{ .URL }}">{{ .Title }}</a>
|
||||||
|
<p>{{ .About }}</p>
|
||||||
|
</div>
|
||||||
|
{{ end }}
|
||||||
|
{{ template "footer" . }}
|
|
@ -18,46 +18,50 @@ type RequestHandler struct {
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TemplateView struct {
|
||||||
|
SiteName string
|
||||||
|
Data interface{}
|
||||||
|
}
|
||||||
|
|
||||||
type SearchData struct {
|
type SearchData struct {
|
||||||
Query string
|
Query string
|
||||||
Pages []types.PageData
|
Pages []types.PageData
|
||||||
}
|
}
|
||||||
|
|
||||||
type AboutData struct {
|
|
||||||
DomainCount int
|
|
||||||
InstanceName string
|
|
||||||
PageCount string
|
|
||||||
TermCount string
|
|
||||||
FilteredLink string
|
|
||||||
RingLink string
|
|
||||||
}
|
|
||||||
|
|
||||||
type ListData struct {
|
type ListData struct {
|
||||||
Title string
|
Title string
|
||||||
URLs []types.PageData
|
URLs []types.PageData
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type AboutData struct {
|
||||||
|
DomainCount int
|
||||||
|
WebringName string
|
||||||
|
PageCount string
|
||||||
|
TermCount string
|
||||||
|
FilteredLink string
|
||||||
|
RingLink string
|
||||||
|
}
|
||||||
|
|
||||||
const useURLTitles = true
|
const useURLTitles = true
|
||||||
|
|
||||||
var indexView = template.Must(template.ParseFiles("html/index-template.html"))
|
var templates = template.Must(template.ParseFiles(
|
||||||
var aboutView = template.Must(template.ParseFiles("html/about-template.html"))
|
"html/head.html", "html/nav.html", "html/footer.html",
|
||||||
var listView = template.Must(template.ParseFiles("html/list-template.html"))
|
"html/about.html", "html/index.html", "html/list.html", "html/search.html", "html/webring.html"))
|
||||||
var searchResultsView = template.Must(template.ParseFiles("html/search-template.html"))
|
|
||||||
|
|
||||||
func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request) {
|
func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request) {
|
||||||
var query string
|
var query string
|
||||||
|
view := &TemplateView{}
|
||||||
|
|
||||||
if req.Method == http.MethodGet {
|
if req.Method == http.MethodGet {
|
||||||
params := req.URL.Query()
|
params := req.URL.Query()
|
||||||
if words, exists := params["q"]; exists && words[0] != "" {
|
if words, exists := params["q"]; exists && words[0] != "" {
|
||||||
query = words[0]
|
query = words[0]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(query) == 0 {
|
if len(query) == 0 {
|
||||||
var empty interface{}
|
view.Data = SearchData{}
|
||||||
err := indexView.Execute(res, empty)
|
h.renderView(res, "index", view)
|
||||||
util.Check(err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,32 +76,34 @@ func (h RequestHandler) searchRoute(res http.ResponseWriter, req *http.Request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
data := SearchData{
|
view.Data = SearchData{
|
||||||
Query: query,
|
Query: query,
|
||||||
Pages: pages,
|
Pages: pages,
|
||||||
}
|
}
|
||||||
err := searchResultsView.Execute(res, data)
|
h.renderView(res, "search", view)
|
||||||
util.Check(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h RequestHandler) aboutRoute(res http.ResponseWriter, req *http.Request) {
|
func (h RequestHandler) aboutRoute(res http.ResponseWriter, req *http.Request) {
|
||||||
|
view := &TemplateView{}
|
||||||
|
|
||||||
pageCount := util.Humanize(database.GetPageCount(h.db))
|
pageCount := util.Humanize(database.GetPageCount(h.db))
|
||||||
wordCount := util.Humanize(database.GetWordCount(h.db))
|
wordCount := util.Humanize(database.GetWordCount(h.db))
|
||||||
domainCount := database.GetDomainCount(h.db)
|
domainCount := database.GetDomainCount(h.db)
|
||||||
|
|
||||||
data := AboutData{
|
view.Data = AboutData{
|
||||||
InstanceName: h.config.General.Name,
|
WebringName: h.config.General.Name,
|
||||||
DomainCount: domainCount,
|
DomainCount: domainCount,
|
||||||
PageCount: pageCount,
|
PageCount: pageCount,
|
||||||
TermCount: wordCount,
|
TermCount: wordCount,
|
||||||
FilteredLink: "/filtered",
|
FilteredLink: "/filtered",
|
||||||
RingLink: h.config.General.URL,
|
RingLink: h.config.General.URL,
|
||||||
}
|
}
|
||||||
err := aboutView.Execute(res, data)
|
h.renderView(res, "about", view)
|
||||||
util.Check(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request) {
|
func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request) {
|
||||||
|
view := &TemplateView{}
|
||||||
|
|
||||||
var URLs []types.PageData
|
var URLs []types.PageData
|
||||||
for _, domain := range util.ReadList(h.config.Crawler.BannedDomains, "\n") {
|
for _, domain := range util.ReadList(h.config.Crawler.BannedDomains, "\n") {
|
||||||
u, err := url.Parse(domain)
|
u, err := url.Parse(domain)
|
||||||
|
@ -108,12 +114,12 @@ func (h RequestHandler) filteredRoute(res http.ResponseWriter, req *http.Request
|
||||||
p := types.PageData{Title: domain, URL: u.String()}
|
p := types.PageData{Title: domain, URL: u.String()}
|
||||||
URLs = append(URLs, p)
|
URLs = append(URLs, p)
|
||||||
}
|
}
|
||||||
data := ListData{
|
|
||||||
|
view.Data = ListData{
|
||||||
Title: "Filtered Domains",
|
Title: "Filtered Domains",
|
||||||
URLs: URLs,
|
URLs: URLs,
|
||||||
}
|
}
|
||||||
err := listView.Execute(res, data)
|
h.renderView(res, "list", view)
|
||||||
util.Check(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request) {
|
func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request) {
|
||||||
|
@ -121,6 +127,12 @@ func (h RequestHandler) randomRoute(res http.ResponseWriter, req *http.Request)
|
||||||
http.Redirect(res, req, link, http.StatusSeeOther)
|
http.Redirect(res, req, link, http.StatusSeeOther)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h RequestHandler) renderView(res http.ResponseWriter, tmpl string, view *TemplateView) {
|
||||||
|
view.SiteName = h.config.General.Name
|
||||||
|
errTemp := templates.ExecuteTemplate(res, tmpl+".html", view)
|
||||||
|
util.Check(errTemp)
|
||||||
|
}
|
||||||
|
|
||||||
func Serve(config types.Config) {
|
func Serve(config types.Config) {
|
||||||
db := database.InitDB(config.Data.Database)
|
db := database.InitDB(config.Data.Database)
|
||||||
handler := RequestHandler{config: config, db: db}
|
handler := RequestHandler{config: config, db: db}
|
||||||
|
@ -131,10 +143,10 @@ func Serve(config types.Config) {
|
||||||
http.HandleFunc("/filtered", handler.filteredRoute)
|
http.HandleFunc("/filtered", handler.filteredRoute)
|
||||||
|
|
||||||
fileserver := http.FileServer(http.Dir("html/assets/"))
|
fileserver := http.FileServer(http.Dir("html/assets/"))
|
||||||
http.Handle("/links/", http.StripPrefix("/links/", fileserver))
|
http.Handle("/assets/", http.StripPrefix("/assets/", fileserver))
|
||||||
|
|
||||||
portstr := fmt.Sprintf(":%d", config.General.Port)
|
portstr := fmt.Sprintf(":%d", config.General.Port)
|
||||||
fmt.Println("listening on", portstr)
|
fmt.Println("Listening on port: ", portstr)
|
||||||
|
|
||||||
http.ListenAndServe(portstr, nil)
|
http.ListenAndServe(portstr, nil)
|
||||||
}
|
}
|
||||||
|
|
58
util/util.go
58
util/util.go
|
@ -1,18 +1,18 @@
|
||||||
package util
|
package util
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"lieu/types"
|
|
||||||
"github.com/jinzhu/inflection"
|
"github.com/jinzhu/inflection"
|
||||||
"github.com/komkom/toml"
|
"github.com/komkom/toml"
|
||||||
|
"lieu/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Inflect(words []string) []string {
|
func Inflect(words []string) []string {
|
||||||
|
@ -30,17 +30,17 @@ func Check(err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func DatabaseDoesNotExist(filepath string) {
|
func DatabaseDoesNotExist(filepath string) {
|
||||||
fmt.Printf("lieu: database %s does not exist\n", filepath)
|
fmt.Printf("lieu: database %s does not exist\n", filepath)
|
||||||
fmt.Println("lieu: try running `lieu ingest` if you have already crawled source data")
|
fmt.Println("lieu: try running `lieu ingest` if you have already crawled source data")
|
||||||
Exit()
|
Exit()
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckFileExists (path string) bool {
|
func CheckFileExists(path string) bool {
|
||||||
_, err := os.Stat(path)
|
_, err := os.Stat(path)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return os.IsExist(err)
|
return os.IsExist(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func Humanize(n int) string {
|
func Humanize(n int) string {
|
||||||
|
@ -64,30 +64,30 @@ func Contains(arr []string, query string) bool {
|
||||||
|
|
||||||
func ReadList(filepath, sep string) []string {
|
func ReadList(filepath, sep string) []string {
|
||||||
data, err := ioutil.ReadFile(filepath)
|
data, err := ioutil.ReadFile(filepath)
|
||||||
if err != nil || len(data) == 0{
|
if err != nil || len(data) == 0 {
|
||||||
return []string{}
|
return []string{}
|
||||||
}
|
}
|
||||||
return strings.Split(strings.TrimSuffix(string(data), sep), sep)
|
return strings.Split(strings.TrimSuffix(string(data), sep), sep)
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckPortOpen(port int) bool {
|
func CheckPortOpen(port int) bool {
|
||||||
tcpaddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%d", port))
|
tcpaddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%d", port))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
l, err := net.ListenTCP("tcp", tcpaddr)
|
l, err := net.ListenTCP("tcp", tcpaddr)
|
||||||
defer l.Close()
|
defer l.Close()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReadConfig() types.Config {
|
func ReadConfig() types.Config {
|
||||||
data, err := ioutil.ReadFile("lieu.toml")
|
data, err := ioutil.ReadFile("lieu.toml")
|
||||||
Check(err)
|
Check(err)
|
||||||
|
|
||||||
var conf types.Config
|
var conf types.Config
|
||||||
decoder := json.NewDecoder(toml.New(bytes.NewBuffer(data)))
|
decoder := json.NewDecoder(toml.New(bytes.NewBuffer(data)))
|
||||||
|
@ -98,8 +98,8 @@ func ReadConfig() types.Config {
|
||||||
return conf
|
return conf
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteMockConfig () {
|
func WriteMockConfig() {
|
||||||
conf := []byte(`[general]
|
conf := []byte(`[general]
|
||||||
name = "Sweet Webring"
|
name = "Sweet Webring"
|
||||||
# used by the precrawl command and linked to in /about route
|
# used by the precrawl command and linked to in /about route
|
||||||
url = "https://example.com/"
|
url = "https://example.com/"
|
||||||
|
@ -127,10 +127,10 @@ boringWords = "data/boring-words.txt"
|
||||||
# domains that won't be output as outgoing links
|
# domains that won't be output as outgoing links
|
||||||
boringDomains = "data/boring-domains.txt"
|
boringDomains = "data/boring-domains.txt"
|
||||||
`)
|
`)
|
||||||
err := ioutil.WriteFile("lieu.toml", conf, 0644)
|
err := ioutil.WriteFile("lieu.toml", conf, 0644)
|
||||||
Check(err)
|
Check(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func Exit () {
|
func Exit() {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
Ładowanie…
Reference in New Issue