kopia lustrzana https://github.com/cblgh/lieu
ignore data/ dir, clarify instructions in README, update go.sum
basebuilder
rodzic
96cf3d915f
commit
4533344652
|
|
@ -1,4 +1,5 @@
|
|||
# Lieu
|
||||
data/
|
||||
searchengine.db
|
||||
|
||||
#~top ignores~
|
||||
|
|
|
|||
11
README.md
11
README.md
|
|
@ -1,4 +1,5 @@
|
|||
# Lieu
|
||||
|
||||
_an alternative search engine_
|
||||
|
||||
Created in response to the environs of apathy concerning the use of hypertext
|
||||
|
|
@ -10,11 +11,13 @@ engine, a way for personal webrings to increase serendipitous connexions.
|
|||
|
||||
|
||||
## Goals
|
||||
|
||||
* Enable serendipitous discovery
|
||||
* Support personal communities
|
||||
* Be reusable, easily
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
$ lieu help
|
||||
Lieu: neighbourhood search engine
|
||||
|
|
@ -28,6 +31,7 @@ Commands
|
|||
|
||||
Example:
|
||||
lieu precrawl > data/webring.txt
|
||||
lieu crawl > data/crawled.txt
|
||||
lieu ingest
|
||||
lieu host
|
||||
```
|
||||
|
|
@ -39,12 +43,13 @@ the files Lieu reads from, as defined in the config file. See below for a
|
|||
typical workflow.
|
||||
|
||||
### Workflow
|
||||
|
||||
* Edit the config
|
||||
* Add domains to crawl in `config.crawler.webring`
|
||||
* **If you have a webpage with links you want to crawl:**
|
||||
* Set the config's `url` field to that page
|
||||
* Populate the list of domains to crawl with `precrawl`: `lieu precrawl > data/webring.txt`
|
||||
* Crawl: `lieu crawl > data/source.txt`
|
||||
* Crawl: `lieu crawl > data/crawled.txt`
|
||||
* Create database: `lieu ingest`
|
||||
* Host engine: `lieu host`
|
||||
|
||||
|
|
@ -52,6 +57,7 @@ After ingesting the data with `lieu ingest`, you can also use lieu to search the
|
|||
corpus in the terminal with `lieu search`.
|
||||
|
||||
## Config
|
||||
|
||||
The config file is written in [TOML](https://toml.io/en/).
|
||||
|
||||
```toml
|
||||
|
|
@ -85,6 +91,7 @@ boringDomains = "data/boring-domains.txt"
|
|||
```
|
||||
|
||||
For your own use, the following config fields should be customized:
|
||||
|
||||
* `name`
|
||||
* `url `
|
||||
* `port`
|
||||
|
|
@ -93,6 +100,7 @@ For your own use, the following config fields should be customized:
|
|||
* `bannedDomains`
|
||||
|
||||
The following config-defined files can stay as-is unless you have specific requirements:
|
||||
|
||||
* `database`
|
||||
* `heuristics`
|
||||
* `wordlist`
|
||||
|
|
@ -102,5 +110,6 @@ For a full rundown of the files and their various jobs, see the [files
|
|||
description](docs/files.md).
|
||||
|
||||
### License
|
||||
|
||||
Source code `AGPL-3.0-or-later`, Inter is available under `SIL OPEN FONT
|
||||
LICENSE Version 1.1`, Noto Serif is licensed as `Apache License, Version 2.0`.
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -66,6 +66,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
|||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
|
|
@ -75,6 +76,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
|
|||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -101,6 +103,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
|
|
@ -114,9 +118,14 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
|
|||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0 h1:u0bliLHgSO64Pb0xbhtwNIHspZc11X8M1bJqBkYl4Co=
|
||||
golang.org/x/tour v0.0.0-20210317163553-0a3a62c5e5c0/go.mod h1:bWzMdWN2SiLomDzvESYfljDnNu60fUM2ATO8j09tZ5Y=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue