Allows the configuration of a proxy (#9)

* Add proxy support, capability to crawl using SOCKS proxies

.gitignore

@@ -224,3 +224,4 @@ pip-log.txt
 
 #Mr Developer
 .mr.developer.cfg
+lieu

crawler/crawler.go

@@ -153,7 +153,29 @@ func collectHeadingText(heading string, e *colly.HTMLElement) {
 	}
 }
 
+func SetupDefaultProxy(config types.Config) error {
+	proxyURL, err := url.Parse(config.General.Proxy)
+	if err != nil {
+		return err
+	}
+
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxyURL),
+		},
+	}
+
+	//colly.SetHTTPClient(httpClient)
+	http.DefaultClient = httpClient
+	return nil
+}
+
 func Precrawl(config types.Config) {
+	// setup proxy
+	err := SetupDefaultProxy(config)
+	if err != nil {
+		log.Fatal(err)
+	}
 	res, err := http.Get(config.General.URL)
 	util.Check(err)
 	defer res.Body.Close()
@@ -189,6 +211,11 @@ func Precrawl(config types.Config) {
 }
 
 func Crawl(config types.Config) {
+	// setup proxy
+	err := SetupDefaultProxy(config)
+	if err != nil {
+		log.Fatal(err)
+	}
 	SUFFIXES := getBannedSuffixes(config.Crawler.BannedSuffixes)
 	links := getWebringLinks(config.Crawler.Webring)
 	domains, pathsites := getDomains(links)
@@ -199,6 +226,7 @@ func Crawl(config types.Config) {
 	c := colly.NewCollector(
 		colly.MaxDepth(3),
 	)
+	c.SetProxy(config.General.Proxy)
 
 	q, _ := queue.New(
 		5, /* threads */

data/crawled.txt

@@ -0,0 +1 @@
+

data/webring.txt

@@ -0,0 +1 @@
+

types/types.go

@@ -20,6 +20,7 @@ type Config struct {
 		Placeholder string `json:placeholder`
 		URL         string `json:url`
 		Port        int    `json:port`
+		Proxy       string `json:proxy`
 	} `json:general`
 	Theme struct {
 		Foreground string `json:"foreground"`

util/util.go

@@ -10,9 +10,10 @@ import (
 	"os"
 	"strings"
 
+	"lieu/types"
+
 	"github.com/jinzhu/inflection"
 	"github.com/komkom/toml"
-	"lieu/types"
 )
 
 func Inflect(words []string) []string {
@@ -140,3 +141,15 @@ boringDomains = "data/boring-domains.txt"
 func Exit() {
 	os.Exit(0)
 }
+
+func DeduplicateSlice(intSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range intSlice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+	return list
+}