added hosts list functionality

Author: wranders

.testdata/hosts.list

@@ -0,0 +1,8 @@
+# This is a comment
+! This is a comment
+; This is a comment
+
+0.0.0.0 example.com
+0.0.0.0 	example.net
+0.0.0.0 		example.com
+0.0.0.0 noop example.com

README.md

@@ -15,7 +15,6 @@ The *filter* plugin is used to block domain name resolution, simliar to
 ```nginx
 filter {
     ACTION TYPE DATA
-    ACTION list TYPE DATA
 }
 ```
 
@@ -29,8 +28,23 @@ filter {
     * Adblock Plus: `||example.com^`
     * DNSMasq Address: `address=/example.com/#`
 
-If the **ACTION** is a `list`, then **DATA** is a `[ file | http | https ]` URL.
-Lists must contain only the **TYPE** specified.
+```nginx
+filter {
+    ACTION list TYPE DATA
+}
+```
+
+* **ACTION**: `[ allow | block ]` What action to take
+* **DATA**: Lists of the following data types
+  * `domain`: A raw domain to match. Subdomains are not matched
+  * `hosts`: A hostsfile formatted list
+  * `regex`: A Go-formatted Regular Expression
+  * `wildcard`: Common wildcard formats
+    * Generic: `*.example.com`
+    * Adblock Plus: `||example.com^`
+    * DNSMasq Address: `address=/example.com/#`
+* **DATA**: A `[ file | http | https ]` URL. Must contain only the **TYPE**
+specified.
 
 ```nginx
 filter {
@@ -88,7 +102,7 @@ filter {
 
 This flexibility is extended to wildcard lists as well. AdblockPlus and DNSMasq
 formats are supported for flexibility and ease of migration from other
-solutions. Hosts, zone, and Unbound configuration files are not supported.
+solutions. Zone and Unbound configuration files are not supported.
 
 ## Examples
 

action.go

@@ -41,8 +41,11 @@ type ActionConfig struct {
 	regex      map[string]*regexp.Regexp
 
 	domainLists   ActionList
+	hostsLists    ActionList
 	regexLists    ActionList
 	wildcardLists ActionList
+
+	hostsRegexp *regexp.Regexp
 }
 
 // NewActionConfig returns an action ready to accept configurations
@@ -52,8 +55,10 @@ func NewActionConfig(action ActionType) ActionConfig {
 		domains:       make(map[string]bool),
 		regex:         make(map[string]*regexp.Regexp),
 		domainLists:   make(ActionList),
+		hostsLists:    make(ActionList),
 		regexLists:    make(ActionList),
 		wildcardLists: make(ActionList),
+		hostsRegexp:   regexp.MustCompile(`\s+|\t+`),
 	}
 }
 
@@ -76,6 +81,18 @@ func (a ActionConfig) AddDomainList(url string) error {
 	return nil
 }
 
+// AddHostsList to match contents
+func (a ActionConfig) AddHostsList(url string) error {
+	if _, ok := a.hostsLists[url]; !ok {
+		loadFunc, err := GetListLoadFunc(url)
+		if err != nil {
+			return err
+		}
+		a.hostsLists[url] = loadFunc
+	}
+	return nil
+}
+
 // AddRegex to match
 func (a ActionConfig) AddRegex(expr string) error {
 	comp, err := regexp.Compile(expr)
@@ -141,6 +158,13 @@ func (a ActionConfig) BuildDomains() map[string]bool {
 		domains[k] = true
 	}
 
+	a.buildDomainsLists(domains)
+	a.buildDomainHostsLists(domains)
+
+	return domains
+}
+
+func (a ActionConfig) buildDomainsLists(domains map[string]bool) {
 	for dom, load := range a.domainLists {
 		file, err := load(dom)
 		if err != nil {
@@ -163,15 +187,43 @@ func (a ActionConfig) BuildDomains() map[string]bool {
 			}
 			if bytes.Contains(line, []byte(" ")) {
 				// Skip formats that contain whitespace characters
-				// Host files should use the stock 'hosts' plugin
 				// Zone files should use the stock 'file' plugin
 				continue
 			}
 			domains[string(line)] = true
 		}
 	}
+}
 
-	return domains
+func (a ActionConfig) buildDomainHostsLists(domains map[string]bool) {
+	for dom, load := range a.hostsLists {
+		file, err := load(dom)
+		if err != nil {
+			log.Errorf(
+				"there was a problem fetching %s hosts list %q; %s",
+				a.configType,
+				dom,
+				err,
+			)
+			continue
+		}
+		defer file.Close()
+		scanner := bufio.NewScanner(file)
+		scanner.Split(bufio.ScanLines)
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			line = bytes.TrimSpace(line)
+			if a.shouldSkip(line) {
+				continue
+			}
+			line = a.hostsRegexp.ReplaceAll(line, []byte(" "))
+			hostsLine := strings.Split(string(line), " ")
+			if len(hostsLine) != 2 {
+				continue
+			}
+			domains[hostsLine[1]] = true
+		}
+	}
 }
 
 // BuildRegExps consolidates individual regular expressions then loads and

action_test.go

@@ -9,6 +9,7 @@ func TestActionList404(t *testing.T) {
 		block list domain https://httpbin.org/status/404
 		block list regex https://httpbin.org/status/404
 		block list wildcard https://httpbin.org/noop
+		block list hosts https://httpbin.org/status/404
 	}`
 	filter := NewTestFilter(t, corefile)
 	filter.Build()
@@ -43,6 +44,20 @@ func TestActionListDomain(t *testing.T) {
 	}
 }
 
+func TestActionListHosts(t *testing.T) {
+	corefile := `filter {
+		block list hosts file://.testdata/hosts.list
+	}`
+	filter := NewTestFilter(t, corefile)
+	filter.Build()
+	if len(filter.blockDomains) != 2 {
+		t.Errorf(
+			"expected two domains; found %d",
+			len(filter.blockDomains),
+		)
+	}
+}
+
 func TestActionListRegex(t *testing.T) {
 	corefile := `filter {
 		block list regex file://.testdata/regex.list

setup.go

@@ -172,6 +172,10 @@ func parseActionList(c *caddy.Controller, f *Filter, a ActionType) error {
 		if err := parseActionListDomain(c, f, a); err != nil {
 			return err
 		}
+	case "hosts":
+		if err := parseActionListHosts(c, f, a); err != nil {
+			return err
+		}
 	case "regex":
 		if err := parseActionListRegex(c, f, a); err != nil {
 			return err
@@ -208,6 +212,23 @@ func parseActionListDomain(c *caddy.Controller, f *Filter, a ActionType) error {
 	return ensureEOL(c)
 }
 
+func parseActionListHosts(c *caddy.Controller, f *Filter, a ActionType) error {
+	if !c.NextArg() {
+		return c.Errf("no %s hosts list specified", a)
+	}
+	switch a {
+	case ActionTypeAllow:
+		if err := f.allowConfig.AddHostsList(c.Val()); err != nil {
+			return err
+		}
+	case ActionTypeBlock:
+		if err := f.blockConfig.AddHostsList(c.Val()); err != nil {
+			return err
+		}
+	}
+	return ensureEOL(c)
+}
+
 func parseActionListRegex(c *caddy.Controller, f *Filter, a ActionType) error {
 	if !c.NextArg() {
 		return c.Errf("no %s regex list specified", a)

setup_test.go

@@ -351,6 +351,35 @@ func TestAllowList(t *testing.T) {
 			}`,
 			false,
 		},
+		{
+			"allow list hosts no url",
+			`filter {
+				allow list hosts
+			}`,
+			true,
+		},
+		{
+			"allow list hosts invalid url",
+			`filter {
+				allow list hosts "file://invalid url"
+			}`,
+			true,
+		},
+		{
+			"allow list hosts from file",
+			`filter {
+				allow list hosts file://.testdata/hosts.list
+			}`,
+			false,
+		},
+		{
+			"allow list hosts duplicate",
+			`filter {
+				allow list hosts file://.testdata/hosts.list
+				allow list hosts file://.testdata/hosts.list
+			}`,
+			false,
+		},
 		{
 			"allow list regex no url",
 			`filter {
@@ -500,6 +529,35 @@ func TestBlockList(t *testing.T) {
 			}`,
 			false,
 		},
+		{
+			"block list hosts no url",
+			`filter {
+				block list hosts
+			}`,
+			true,
+		},
+		{
+			"block list hosts invalid url",
+			`filter {
+				block list hosts "file://invalid url"
+			}`,
+			true,
+		},
+		{
+			"block list hosts from file",
+			`filter {
+				block list hosts file://.testdata/hosts.list
+			}`,
+			false,
+		},
+		{
+			"block list hosts duplicate",
+			`filter {
+				block list hosts file://.testdata/hosts.list
+				block list hosts file://.testdata/hosts.list
+			}`,
+			false,
+		},
 		{
 			"block list regex no url",
 			`filter {