Skip to content

Commit

Permalink
feat: support multiple hosts files
Browse files Browse the repository at this point in the history
  • Loading branch information
ThinkChaos committed Jul 7, 2023
1 parent 5e4c155 commit cfc3699
Show file tree
Hide file tree
Showing 29 changed files with 1,491 additions and 807 deletions.
3 changes: 2 additions & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ issues:
# Exclude some linters from running on tests files.
- path: _test\.go
linters:
- gochecknoglobals
- dupl
- funlen
- gochecknoglobals
- gosec
71 changes: 31 additions & 40 deletions config/blocking.go
Original file line number Diff line number Diff line change
@@ -1,41 +1,47 @@
package config

import (
"strings"

. "github.com/0xERR0R/blocky/config/migration" //nolint:revive,stylecheck
"github.com/0xERR0R/blocky/log"
"github.com/sirupsen/logrus"
)

// BlockingConfig configuration for query blocking
type BlockingConfig struct {
BlackLists map[string][]string `yaml:"blackLists"`
WhiteLists map[string][]string `yaml:"whiteLists"`
ClientGroupsBlock map[string][]string `yaml:"clientGroupsBlock"`
BlockType string `yaml:"blockType" default:"ZEROIP"`
BlockTTL Duration `yaml:"blockTTL" default:"6h"`
DownloadTimeout Duration `yaml:"downloadTimeout" default:"60s"`
DownloadAttempts uint `yaml:"downloadAttempts" default:"3"`
DownloadCooldown Duration `yaml:"downloadCooldown" default:"1s"`
RefreshPeriod Duration `yaml:"refreshPeriod" default:"4h"`
ProcessingConcurrency uint `yaml:"processingConcurrency" default:"4"`
StartStrategy StartStrategyType `yaml:"startStrategy" default:"blocking"`
MaxErrorsPerFile int `yaml:"maxErrorsPerFile" default:"5"`
BlackLists map[string][]BytesSource `yaml:"blackLists"`
WhiteLists map[string][]BytesSource `yaml:"whiteLists"`
ClientGroupsBlock map[string][]string `yaml:"clientGroupsBlock"`
BlockType string `yaml:"blockType" default:"ZEROIP"`
BlockTTL Duration `yaml:"blockTTL" default:"6h"`
Loading SourceLoadingConfig `yaml:"loading"`

// Deprecated options
Deprecated struct {
FailStartOnListError *bool `yaml:"failStartOnListError"`
DownloadTimeout *Duration `yaml:"downloadTimeout"`
DownloadAttempts *uint `yaml:"downloadAttempts"`
DownloadCooldown *Duration `yaml:"downloadCooldown"`
RefreshPeriod *Duration `yaml:"refreshPeriod"`
FailStartOnListError *bool `yaml:"failStartOnListError"`
ProcessingConcurrency *uint `yaml:"processingConcurrency"`
StartStrategy *StartStrategyType `yaml:"startStrategy"`
MaxErrorsPerFile *int `yaml:"maxErrorsPerFile"`
} `yaml:",inline"`
}

func (c *BlockingConfig) migrate(logger *logrus.Entry) bool {
return Migrate(logger, "blocking", c.Deprecated, map[string]Migrator{
"failStartOnListError": Apply(To("startStrategy", c), func(oldValue bool) {
if oldValue && c.StartStrategy != StartStrategyTypeFast {
c.StartStrategy = StartStrategyTypeFailOnError
"downloadTimeout": Move(To("loading.downloads.timeout", &c.Loading.Downloads)),
"downloadAttempts": Move(To("loading.downloads.attempts", &c.Loading.Downloads)),
"downloadCooldown": Move(To("loading.downloads.cooldown", &c.Loading.Downloads)),
"refreshPeriod": Move(To("loading.refreshPeriod", &c.Loading)),
"failStartOnListError": Apply(To("loading.strategy", &c.Loading), func(oldValue bool) {
if oldValue {
c.Loading.Strategy = StartStrategyTypeFailOnError
}
}),
"processingConcurrency": Move(To("loading.concurrency", &c.Loading)),
"startStrategy": Move(To("loading.strategy", &c.Loading)),
"maxErrorsPerFile": Move(To("loading.maxErrorsPerSource", &c.Loading)),
})
}

Expand All @@ -44,7 +50,7 @@ func (c *BlockingConfig) IsEnabled() bool {
return len(c.ClientGroupsBlock) != 0
}

// IsEnabled implements `config.Configurable`.
// LogConfig implements `config.Configurable`.
func (c *BlockingConfig) LogConfig(logger *logrus.Entry) {
logger.Info("clientGroupsBlock:")

Expand All @@ -58,17 +64,8 @@ func (c *BlockingConfig) LogConfig(logger *logrus.Entry) {
logger.Infof("blockTTL = %s", c.BlockTTL)
}

logger.Infof("downloadTimeout = %s", c.DownloadTimeout)

logger.Infof("startStrategy = %s", c.StartStrategy)

logger.Infof("maxErrorsPerFile = %d", c.MaxErrorsPerFile)

if c.RefreshPeriod > 0 {
logger.Infof("refresh = every %s", c.RefreshPeriod)
} else {
logger.Debug("refresh = disabled")
}
logger.Info("loading:")
log.WithIndent(logger, " ", c.Loading.LogConfig)

logger.Info("blacklist:")
log.WithIndent(logger, " ", func(logger *logrus.Entry) {
Expand All @@ -81,18 +78,12 @@ func (c *BlockingConfig) LogConfig(logger *logrus.Entry) {
})
}

func (c *BlockingConfig) logListGroups(logger *logrus.Entry, listGroups map[string][]string) {
for group, links := range listGroups {
func (c *BlockingConfig) logListGroups(logger *logrus.Entry, listGroups map[string][]BytesSource) {
for group, sources := range listGroups {
logger.Infof("%s:", group)

for _, link := range links {
if idx := strings.IndexRune(link, '\n'); idx != -1 && idx < len(link) { // found and not last char
link = link[:idx] // first line only

logger.Infof(" - %s [...]", link)
} else {
logger.Infof(" - %s", link)
}
for _, source := range sources {
logger.Infof(" - %s", source)
}
}
}
27 changes: 3 additions & 24 deletions config/blocking_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"github.com/creasty/defaults"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)

var _ = Describe("BlockingConfig", func() {
Expand All @@ -18,13 +17,12 @@ var _ = Describe("BlockingConfig", func() {
cfg = BlockingConfig{
BlockType: "ZEROIP",
BlockTTL: Duration(time.Minute),
BlackLists: map[string][]string{
"gr1": {"/a/file/path"},
BlackLists: map[string][]BytesSource{
"gr1": NewBytesSources("/a/file/path"),
},
ClientGroupsBlock: map[string][]string{
"default": {"gr1"},
},
RefreshPeriod: Duration(time.Hour),
}
})

Expand Down Expand Up @@ -59,26 +57,7 @@ var _ = Describe("BlockingConfig", func() {

Expect(hook.Calls).ShouldNot(BeEmpty())
Expect(hook.Messages[0]).Should(Equal("clientGroupsBlock:"))
Expect(hook.Messages).Should(ContainElement(ContainSubstring("refresh = every 1 hour")))
})
When("refresh is disabled", func() {
It("should reflect that", func() {
cfg.RefreshPeriod = Duration(-1)

logger.Logger.Level = logrus.InfoLevel

cfg.LogConfig(logger)

Expect(hook.Calls).ShouldNot(BeEmpty())
Expect(hook.Messages).ShouldNot(ContainElement(ContainSubstring("refresh = disabled")))

logger.Logger.Level = logrus.TraceLevel

cfg.LogConfig(logger)

Expect(hook.Calls).ShouldNot(BeEmpty())
Expect(hook.Messages).Should(ContainElement(ContainSubstring("refresh = disabled")))
})
Expect(hook.Messages).Should(ContainElement(Equal("blockType = ZEROIP")))
})
})
})
111 changes: 111 additions & 0 deletions config/bytes_source.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
//go:generate go run github.com/abice/go-enum -f=$GOFILE --marshal --names --values
package config

import (
"fmt"
"strings"
)

const maxTextSourceDisplayLen = 12

// var BytesSourceNone = BytesSource{}

// BytesSourceType supported BytesSource types. ENUM(
// text=1 // Inline YAML block.
// http // HTTP(S).
// file // Local file.
// )
type BytesSourceType uint16

type BytesSource struct {
Type BytesSourceType
From string
}

func (s BytesSource) String() string {
switch s.Type {
case BytesSourceTypeText:
break

case BytesSourceTypeHttp:
return s.From

case BytesSourceTypeFile:
return fmt.Sprintf("file://%s", s.From)

default:
return fmt.Sprintf("unknown source (%s: %s)", s.Type, s.From)
}

text := s.From
truncated := false

if idx := strings.IndexRune(text, '\n'); idx != -1 {
text = text[:idx] // first line only
truncated = idx < len(text) // don't count removing last char
}

if len(text) > maxTextSourceDisplayLen { // truncate
text = text[:maxTextSourceDisplayLen]
truncated = true
}

if truncated {
return fmt.Sprintf("%s...", text[:maxTextSourceDisplayLen])
}

return text
}

// UnmarshalText implements `encoding.TextUnmarshaler`.
func (s *BytesSource) UnmarshalText(data []byte) error {
source := string(data)

switch {
// Inline definition in YAML (with literal style Block Scalar)
case strings.ContainsAny(source, "\n"):
*s = BytesSource{Type: BytesSourceTypeText, From: source}

// HTTP(S)
case strings.HasPrefix(source, "http"):
*s = BytesSource{Type: BytesSourceTypeHttp, From: source}

// Probably path to a local file
default:
*s = BytesSource{Type: BytesSourceTypeFile, From: strings.TrimPrefix(source, "file://")}
}

return nil
}

func newBytesSource(source string) BytesSource {
var res BytesSource

// UnmarshalText never returns an error
_ = res.UnmarshalText([]byte(source))

return res
}

func NewBytesSources(sources ...string) []BytesSource {
res := make([]BytesSource, 0, len(sources))

for _, source := range sources {
res = append(res, newBytesSource(source))
}

return res
}

func TextBytesSource(lines ...string) BytesSource {
return BytesSource{Type: BytesSourceTypeText, From: inlineList(lines...)}
}

func inlineList(lines ...string) string {
res := strings.Join(lines, "\n")

// ensure at least one line ending so it's parsed as an inline block
res += "\n"

return res
}
101 changes: 101 additions & 0 deletions config/bytes_source_enum.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit cfc3699

Please sign in to comment.