feat(lists): add support for wildcard lists using a custom Trie (0xER…

…R0R#1233)

.gitignore

@@ -12,5 +12,6 @@ todo.txt
 node_modules
 package-lock.json
 vendor/
+coverage.html
 coverage.txt
 coverage/

Makefile

@@ -45,7 +45,7 @@ else
 	go generate ./...
 endif
 
-build: fmt generate ## Build binary
+build: generate ## Build binary
 	go build $(GO_BUILD_FLAGS) -ldflags="$(GO_BUILD_LD_FLAGS)" -o $(GO_BUILD_OUTPUT)
 ifdef BIN_USER
 	$(info setting owner of $(GO_BUILD_OUTPUT) to $(BIN_USER))
@@ -58,6 +58,7 @@ endif
 
 test: ## run tests
 	go run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!e2e" --coverprofile=coverage.txt --covermode=atomic --cover -r ${GINKGO_PROCS}
+	go tool cover -html coverage.txt -o coverage.html
 
 e2e-test: ## run e2e tests
 	docker buildx build \
@@ -81,7 +82,7 @@ fmt: ## gofmt and goimports all go files
 	go run mvdan.cc/gofumpt -l -w -extra .
 	find . -name '*.go' -exec go run golang.org/x/tools/cmd/goimports -w {} +
 
-docker-build: generate ## Build docker image 
+docker-build: generate ## Build docker image
 	docker buildx build \
 		--build-arg VERSION=${VERSION} \
 		--build-arg BUILD_TIME=${BUILD_TIME} \

cache/stringcache/chained_grouped_cache.go

@@ -56,10 +56,14 @@ type chainedGroupFactory struct {
 	cacheFactories []GroupFactory
 }
 
-func (c *chainedGroupFactory) AddEntry(entry string) {
+func (c *chainedGroupFactory) AddEntry(entry string) bool {
 	for _, factory := range c.cacheFactories {
-		factory.AddEntry(entry)
+		if factory.AddEntry(entry) {
+			return true
+		}
 	}
+
+	return false
 }
 
 func (c *chainedGroupFactory) Count() int {

cache/stringcache/chained_grouped_cache_test.go

@@ -24,6 +24,10 @@ var _ = Describe("Chained grouped cache", func() {
 			It("should not find any string", func() {
 				Expect(cache.Contains("searchString", []string{"someGroup"})).Should(BeEmpty())
 			})
+
+			It("should not add entries", func() {
+				Expect(cache.Refresh("group").AddEntry("test")).Should(BeFalse())
+			})
 		})
 	})
 	Describe("Delegation", func() {
@@ -44,12 +48,12 @@ var _ = Describe("Chained grouped cache", func() {
 			})
 
 			It("factory has 4 elements (both caches)", func() {
-				Expect(factory.Count()).Should(BeNumerically("==", 4))
+				Expect(factory.Count()).Should(BeNumerically("==", 2))
 			})
 
 			It("should have element count of 4", func() {
 				factory.Finish()
-				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 4))
+				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 2))
 			})
 
 			It("should find strings", func() {
@@ -80,25 +84,38 @@ var _ = Describe("Chained grouped cache", func() {
 			})
 
 			It("should contain 4 elements in 2 groups", func() {
-				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 4))
-				Expect(cache.ElementCount("group2")).Should(BeNumerically("==", 4))
+				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 2))
+				Expect(cache.ElementCount("group2")).Should(BeNumerically("==", 2))
 				Expect(cache.Contains("g1", []string{"group1", "group2"})).Should(ConsistOf("group1"))
 				Expect(cache.Contains("g2", []string{"group1", "group2"})).Should(ConsistOf("group2"))
 				Expect(cache.Contains("both", []string{"group1", "group2"})).Should(ConsistOf("group1", "group2"))
 			})
 
-			It("Should replace group content on refresh", func() {
+			It("should replace group content on refresh", func() {
 				factory = cache.Refresh("group1")
 				factory.AddEntry("newString")
 				factory.Finish()
 
-				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 2))
-				Expect(cache.ElementCount("group2")).Should(BeNumerically("==", 4))
+				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 1))
+				Expect(cache.ElementCount("group2")).Should(BeNumerically("==", 2))
 				Expect(cache.Contains("g1", []string{"group1", "group2"})).Should(BeEmpty())
 				Expect(cache.Contains("newString", []string{"group1", "group2"})).Should(ConsistOf("group1"))
 				Expect(cache.Contains("g2", []string{"group1", "group2"})).Should(ConsistOf("group2"))
 				Expect(cache.Contains("both", []string{"group1", "group2"})).Should(ConsistOf("group2"))
 			})
+
+			It("should replace empty groups on refresh", func() {
+				factory = cache.Refresh("group")
+				factory.AddEntry("begone")
+				factory.Finish()
+
+				Expect(cache.ElementCount("group")).Should(BeNumerically("==", 1))
+
+				factory = cache.Refresh("group")
+				factory.Finish()
+
+				Expect(cache.ElementCount("group")).Should(BeNumerically("==", 0))
+			})
 		})
 	})
 })

cache/stringcache/grouped_cache_interface.go

@@ -15,7 +15,7 @@ type GroupedStringCache interface {
 
 type GroupFactory interface {
 	// AddEntry adds a new string to the factory to be added later to the cache groups.
-	AddEntry(entry string)
+	AddEntry(entry string) bool
 
 	// Count returns amount of processed string in the factory
 	Count() int

cache/stringcache/in_memory_grouped_cache.go

@@ -24,6 +24,13 @@ func NewInMemoryGroupedRegexCache() *InMemoryGroupedCache {
 	}
 }
 
+func NewInMemoryGroupedWildcardCache() *InMemoryGroupedCache {
+	return &InMemoryGroupedCache{
+		caches:    make(map[string]stringCache),
+		factoryFn: newWildcardCacheFactory,
+	}
+}
+
 func (c *InMemoryGroupedCache) ElementCount(group string) int {
 	c.lock.RLock()
 	cache, found := c.caches[group]
@@ -57,8 +64,13 @@ func (c *InMemoryGroupedCache) Refresh(group string) GroupFactory {
 		factory: c.factoryFn(),
 		finishFn: func(sc stringCache) {
 			c.lock.Lock()
-			c.caches[group] = sc
-			c.lock.Unlock()
+			defer c.lock.Unlock()
+
+			if sc != nil {
+				c.caches[group] = sc
+			} else {
+				delete(c.caches, group)
+			}
 		},
 	}
 }
@@ -68,8 +80,8 @@ type inMemoryGroupFactory struct {
 	finishFn func(stringCache)
 }
 
-func (c *inMemoryGroupFactory) AddEntry(entry string) {
-	c.factory.addEntry(entry)
+func (c *inMemoryGroupFactory) AddEntry(entry string) bool {
+	return c.factory.addEntry(entry)
 }
 
 func (c *inMemoryGroupFactory) Count() int {

cache/stringcache/in_memory_grouped_cache_test.go

@@ -46,8 +46,8 @@ var _ = Describe("In-Memory grouped cache", func() {
 				cache = stringcache.NewInMemoryGroupedStringCache()
 				factory = cache.Refresh("group1")
 
-				factory.AddEntry("string1")
-				factory.AddEntry("string2")
+				Expect(factory.AddEntry("string1")).Should(BeTrue())
+				Expect(factory.AddEntry("string2")).Should(BeTrue())
 			})
 
 			It("cache should still have 0 element, since finish was not executed", func() {
@@ -69,36 +69,40 @@ var _ = Describe("In-Memory grouped cache", func() {
 				Expect(cache.Contains("string2", []string{"group1", "someOtherGroup"})).Should(ConsistOf("group1"))
 			})
 		})
-		When("String grouped cache is used", func() {
+		When("Regex grouped cache is used", func() {
 			BeforeEach(func() {
-				cache = stringcache.NewInMemoryGroupedStringCache()
+				cache = stringcache.NewInMemoryGroupedRegexCache()
 				factory = cache.Refresh("group1")
 
-				factory.AddEntry("string1")
-				factory.AddEntry("/string2/")
+				Expect(factory.AddEntry("string1")).Should(BeFalse())
+				Expect(factory.AddEntry("/string2/")).Should(BeTrue())
 				factory.Finish()
 			})
 
-			It("should ignore regex", func() {
+			It("should ignore non-regex", func() {
 				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 1))
-				Expect(cache.Contains("string1", []string{"group1"})).Should(ConsistOf("group1"))
+				Expect(cache.Contains("string1", []string{"group1"})).Should(BeEmpty())
+				Expect(cache.Contains("string2", []string{"group1"})).Should(ConsistOf("group1"))
+				Expect(cache.Contains("shouldalsomatchstring2", []string{"group1"})).Should(ConsistOf("group1"))
 			})
 		})
-		When("Regex grouped cache is used", func() {
+		When("Wildcard grouped cache is used", func() {
 			BeforeEach(func() {
-				cache = stringcache.NewInMemoryGroupedRegexCache()
+				cache = stringcache.NewInMemoryGroupedWildcardCache()
 				factory = cache.Refresh("group1")
 
-				factory.AddEntry("string1")
-				factory.AddEntry("/string2/")
+				Expect(factory.AddEntry("string1")).Should(BeFalse())
+				Expect(factory.AddEntry("/string2/")).Should(BeFalse())
+				Expect(factory.AddEntry("*.string3")).Should(BeTrue())
 				factory.Finish()
 			})
 
-			It("should ignore non-regex", func() {
+			It("should ignore non-wildcard", func() {
 				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 1))
 				Expect(cache.Contains("string1", []string{"group1"})).Should(BeEmpty())
-				Expect(cache.Contains("string2", []string{"group1"})).Should(ConsistOf("group1"))
-				Expect(cache.Contains("shouldalsomatchstring2", []string{"group1"})).Should(ConsistOf("group1"))
+				Expect(cache.Contains("string2", []string{"group1"})).Should(BeEmpty())
+				Expect(cache.Contains("string3", []string{"group1"})).Should(ConsistOf("group1"))
+				Expect(cache.Contains("shouldalsomatch.string3", []string{"group1"})).Should(ConsistOf("group1"))
 			})
 		})
 	})
@@ -109,13 +113,13 @@ var _ = Describe("In-Memory grouped cache", func() {
 				cache = stringcache.NewInMemoryGroupedStringCache()
 				factory = cache.Refresh("group1")
 
-				factory.AddEntry("g1")
-				factory.AddEntry("both")
+				Expect(factory.AddEntry("g1")).Should(BeTrue())
+				Expect(factory.AddEntry("both")).Should(BeTrue())
 				factory.Finish()
 
 				factory = cache.Refresh("group2")
-				factory.AddEntry("g2")
-				factory.AddEntry("both")
+				Expect(factory.AddEntry("g2")).Should(BeTrue())
+				Expect(factory.AddEntry("both")).Should(BeTrue())
 				factory.Finish()
 			})
 
@@ -129,7 +133,7 @@ var _ = Describe("In-Memory grouped cache", func() {
 
 			It("Should replace group content on refresh", func() {
 				factory = cache.Refresh("group1")
-				factory.AddEntry("newString")
+				Expect(factory.AddEntry("newString")).Should(BeTrue())
 				factory.Finish()
 
 				Expect(cache.ElementCount("group1")).Should(BeNumerically("==", 1))