Add union and intersection

Author: korthaj

example_test.go

@@ -50,3 +50,30 @@ func Example_falsePositives() {
 	fmt.Println(count, "mistakes were made.")
 	// Output: 1 mistakes were made.
 }
+
+// Compute the intersection and union of two filters.
+func ExampleFilter_And() {
+	// Create two Bloom filter with room for n elements
+	// at a false-positives rate less than 1/p.
+	n := 1000
+	p := 100
+	f1, f2 := bloom.New(n, p), bloom.New(n, p)
+
+	// Add "0", "1", …, "499" to f1
+	for i := 0; i < n/2; i++ {
+		f1.Add(strconv.Itoa(i))
+	}
+
+	// Add "250", "251", …, "749" to f2
+	for i := n / 4; i < 3*n/4; i++ {
+		f2.Add(strconv.Itoa(i))
+	}
+
+	// Compute the approximate size of f1 ∩ f2 and f1 ∪ f2.
+	fmt.Println("f1 ∩ f2:", f1.And(f2).Count())
+	fmt.Println("f1 ∪ f2:", f1.Or(f2).Count())
+	// Output:
+	// f1 ∩ f2: 276
+	// f1 ∪ f2: 758
+
+}

filter.go

@@ -117,7 +117,80 @@ func (f *Filter) Test(s string) bool {
 	return f.TestByte(b)
 }
 
-// Count returns an estimate of the number of unique elements added to this filter.
+// Count returns an estimate of the number of unique elements in this filter.
 func (f *Filter) Count() int64 {
 	return f.count
 }
+
+// And returns a new Bloom filter that consists of all elements
+// that belong to both f1 and f2. It requires two filters with
+// the same size and the same false-positives rate.
+//
+// The false-positive rate of the resulting filter is at most
+// the false-positive rate of f1 and f2, but may be larger than
+// the rate of the filter created from scratch using the intersection
+// of the two sets.
+func (f1 *Filter) And(f2 *Filter) *Filter {
+	if len(f1.data) != len(f2.data) || f1.lookups != f2.lookups {
+		panic("operation requires filters of the same type")
+	}
+	len := len(f1.data)
+	res := &Filter{
+		data:    make([]uint64, len),
+		lookups: f1.lookups,
+	}
+	bitCount := 0
+	for i := 0; i < len; i++ {
+		w := f1.data[i] & f2.data[i]
+		res.data[i] = w
+		bitCount += count(w)
+	}
+	// Estimate the number of elements from the bitCount.
+	m := 64 * float64(len)
+	bits := float64(bitCount)
+	n := m / float64(f1.lookups) * math.Log(m/(m-bits))
+	res.count = int64(n)
+	return res
+}
+
+// Or returns a new Bloom filter that consists of all elements
+// that belong to either f1 or f2. It requires two filters with
+// the same size and the same false-positives rate.
+//
+// The resulting filter is the same as the filter created from scratch
+// using the union of the two sets.
+func (f1 *Filter) Or(f2 *Filter) *Filter {
+	if len(f1.data) != len(f2.data) || f1.lookups != f2.lookups {
+		panic("operation requires filters of the same type")
+	}
+	len := len(f1.data)
+	res := &Filter{
+		data:    make([]uint64, len),
+		lookups: f1.lookups,
+	}
+	bitCount := 0
+	for i := 0; i < len; i++ {
+		w := f1.data[i] | f2.data[i]
+		res.data[i] = w
+		bitCount += count(w)
+	}
+	// Estimate the number of elements from the bitCount.
+	m := 64 * float64(len)
+	n := m / float64(f1.lookups) * math.Log(m/(m-float64(bitCount)))
+	res.count = int64(n)
+	return res
+}
+
+// count returns the number of nonzero bits in w.
+func count(w uint64) int {
+	// Adapted from github.com/yourbasic/bit/funcs.go.
+	const maxw = 1<<64 - 1
+	const bpw = 64
+	w -= (w >> 1) & (maxw / 3)
+	w = w&(maxw/15*3) + (w>>2)&(maxw/15*3)
+	w += w >> 4
+	w &= maxw / 255 * 15
+	w *= maxw / 255
+	w >>= (bpw/8 - 1) * 8
+	return int(w)
+}

filter_test.go

@@ -58,23 +58,64 @@ func TestFilter(t *testing.T) {
 	}
 }
 
+func TestAndOr(t *testing.T) {
+	s1 := "asöldkgjaösldkgaösldkasldgjkaösldkgjöasgkdjg"
+	s2 := "elasödlnkgaölsdkfgaölsdkjfaölsdkgaölskgnaösl"
+	s3 := "aölsdgkaösldkgaösldkgjaölsdkjgaölsdkgjaösldk"
+	for n := 0; n < 100; n++ {
+		for p := 1; p <= 128; p *= 2 {
+			f1, f2 := New(n, p), New(n, p)
+			f1.Add(s1)
+			f1.Add(s2)
+			f2.Add(s2)
+			f2.Add(s3)
+			and, or := f1.And(f2), f1.Or(f2)
+			member := and.Test(s1)
+			if member {
+				t.Errorf("and.Test(s1) = %v; want false\n", member)
+			}
+			member = and.Test(s2)
+			if !member {
+				t.Errorf("and.Test(s2) = %v; want true\n", member)
+			}
+			member = and.Test(s3)
+			if member {
+				t.Errorf("and.Test(s3) = %v; want false\n", member)
+			}
+			member = or.Test(s1)
+			if !member {
+				t.Errorf("or.Test(s1) = %v; want true\n", member)
+			}
+			member = or.Test(s2)
+			if !member {
+				t.Errorf("or.Test(s2) = %v; want true\n", member)
+			}
+			member = or.Test(s3)
+			if !member {
+				t.Errorf("or.Test(s3) = %v; want true\n", member)
+			}
+		}
+	}
+}
+
+var fox string = "The quick brown fox jumps over the lazy dog."
+
 func BenchmarkAdd(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := "The quick brown fox jumps over the lazy dog."
 	for i := 0; i < b.N; i++ {
-		filter.Add(s)
+		filter.Add(fox)
 	}
 }
 
 func BenchmarkAddByte(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := []byte("The quick brown fox jumps over the lazy dog.")
+	bytes := []byte(fox)
 	for i := 0; i < b.N; i++ {
-		filter.AddByte(s)
+		filter.AddByte(bytes)
 	}
 }
 
@@ -83,16 +124,27 @@ func BenchmarkTest(b *testing.B) {
 	filter := New(1<<30, 200)
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		filter.Test("The quick brown fox jumps over the lazy dog.")
+		filter.Test(fox)
 	}
 }
 
 func BenchmarkTestByte(b *testing.B) {
 	b.StopTimer()
 	filter := New(1<<30, 200)
 	b.StartTimer()
-	s := []byte("The quick brown fox jumps over the lazy dog.")
+	bytes := []byte(fox)
+	for i := 0; i < b.N; i++ {
+		filter.TestByte(bytes)
+	}
+}
+
+func BenchmarkTestAnd(b *testing.B) {
+	n := 1000
+	b.StopTimer()
+	f1 := New(n, 200)
+	f2 := New(n, 200)
+	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		filter.TestByte(s)
+		_ = f1.And(f2)
 	}
 }