more trying

Author: adienes

lazy_iterate.jl

@@ -3,48 +3,109 @@ using BitIntegers
 using Base: count_ones
 import Base: in!, iterate, IteratorSize
 
-function _hash_bloom(x, seeds, T)
+
+
+mutable struct BloomFilterNode{T,K,L}
+    memory::T
+    seeds::NTuple{K, UInt64}
+    const next::L
+    function BloomFilterNode{T,K}(seeds, next=nothing) where {T<:Unsigned,K}
+        new{T,K,typeof(next)}(zero(T), seeds, next)
+    end
+end
+
+struct BloomFilterChain{T,K,L}
+    head::L
+    function BloomFilterChain{T,K}(head) where {T<:Unsigned,K}
+        new{T,K,typeof(head)}(head)
+    end
+end
+
+function BloomFilterChain{T,K}() where {T,K}
+    seeds = ntuple(_ -> rand(UInt64), K)
+    head = BloomFilterNode{T,K}(seeds)
+    return BloomFilterChain{T,K}(head)
+end
+
+function grow_chain(f::BloomFilterChain{T,K}) where {T,K}
+    # wT = widen(T)
+    wT = T
+    new_seeds = ntuple(_ -> rand(UInt64), K)
+    head = BloomFilterNode{wT,K}(new_seeds, f.head)
+    return BloomFilterChain{wT,K}(head)
+end
+function into(f::BloomFilterChain{T,K}, x; p = 0.05) where {T,K}
+    
+    # wT = widen(T)
+    wT = T
+    new_seeds = ntuple(_ -> rand(UInt64), K)
+    head = BloomFilterNode{wT,K}(new_seeds, f.head)
+    return BloomFilterChain{wT,K}(head)
+end
+
+
+function _hash_bloom(x, seeds, ::Type{T}) where T
     out = zero(T)
+    bits = 8sizeof(T)
     for h in seeds
-        out |= one(T)<<(hash(x, h) % (8sizeof(T)))
+        out |= one(T)<<(hash(x, h) & (bits-1))
     end
     return out
 end
 
-mutable struct BloomFilter{T,K}
-    memory::T
-    const seeds::NTuple{K, UInt64}
-    const p::Float64 # max tolerable false positive rate
-    function BloomFilter{T,K}(p, seeds) where {T<:Unsigned,K}
-        new{T,K}(zero(T), seeds, p)
-    end
+function _in(x, f::BloomFilterNode{T,K,Nothing}) where {T,K}
+    h = _hash_bloom(x, f.seeds, T)
+    return (f.memory & h) == h
 end
 
-function BloomFilter(; k=5, p=1/16)
-    return BloomFilter{UInt64, k}(p, ntuple(_ -> rand(UInt), k))
+function _in(x, f::BloomFilterNode{T}) where {T}
+    h = _hash_bloom(x, f.seeds, T)
+    return ((f.memory & h) == h) || _in(x, f.next)
 end
 
-function in!(x, f::BloomFilter{T}) where {T}
+function _in!(x, f::BloomFilterNode{T,K,Nothing}) where {T,K}
     h = _hash_bloom(x, f.seeds, T)
-    r = (f.memory & h) == h
+    r = ((f.memory & h) == h)
     f.memory |= h
     return r
 end
 
-function false_positive_rate(f::BloomFilter{T,K}) where {T,K}
+function _in!(x, f::BloomFilterNode{T}) where {T}
+    h = _hash_bloom(x, f.seeds, T)
+    r = ((f.memory & h) == h)
+    f.memory |= h
+    return r || _in(x, f.next)
+end
+
+in!(x, f::BloomFilterChain{T}) where {T} = _in!(x, f.head)
+
+_false_positive_rate(::Nothing, k) = 0
+function _false_positive_rate(f::BloomFilterNode{T}, k) where {T}
     pop = count_ones(f.memory)
     bits = 8sizeof(T)
-    return (pop/bits)^K
+
+    p1 = (pop/bits)^k
+    p2 = _false_positive_rate(f.next, k)
+    return  p1 + p2 - p1 * p2
 end
 
-struct Unique
-    f::Set
-    g
+_false_positive_rate(f::BloomFilterChain{T,K}) where {T,K} = _false_positive_rate(f.head, K)
+
+
+
+
+
+
+
+myunique(it) = Unique{typeof(it)}(BloomFilterChain{UInt128,3}(), it)
+struct Unique{T}
+    f::BloomFilterChain{UInt128,3}
+    g::T
 end
 
 function iterate(u::Unique)
     x, next = iterate(u.g)
-    push!(u.f, x)
+    in!(x, u.f)
     return (x, next)
 end