Skip to content

Commit

Permalink
Addressing grafana#28 added some randomized tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@petethepig
petethepig committed Mar 11, 2021
1 parent 1fc03bb commit 8a7134c
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 10 deletions.
95 changes: 95 additions & 0 deletions pkg/storage/segment/fuzz_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package segment

import (
"math/big"
"math/rand"
"time"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pyroscope-io/pyroscope/pkg/testing"
)

type datapoint struct {
t time.Time
samples uint64
r *big.Rat
}

type storageMock struct {
resolution time.Duration
data []datapoint
}

func newMock(resolution time.Duration) *storageMock {
return &storageMock{
resolution: resolution,
data: []datapoint{},
}
}

func (sm *storageMock) Put(st, et time.Time, samples uint64) {
st, et = normalize(st, et)
fullDur := et.Sub(st) / sm.resolution
for t := st; t.Before(et); t = t.Add(sm.resolution) {
d := datapoint{
t: t,
samples: samples,
r: big.NewRat(int64(samples), int64(fullDur)),
}

sm.data = append(sm.data, d)
}
}

func (sm *storageMock) Get(st, et time.Time, cb func(depth int, samples uint64, t time.Time, r *big.Rat)) {
st, et = normalize(st, et)
for _, d := range sm.data {
if !d.t.Before(st) && !d.t.Add(sm.resolution).After(et) {
cb(0, 1, d.t, d.r)
}
}
}

var _ = Describe("segment", func() {
Context("Segment", func() {
It("works", func(done Done) {
s := New(10*time.Second, 10)
m := newMock(10 * time.Second)

r := rand.New(rand.NewSource(123))

for k := 0; k < 100; k++ {
for i := 0; i < r.Intn(1000); i++ {
sti := r.Intn(100) * 10
st := testing.SimpleTime(sti)
et := testing.SimpleTime(sti + 10)
samples := uint64(r.Intn(100))

m.Put(st, et, samples)
s.Put(st, et, samples, func(depth int, t time.Time, r *big.Rat, addons []Addon) {

})
}
mSum := big.NewRat(0, 1)
sSum := big.NewRat(0, 1)
for i := 0; i < r.Intn(100); i++ {
sti := r.Intn(100) * 10
st := testing.SimpleTime(sti)
et := testing.SimpleTime(sti + 160)
// et := testing.SimpleTime(sti + r.Intn(100)*10)

m.Get(st, et, func(depth int, samples uint64, t time.Time, r *big.Rat) {
mSum.Add(mSum, r.Mul(r, big.NewRat(int64(samples), 1)))
})

s.Get(st, et, func(depth int, samples uint64, t time.Time, r *big.Rat) {
sSum.Add(sSum, r.Mul(r, big.NewRat(int64(samples), 1)))
})
}
Expect(mSum.Cmp(sSum)).To(Equal(0))
}
close(done)
}, 5)
})
})
26 changes: 18 additions & 8 deletions pkg/storage/segment/segment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ func (sn *streeNode) relationship(st, et time.Time) rel {
return relationship(sn.time, t2, st, et)
}

func (sn *streeNode) endTime() time.Time {
return sn.time.Add(durations[sn.depth])
}

func (sn *streeNode) overlapRead(st, et time.Time) *big.Rat {
t2 := sn.time.Add(durations[sn.depth])
return overlapRead(sn.time, t2, st, et, durations[0])
Expand Down Expand Up @@ -66,7 +70,7 @@ func (sn *streeNode) findAddons() []Addon {
return res
}

func (sn *streeNode) put(st, et time.Time, _ uint64, cb func(n *streeNode, depth int, dt time.Time, r *big.Rat, addons []Addon)) {
func (sn *streeNode) put(st, et time.Time, samples uint64, cb func(n *streeNode, depth int, dt time.Time, r *big.Rat, addons []Addon)) {
nodes := []*streeNode{sn}

for len(nodes) > 0 {
Expand Down Expand Up @@ -94,18 +98,22 @@ func (sn *streeNode) put(st, et time.Time, _ uint64, cb func(n *streeNode, depth
}
var addons []Addon

r := sn.overlapWrite(st, et)
fv, _ := r.Float64()
sn.samples += samples * uint64(fv)

// relationship overlap read overlap write
// inside rel = iota // | S E | <1 1/1
// match // matching ranges 1/1 1/1
// outside // | | S E 0/1 0/1
// overlap // | S | E <1 <1
// contain // S | | E 1/1 <1

if rel == match || rel == contain || childrenCount > 1 || sn.present {
if !sn.present {
addons = sn.findAddons()
}
// TODO: calculate proper r
r := sn.overlapWrite(st, et)

cb(sn, sn.depth, sn.time, r, addons)
sn.present = true
}
Expand All @@ -119,6 +127,9 @@ func normalize(st, et time.Time) (time.Time, time.Time) {
if et2.Equal(et) {
return st, et
}
// if st.Equal(et2) {
// et2.Add(durations[0])
// }
return st, et2.Add(durations[0])
}

Expand Down Expand Up @@ -213,7 +224,7 @@ func (s *Segment) growTree(st, et time.Time) {
var prevVal *streeNode
if s.root != nil {
st = minTime(st, s.root.time)
et = maxTime(et, s.root.time)
et = maxTime(et, s.root.endTime())
} else {
st = st.Truncate(s.durations[0])
s.root = newNode(st, 0, s.multiplier)
Expand All @@ -228,7 +239,7 @@ func (s *Segment) growTree(st, et time.Time) {

prevVal = s.root
newDepth := prevVal.depth + 1
s.root = newNode(st.Truncate(s.durations[newDepth]), newDepth, s.multiplier)
s.root = newNode(prevVal.time.Truncate(s.durations[newDepth]), newDepth, s.multiplier)
if prevVal != nil {
s.root.samples = prevVal.samples
s.root.replace(prevVal)
Expand All @@ -250,14 +261,13 @@ func (s *Segment) Put(st, et time.Time, samples uint64, cb func(depth int, t tim
s.growTree(st, et)
v := newVis()
s.root.put(st, et, samples, func(sn *streeNode, depth int, tm time.Time, r *big.Rat, addons []Addon) {
sn.samples += samples * uint64(r.Num().Int64()) / uint64(r.Denom().Int64())
v.add(sn, r, true)
cb(depth, tm, r, addons)
})
v.print(fmt.Sprintf("/tmp/0-put-%s-%s.html", st.String(), et.String()))
}

func (s *Segment) Get(st, et time.Time, cb func(depth int, t time.Time, r *big.Rat)) {
func (s *Segment) Get(st, et time.Time, cb func(depth int, samples uint64, t time.Time, r *big.Rat)) {
s.m.RLock()
defer s.m.RUnlock()

Expand All @@ -270,7 +280,7 @@ func (s *Segment) Get(st, et time.Time, cb func(depth int, t time.Time, r *big.R
s.root.get(st, et, func(sn *streeNode, depth int, t time.Time, r *big.Rat) {
// TODO: pass m / d from .get() ?
v.add(sn, r, true)
cb(depth, t, r)
cb(depth, sn.samples, t, r)
})
v.print(fmt.Sprintf("/tmp/0-get-%s-%s.html", st.String(), et.String()))
}
Expand Down
50 changes: 49 additions & 1 deletion pkg/storage/segment/segment_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package segment
import (
"bufio"
"bytes"
"log"
"math/big"
"strings"

Expand All @@ -18,7 +19,7 @@ import (

func doGet(s *Segment, st, et time.Time) []time.Time {
res := []time.Time{}
s.Get(st, et, func(d int, t time.Time, r *big.Rat) {
s.Get(st, et, func(d int, samples uint64, t time.Time, r *big.Rat) {
res = append(res, t)
})
return res
Expand All @@ -36,6 +37,20 @@ func strip(val string) string {
return ret
}

func expectChildrenSamplesAddUpToParentSamples(tn *streeNode) {
childrenSum := uint64(0)
if len(tn.children) == 0 {
return
}
for _, v := range tn.children {
if v != nil {
expectChildrenSamplesAddUpToParentSamples(v)
childrenSum += v.samples
}
}
Expect(childrenSum).To(Equal(tn.samples))
}

var _ = Describe("stree", func() {
r := 10 * time.Second
m := 10
Expand Down Expand Up @@ -80,6 +95,35 @@ var _ = Describe("stree", func() {
})

Context("Put", func() {
Context("When inserts are far apart", func() {
Context("When second insert is far in the future", func() {
It("sets root properly", func() {
log.Println("---")
s := New(r, m)
s.Put(testing.SimpleTime(1330),
testing.SimpleTime(1339), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
Expect(s.root).ToNot(BeNil())
Expect(s.root.depth).To(Equal(0))
s.Put(testing.SimpleTime(1110),
testing.SimpleTime(1119), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
expectChildrenSamplesAddUpToParentSamples(s.root)
})
})
Context("When second insert is far in the past", func() {
It("sets root properly", func() {
log.Println("---")
s := New(r, m)
s.Put(testing.SimpleTime(2030),
testing.SimpleTime(2039), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
Expect(s.root).ToNot(BeNil())
Expect(s.root.depth).To(Equal(0))
s.Put(testing.SimpleTime(0),
testing.SimpleTime(9), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
expectChildrenSamplesAddUpToParentSamples(s.root)
})
})
})

Context("When empty", func() {
It("sets root properly", func() {
s := New(r, m)
Expand Down Expand Up @@ -113,6 +157,7 @@ var _ = Describe("stree", func() {
Expect(s.root.depth).To(Equal(0))
s.Put(testing.SimpleTime(10),
testing.SimpleTime(19), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
expectChildrenSamplesAddUpToParentSamples(s.root)
})

It("sets root properly", func() {
Expand All @@ -125,6 +170,7 @@ var _ = Describe("stree", func() {
testing.SimpleTime(29), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
Expect(s.root).ToNot(BeNil())
Expect(s.root.depth).To(Equal(1))
expectChildrenSamplesAddUpToParentSamples(s.root)
})

It("sets root properly", func() {
Expand All @@ -144,6 +190,7 @@ var _ = Describe("stree", func() {
Expect(s.root).ToNot(BeNil())
Expect(s.root.depth).To(Equal(1))
spew.Dump(s.root)
expectChildrenSamplesAddUpToParentSamples(s.root)
})

It("sets root properly", func() {
Expand Down Expand Up @@ -190,6 +237,7 @@ var _ = Describe("stree", func() {
s.Put(testing.SimpleTime(100),
testing.SimpleTime(109), 1, func(de int, t time.Time, r *big.Rat, a []Addon) {})
spew.Dump(s.root)
expectChildrenSamplesAddUpToParentSamples(s.root)
Expect(s.root).ToNot(BeNil())
Expect(s.root.depth).To(Equal(2))
Expect(s.root.present).To(BeTrue())
Expand Down
2 changes: 1 addition & 1 deletion pkg/storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ func (s *Storage) Get(startTime, endTime time.Time, key *Key) (*tree.Tree, *segm

tl.PopulateTimeline(startTime, endTime, st)

st.Get(startTime, endTime, func(depth int, t time.Time, r *big.Rat) {
st.Get(startTime, endTime, func(depth int, samples uint64, t time.Time, r *big.Rat) {
k := skk.TreeKey(depth, t)
tr := s.trees.Get(k).(*tree.Tree)
// TODO: these clones are probably are not the most efficient way of doing this
Expand Down
4 changes: 4 additions & 0 deletions pkg/testing/time.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@ func ParseTime(str string) time.Time {
func SimpleTime(i int) time.Time {
return time.Time{}.Add(time.Duration(i) * time.Second).UTC()
}

func PrintTime(t time.Time) int {
return int(t.Sub(time.Time{}) / time.Second)
}

0 comments on commit 8a7134c

Please sign in to comment.