Skip to content

Commit

Permalink
Switch task.List from uint64 slice to a block list structure
Browse files Browse the repository at this point in the history
  • Loading branch information
@ashwin95r
ashwin95r committed Feb 9, 2017
1 parent 11c63f3 commit 3ca38ad
Show file tree
Hide file tree
Showing 15 changed files with 909 additions and 469 deletions.
403 changes: 313 additions & 90 deletions algo/uidlist.go
View file

Large diffs are not rendered by default.

92 changes: 59 additions & 33 deletions algo/uidlist_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,29 @@
package algo

import (
"fmt"
"math/rand"
"sort"
"testing"

"github.com/dgraph-io/dgraph/task"
"github.com/stretchr/testify/require"
)

func newList(data []uint64) *task.List {
return &task.List{Uids: data}
return SortedListToBlock(data)
}

func TestMergeSorted1(t *testing.T) {
input := []*task.List{
newList([]uint64{55}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{55})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{55})
}

func TestMergeSorted2(t *testing.T) {
input := []*task.List{
newList([]uint64{1, 3, 6, 8, 10}),
newList([]uint64{2, 4, 5, 7, 15}),
}
require.Equal(t, MergeSorted(input).Uids,
require.Equal(t, BlockToList(MergeSorted(input)),
[]uint64{1, 2, 3, 4, 5, 6, 7, 8, 10, 15})
}

Expand All @@ -51,23 +48,23 @@ func TestMergeSorted3(t *testing.T) {
newList([]uint64{1, 3, 6, 8, 10}),
newList([]uint64{}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{1, 3, 6, 8, 10})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{1, 3, 6, 8, 10})
}

func TestMergeSorted4(t *testing.T) {
input := []*task.List{
newList([]uint64{}),
newList([]uint64{1, 3, 6, 8, 10}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{1, 3, 6, 8, 10})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{1, 3, 6, 8, 10})
}

func TestMergeSorted5(t *testing.T) {
input := []*task.List{
newList([]uint64{}),
newList([]uint64{}),
}
require.Empty(t, MergeSorted(input).Uids)
require.Empty(t, BlockToList(MergeSorted(input)))
}

func TestMergeSorted6(t *testing.T) {
Expand All @@ -76,7 +73,7 @@ func TestMergeSorted6(t *testing.T) {
newList([]uint64{12, 14, 15, 15, 16, 16, 17, 25}),
newList([]uint64{1, 2}),
}
require.Equal(t, MergeSorted(input).Uids,
require.Equal(t, BlockToList(MergeSorted(input)),
[]uint64{1, 2, 11, 12, 13, 14, 15, 16, 17, 18, 20, 25})
}

Expand All @@ -87,54 +84,54 @@ func TestMergeSorted7(t *testing.T) {
newList([]uint64{1, 2}),
newList([]uint64{}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{1, 2, 3, 4, 5, 6, 7})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{1, 2, 3, 4, 5, 6, 7})
}

func TestMergeSorted8(t *testing.T) {
input := []*task.List{}
require.Empty(t, MergeSorted(input).Uids)
require.Empty(t, BlockToList(MergeSorted(input)))
}

func TestMergeSorted9(t *testing.T) {
input := []*task.List{
newList([]uint64{1, 1, 1}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{1})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{1})
}

func TestMergeSorted10(t *testing.T) {
input := []*task.List{
newList([]uint64{1, 2, 3, 3, 6}),
newList([]uint64{4, 8, 9}),
}
require.Equal(t, MergeSorted(input).Uids, []uint64{1, 2, 3, 4, 6, 8, 9})
require.Equal(t, BlockToList(MergeSorted(input)), []uint64{1, 2, 3, 4, 6, 8, 9})
}

func TestIntersectSorted1(t *testing.T) {
input := []*task.List{
newList([]uint64{1, 2, 3}),
newList([]uint64{2, 3, 4, 5}),
}
require.Equal(t, IntersectSorted(input).Uids, []uint64{2, 3})
require.Equal(t, BlockToList(IntersectSorted(input)), []uint64{2, 3})
}

func TestIntersectSorted2(t *testing.T) {
input := []*task.List{
newList([]uint64{1, 2, 3}),
}
require.Equal(t, IntersectSorted(input).Uids, []uint64{1, 2, 3})
require.Equal(t, BlockToList(IntersectSorted(input)), []uint64{1, 2, 3})
}

func TestIntersectSorted3(t *testing.T) {
input := []*task.List{}
require.Empty(t, IntersectSorted(input).Uids)
require.Empty(t, BlockToList(IntersectSorted(input)))
}

func TestIntersectSorted4(t *testing.T) {
input := []*task.List{
newList([]uint64{100, 101}),
}
require.Equal(t, IntersectSorted(input).Uids, []uint64{100, 101})
require.Equal(t, BlockToList(IntersectSorted(input)), []uint64{100, 101})
}

func TestIntersectSorted5(t *testing.T) {
Expand All @@ -143,7 +140,7 @@ func TestIntersectSorted5(t *testing.T) {
newList([]uint64{2, 3, 4, 5}),
newList([]uint64{4, 5, 6}),
}
require.Empty(t, IntersectSorted(input).Uids)
require.Empty(t, BlockToList(IntersectSorted(input)))
}

func TestIntersectSorted6(t *testing.T) {
Expand All @@ -152,7 +149,7 @@ func TestIntersectSorted6(t *testing.T) {
newList([]uint64{2, 3, 4, 13}),
newList([]uint64{4, 5, 6}),
}
require.Empty(t, IntersectSorted(input).Uids)
require.Empty(t, BlockToList(IntersectSorted(input)))
}

func TestSubSorted1(t *testing.T) {
Expand All @@ -161,7 +158,7 @@ func TestSubSorted1(t *testing.T) {
newList([]uint64{2, 3, 4, 5}),
}
Difference(input[0], input[1])
require.Equal(t, []uint64{1}, input[0].Uids)
require.Equal(t, []uint64{1}, BlockToList(input[0]))
}

func TestSubSorted6(t *testing.T) {
Expand All @@ -170,69 +167,96 @@ func TestSubSorted6(t *testing.T) {
newList([]uint64{2, 3, 4, 13}),
}
Difference(input[0], input[1])
require.Equal(t, []uint64{10, 12}, input[0].Uids)
require.Equal(t, []uint64{10, 12}, BlockToList(input[0]))
}

func TestIterator1(t *testing.T) {
u := newList([]uint64{1, 2, 3, 4, 43, 234, 2344})

it := NewListIterator(u)
var res []uint64
for ; it.Valid(); it.Next() {
res = append(res, it.Val())
}

require.Equal(t, []uint64{1, 2, 3, 4, 43, 234, 2344}, res)
}

func TestIterator2(t *testing.T) {
var ls, res []uint64
for i := 0; i < 1000; i++ {
ls = append(ls, uint64(i*2))
}
u := newList(ls)
it := NewListIterator(u)
for ; it.Valid(); it.Next() {
res = append(res, it.Val())
}
require.Equal(t, ls, res)
}

func TestUIDListIntersect1(t *testing.T) {
u := newList([]uint64{1, 2, 3})
v := newList([]uint64{})
IntersectWith(u, v)
require.Empty(t, u.Uids)

require.Empty(t, BlockToList(u))
}

func TestUIDListIntersect2(t *testing.T) {
u := newList([]uint64{1, 2, 3})
v := newList([]uint64{1, 2, 3, 4, 5})
IntersectWith(u, v)
require.Equal(t, u.Uids, []uint64{1, 2, 3})
require.Equal(t, []uint64{1, 2, 3}, BlockToList(u))
}

func TestUIDListIntersect3(t *testing.T) {
u := newList([]uint64{1, 2, 3})
v := newList([]uint64{2})
IntersectWith(u, v)
require.Equal(t, u.Uids, []uint64{2})
require.Equal(t, []uint64{2}, BlockToList(u))
}

func TestUIDListIntersect4(t *testing.T) {
u := newList([]uint64{1, 2, 3})
v := newList([]uint64{0, 5})
IntersectWith(u, v)
require.Empty(t, u.Uids)
require.Empty(t, BlockToList(u))
}

func TestUIDListIntersect5(t *testing.T) {
u := newList([]uint64{1, 2, 3})
v := newList([]uint64{3, 5})
IntersectWith(u, v)
require.Equal(t, u.Uids, []uint64{3})
require.Equal(t, []uint64{3}, BlockToList(u))
}

func TestUIDListIntersectDupFirst(t *testing.T) {
u := newList([]uint64{1, 1, 2, 3})
v := newList([]uint64{1, 2})
IntersectWith(u, v)
require.Equal(t, []uint64{1, 2}, u.Uids)
require.Equal(t, []uint64{1, 2}, BlockToList(u))
}

func TestUIDListIntersectDupBoth(t *testing.T) {
u := newList([]uint64{1, 1, 2, 3, 5})
v := newList([]uint64{1, 1, 2, 4})
IntersectWith(u, v)
require.Equal(t, []uint64{1, 1, 2}, u.Uids)
require.Equal(t, []uint64{1, 1, 2}, BlockToList(u))
}

func TestUIDListIntersectDupSecond(t *testing.T) {
u := newList([]uint64{1, 2, 3, 5})
v := newList([]uint64{1, 1, 2, 4})
IntersectWith(u, v)
require.Equal(t, []uint64{1, 2}, u.Uids)
require.Equal(t, []uint64{1, 2}, BlockToList(u))
}

func TestApplyFilterUint(t *testing.T) {
u := newList([]uint64{1, 2, 3, 4, 5})
ApplyFilter(u, func(a uint64, idx int) bool { return (a % 2) == 1 })
require.Equal(t, u.Uids, []uint64{1, 3, 5})
l := []uint64{1, 2, 3, 4, 5}
u := newList(l)
ApplyFilter(u, func(a uint64, idx int) bool { return (l[idx] % 2) == 1 })
require.Equal(t, []uint64{1, 3, 5}, BlockToList(u))
}

// sort interface for []uint64
Expand All @@ -248,6 +272,7 @@ func (xs uint64Slice) Swap(i, j int) {
xs[i], xs[j] = xs[j], xs[i]
}

/*
// Benchmarks for IntersectWith
// random data : u and v having data within range [0, limit)
// where limit = N * sizeof-list ; for different N
Expand Down Expand Up @@ -292,3 +317,4 @@ func BenchmarkListIntersectRandom(b *testing.B) {
randomTests(10000, 0.01)
randomTests(1000000, 0.01)
}
*/
16 changes: 9 additions & 7 deletions posting/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (

"github.com/dgryski/go-farm"

"github.com/dgraph-io/dgraph/algo"
"github.com/dgraph-io/dgraph/store"
"github.com/dgraph-io/dgraph/task"
"github.com/dgraph-io/dgraph/types"
Expand Down Expand Up @@ -562,25 +563,26 @@ func (l *List) Uids(opt ListOptions) *task.List {
l.RLock()
defer l.RUnlock()

result := make([]uint64, 0, 10)
var intersectIdx int // Indexes into opt.Intersect if it exists.
res := new(task.List)
wit := algo.NewWriteIterator(res)
l.iterate(opt.AfterUID, func(p *types.Posting) bool {
if postingType(p) != valueUid {
return false
}
uid := p.Uid
if opt.Intersect != nil {
intersectUidsLen := len(opt.Intersect.Uids)
for ; intersectIdx < intersectUidsLen && opt.Intersect.Uids[intersectIdx] < uid; intersectIdx++ {
it := algo.NewListIterator(opt.Intersect)
for ; it.Valid() && it.Val() < uid; it.Next() {
}
if intersectIdx >= intersectUidsLen || opt.Intersect.Uids[intersectIdx] > uid {
if !it.Valid() || it.Val() > uid {
return true
}
}
result = append(result, uid)
wit.Append(uid)
return true
})
return &task.List{Uids: result}
wit.End()
return res
}

func (l *List) Value() (rval types.Val, rerr error) {
Expand Down
11 changes: 7 additions & 4 deletions query/outputnode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
geom "github.com/twpayne/go-geom"
"github.com/twpayne/go-geom/encoding/geojson"

"github.com/dgraph-io/dgraph/algo"
"github.com/dgraph-io/dgraph/query/graph"
"github.com/dgraph-io/dgraph/types"
"github.com/dgraph-io/dgraph/x"
Expand Down Expand Up @@ -147,8 +148,9 @@ func (sg *SubGraph) ToProtocolBuffer(l *Latency) (*graph.Node, error) {
}

n := seedNode.New("_root_")
// At the root, we always have one list in UidMatrix.
for _, uid := range sg.uidMatrix[0].Uids {
it := algo.NewListIterator(sg.uidMatrix[0])
for ; it.Valid(); it.Next() {
uid := it.Val()
// For the root, the name is stored in Alias, not Attr.
n1 := seedNode.New(sg.Params.Alias)
if sg.Params.GetUID || sg.Params.isDebug {
Expand Down Expand Up @@ -338,8 +340,9 @@ func processNodeUids(n *fastJsonNode, sg *SubGraph) error {
if sg.uidMatrix == nil {
return nil
}
// At the root, we always have one list in UidMatrix.
for _, uid := range sg.uidMatrix[0].Uids {
it := algo.NewListIterator(sg.uidMatrix[0])
for ; it.Valid(); it.Next() {
uid := it.Val()
n1 := seedNode.New(sg.Params.Alias)
if sg.Params.GetUID || sg.Params.isDebug {
n1.SetUID(uid)
Expand Down
Loading

0 comments on commit 3ca38ad

Please sign in to comment.