Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Perform incremental rollups instead of rollups at snapshot #4410

Merged
merged 17 commits into from
Jan 9, 2020
Prev Previous commit
addr review comments; minor cleanups
  • Loading branch information
Paras Shah committed Jan 9, 2020
commit c9aa143317a1fcbfcb771de2448ccac1f3c9f37c
13 changes: 4 additions & 9 deletions worker/draft.go
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ func (n *node) processRollups() {
if readTs <= last {
break // Break out of the select case.
}
if err := n.rollupLists(readTs); err != nil {
if err := n.calcTabletSizes(readTs); err != nil {
// If we encounter error here, we don't need to do anything about
// it. Just let the user know.
glog.Errorf("Error while rolling up lists at %d: %v\n", readTs, err)
Expand Down Expand Up @@ -1008,13 +1008,11 @@ func listWrap(kv *bpb.KV) *bpb.KVList {
return &bpb.KVList{Kv: []*bpb.KV{kv}}
}

// rollupLists would consolidate all the deltas that constitute one posting
// list, and write back a complete posting list.
func (n *node) rollupLists(readTs uint64) error {
// calcTabletSizes updates the tablet sizes for the keys.
func (n *node) calcTabletSizes(readTs uint64) error {
// We can now discard all invalid versions of keys below this ts.
pstore.SetDiscardTs(readTs)

// We're doing rollups. We should use this opportunity to calculate the tablet sizes.
if !n.AmLeader() {
// Only leader needs to calculate the tablet sizes.
return nil
Expand Down Expand Up @@ -1051,7 +1049,7 @@ func (n *node) rollupLists(readTs uint64) error {
return false
}
}
var numKeys uint64

stream.KeyToList = func(key []byte, itr *badger.Iterator) (*bpb.KVList, error) {
return nil, nil // no-op
}
Expand All @@ -1062,9 +1060,6 @@ func (n *node) rollupLists(readTs uint64) error {
return err
}

// For all the keys, let's see if they're in the LRU cache. If so, we can roll them up.
glog.Infof("Rolled up %d keys. Done", atomic.LoadUint64(&numKeys))

// Only leader sends the tablet size updates to Zero. No one else does.
// doSendMembership is also being concurrently called from another goroutine.
go func() {
Expand Down