Add roaring bitmaps to TSI index files.

Gopkg.toml

@@ -8,7 +8,8 @@
 
 [[constraint]]
   name = "github.com/RoaringBitmap/roaring"
-  version = "0.4.3"
+  source = "https://github.com/influxdata/roaring.git"
+  revision = "ec86e26aba5545a1819e1ad68e9faa0f1745fff5"
 
 [[constraint]]
   name = "github.com/boltdb/bolt"

cmd/influx_inspect/dumptsi/dumptsi.go

@@ -353,7 +353,10 @@ func (cmd *Command) printTagValueSeries(sfile *tsdb.SeriesFile, fs *tsi1.FileSet
 
 	// Iterate over each series.
 	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
-	itr := fs.TagValueSeriesIDIterator(name, key, value)
+	itr, err := fs.TagValueSeriesIDIterator(name, key, value)
+	if err != nil {
+		return err
+	}
 	for {
 		e, err := itr.Next()
 		if err != nil {

tsdb/index/tsi1/file_set.go

@@ -395,15 +395,17 @@ func (fs *FileSet) TagValueIterator(name, key []byte) TagValueIterator {
 }
 
 // TagValueSeriesIDIterator returns a series iterator for a single tag value.
-func (fs *FileSet) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
+func (fs *FileSet) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
 	a := make([]tsdb.SeriesIDIterator, 0, len(fs.files))
 	for _, f := range fs.files {
-		itr := f.TagValueSeriesIDIterator(name, key, value)
-		if itr != nil {
+		itr, err := f.TagValueSeriesIDIterator(name, key, value)
+		if err != nil {
+			return nil, err
+		} else if itr != nil {
 			a = append(a, itr)
 		}
 	}
-	return tsdb.MergeSeriesIDIterators(a...)
+	return tsdb.MergeSeriesIDIterators(a...), nil
 }
 
 // MeasurementsSketches returns the merged measurement sketches for the FileSet.
@@ -453,7 +455,7 @@ type File interface {
 	// Series iteration.
 	MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator
 	TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator
-	TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator
+	TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error)
 
 	// Sketches for cardinality estimation
 	MergeMeasurementsSketches(s, t estimator.Sketch) error

tsdb/index/tsi1/index.go

@@ -838,8 +838,10 @@ func (i *Index) TagKeySeriesIDIterator(name, key []byte) (tsdb.SeriesIDIterator,
 func (i *Index) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
 	a := make([]tsdb.SeriesIDIterator, 0, len(i.partitions))
 	for _, p := range i.partitions {
-		itr := p.TagValueSeriesIDIterator(name, key, value)
-		if itr != nil {
+		itr, err := p.TagValueSeriesIDIterator(name, key, value)
+		if err != nil {
+			return nil, err
+		} else if itr != nil {
 			a = append(a, itr)
 		}
 	}

tsdb/index/tsi1/index_file.go

@@ -331,20 +331,31 @@ func (f *IndexFile) TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterat
 
 // TagValueSeriesIDIterator returns a series iterator for a tag value and a flag
 // indicating if a tombstone exists on the measurement, key, or value.
-func (f *IndexFile) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
+func (f *IndexFile) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
 	tblk := f.tblks[string(name)]
 	if tblk == nil {
-		return nil
+		return nil, nil
 	}
 
 	// Find value element.
-	n, data := tblk.TagValueSeriesData(key, value)
-	if n == 0 {
-		return nil
+	var valueElem TagBlockValueElem
+	if !tblk.DecodeTagValueElem(key, value, &valueElem) {
+		return nil, nil
+	}
+
+	// Return iterator based on uvarint encoding, if set.
+	if valueElem.SeriesN() == 0 {
+		return nil, nil
+	} else if valueElem.SeriesData() != nil {
+		return &rawSeriesIDIterator{n: valueElem.SeriesN(), data: valueElem.SeriesData()}, nil
 	}
 
-	// Create an iterator over value's series.
-	return &rawSeriesIDIterator{n: n, data: data}
+	// Otherwise return iterator over roaring.
+	ss, err := valueElem.SeriesIDSet()
+	if err != nil {
+		return nil, err
+	}
+	return tsdb.NewSeriesIDSetIterator(ss), nil
 }
 
 // TagKey returns a tag key.

tsdb/index/tsi1/index_files.go

@@ -139,16 +139,18 @@ func (p IndexFiles) MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterat
 }
 
 // TagValueSeriesIDIterator returns an iterator that merges series across all files.
-func (p IndexFiles) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
+func (p IndexFiles) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
 	a := make([]tsdb.SeriesIDIterator, 0, len(p))
 
 	for i := range p {
-		itr := p[i].TagValueSeriesIDIterator(name, key, value)
-		if itr != nil {
+		itr, err := p[i].TagValueSeriesIDIterator(name, key, value)
+		if err != nil {
+			return nil, err
+		} else if itr != nil {
 			a = append(a, itr)
 		}
 	}
-	return tsdb.MergeSeriesIDIterators(a...)
+	return tsdb.MergeSeriesIDIterators(a...), nil
 }
 
 // CompactTo merges all index files and writes them to w.
@@ -185,6 +187,13 @@ func (p IndexFiles) CompactTo(w io.Writer, sfile *tsdb.SeriesFile, m, k uint64,
 		return n, err
 	}
 
+	// Ensure block is word aligned.
+	if offset := (n) % 8; offset != 0 {
+		if err := writeTo(bw, make([]byte, 8-offset), &n); err != nil {
+			return n, err
+		}
+	}
+
 	// Write measurement block.
 	t.MeasurementBlock.Offset = n
 	if err := p.writeMeasurementBlockTo(bw, &info, &n); err != nil {
@@ -289,6 +298,13 @@ func (p IndexFiles) writeTagsetTo(w io.Writer, name []byte, info *indexCompactIn
 	default:
 	}
 
+	// Ensure block is word aligned.
+	if offset := (*n) % 8; offset != 0 {
+		if err := writeTo(w, make([]byte, 8-offset), n); err != nil {
+			return err
+		}
+	}
+
 	kitr, err := p.TagKeyIterator(name)
 	if err != nil {
 		return err
@@ -309,8 +325,10 @@ func (p IndexFiles) writeTagsetTo(w io.Writer, name []byte, info *indexCompactIn
 
 			// Merge all series together.
 			if err := func() error {
-				sitr := p.TagValueSeriesIDIterator(name, ke.Key(), ve.Value())
-				if sitr != nil {
+				sitr, err := p.TagValueSeriesIDIterator(name, ke.Key(), ve.Value())
+				if err != nil {
+					return err
+				} else if sitr != nil {
 					defer sitr.Close()
 					for {
 						se, err := sitr.Next()

tsdb/index/tsi1/log_file.go

@@ -451,28 +451,28 @@ func (f *LogFile) DeleteTagKey(name, key []byte) error {
 }
 
 // TagValueSeriesIDIterator returns a series iterator for a tag value.
-func (f *LogFile) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
+func (f *LogFile) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 
 	mm, ok := f.mms[string(name)]
 	if !ok {
-		return nil
+		return nil, nil
 	}
 
 	tk, ok := mm.tagSet[string(key)]
 	if !ok {
-		return nil
+		return nil, nil
 	}
 
 	tv, ok := tk.tagValues[string(value)]
 	if !ok {
-		return nil
+		return nil, nil
 	} else if tv.cardinality() == 0 {
-		return nil
+		return nil, nil
 	}
 
-	return tsdb.NewSeriesIDSetIterator(tv.seriesIDSet())
+	return tsdb.NewSeriesIDSetIterator(tv.seriesIDSet()), nil
 }
 
 // MeasurementN returns the total number of measurements.
@@ -846,6 +846,13 @@ func (f *LogFile) CompactTo(w io.Writer, m, k uint64, cancel <-chan struct{}) (n
 		return n, err
 	}
 
+	// Ensure block is word aligned.
+	if offset := n % 8; offset != 0 {
+		if err := writeTo(bw, make([]byte, 8-offset), &n); err != nil {
+			return n, err
+		}
+	}
+
 	// Write measurement block.
 	t.MeasurementBlock.Offset = n
 	if err := f.writeMeasurementBlockTo(bw, names, info, &n); err != nil {
@@ -924,6 +931,13 @@ func (f *LogFile) writeTagsetTo(w io.Writer, name string, info *logFileCompactIn
 	default:
 	}
 
+	// Ensure block is word aligned.
+	if offset := (*n) % 8; offset != 0 {
+		if err := writeTo(w, make([]byte, 8-offset), n); err != nil {
+			return err
+		}
+	}
+
 	enc := NewTagBlockEncoder(w)
 	var valueN int
 	for _, k := range mm.keys() {