Create an index over JSONL files. (Velocidex#713)

This allows arbitrary seeking within the result sets and provides an
infinite result table experience.

It is now possible to page to any point in a huge JSONL table quickly.

actions/vql.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"runtime"
 	"runtime/debug"
 	"strings"
 	"time"
@@ -132,6 +134,13 @@ func (self VQLClientAction) StartQuery(
 	scope := manager.BuildScope(builder)
 	defer scope.Close()
 
+	if runtime.GOARCH == "386" &&
+		os.Getenv("PROCESSOR_ARCHITECTURE") == "AMD64" {
+		scope.Log("You are running a 32 bit built binary on Windows x64. " +
+			"This configuration is not supported and may result in " +
+			"incorrect or missed results or even crashes.")
+	}
+
 	scope.Log("Starting query execution.")
 
 	vfilter.InstallThrottler(scope, vfilter.NewTimeThrottler(float64(rate)))

api/api.go

@@ -41,6 +41,7 @@ import (
 	"google.golang.org/grpc/status"
 	"www.velocidex.com/golang/velociraptor/acls"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
+	"www.velocidex.com/golang/velociraptor/api/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
@@ -60,6 +61,7 @@ import (
 )
 
 type ApiServer struct {
+	proto.UnimplementedAPIServer
 	config     *config_proto.Config
 	server_obj *server.Server
 	ca_pool    *x509.CertPool

api/csv.go

@@ -19,7 +19,6 @@ package api
 
 import (
 	context "golang.org/x/net/context"
-	file_store "www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
 	"www.velocidex.com/golang/velociraptor/paths"
@@ -70,13 +69,33 @@ func getTable(
 	result := &api_proto.GetTableResponse{}
 
 	if path_manager != nil {
-		row_chan, err := file_store.GetTimeRange(ctx, config_obj,
-			path_manager, 0, 0)
+		rs_reader, err := result_sets.NewResultSetReader(config_obj, path_manager)
+		if err != nil {
+			return nil, err
+		}
+		defer rs_reader.Close()
+
+		// Let the browser know how many rows we have in total.
+		result.TotalRows = rs_reader.TotalRows
+
+		// FIXME: Backwards compatibility: Just give a few
+		// rows if the result set does not have an index. This
+		// is the same as the previous behavior but for new
+		// collections, an index is created and we respect the
+		// number of rows the callers asked for. Eventually
+		// this will not be needed.
+		if result.TotalRows < 0 {
+			in.Rows = 100
+		}
+
+		// Seek to the row we need.
+		err = rs_reader.SeekToRow(int64(in.StartRow))
 		if err != nil {
 			return nil, err
 		}
 
-		for row := range row_chan {
+		// Unpack the rows into the output protobuf
+		for row := range rs_reader.Rows(ctx) {
 			if result.Columns == nil {
 				result.Columns = row.Keys()
 			}