Refactored result sets into two types (Velocidex#1150)

1. The standard result sets are used to store simple queries
2. The Timed result sets are used for event queries.

Timed result sets keep a time index and split files into daily
chunks. Time indexes help to seek in the file for a specific time.

The two results sets are distinct and have a different API interface.

actions/events_test.go

@@ -28,6 +28,8 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/repository"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/velociraptor/vtesting"
+
+	_ "www.velocidex.com/golang/velociraptor/result_sets/timed"
 )
 
 var (

api/builder.go

@@ -15,6 +15,8 @@ import (
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/server"
 	"www.velocidex.com/golang/velociraptor/services"
+
+	_ "www.velocidex.com/golang/velociraptor/result_sets/timed"
 )
 
 // Builder builds a new GUI and Frontend server based on configuration

api/csv.go

@@ -25,10 +25,10 @@ import (
 	file_store "www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
-	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/paths/artifacts"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/timelines"
 
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
@@ -170,38 +170,31 @@ func getEventTableWithPathManager(
 
 	rows := uint64(0)
 	if in.Rows == 0 {
-		in.Rows = 500
+		in.Rows = 10
 	}
 
 	result := &api_proto.GetTableResponse{}
 
 	file_store_factory := file_store.GetFileStore(config_obj)
 	rs_reader, err := result_sets.NewTimedResultSetReader(ctx,
-		file_store_factory, path_manager, in.StartTime, in.EndTime)
+		file_store_factory, path_manager)
 	if err != nil {
 		return nil, err
 	}
 	defer rs_reader.Close()
 
-	// Let the browser know how many rows we have in total.
-	result.TotalRows = rs_reader.TotalRows()
-
-	// FIXME: Backwards compatibility: Just give a few
-	// rows if the result set does not have an index. This
-	// is the same as the previous behavior but for new
-	// collections, an index is created and we respect the
-	// number of rows the callers asked for. Eventually
-	// this will not be needed.
-	if result.TotalRows < 0 {
-		in.Rows = 100
-	}
-
-	// Seek to the row we need.
-	err = rs_reader.SeekToRow(int64(in.StartRow))
+	err = rs_reader.SeekToTime(time.Unix(int64(in.StartTime), 0))
 	if err != nil {
 		return nil, err
 	}
 
+	if in.EndTime != 0 {
+		err = rs_reader.SeekToTime(time.Unix(int64(in.EndTime), 0))
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Unpack the rows into the output protobuf
 	for row := range rs_reader.Rows(ctx) {
 		if result.Columns == nil {
@@ -253,9 +246,9 @@ func getTimeline(
 	}
 	for item := range reader.Read(ctx) {
 		if result.StartTime == 0 {
-			result.StartTime = item.Time
+			result.StartTime = item.Time.UnixNano()
 		}
-		result.EndTime = item.Time
+		result.EndTime = item.Time.UnixNano()
 		result.Rows = append(result.Rows, &api_proto.Row{
 			Cell: []string{
 				item.Source,

api/download.go

@@ -49,13 +49,13 @@ import (
 	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
-	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
 	"www.velocidex.com/golang/velociraptor/flows"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/paths/artifacts"
+	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -158,10 +158,11 @@ func vfsFileDownloadHandler(
 	})
 }
 
-func getRSReader(
+func getRows(
 	ctx context.Context,
 	config_obj *config_proto.Config,
-	request *api_proto.GetTableRequest) (result_sets.ResultSetReader, string, error) {
+	request *api_proto.GetTableRequest) (
+	rows <-chan *ordereddict.Dict, close func(), log_path string, err error) {
 	file_store_factory := file_store.GetFileStore(config_obj)
 
 	// We want an event table.
@@ -170,35 +171,34 @@ func getRSReader(
 			config_obj, request.ClientId, request.FlowId,
 			request.Artifact)
 		if err != nil {
-			return nil, "", err
+			return nil, nil, "", err
 		}
 
 		log_path, err := path_manager.GetPathForWriting()
 		if err != nil {
-			return nil, "", err
+			return nil, nil, "", err
 		}
 
 		rs_reader, err := result_sets.NewTimedResultSetReader(
-			ctx, file_store_factory, path_manager,
-			request.StartTime, request.EndTime)
+			ctx, file_store_factory, path_manager)
 
-		return rs_reader, log_path, err
+		return rs_reader.Rows(ctx), rs_reader.Close, log_path, err
 
 	} else {
 		path_manager, err := getPathManager(config_obj, request)
 		if err != nil {
-			return nil, "", err
+			return nil, nil, "", err
 		}
 
 		log_path, err := path_manager.GetPathForWriting()
 		if err != nil {
-			return nil, "", err
+			return nil, nil, "", err
 		}
 
 		rs_reader, err := result_sets.NewResultSetReader(
 			file_store_factory, path_manager)
 
-		return rs_reader, log_path, err
+		return rs_reader.Rows(ctx), rs_reader.Close, log_path, err
 	}
 }
 
@@ -246,13 +246,13 @@ func downloadTable(config_obj *config_proto.Config) http.Handler {
 			return
 		}
 
-		rs_reader, log_path, err := getRSReader(r.Context(),
-			config_obj, request)
+		row_chan, closer, log_path, err := getRows(
+			r.Context(), config_obj, request)
 		if err != nil {
 			returnError(w, 400, "Invalid request")
 			return
 		}
-		defer rs_reader.Close()
+		defer closer()
 
 		transform := getTransformer(config_obj, request)
 
@@ -288,7 +288,7 @@ func downloadTable(config_obj *config_proto.Config) http.Handler {
 
 			scope := vql_subsystem.MakeScope()
 			csv_writer := csv.GetCSVAppender(scope, w, true /* write_headers */)
-			for row := range rs_reader.Rows(r.Context()) {
+			for row := range row_chan {
 				csv_writer.Write(
 					filterColumns(request.Columns, transform(row)))
 			}
@@ -314,7 +314,7 @@ func downloadTable(config_obj *config_proto.Config) http.Handler {
 				"remote":  r.RemoteAddr,
 			}).Info("DownloadTable")
 
-			for row := range rs_reader.Rows(r.Context()) {
+			for row := range row_chan {
 				serialized, err := json.Marshal(
 					filterColumns(request.Columns, transform(row)))
 				if err != nil {

api/events.go

@@ -220,7 +220,7 @@ func listAvailableEventTimestampFiles(
 	ctx context.Context, path_manager api.PathManager) ([]int32, error) {
 	result := []int32{}
 
-	for prop := range path_manager.GeneratePaths(ctx) {
+	for _, prop := range path_manager.GetAvailableFiles(ctx) {
 		if prop.Size == 0 {
 			continue
 		}

api/hunts.go

@@ -13,10 +13,10 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	file_store "www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
-	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
 	"www.velocidex.com/golang/velociraptor/flows"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
+	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"

artifacts/testdata/server/clients/C.4f5e52adf0a337a9/artifacts/Windows.Applications.ChocolateyPackages/F.BN2HP3OSS3LK6.json

@@ -0,0 +1 @@
+{"FullPath":"\\C:\\ProgramData\\chocolatey\\lib\\wmiexplorer\\wmiexplorer.nuspec","Name":"wmiexplorer","Version":"2.0.0.0","Summary":"WMI Explorer is a utility intended to provide the ability to browse and view WMI namespaces/classes/instances/properties in a single pane of view.","Authors":"Vinay Pamnani","License":"https://wmie.codeplex.com/license"}

artifacts/testdata/server/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads.json

@@ -0,0 +1 @@
+{"Timestamp":1573198259,"started":"2019-11-08 07:30:59.920512962 +0000 UTC","vfs_path":"/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/1.zip","expected_size":1319}

artifacts/testdata/server/clients/C.4f5e52adf0a337a9/monitoring/Windows.Events.ProcessCreation/2019-10-25.json

@@ -1,6 +1,6 @@
-{ "_ts": 1571988000, "Timestamp": "2019-10-25T07:18:20Z", "PPID": 4700, "PID": 6884, "Name": "velociraptor.exe", "CommandLine": "velociraptor.exe  --config client.config.yaml client -v --debug", "ParentInfo": "\"C:\\WINDOWS\\system32\\cmd.exe\" " }
-{ "_ts": 1571988100, "Timestamp": "2019-10-25T07:20:32Z", "PPID": 3800, "PID": 5276, "Name": "PsExec.exe", "CommandLine": "bin\\PsExec.exe  -s cmd.exe", "ParentInfo": "\"C:\\WINDOWS\\system32\\cmd.exe\" " }
-{ "_ts": 1571988100, "Timestamp": "2019-10-25T07:20:33Z", "PPID": 604, "PID": 7176, "Name": "PSEXESVC.exe", "CommandLine": "C:\\WINDOWS\\PSEXESVC.exe", "ParentInfo": "" }
-{ "_ts": 1571988100, "Timestamp": "2019-10-25T07:20:33Z", "PPID": 7176, "PID": 6848, "Name": "cmd.exe", "CommandLine": "\"cmd.exe\" ", "ParentInfo": "C:\\WINDOWS\\PSEXESVC.exe" }
-{ "_ts": 1571988100, "Timestamp": "2019-10-25T07:20:33Z", "PPID": 6848, "PID": 5660, "Name": "conhost.exe", "CommandLine": "\\??\\C:\\WINDOWS\\system32\\conhost.exe 0x4", "ParentInfo": "\"cmd.exe\" " }
-{ "_ts": 1571988100, "Timestamp": "2019-10-25T07:20:36Z", "PPID": 6848, "PID": 5280, "Name": "whoami.exe", "CommandLine": "", "ParentInfo": "\"cmd.exe\" " }
+{"_ts":1571988000,"Timestamp":"2019-10-25T07:18:20Z","PPID":4700,"PID":6884,"Name":"velociraptor.exe","CommandLine":"velociraptor.exe  --config client.config.yaml client -v --debug","ParentInfo":"\"C:\\WINDOWS\\system32\\cmd.exe\" "}
+{"_ts":1571988100,"Timestamp":"2019-10-25T07:20:32Z","PPID":3800,"PID":5276,"Name":"PsExec.exe","CommandLine":"bin\\PsExec.exe  -s cmd.exe","ParentInfo":"\"C:\\WINDOWS\\system32\\cmd.exe\" "}
+{"_ts":1571988100,"Timestamp":"2019-10-25T07:20:33Z","PPID":604,"PID":7176,"Name":"PSEXESVC.exe","CommandLine":"C:\\WINDOWS\\PSEXESVC.exe","ParentInfo":""}
+{"_ts":1571988100,"Timestamp":"2019-10-25T07:20:33Z","PPID":7176,"PID":6848,"Name":"cmd.exe","CommandLine":"\"cmd.exe\" ","ParentInfo":"C:\\WINDOWS\\PSEXESVC.exe"}
+{"_ts":1571988100,"Timestamp":"2019-10-25T07:20:33Z","PPID":6848,"PID":5660,"Name":"conhost.exe","CommandLine":"\\??\\C:\\WINDOWS\\system32\\conhost.exe 0x4","ParentInfo":"\"cmd.exe\" "}
+{"_ts":1571988100,"Timestamp":"2019-10-25T07:20:36Z","PPID":6848,"PID":5280,"Name":"whoami.exe","CommandLine":"","ParentInfo":"\"cmd.exe\" "}

artifacts/testdata/server/testcases/results.out.yaml

@@ -1,4 +1,13 @@
-SELECT Authors, FullPath, License, Name, Summary, Version FROM source( client_id='C.4f5e52adf0a337a9', flow_id="F.BN2HP3OSS3LK6", artifact='Windows.Applications.ChocolateyPackages')[]SELECT Path, Name FROM source( hunt_id="H.49ba8939", artifact="Windows.Network.NetstatEnriched/Netstat") LIMIT 1[
+SELECT Authors, FullPath, License, Name, Summary, Version FROM source( client_id='C.4f5e52adf0a337a9', flow_id="F.BN2HP3OSS3LK6", artifact='Windows.Applications.ChocolateyPackages')[
+ {
+  "Authors": "Vinay Pamnani",
+  "FullPath": "\\C:\\ProgramData\\chocolatey\\lib\\wmiexplorer\\wmiexplorer.nuspec",
+  "License": "https://wmie.codeplex.com/license",
+  "Name": "wmiexplorer",
+  "Summary": "WMI Explorer is a utility intended to provide the ability to browse and view WMI namespaces/classes/instances/properties in a single pane of view.",
+  "Version": "2.0.0.0"
+ }
+]SELECT Path, Name FROM source( hunt_id="H.49ba8939", artifact="Windows.Network.NetstatEnriched/Netstat") LIMIT 1[
  {
   "Path": "C:\\Windows\\System32\\svchost.exe",
   "Name": "svchost.exe"

file_store/api/file_store.go

@@ -49,4 +49,5 @@ type FileStore interface {
 	ListDirectory(dirname string) ([]os.FileInfo, error)
 	Walk(root string, cb filepath.WalkFunc) error
 	Delete(filename string) error
+	Move(src, dest string) error
 }

file_store/api/queues.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"time"
 
 	"github.com/Velocidex/ordereddict"
 )
@@ -16,7 +17,7 @@ type QueueManager interface {
 
 type ResultSetFileProperties struct {
 	Path               string
-	StartTime, EndTime int64
+	StartTime, EndTime time.Time
 	Size               int64
 }
 
@@ -30,5 +31,5 @@ type PathManager interface {
 	GetQueueName() string
 
 	// Generate paths for reading linked result sets.
-	GeneratePaths(ctx context.Context) <-chan *ResultSetFileProperties
+	GetAvailableFiles(ctx context.Context) []*ResultSetFileProperties
 }

file_store/api/testsuite.go

@@ -289,13 +289,7 @@ func (self MockPathManager) GetQueueName() string {
 	return self.ArtifactName
 }
 
-func (self MockPathManager) GeneratePaths(ctx context.Context) <-chan *ResultSetFileProperties {
-	output := make(chan *ResultSetFileProperties)
-
-	go func() {
-		defer close(output)
-
-	}()
-
-	return output
+func (self MockPathManager) GetAvailableFiles(
+	ctx context.Context) []*ResultSetFileProperties {
+	return nil
 }

file_store/directory/directory.go

@@ -96,6 +96,13 @@ func NewDirectoryFileStore(config_obj *config_proto.Config) *DirectoryFileStore
 	return &DirectoryFileStore{config_obj}
 }
 
+func (self *DirectoryFileStore) Move(src string, dest string) error {
+	src_path := self.FilenameToFileStorePath(src)
+	dest_path := self.FilenameToFileStorePath(dest)
+
+	return os.Rename(src_path, dest_path)
+}
+
 func (self *DirectoryFileStore) ListDirectory(dirname string) (
 	[]os.FileInfo, error) {
 

file_store/directory/queue.go

@@ -36,7 +36,7 @@ import (
 	"github.com/Velocidex/ordereddict"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
-	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
+	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/utils"
 )
 
@@ -286,8 +286,8 @@ func (self *DirectoryQueueManager) Debug() *ordereddict.Dict {
 func (self *DirectoryQueueManager) PushEventRows(
 	path_manager api.PathManager, dict_rows []*ordereddict.Dict) error {
 
-	rs_writer, err := result_sets.NewResultSetWriter(
-		self.FileStore, path_manager, nil, false /* truncate */)
+	rs_writer, err := result_sets.NewTimedResultSetWriter(
+		self.FileStore, path_manager, nil)
 	if err != nil {
 		return err
 	}

file_store/directory/queue_test.go

@@ -22,6 +22,9 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/journal"
 	"www.velocidex.com/golang/velociraptor/services/repository"
 	"www.velocidex.com/golang/velociraptor/utils"
+
+	_ "www.velocidex.com/golang/velociraptor/result_sets/simple"
+	_ "www.velocidex.com/golang/velociraptor/result_sets/timed"
 )
 
 var (