Refactor of data store and filestore APIs (Velocidex#1177)

Makefile

@@ -8,7 +8,7 @@ test:
 	go test -v --tags server_vql ./...
 
 golden:
-	./output/velociraptor -v --config artifacts/testdata/windows/test.config.yaml golden artifacts/testdata/server/testcases/ --env srcDir=`pwd` --filter=
+	./output/velociraptor -v --config artifacts/testdata/windows/test.config.yaml golden artifacts/testdata/server/testcases/ --env srcDir=`pwd` --filter=${GOLDEN}
 
 references:
 	./output/velociraptor vql export docs/references/vql.yaml > docs/references/vql.yaml.tmp
@@ -57,5 +57,8 @@ debug:
 debug_client:
 	dlv debug --build-flags="-tags 'server_vql extras'" ./bin/ -- client -v
 
+debug_golden:
+	dlv debug --build-flags="-tags 'server_vql extras'" ./bin/ -- --config artifacts/testdata/windows/test.config.yaml golden artifacts/testdata/server/testcases/ --env srcDir=`pwd` --disable_alarm --filter=${GOLDEN}
+
 lint:
 	golangci-lint run

actions/proto/vql.proto

@@ -2,7 +2,6 @@
 syntax = "proto3";
 
 import "proto/semantic.proto";
-import "proto/flow_metadata.proto";
 import "artifacts/proto/artifact.proto";
 
 package proto;

api/api.go

@@ -54,6 +54,7 @@ import (
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
 	"www.velocidex.com/golang/velociraptor/grpc_client"
 	"www.velocidex.com/golang/velociraptor/logging"
+	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/search"
 	"www.velocidex.com/golang/velociraptor/server"
 	"www.velocidex.com/golang/velociraptor/services"
@@ -406,7 +407,7 @@ func (self *ApiServer) SetGUIOptions(
 
 func (self *ApiServer) VFSListDirectory(
 	ctx context.Context,
-	in *flows_proto.VFSListRequest) (*flows_proto.VFSListResponse, error) {
+	in *api_proto.VFSListRequest) (*api_proto.VFSListResponse, error) {
 
 	defer Instrument("VFSListDirectory")()
 
@@ -419,13 +420,13 @@ func (self *ApiServer) VFSListDirectory(
 	}
 
 	result, err := vfsListDirectory(
-		self.config, in.ClientId, in.VfsPath)
+		self.config, in.ClientId, in.VfsComponents)
 	return result, err
 }
 
 func (self *ApiServer) VFSStatDirectory(
 	ctx context.Context,
-	in *flows_proto.VFSListRequest) (*flows_proto.VFSListResponse, error) {
+	in *api_proto.VFSListRequest) (*api_proto.VFSListResponse, error) {
 
 	defer Instrument("VFSStatDirectory")()
 
@@ -438,13 +439,13 @@ func (self *ApiServer) VFSStatDirectory(
 	}
 
 	result, err := vfsStatDirectory(
-		self.config, in.ClientId, in.VfsPath)
+		self.config, in.ClientId, in.VfsComponents)
 	return result, err
 }
 
 func (self *ApiServer) VFSStatDownload(
 	ctx context.Context,
-	in *flows_proto.VFSStatDownloadRequest) (*flows_proto.VFSDownloadInfo, error) {
+	in *api_proto.VFSStatDownloadRequest) (*flows_proto.VFSDownloadInfo, error) {
 
 	defer Instrument("VFSStatDownload")()
 
@@ -457,7 +458,7 @@ func (self *ApiServer) VFSStatDownload(
 	}
 
 	result, err := vfsStatDownload(
-		self.config, in.ClientId, in.Accessor, in.Path)
+		self.config, in.ClientId, in.Accessor, in.Components)
 	return result, err
 }
 
@@ -477,7 +478,7 @@ func (self *ApiServer) VFSRefreshDirectory(
 	}
 
 	result, err := vfsRefreshDirectory(
-		self, ctx, in.ClientId, in.VfsPath, in.Depth)
+		self, ctx, in.ClientId, in.VfsComponents, in.Depth)
 	return result, err
 }
 
@@ -496,8 +497,10 @@ func (self *ApiServer) VFSGetBuffer(
 			"User is not allowed to view the VFS.")
 	}
 
+	path_spec := paths.NewClientPathManager(
+		in.ClientId).FSItem(in.Components)
 	result, err := vfsGetBuffer(
-		self.config, in.ClientId, in.VfsPath, in.Offset, in.Length)
+		self.config, in.ClientId, path_spec, in.Offset, in.Length)
 
 	return result, err
 }

api/clients.go

@@ -29,7 +29,6 @@ import (
 	"google.golang.org/grpc/status"
 	"www.velocidex.com/golang/velociraptor/acls"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/flows"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
@@ -104,7 +103,7 @@ func (self *ApiServer) GetClient(
 	}
 
 	if in.UpdateMru {
-		err = updateMRU(self.config, user_name, in.ClientId)
+		err = search.UpdateMRU(self.config, user_name, in.ClientId)
 		if err != nil {
 			return nil, err
 		}
@@ -167,18 +166,3 @@ func (self *ApiServer) GetClientFlows(
 	return flows.GetFlows(self.config, in.ClientId,
 		in.IncludeArchived, filter, in.Offset, in.Count)
 }
-
-func updateMRU(
-	config_obj *config_proto.Config,
-	user_name string, client_id string) error {
-	path_manager := &paths.UserPathManager{user_name}
-	db, err := datastore.GetDB(config_obj)
-	if err != nil {
-		return err
-	}
-
-	err = db.SetIndex(config_obj, path_manager.MRU(),
-		client_id, []string{"mru"})
-
-	return err
-}

api/csv.go

@@ -27,7 +27,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/paths/artifacts"
-	"www.velocidex.com/golang/velociraptor/reporting"
 	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/timelines"
 
@@ -47,14 +46,14 @@ func getTable(
 	}
 
 	result := &api_proto.GetTableResponse{}
-	path_manager, err := getPathManager(config_obj, in)
+	path_spec, err := getPathSpec(config_obj, in)
 	if err != nil {
 		return result, err
 	}
 
 	file_store_factory := file_store.GetFileStore(config_obj)
 	rs_reader, err := result_sets.NewResultSetReader(
-		file_store_factory, path_manager)
+		file_store_factory, path_spec)
 	if err != nil {
 		return result, nil
 	}
@@ -103,17 +102,22 @@ func getTable(
 	return result, nil
 }
 
-func getPathManager(
+func getPathSpec(
 	config_obj *config_proto.Config,
-	in *api_proto.GetTableRequest) (api.PathManager, error) {
+	in *api_proto.GetTableRequest) (api.FSPathSpec, error) {
 
 	if in.FlowId != "" && in.Artifact != "" {
-		return artifacts.NewArtifactPathManager(
+		path_manager, err := artifacts.NewArtifactPathManager(
 			config_obj, in.ClientId, in.FlowId, in.Artifact)
+		if err != nil {
+			return nil, err
+		}
+		return path_manager.Path(), nil
 
 	} else if in.FlowId != "" && in.Type != "" {
 		flow_path_manager := paths.NewFlowPathManager(
 			in.ClientId, in.FlowId)
+
 		switch in.Type {
 		case "log":
 			return flow_path_manager.Log(), nil
@@ -127,8 +131,8 @@ func getPathManager(
 		return paths.NewHuntPathManager(in.HuntId).ClientErrors(), nil
 
 	} else if in.NotebookId != "" && in.CellId != "" {
-		return reporting.NewNotebookPathManager(in.NotebookId).Cell(
-			in.CellId).QueryStorage(in.TableId), nil
+		return paths.NewNotebookPathManager(in.NotebookId).Cell(
+			in.CellId).QueryStorage(in.TableId).Path(), nil
 	}
 
 	return nil, errors.New("Invalid request")
@@ -225,8 +229,10 @@ func getTimeline(
 		return nil, errors.New("NotebookId must be specified")
 	}
 
-	path_manager := reporting.NewNotebookPathManager(in.NotebookId).Timeline(in.Timeline)
-	reader, err := timelines.NewSuperTimelineReader(config_obj, path_manager, in.SkipComponents)
+	path_manager := paths.NewNotebookPathManager(in.NotebookId).
+		SuperTimeline(in.Timeline)
+	reader, err := timelines.NewSuperTimelineReader(
+		config_obj, path_manager, in.SkipComponents)
 	if err != nil {
 		return nil, err
 	}