Redeisgned hunt results screen. (Velocidex#726)

* Redeisgned hunt results screen.

Removed un paged hunt-results and added hunt clients with a flow
inspector popup. This way users can inspect the results from each hunt
using paged table.

* Refactored result set writer to be part of the file store.

Now it is also used by the queue manager making it possible to seek
inside event log files.

README.md

@@ -48,45 +48,25 @@ To build from source, make sure you have a recent Golang installed
 from https://golang.org/dl/ (Currently at least Go 1.14):
 
 ```bash
-
     $ git clone https://github.com/Velocidex/velociraptor.git
     $ cd velociraptor
 
     # This will build the GUI elements. You will need to have node
-    # installed first. For example on Windows get it from
-    # https://nodejs.org/en/download/ . You also need to have JAVA
-    # installed from https://www.java.com because the js compiler
-    # needs it.
-    $ cd gui/static/
+    # installed first. For example get it from
+    # https://nodejs.org/en/download/.
+    $ cd gui/velociraptor/
     $ npm install
 
-    # If gulp is not on your path you need to run it using node:
-    # node node_modules\gulp\bin\gulp.js compile
-    $ gulp compile
-    $ cd -
-
-    # This builds a release (i.e. it will embed the GUI files in the
-    # binary). If you dont care about the GUI a simple "make" will
-    # build a bare debug binary.
-    $ go run make.go -v release
-    $ go run make.go -v windows
-```
-
-If you want to rebuild the protobuf you will need to install protobuf
-compiler (This is only necessary when editing any `*.proto` file):
-
-```bash
+    # This will build the webpack bundle
+    $ make build
 
-   $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protoc-3.13.0-linux-x86_64.zip
-   $ unzip protoc-3.13.0-linux-x86_64.zip
-   $ sudo mv include/google/ /usr/local/include/
-   $ sudo mv bin/protoc /usr/local/bin/
-   $ go get -u github.com/golang/protobuf/protoc-gen-go/
-   $ go install github.com/golang/protobuf/protoc-gen-go/
-   $ go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-   $ go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-   $ ./make_proto.sh
+    # To build a dev binary just run make.
+    # NOTE: Make sure ~/go/bin is on your path - this is required to find the Golang tools we need.
+    $ make
 
+    # To build production binaries
+    $ make linux
+    $ make windows
 ```
 
 ## Getting the latest version

api/artifacts.go

@@ -43,7 +43,7 @@ import (
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
-	"www.velocidex.com/golang/velociraptor/result_sets"
+	"www.velocidex.com/golang/velociraptor/paths/artifacts"
 	"www.velocidex.com/golang/velociraptor/services"
 	users "www.velocidex.com/golang/velociraptor/users"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -372,7 +372,7 @@ func (self *ApiServer) ListAvailableEventResults(
 			"User is not allowed to view results.")
 	}
 
-	path_manager := result_sets.NewMonitoringArtifactPathManager(in.ClientId)
+	path_manager := artifacts.NewMonitoringArtifactPathManager(in.ClientId)
 	file_store_factory := file_store.GetFileStore(self.config)
 
 	seen := make(map[string]*api_proto.AvailableEvent)

api/csv.go

@@ -19,11 +19,13 @@ package api
 
 import (
 	context "golang.org/x/net/context"
+	file_store "www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
+	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
 	"www.velocidex.com/golang/velociraptor/paths"
+	"www.velocidex.com/golang/velociraptor/paths/artifacts"
 	"www.velocidex.com/golang/velociraptor/reporting"
-	"www.velocidex.com/golang/velociraptor/result_sets"
 
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
@@ -43,51 +45,55 @@ func getTable(
 	result := &api_proto.GetTableResponse{}
 
 	path_manager := getPathManager(config_obj, in)
-	if path_manager != nil {
-		rs_reader, err := result_sets.NewResultSetReader(config_obj, path_manager)
-		if err != nil {
-			return nil, err
-		}
-		defer rs_reader.Close()
-
-		// Let the browser know how many rows we have in total.
-		result.TotalRows = rs_reader.TotalRows
-
-		// FIXME: Backwards compatibility: Just give a few
-		// rows if the result set does not have an index. This
-		// is the same as the previous behavior but for new
-		// collections, an index is created and we respect the
-		// number of rows the callers asked for. Eventually
-		// this will not be needed.
-		if result.TotalRows < 0 {
-			in.Rows = 100
+	if path_manager == nil {
+		return result, nil
+	}
+
+	file_store_factory := file_store.GetFileStore(config_obj)
+	rs_reader, err := result_sets.NewResultSetReader(
+		file_store_factory, path_manager)
+	if err != nil {
+		return nil, err
+	}
+	defer rs_reader.Close()
+
+	// Let the browser know how many rows we have in total.
+	result.TotalRows = rs_reader.TotalRows()
+
+	// FIXME: Backwards compatibility: Just give a few
+	// rows if the result set does not have an index. This
+	// is the same as the previous behavior but for new
+	// collections, an index is created and we respect the
+	// number of rows the callers asked for. Eventually
+	// this will not be needed.
+	if result.TotalRows < 0 {
+		in.Rows = 100
+	}
+
+	// Seek to the row we need.
+	err = rs_reader.SeekToRow(int64(in.StartRow))
+	if err != nil {
+		return nil, err
+	}
+
+	// Unpack the rows into the output protobuf
+	for row := range rs_reader.Rows(ctx) {
+		if result.Columns == nil {
+			result.Columns = row.Keys()
 		}
 
-		// Seek to the row we need.
-		err = rs_reader.SeekToRow(int64(in.StartRow))
-		if err != nil {
-			return nil, err
+		row_data := make([]string, 0, len(result.Columns))
+		for _, key := range result.Columns {
+			value, _ := row.Get(key)
+			row_data = append(row_data, csv.AnyToString(value))
 		}
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Cell: row_data,
+		})
 
-		// Unpack the rows into the output protobuf
-		for row := range rs_reader.Rows(ctx) {
-			if result.Columns == nil {
-				result.Columns = row.Keys()
-			}
-
-			row_data := make([]string, 0, len(result.Columns))
-			for _, key := range result.Columns {
-				value, _ := row.Get(key)
-				row_data = append(row_data, csv.AnyToString(value))
-			}
-			result.Rows = append(result.Rows, &api_proto.Row{
-				Cell: row_data,
-			})
-
-			rows += 1
-			if rows > in.Rows {
-				break
-			}
+		rows += 1
+		if rows > in.Rows {
+			break
 		}
 	}
 
@@ -98,7 +104,7 @@ func getPathManager(
 	config_obj *config_proto.Config,
 	in *api_proto.GetTableRequest) api.PathManager {
 	if in.FlowId != "" && in.Artifact != "" {
-		return result_sets.NewArtifactPathManager(
+		return artifacts.NewArtifactPathManager(
 			config_obj, in.ClientId, in.FlowId, in.Artifact)
 
 	} else if in.FlowId != "" && in.Type != "" {

api/download.go

@@ -45,11 +45,11 @@ import (
 	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
+	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
-	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
@@ -190,8 +190,9 @@ func downloadTable(config_obj *config_proto.Config) http.Handler {
 			download_name = strings.TrimSuffix(download_name, ".json")
 			download_name += ".csv"
 
+			file_store_factory := file_store.GetFileStore(config_obj)
 			rs_reader, err := result_sets.NewResultSetReader(
-				config_obj, path_manager)
+				file_store_factory, path_manager)
 			if err != nil {
 				returnError(w, 400, err.Error())
 				return

api/hunts.go

@@ -0,0 +1,74 @@
+package api
+
+import (
+	context "golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"www.velocidex.com/golang/velociraptor/acls"
+	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+	file_store "www.velocidex.com/golang/velociraptor/file_store"
+	"www.velocidex.com/golang/velociraptor/file_store/csv"
+	"www.velocidex.com/golang/velociraptor/file_store/result_sets"
+	"www.velocidex.com/golang/velociraptor/flows"
+	"www.velocidex.com/golang/velociraptor/paths"
+	"www.velocidex.com/golang/velociraptor/utils"
+)
+
+func (self *ApiServer) GetHuntFlows(
+	ctx context.Context,
+	in *api_proto.GetTableRequest) (*api_proto.GetTableResponse, error) {
+
+	user_name := GetGRPCUserInfo(self.config, ctx).Name
+	permissions := acls.READ_RESULTS
+	perm, err := acls.CheckAccess(self.config, user_name, permissions)
+	if !perm || err != nil {
+		return nil, status.Error(codes.PermissionDenied,
+			"User is not allowed to view hunt results.")
+	}
+
+	hunt_path_manager := paths.NewHuntPathManager(in.HuntId).Clients()
+	file_store_factory := file_store.GetFileStore(self.config)
+	rs_reader, err := result_sets.NewResultSetReader(
+		file_store_factory, hunt_path_manager)
+	if err != nil {
+		return nil, err
+	}
+	defer rs_reader.Close()
+
+	// Seek to the row we need.
+	err = rs_reader.SeekToRow(int64(in.StartRow))
+	if err != nil {
+		return nil, err
+	}
+
+	result := &api_proto.GetTableResponse{
+		TotalRows: rs_reader.TotalRows(),
+		Columns: []string{
+			"ClientId", "FlowId", "StartedTime", "State", "Duration",
+			"TotalBytes", "TotalRows",
+		}}
+
+	for row := range rs_reader.Rows(ctx) {
+		client_id := utils.GetString(row, "ClientId")
+		flow_id := utils.GetString(row, "FlowId")
+		flow, err := flows.LoadCollectionContext(self.config, client_id, flow_id)
+		if err != nil {
+			continue
+		}
+
+		row_data := []string{
+			client_id, flow_id,
+			csv.AnyToString(flow.StartTime / 1000),
+			flow.State.String(),
+			csv.AnyToString(flow.ExecutionDuration / 1000000000),
+			csv.AnyToString(flow.TotalUploadedBytes),
+			csv.AnyToString(flow.TotalCollectedRows)}
+
+		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+
+		if uint64(len(result.Rows)) > in.Rows {
+			break
+		}
+	}
+	return result, nil
+}