Fixed tests.

Updated VFS accessors to encode results in JSON

accessors/vfs/vfs.go

@@ -2,7 +2,6 @@ package vfs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 
 	"github.com/Velocidex/ordereddict"
@@ -11,14 +10,17 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
+	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
 
 var (
 	ErrNotFound     = errors.New("file not found")
 	ErrNotAvailable = errors.New("File content not available")
+	ErrInvalidRow   = errors.New("Stored row is invalid")
 )
 
 type VFSFileSystemAccessor struct {
@@ -97,14 +99,18 @@ func (self VFSFileSystemAccessor) LstatWithOSPath(filename *accessors.OSPath) (
 	if err != nil {
 		return nil, err
 	}
+
 	// Find the row that matches this filename
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
 			continue
 		}
 
-		if r.Cell[5] == filename.Basename() {
-			return rowCellToFSInfo(r.Cell)
+		name, ok := row[5].(string)
+		if ok && name == filename.Basename() {
+			return rowCellToFSInfo(row)
 		}
 	}
 
@@ -147,11 +153,13 @@ func (self VFSFileSystemAccessor) ReadDirWithOSPath(
 
 	result := []accessors.FileInfo{}
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
 			continue
 		}
 
-		fs_info, err := rowCellToFSInfo(r.Cell)
+		fs_info, err := rowCellToFSInfo(row)
 		if err != nil {
 			continue
 		}
@@ -191,14 +199,21 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (
 
 	// Find the row that matches this filename
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
+			continue
+		}
+
+		name, ok := row[5].(string)
+		if !ok {
 			continue
 		}
 
-		if r.Cell[5] == filename.Basename() {
+		if name == filename.Basename() {
 			// Check if it has a download link
 			record := &flows_proto.VFSDownloadInfo{}
-			err = json.Unmarshal([]byte(r.Cell[0]), record)
+			err = utils.ParseIntoProtobuf(row[0], record)
 			if err != nil || record.Name == "" {
 				return nil, ErrNotAvailable
 			}
@@ -211,23 +226,31 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (
 	return nil, ErrNotFound
 }
 
-func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {
-	components := []string{}
-	err := json.Unmarshal([]byte(cell[2]), &components)
-	if err != nil {
-		return nil, err
+func rowCellToFSInfo(cell []interface{}) (accessors.FileInfo, error) {
+	components := utils.ConvertToStringSlice(cell[2])
+	if len(components) == 0 {
+		return nil, ErrInvalidRow
 	}
 
-	size := int64(0)
-	_ = json.Unmarshal([]byte(cell[6]), &size)
+	size, ok := utils.ToInt64(cell[6])
+	if !ok {
+		return nil, ErrInvalidRow
+	}
 
-	is_dir := false
-	if len(cell[7]) > 1 && cell[7][0] == 'd' {
-		is_dir = true
+	mode, ok := cell[7].(string)
+	if !ok {
+		return nil, ErrInvalidRow
 	}
 
+	is_dir := len(mode) > 1 && mode[0] == 'd'
+
 	// The Accessor + components is the path of the item
-	path := accessors.MustNewGenericOSPath(cell[3]).Append(components...)
+	ospath, ok := cell[3].(string)
+	if !ok {
+		return nil, ErrInvalidRow
+	}
+
+	path := accessors.MustNewGenericOSPath(ospath).Append(components...)
 	fs_info := &accessors.VirtualFileInfo{
 		Path:   path,
 		IsDir_: is_dir,
@@ -237,7 +260,7 @@ func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {
 
 	// The download pointer allows us to fetch the file itself.
 	record := &flows_proto.VFSDownloadInfo{}
-	err = json.Unmarshal([]byte(cell[0]), record)
+	err := utils.ParseIntoProtobuf(cell[0], record)
 	if err == nil {
 		fs_info.Data_.Set("DownloadInfo", record)
 	}

api/flows.go

@@ -79,20 +79,27 @@ func (self *ApiServer) GetClientFlows(
 		if flow.Request == nil {
 			continue
 		}
-		row_data := []string{
+		row_data := []interface{}{
 			flow.State.String(),
 			flow.SessionId,
-			json.AnyToString(flow.Request.Artifacts, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.CreateTime, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.ActiveTime, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Request.Creator, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.TotalUploadedBytes, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.TotalCollectedRows, vjson.DefaultEncOpts()),
-			json.MustMarshalProtobufString(flow, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Request.Urgent, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.ArtifactsWithResults, vjson.DefaultEncOpts()),
+			flow.Request.Artifacts,
+			flow.CreateTime,
+			flow.ActiveTime,
+			flow.Request.Creator,
+			flow.TotalUploadedBytes,
+			flow.TotalCollectedRows,
+			flow,
+			flow.Request.Urgent,
+			flow.ArtifactsWithResults,
 		}
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 	}
 
 	return result, nil

api/hunts.go

@@ -74,18 +74,26 @@ func (self *ApiServer) GetHuntFlows(
 			continue
 		}
 
-		row_data := []string{
+		row_data := []interface{}{
 			flow.Context.ClientId,
 			services.GetHostname(ctx, org_config_obj, flow.Context.ClientId),
 			flow.Context.SessionId,
-			json.AnyToString(flow.Context.StartTime/1000, vjson.DefaultEncOpts()),
+			flow.Context.StartTime / 1000,
 			flow.Context.State.String(),
-			json.AnyToString(flow.Context.ExecutionDuration/1000000000,
-				vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Context.TotalUploadedBytes, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Context.TotalCollectedRows, vjson.DefaultEncOpts())}
+			flow.Context.ExecutionDuration / 1000000000,
+			flow.Context.TotalUploadedBytes,
+			flow.Context.TotalCollectedRows,
+		}
+
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
 
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 
 		if uint64(len(result.Rows)) > in.Rows {
 			break
@@ -142,19 +150,25 @@ func (self *ApiServer) GetHuntTable(
 			total_clients_scheduled = hunt.Stats.TotalClientsScheduled
 		}
 
-		row_data := []string{
+		row_data := []interface{}{
 			fmt.Sprintf("%v", hunt.State),
-			json.AnyToString(hunt.Tags, vjson.DefaultEncOpts()),
+			hunt.Tags,
 			hunt.HuntId,
 			hunt.HuntDescription,
-			json.AnyToString(hunt.CreateTime, vjson.DefaultEncOpts()),
-			json.AnyToString(hunt.StartTime, vjson.DefaultEncOpts()),
-			json.AnyToString(hunt.Expires, vjson.DefaultEncOpts()),
-			fmt.Sprintf("%v", total_clients_scheduled),
+			hunt.CreateTime,
+			hunt.StartTime,
+			hunt.Expires,
+			total_clients_scheduled,
 			hunt.Creator,
 		}
-
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 
 		if uint64(len(result.Rows)) > in.Rows {
 			break

api/proto/csv.proto

@@ -88,10 +88,11 @@ message GetTableRequest {
 }
 
 message Row {
-    // Deprecated.
-    repeated string cell = 1;
+    // Deprecated - Old code serializes each cell separately.
+    // repeated string cell = 1;
 
-    // The row encoded as a JSON array
+    // The row encoded as a JSON array: Current code encodes the
+    // entire JSON []interface{} into a single JSON string.
     string json = 2;
 }
 

api/tables/table.go

@@ -372,12 +372,8 @@ func ConvertRowsToTableResponse(
 				column_known[key] = true
 			}
 
-			value, pres := row.Get(key)
-			if pres {
-				data[key] = value
-			} else {
-				data[key] = nil
-			}
+			value, _ := row.Get(key)
+			data[key] = value
 		}
 
 		json_out := make([]interface{}, 0, len(result.Columns))