Skip to content

Commit

Permalink
Bugfix: Updated table API to use JSON (#3812)
Browse files Browse the repository at this point in the history 
This preserves the column types more accurately to correspond with
internal data types.
  • Loading branch information
@scudette
scudette committed Oct 13, 2024
1 parent da30624 commit 42b9b26
Show file tree
Hide file tree
Showing 16 changed files with 402 additions and 396 deletions.
65 changes: 44 additions & 21 deletions accessors/vfs/vfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package vfs

import (
"context"
"encoding/json"
"errors"

"github.com/Velocidex/ordereddict"
Expand All @@ -11,14 +10,17 @@ import (
api_proto "www.velocidex.com/golang/velociraptor/api/proto"
config_proto "www.velocidex.com/golang/velociraptor/config/proto"
flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
"www.velocidex.com/golang/velociraptor/json"
"www.velocidex.com/golang/velociraptor/services"
"www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
"www.velocidex.com/golang/vfilter"
)

var (
ErrNotFound = errors.New("file not found")
ErrNotAvailable = errors.New("File content not available")
ErrInvalidRow = errors.New("Stored row is invalid")
)

type VFSFileSystemAccessor struct {
Expand Down Expand Up @@ -97,14 +99,18 @@ func (self VFSFileSystemAccessor) LstatWithOSPath(filename *accessors.OSPath) (
if err != nil {
return nil, err
}

// Find the row that matches this filename
for _, r := range res.Rows {
if len(r.Cell) < 12 {
var row []interface{}
_ = json.Unmarshal([]byte(r.Json), &row)
if len(row) < 12 {
continue
}

if r.Cell[5] == filename.Basename() {
return rowCellToFSInfo(r.Cell)
name, ok := row[5].(string)
if ok && name == filename.Basename() {
return rowCellToFSInfo(row)
}
}

Expand Down Expand Up @@ -147,11 +153,13 @@ func (self VFSFileSystemAccessor) ReadDirWithOSPath(

result := []accessors.FileInfo{}
for _, r := range res.Rows {
if len(r.Cell) < 12 {
var row []interface{}
_ = json.Unmarshal([]byte(r.Json), &row)
if len(row) < 12 {
continue
}

fs_info, err := rowCellToFSInfo(r.Cell)
fs_info, err := rowCellToFSInfo(row)
if err != nil {
continue
}
Expand Down Expand Up @@ -191,14 +199,21 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (

// Find the row that matches this filename
for _, r := range res.Rows {
if len(r.Cell) < 12 {
var row []interface{}
_ = json.Unmarshal([]byte(r.Json), &row)
if len(row) < 12 {
continue
}

name, ok := row[5].(string)
if !ok {
continue
}

if r.Cell[5] == filename.Basename() {
if name == filename.Basename() {
// Check if it has a download link
record := &flows_proto.VFSDownloadInfo{}
err = json.Unmarshal([]byte(r.Cell[0]), record)
err = utils.ParseIntoProtobuf(row[0], record)
if err != nil || record.Name == "" {
return nil, ErrNotAvailable
}
Expand All @@ -211,23 +226,31 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (
return nil, ErrNotFound
}

func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {
components := []string{}
err := json.Unmarshal([]byte(cell[2]), &components)
if err != nil {
return nil, err
func rowCellToFSInfo(cell []interface{}) (accessors.FileInfo, error) {
components := utils.ConvertToStringSlice(cell[2])
if len(components) == 0 {
return nil, ErrInvalidRow
}

size := int64(0)
_ = json.Unmarshal([]byte(cell[6]), &size)
size, ok := utils.ToInt64(cell[6])
if !ok {
return nil, ErrInvalidRow
}

is_dir := false
if len(cell[7]) > 1 && cell[7][0] == 'd' {
is_dir = true
mode, ok := cell[7].(string)
if !ok {
return nil, ErrInvalidRow
}

is_dir := len(mode) > 1 && mode[0] == 'd'

// The Accessor + components is the path of the item
path := accessors.MustNewGenericOSPath(cell[3]).Append(components...)
ospath, ok := cell[3].(string)
if !ok {
return nil, ErrInvalidRow
}

path := accessors.MustNewGenericOSPath(ospath).Append(components...)
fs_info := &accessors.VirtualFileInfo{
Path: path,
IsDir_: is_dir,
Expand All @@ -237,7 +260,7 @@ func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {

// The download pointer allows us to fetch the file itself.
record := &flows_proto.VFSDownloadInfo{}
err = json.Unmarshal([]byte(cell[0]), record)
err := utils.ParseIntoProtobuf(cell[0], record)
if err == nil {
fs_info.Data_.Set("DownloadInfo", record)
}
Expand Down
29 changes: 18 additions & 11 deletions api/flows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,20 +79,27 @@ func (self *ApiServer) GetClientFlows(
if flow.Request == nil {
continue
}
row_data := []string{
row_data := []interface{}{
flow.State.String(),
flow.SessionId,
json.AnyToString(flow.Request.Artifacts, vjson.DefaultEncOpts()),
json.AnyToString(flow.CreateTime, vjson.DefaultEncOpts()),
json.AnyToString(flow.ActiveTime, vjson.DefaultEncOpts()),
json.AnyToString(flow.Request.Creator, vjson.DefaultEncOpts()),
json.AnyToString(flow.TotalUploadedBytes, vjson.DefaultEncOpts()),
json.AnyToString(flow.TotalCollectedRows, vjson.DefaultEncOpts()),
json.MustMarshalProtobufString(flow, vjson.DefaultEncOpts()),
json.AnyToString(flow.Request.Urgent, vjson.DefaultEncOpts()),
json.AnyToString(flow.ArtifactsWithResults, vjson.DefaultEncOpts()),
flow.Request.Artifacts,
flow.CreateTime,
flow.ActiveTime,
flow.Request.Creator,
flow.TotalUploadedBytes,
flow.TotalCollectedRows,
flow,
flow.Request.Urgent,
flow.ArtifactsWithResults,
}
result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
opts := vjson.DefaultEncOpts()
serialized, err := json.MarshalWithOptions(row_data, opts)
if err != nil {
continue
}
result.Rows = append(result.Rows, &api_proto.Row{
Json: string(serialized),
})
}

return result, nil
Expand Down
44 changes: 29 additions & 15 deletions api/hunts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,26 @@ func (self *ApiServer) GetHuntFlows(
continue
}

row_data := []string{
row_data := []interface{}{
flow.Context.ClientId,
services.GetHostname(ctx, org_config_obj, flow.Context.ClientId),
flow.Context.SessionId,
json.AnyToString(flow.Context.StartTime/1000, vjson.DefaultEncOpts()),
flow.Context.StartTime / 1000,
flow.Context.State.String(),
json.AnyToString(flow.Context.ExecutionDuration/1000000000,
vjson.DefaultEncOpts()),
json.AnyToString(flow.Context.TotalUploadedBytes, vjson.DefaultEncOpts()),
json.AnyToString(flow.Context.TotalCollectedRows, vjson.DefaultEncOpts())}
flow.Context.ExecutionDuration / 1000000000,
flow.Context.TotalUploadedBytes,
flow.Context.TotalCollectedRows,
}

opts := vjson.DefaultEncOpts()
serialized, err := json.MarshalWithOptions(row_data, opts)
if err != nil {
continue
}

result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
result.Rows = append(result.Rows, &api_proto.Row{
Json: string(serialized),
})

if uint64(len(result.Rows)) > in.Rows {
break
Expand Down Expand Up @@ -142,19 +150,25 @@ func (self *ApiServer) GetHuntTable(
total_clients_scheduled = hunt.Stats.TotalClientsScheduled
}

row_data := []string{
row_data := []interface{}{
fmt.Sprintf("%v", hunt.State),
json.AnyToString(hunt.Tags, vjson.DefaultEncOpts()),
hunt.Tags,
hunt.HuntId,
hunt.HuntDescription,
json.AnyToString(hunt.CreateTime, vjson.DefaultEncOpts()),
json.AnyToString(hunt.StartTime, vjson.DefaultEncOpts()),
json.AnyToString(hunt.Expires, vjson.DefaultEncOpts()),
fmt.Sprintf("%v", total_clients_scheduled),
hunt.CreateTime,
hunt.StartTime,
hunt.Expires,
total_clients_scheduled,
hunt.Creator,
}

result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
opts := vjson.DefaultEncOpts()
serialized, err := json.MarshalWithOptions(row_data, opts)
if err != nil {
continue
}
result.Rows = append(result.Rows, &api_proto.Row{
Json: string(serialized),
})

if uint64(len(result.Rows)) > in.Rows {
break
Expand Down
14 changes: 8 additions & 6 deletions api/proto/csv.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion api/proto/csv.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,12 @@ message GetTableRequest {
}

message Row {
repeated string cell = 1;
// Deprecated - Old code serializes each cell separately.
// repeated string cell = 1;

// The row encoded as a JSON array: Current code encodes the
// entire JSON []interface{} into a single JSON string.
string json = 2;
}

message GetTableResponse {
Expand Down
28 changes: 14 additions & 14 deletions api/tables/table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ func ConvertRowsToTableResponse(
var rows uint64
column_known := make(map[string]bool)
for row := range in {
data := make(map[string]string)
data := make(map[string]interface{})
for _, key := range row.Keys() {
// Do we already know about this column?
_, pres := column_known[key]
Expand All @@ -372,23 +372,23 @@ func ConvertRowsToTableResponse(
column_known[key] = true
}

value, pres := row.Get(key)
if pres {
data[key] = json.AnyToString(value, opts)
} else {
data[key] = "null"
}
value, _ := row.Get(key)
data[key] = value
}

row_proto := &api_proto.Row{}
json_out := make([]interface{}, 0, len(result.Columns))
for _, k := range result.Columns {
value, pres := data[k]
if !pres {
value = "null"
}
row_proto.Cell = append(row_proto.Cell, value)
value, _ := data[k]
json_out = append(json_out, value)
}
result.Rows = append(result.Rows, row_proto)
serialized, err := json.MarshalWithOptions(json_out, opts)
if err != nil {
continue
}

result.Rows = append(result.Rows, &api_proto.Row{
Json: string(serialized),
})

rows += 1
if rows >= limit {
Expand Down
28 changes: 14 additions & 14 deletions api/tables/timelines.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func ConvertTimelineRowsToTableResponse(
}
result.EndTime = timestamp.UnixNano()

data := make(map[string]string)
data := make(map[string]interface{})
for _, key := range row.Keys() {
// Do we already know about this column?
_, pres := column_known[key]
Expand All @@ -87,23 +87,23 @@ func ConvertTimelineRowsToTableResponse(
column_known[key] = true
}

value, pres := row.Get(key)
if pres {
data[key] = json.AnyToString(value, opts)
} else {
data[key] = "null"
}
value, _ := row.Get(key)
data[key] = value
}

row_proto := &api_proto.Row{}
json_out := make([]interface{}, 0, len(result.Columns))
for _, k := range result.Columns {
value, pres := data[k]
if !pres {
value = "null"
}
row_proto.Cell = append(row_proto.Cell, value)
value, _ := data[k]
json_out = append(json_out, value)
}
result.Rows = append(result.Rows, row_proto)
serialized, err := json.MarshalWithOptions(json_out, opts)
if err != nil {
continue
}

result.Rows = append(result.Rows, &api_proto.Row{
Json: string(serialized),
})

rows += 1
if rows >= limit {
Expand Down
Loading

0 comments on commit 42b9b26

Please sign in to comment.