From 01979cd6f8a11651e4b55aaba95760d96a5dbb62 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 11 Nov 2022 00:00:30 +1000 Subject: [PATCH] Propagate user's prefered timezone for export tables (#2232) When the user sets their preferred timezone in the user preferences the UI adjusts the output. This PR also ensures this preference is propagated to the download table API so the encoded timestamps are also presented in the correct timezone. The user may always adjust the timezone back to UTC to receive their times in UTC again. --- api/csv.go | 13 +++-- api/download.go | 32 ++++++++++- api/hunts.go | 10 ++-- api/proto/csv.pb.go | 55 +++++++++++-------- api/proto/csv.proto | 3 + api/vql.go | 4 +- bin/query.go | 5 +- config/proto/config.pb.go | 17 +++--- file_store/csv/utils.go | 21 +++++-- file_store/csv/writer.go | 17 ++++-- file_store/csv/writer_test.go | 3 +- go.mod | 3 +- go.sum | 7 +-- .../src/components/core/paged-table.js | 8 +++ json/json.go | 2 +- reporting/container.go | 4 +- vql/functions/encode.go | 3 +- vql/parsers/csv/csv.go | 4 +- vql/tools/collector/collector.go | 3 +- 19 files changed, 145 insertions(+), 69 deletions(-) diff --git a/api/csv.go b/api/csv.go index 9438578c15c..6b57c28fdd8 100644 --- a/api/csv.go +++ b/api/csv.go @@ -93,6 +93,8 @@ func getTable( return nil, err } + opts := getJsonOptsForTimezone(in.Timezone) + // Unpack the rows into the output protobuf for row := range rs_reader.Rows(ctx) { if result.Columns == nil { @@ -102,7 +104,7 @@ func getTable( row_data := make([]string, 0, len(result.Columns)) for _, key := range result.Columns { value, _ := row.Get(key) - row_data = append(row_data, csv.AnyToString(value)) + row_data = append(row_data, csv.AnyToString(value, opts)) } result.Rows = append(result.Rows, &api_proto.Row{ Cell: row_data, @@ -262,6 +264,8 @@ func getEventTableWithPathManager( rs_reader.SetMaxTime(time.Unix(int64(in.EndTime), 0)) } + opts := getJsonOptsForTimezone(in.Timezone) + // Unpack the rows into the output protobuf for row := range rs_reader.Rows(ctx) { if result.Columns == nil { @@ -271,7 +275,7 @@ func getEventTableWithPathManager( row_data := make([]string, 0, len(result.Columns)) for _, key := range result.Columns { value, _ := row.Get(key) - row_data = append(row_data, csv.AnyToString(value)) + row_data = append(row_data, csv.AnyToString(value, opts)) } result.Rows = append(result.Rows, &api_proto.Row{ Cell: row_data, @@ -315,6 +319,7 @@ func getTimeline( } rows := uint64(0) + opts := getJsonOptsForTimezone(in.Timezone) for item := range reader.Read(ctx) { if result.StartTime == 0 { result.StartTime = item.Time.UnixNano() @@ -323,8 +328,8 @@ func getTimeline( result.Rows = append(result.Rows, &api_proto.Row{ Cell: []string{ item.Source, - csv.AnyToString(item.Time), - csv.AnyToString(item.Row)}, + csv.AnyToString(item.Time, opts), + csv.AnyToString(item.Row, opts)}, }) rows += 1 diff --git a/api/download.go b/api/download.go index c0b5cfe82b5..46f161dcd00 100644 --- a/api/download.go +++ b/api/download.go @@ -52,6 +52,7 @@ import ( "www.velocidex.com/golang/velociraptor/file_store/path_specs" "www.velocidex.com/golang/velociraptor/flows" "www.velocidex.com/golang/velociraptor/json" + vjson "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" "www.velocidex.com/golang/velociraptor/paths/artifacts" @@ -400,6 +401,7 @@ func downloadTable() http.Handler { return } + opts := getJsonOptsForTimezone(request.Timezone) switch request.DownloadFormat { case "csv": download_name = strings.TrimSuffix(download_name, ".json") @@ -421,7 +423,8 @@ func downloadTable() http.Handler { scope := vql_subsystem.MakeScope() csv_writer := csv.GetCSVAppender( - org_config_obj, scope, w, true /* write_headers */) + org_config_obj, scope, w, + csv.WriteHeaders, opts) for row := range row_chan { csv_writer.Write( filterColumns(request.Columns, transform(row))) @@ -449,8 +452,9 @@ func downloadTable() http.Handler { }).Info("DownloadTable") for row := range row_chan { - serialized, err := json.Marshal( - filterColumns(request.Columns, transform(row))) + serialized, err := json.MarshalWithOptions( + filterColumns(request.Columns, transform(row)), + getJsonOptsForTimezone(request.Timezone)) if err != nil { return } @@ -540,3 +544,25 @@ func filterColumns(columns []string, row *ordereddict.Dict) *ordereddict.Dict { } return new_row } + +func getJsonOptsForTimezone(timezone string) *json.EncOpts { + if timezone == "" { + return vjson.NoEncOpts + } + + loc := time.UTC + if timezone != "" { + loc, _ = time.LoadLocation(timezone) + } + + return vjson.NewEncOpts(). + WithCallback(time.Time{}, + func(v interface{}, opts *json.EncOpts) ([]byte, error) { + switch t := v.(type) { + case time.Time: + return t.In(loc).MarshalJSON() + } + return nil, json.EncoderCallbackSkip + }) + +} diff --git a/api/hunts.go b/api/hunts.go index b95a57d9ea9..d9436ab4f3a 100644 --- a/api/hunts.go +++ b/api/hunts.go @@ -14,6 +14,7 @@ import ( api_proto "www.velocidex.com/golang/velociraptor/api/proto" "www.velocidex.com/golang/velociraptor/file_store/csv" "www.velocidex.com/golang/velociraptor/json" + vjson "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/services/hunt_dispatcher" @@ -72,11 +73,12 @@ func (self *ApiServer) GetHuntFlows( flow.Context.ClientId, services.GetHostname(ctx, org_config_obj, flow.Context.ClientId), flow.Context.SessionId, - csv.AnyToString(flow.Context.StartTime / 1000), + csv.AnyToString(flow.Context.StartTime/1000, vjson.NoEncOpts), flow.Context.State.String(), - csv.AnyToString(flow.Context.ExecutionDuration / 1000000000), - csv.AnyToString(flow.Context.TotalUploadedBytes), - csv.AnyToString(flow.Context.TotalCollectedRows)} + csv.AnyToString(flow.Context.ExecutionDuration/1000000000, + vjson.NoEncOpts), + csv.AnyToString(flow.Context.TotalUploadedBytes, vjson.NoEncOpts), + csv.AnyToString(flow.Context.TotalCollectedRows, vjson.NoEncOpts)} result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data}) diff --git a/api/proto/csv.pb.go b/api/proto/csv.pb.go index 83d794fd443..bd76785e035 100644 --- a/api/proto/csv.pb.go +++ b/api/proto/csv.pb.go @@ -64,6 +64,8 @@ type GetTableRequest struct { // The org id may be specified in the query string - The protobuf // is normally parsed from the query string directly. OrgId string `protobuf:"bytes,23,opt,name=org_id,json=orgId,proto3" json:"org_id,omitempty"` + // The required timezone to export in. + Timezone string `protobuf:"bytes,24,opt,name=timezone,proto3" json:"timezone,omitempty"` } func (x *GetTableRequest) Reset() { @@ -252,6 +254,13 @@ func (x *GetTableRequest) GetOrgId() string { return "" } +func (x *GetTableRequest) GetTimezone() string { + if x != nil { + return x.Timezone + } + return "" +} + type Row struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -393,7 +402,7 @@ var file_csv_proto_rawDesc = []byte{ 0x74, 0x6f, 0x1a, 0x14, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xac, 0x05, 0x0a, 0x0f, 0x47, 0x65, 0x74, + 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc8, 0x05, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x72, 0x6f, 0x77, 0x18, 0x03, 0x20, @@ -436,27 +445,29 @@ var file_csv_proto_rawDesc = []byte{ 0x0c, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x15, 0x0a, 0x06, 0x6f, 0x72, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x6f, 0x72, 0x67, 0x49, 0x64, 0x22, 0x19, 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x12, - 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, - 0x6c, 0x6c, 0x22, 0xf0, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x42, 0x13, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x0d, - 0x12, 0x0b, 0x54, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x52, 0x07, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, - 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, - 0x72, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, - 0x6c, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x34, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, - 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, - 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, - 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, - 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x05, 0x6f, 0x72, 0x67, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x7a, + 0x6f, 0x6e, 0x65, 0x18, 0x18, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x7a, + 0x6f, 0x6e, 0x65, 0x22, 0x19, 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, + 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, 0xf0, + 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x42, 0x13, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x0d, 0x12, 0x0b, 0x54, 0x68, + 0x65, 0x20, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, + 0x77, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x77, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x52, 0x6f, 0x77, + 0x73, 0x12, 0x34, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, + 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, + 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/csv.proto b/api/proto/csv.proto index 72eb858c8f8..8cca1ee8c35 100644 --- a/api/proto/csv.proto +++ b/api/proto/csv.proto @@ -62,6 +62,9 @@ message GetTableRequest { // The org id may be specified in the query string - The protobuf // is normally parsed from the query string directly. string org_id = 23; + + // The required timezone to export in. + string timezone = 24; } message Row { diff --git a/api/vql.go b/api/vql.go index 9ce971a8814..2e6c2ca423d 100644 --- a/api/vql.go +++ b/api/vql.go @@ -23,6 +23,7 @@ import ( api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/csv" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/vql/acl_managers" @@ -69,7 +70,8 @@ func RunVQL( if !pres { value = "" } - new_row.Cell = append(new_row.Cell, csv.AnyToString(value)) + new_row.Cell = append(new_row.Cell, + csv.AnyToString(value, json.NoEncOpts)) } result.Rows = append(result.Rows, new_row) diff --git a/bin/query.go b/bin/query.go index 70619e3b404..f8505a7bccd 100644 --- a/bin/query.go +++ b/bin/query.go @@ -127,7 +127,7 @@ func outputCSV(ctx context.Context, 10, *max_wait) csv_writer := csv.GetCSVAppender(config_obj, - scope, &StdoutWrapper{out}, true /* write_headers */) + scope, &StdoutWrapper{out}, csv.WriteHeaders, json.NoEncOpts) defer csv_writer.Close() for result := range result_chan { @@ -241,7 +241,8 @@ func doRemoteQuery( scope := vql_subsystem.MakeScope() csv_writer := csv.GetCSVAppender(config_obj, - scope, &StdoutWrapper{os.Stdout}, true /* write_headers */) + scope, &StdoutWrapper{os.Stdout}, + csv.WriteHeaders, json.NoEncOpts) defer csv_writer.Close() for _, row := range rows { diff --git a/config/proto/config.pb.go b/config/proto/config.pb.go index b4b8bc3ec06..448c1a43aff 100644 --- a/config/proto/config.pb.go +++ b/config/proto/config.pb.go @@ -1251,7 +1251,7 @@ type GUIConfig struct { // available in the GUI to a small subset (e.g. only certain // packs). ArtifactSearchFilter string `protobuf:"bytes,18,opt,name=artifact_search_filter,json=artifactSearchFilter,proto3" json:"artifact_search_filter,omitempty"` - // SAML info deprecated - will be moved to a sample authenticator. + // SAML info deprecated - will be moved to a saml authenticator. SamlCertificate string `protobuf:"bytes,12,opt,name=saml_certificate,json=samlCertificate,proto3" json:"saml_certificate,omitempty"` SamlPrivateKey string `protobuf:"bytes,13,opt,name=saml_private_key,json=samlPrivateKey,proto3" json:"saml_private_key,omitempty"` SamlIdpMetadataUrl string `protobuf:"bytes,14,opt,name=saml_idp_metadata_url,json=samlIdpMetadataUrl,proto3" json:"saml_idp_metadata_url,omitempty"` @@ -1739,13 +1739,12 @@ type FrontendResourceControl struct { // How quickly do we enroll clients (default 100/s, -1 to disable enrollments) EnrollmentsPerSecond int64 `protobuf:"varint,3,opt,name=enrollments_per_second,json=enrollmentsPerSecond,proto3" json:"enrollments_per_second,omitempty"` // The maximum number of concurrent client connections we can - // process. The actual concurrency level can be dynamically - // adjusted to try to control the target_heap_size below. As the - // heap size increases, the frontend will reduce the concurrency - // level in order to reduce memory pressure. Low concurrency - // levels increase average latency and in the worse case cause - // clients to time out. When clients timeout, they will back off - // and try to send data later. + // process. Concurrency limits helps to ensure the server is not + // overloaded serving too many clients at the same time. + // Concurrency refers to the actual serving time of a client + // (i.e. time taken to read the response and write to the + // datastore), not the total number of clients served by + // server. Default is number of cores * 2. Concurrency uint64 `protobuf:"varint,9,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // Aim for this heap size (default 2Gb). If actual memory usage // approaches this maximum, the frontend will begin to limit @@ -3524,7 +3523,7 @@ type Config struct { Defaults *Defaults `protobuf:"bytes,33,opt,name=defaults,proto3" json:"defaults,omitempty"` // The Operating System of the analysis target. Only useful in conjunction // with the `device' parameter in case the host's operating system differs - // from the operating system used on the device. + // from the operating system used on the device. DEPRECATED! AnalysisTarget string `protobuf:"bytes,34,opt,name=analysis_target,json=analysisTarget,proto3" json:"analysis_target,omitempty"` // The list of data sources which Velociraptor should map instead of the // host's own file system. diff --git a/file_store/csv/utils.go b/file_store/csv/utils.go index 92001310f92..512fbc22b7a 100644 --- a/file_store/csv/utils.go +++ b/file_store/csv/utils.go @@ -29,9 +29,15 @@ import ( "github.com/Velocidex/ordereddict" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/vfilter" ) +const ( + WriteHeaders = true + DoNotWriteHeader = false +) + type CSVWriter struct { row_chan chan vfilter.Row wg sync.WaitGroup @@ -89,7 +95,8 @@ func GetCSVReader(ctx context.Context, fd api.FileReader) CSVReader { func GetCSVAppender( config_obj *config_proto.Config, - scope vfilter.Scope, fd io.Writer, write_headers bool) *CSVWriter { + scope vfilter.Scope, fd io.Writer, + write_headers bool, opts *json.EncOpts) *CSVWriter { result := &CSVWriter{ row_chan: make(chan vfilter.Row), wg: sync.WaitGroup{}, @@ -142,7 +149,7 @@ func GetCSVAppender( item, _ := scope.Associative(row, column) csv_row = append(csv_row, item) } - err := w.WriteAny(csv_row) + err := w.WriteAny(csv_row, opts) if err != nil { return } @@ -160,25 +167,27 @@ func GetCSVAppender( func GetCSVWriter( config_obj *config_proto.Config, - scope vfilter.Scope, fd api.FileWriter) (*CSVWriter, error) { + scope vfilter.Scope, fd api.FileWriter, + opts *json.EncOpts) (*CSVWriter, error) { // Seek to the end of the file. length, err := fd.Size() if err != nil { return nil, err } - return GetCSVAppender(config_obj, scope, fd, length == 0), nil + return GetCSVAppender(config_obj, scope, fd, length == 0, opts), nil } func EncodeToCSV( config_obj *config_proto.Config, - scope vfilter.Scope, v interface{}) (string, error) { + scope vfilter.Scope, v interface{}, + opts *json.EncOpts) (string, error) { slice := reflect.ValueOf(v) if slice.Type().Kind() != reflect.Slice { return "", errors.New("EncodeToCSV - should be a list of rows") } buffer := &bytes.Buffer{} - writer := GetCSVAppender(config_obj, scope, buffer, true) + writer := GetCSVAppender(config_obj, scope, buffer, true, opts) for i := 0; i < slice.Len(); i++ { value := slice.Index(i).Interface() diff --git a/file_store/csv/writer.go b/file_store/csv/writer.go index e8d39803f68..0286b3aeb6e 100644 --- a/file_store/csv/writer.go +++ b/file_store/csv/writer.go @@ -54,7 +54,7 @@ func NewWriter(w io.Writer) *Writer { } } -func AnyToString(item vfilter.Any) string { +func AnyToString(item vfilter.Any, opts *json.EncOpts) string { value := "" switch t := item.(type) { @@ -65,7 +65,14 @@ func AnyToString(item vfilter.Any) string { value = strconv.FormatFloat(t, 'f', -1, 64) case time.Time: - value = t.Format(time.RFC3339Nano) + // Use the encoding options to control how to serialize the + // time into the correct timezone. + serialized, err := json.MarshalIndentWithOptions(t, opts) + if err != nil || len(serialized) < 10 { + return "" + } + // Strip the quote marks so it is a bare string value. + return string(serialized[1 : len(serialized)-1]) case int, int16, int32, int64, uint16, uint32, uint64, bool: value = fmt.Sprintf("%v", item) @@ -87,7 +94,7 @@ func AnyToString(item vfilter.Any) string { } default: - serialized, err := json.MarshalIndent(item) + serialized, err := json.MarshalIndentWithOptions(item, opts) if err != nil { return "" } @@ -101,11 +108,11 @@ func AnyToString(item vfilter.Any) string { return value } -func (w *Writer) WriteAny(record []interface{}) error { +func (w *Writer) WriteAny(record []interface{}, opts *json.EncOpts) error { row := []string{} for _, item := range record { - row = append(row, AnyToString(item)) + row = append(row, AnyToString(item, opts)) } return w.Write(row) diff --git a/file_store/csv/writer_test.go b/file_store/csv/writer_test.go index 7ddba7f6443..c7d0a68d9cc 100644 --- a/file_store/csv/writer_test.go +++ b/file_store/csv/writer_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/utils" ) @@ -110,7 +111,7 @@ func TestWriteAny(t *testing.T) { f := NewWriter(b) f.UseCRLF = tt.UseCRLF for _, item := range tt.Input { - err := f.WriteAny(item) + err := f.WriteAny(item, json.NoEncOpts) if err != nil { t.Errorf("Unexpected error: %s\n", err) } diff --git a/go.mod b/go.mod index 89cdeb4729e..7589d8dc3e5 100644 --- a/go.mod +++ b/go.mod @@ -110,7 +110,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Masterminds/sprig/v3 v3.2.2 github.com/Velocidex/file-rotatelogs v0.0.0-20211221020724-d12e4dae4e11 - github.com/Velocidex/ordereddict v0.0.0-20220428153415-da46091cd216 + github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd github.com/clayscode/Go-Splunk-HTTP/splunk/v2 v2.0.1-0.20221027171526-76a36be4fa02 github.com/coreos/go-oidc/v3 v3.4.0 github.com/evanphx/json-patch/v5 v5.6.0 @@ -186,7 +186,6 @@ require ( go.uber.org/goleak v1.2.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.0.0-20221017184919-83659145692c // indirect - golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a9f9b27c976..91ad0146ce0 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,9 @@ github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9 github.com/Velocidex/ordereddict v0.0.0-20200723153557-9460a6764ab8/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo= github.com/Velocidex/ordereddict v0.0.0-20211223082514-572009c595d0/go.mod h1:USioSRAHYrnbOEZvFUX5Puw3BBWl42kVg8D9xuZOEwA= github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844/go.mod h1:Y5Tfx5SKGOzkulpqfonrdILSPIuNg+GqKE/DhVJgnpg= -github.com/Velocidex/ordereddict v0.0.0-20220428153415-da46091cd216 h1:dVwtzsWggC8DiClwbwck+qj6+S0WatVIYyqlwdC8elc= github.com/Velocidex/ordereddict v0.0.0-20220428153415-da46091cd216/go.mod h1:XJDUbaGh2U9e0z78L5O2OXf1hE1wSxnJ7nSlQmA+bIs= +github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd h1:GA/Aogkc2wf0RCvjiSXIILg2WqGoSb5OsozwX3gvZFY= +github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= github.com/Velocidex/pkcs7 v0.0.0-20210524015001-8d1eee94a157 h1:cNRL6O5MZdKi4i0aQxW6+7RoT34QMHFuRKpigCIHBG8= github.com/Velocidex/pkcs7 v0.0.0-20210524015001-8d1eee94a157/go.mod h1:/fy/Eg4TQz9KkJduvZfGCnbWTQ/LKaknS2wYB52cU6c= github.com/Velocidex/saml v0.0.0-20221019055034-272f55e26c8d h1:PVZCv5VhxV2i9FXWi/XODO33WdlmaytpQSunDUxTOxI= @@ -942,8 +943,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1200,8 +1199,6 @@ www.velocidex.com/golang/oleparse v0.0.0-20220617011920-94df2342d0b7/go.mod h1:R www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b h1:NrnjFXwjUi7vdLEDKgSxu6cs304UJLZE/H7pSXXakVA= www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= www.velocidex.com/golang/vfilter v0.0.0-20220103082604-85bb38175cb7/go.mod h1:eEFMhAmoFHWGCKF39j+iOhTH8REpqBndc3OsdPsxqo8= -www.velocidex.com/golang/vfilter v0.0.0-20221020070405-7af3cd80b934 h1:vUdXxTpIjiZmSvAp3dPEZs4ZcBpQKJWh3lfmZVqYKNQ= -www.velocidex.com/golang/vfilter v0.0.0-20221020070405-7af3cd80b934/go.mod h1:R3nLf1iHcc7eezqqc68KF+SUOXaAJeFz3TV+j8xorfY= www.velocidex.com/golang/vfilter v0.0.0-20221101121437-3c06b865adbf h1:9QCjJRFZWaXrUhcUFzld1EhgHSXywn1dpEqq25dx55Q= www.velocidex.com/golang/vfilter v0.0.0-20221101121437-3c06b865adbf/go.mod h1:R3nLf1iHcc7eezqqc68KF+SUOXaAJeFz3TV+j8xorfY= www.velocidex.com/golang/vtypes v0.0.0-20220816192452-6a27ae078f12 h1:8azOLd/l6sPy1/ug03ueA7jLfsVwE1sI3oHg9q/nkqQ= diff --git a/gui/velociraptor/src/components/core/paged-table.js b/gui/velociraptor/src/components/core/paged-table.js index 1056da983d5..d9170c5ff77 100644 --- a/gui/velociraptor/src/components/core/paged-table.js +++ b/gui/velociraptor/src/components/core/paged-table.js @@ -27,6 +27,7 @@ import ClientLink from '../clients/client-link.js'; import HexView from '../utils/hex.js'; import TableTransformDialog from './table-transform-dialog.js'; import T from '../i8n/i8n.js'; +import UserConfig from '../core/user.js'; import { InspectRawJson, ColumnToggleList, @@ -85,6 +86,8 @@ const pageListRenderer = ({ class VeloPagedTable extends Component { + static contextType = UserConfig; + static propTypes = { // Params to the GetTable API call. params: PropTypes.object, @@ -315,6 +318,9 @@ class VeloPagedTable extends Component { ); render() { + let timezone = (this.context.traits && + this.context.traits.timezone) || "UTC"; + if (_.isEmpty(this.state.columns) && this.state.loading) { return <> @@ -462,6 +468,7 @@ class VeloPagedTable extends Component { title={T("Download JSON")} href={api.href("/api/v1/DownloadTable", Object.assign(downloads, { + timezone: timezone, download_format: "json", }))}> @@ -471,6 +478,7 @@ class VeloPagedTable extends Component { title={T("Download CSV")} href={api.href("/api/v1/DownloadTable", Object.assign(downloads, { + timezone: timezone, download_format: "csv", }))}> diff --git a/json/json.go b/json/json.go index 0fbb7c455ee..e03b04f15d8 100644 --- a/json/json.go +++ b/json/json.go @@ -15,7 +15,7 @@ import ( ) var ( - NoEncOpts *json.EncOpts = nil + NoEncOpts *json.EncOpts = &json.EncOpts{} ) type EncOpts = json.EncOpts diff --git a/reporting/container.go b/reporting/container.go index b00b4b51406..9ade457173d 100644 --- a/reporting/container.go +++ b/reporting/container.go @@ -200,7 +200,9 @@ func (self *Container) WriteResultSet( } csv_writer = csv.GetCSVAppender(config_obj, - scope, csv_fd, true /* write_headers */) + scope, csv_fd, + true, /* write_headers */ + json.NoEncOpts) // Preserve the error for our caller. defer func() { diff --git a/vql/functions/encode.go b/vql/functions/encode.go index e5712e91046..7edf0070456 100644 --- a/vql/functions/encode.go +++ b/vql/functions/encode.go @@ -100,7 +100,8 @@ func (self *EncodeFunction) Call(ctx context.Context, buff := bytes.NewBuffer([]byte{}) csv_writer := csv.GetCSVAppender(config_obj, scope, buff, - true /* write_headers */) + true, /* write_headers */ + json.NoEncOpts) result_rows_value := reflect.ValueOf(result) for i := 0; i < result_rows_value.Len(); i++ { diff --git a/vql/parsers/csv/csv.go b/vql/parsers/csv/csv.go index 1064602c101..ad5d68622a8 100644 --- a/vql/parsers/csv/csv.go +++ b/vql/parsers/csv/csv.go @@ -27,6 +27,7 @@ import ( "www.velocidex.com/golang/velociraptor/accessors" "www.velocidex.com/golang/velociraptor/acls" "www.velocidex.com/golang/velociraptor/file_store/csv" + "www.velocidex.com/golang/velociraptor/json" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" vfilter "www.velocidex.com/golang/vfilter" "www.velocidex.com/golang/vfilter/arg_parser" @@ -263,7 +264,8 @@ func (self WriteCSVPlugin) Call( defer file.Close() config_obj, _ := vql_subsystem.GetServerConfig(scope) - writer = csv.GetCSVAppender(config_obj, scope, file, true) + writer = csv.GetCSVAppender( + config_obj, scope, file, true, json.NoEncOpts) defer writer.Close() default: diff --git a/vql/tools/collector/collector.go b/vql/tools/collector/collector.go index 1d1ec987ba6..2b5c7ce9d11 100644 --- a/vql/tools/collector/collector.go +++ b/vql/tools/collector/collector.go @@ -330,7 +330,8 @@ func AddSpecProtobuf( case "csv": if !is_str { value_str, err = csv.EncodeToCSV( - config_obj, scope, value_any) + config_obj, scope, value_any, + json.NoEncOpts) if err != nil { scope.Log("Invalid CSV for %v", parameter_definition.Name)