diff --git a/api/api.go b/api/api.go index 43bfdc6b695..865137403c5 100644 --- a/api/api.go +++ b/api/api.go @@ -582,7 +582,10 @@ func (self *ApiServer) GetTable( var result *api_proto.GetTableResponse // We want an event table. - if in.Type == "CLIENT_EVENT_LOGS" || in.Type == "SERVER_EVENT_LOGS" { + if in.Type == "TIMELINE" { + result, err = getTimeline(ctx, self.config, in) + + } else if in.Type == "CLIENT_EVENT_LOGS" || in.Type == "SERVER_EVENT_LOGS" { result, err = getEventTableLogs(ctx, self.config, in) } else if in.Type == "CLIENT_EVENT" || in.Type == "SERVER_EVENT" { diff --git a/api/csv.go b/api/csv.go index 07db63a4d54..c608a9e6d87 100644 --- a/api/csv.go +++ b/api/csv.go @@ -18,6 +18,8 @@ package api import ( + "time" + errors "github.com/pkg/errors" context "golang.org/x/net/context" file_store "www.velocidex.com/golang/velociraptor/file_store" @@ -27,6 +29,7 @@ import ( "www.velocidex.com/golang/velociraptor/paths" "www.velocidex.com/golang/velociraptor/paths/artifacts" "www.velocidex.com/golang/velociraptor/reporting" + "www.velocidex.com/golang/velociraptor/timelines" api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" @@ -222,3 +225,49 @@ func getEventTableWithPathManager( return result, nil } + +func getTimeline( + ctx context.Context, + config_obj *config_proto.Config, + in *api_proto.GetTableRequest) (*api_proto.GetTableResponse, error) { + + if in.NotebookId == "" { + return nil, errors.New("NotebookId must be specified") + } + + path_manager := reporting.NewNotebookPathManager(in.NotebookId).Timeline(in.Timeline) + reader, err := timelines.NewSuperTimelineReader(config_obj, path_manager, in.SkipComponents) + if err != nil { + return nil, err + } + defer reader.Close() + + if in.StartTime != 0 { + ts := time.Unix(0, int64(in.StartTime)) + reader.SeekToTime(ts) + } + + rows := uint64(0) + result := &api_proto.GetTableResponse{ + Columns: []string{"_Source", "Time", "Data"}, + } + for item := range reader.Read(ctx) { + if result.StartTime == 0 { + result.StartTime = item.Time + } + result.EndTime = item.Time + result.Rows = append(result.Rows, &api_proto.Row{ + Cell: []string{ + item.Source, + csv.AnyToString(item.Time), + csv.AnyToString(item.Row)}, + }) + + rows += 1 + if rows > in.Rows { + break + } + } + + return result, nil +} diff --git a/api/notebooks.go b/api/notebooks.go index e0d84a3b0eb..ba60c1f8879 100644 --- a/api/notebooks.go +++ b/api/notebooks.go @@ -687,6 +687,8 @@ func (self *ApiServer) updateNotebookCell( return nil, err } + tmpl.SetEnv("NotebookId", in.NotebookId) + // Register a progress reporter so we can monitor how the // template rendering is going. tmpl.Progress = &progressReporter{ diff --git a/api/proto/api.pb.gw.go b/api/proto/api.pb.gw.go index a7470a02897..ef3800f5482 100644 --- a/api/proto/api.pb.gw.go +++ b/api/proto/api.pb.gw.go @@ -22,8 +22,8 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" - proto_1 "www.velocidex.com/golang/velociraptor/artifacts/proto" - proto_3 "www.velocidex.com/golang/velociraptor/flows/proto" + proto_0 "www.velocidex.com/golang/velociraptor/artifacts/proto" + proto_5 "www.velocidex.com/golang/velociraptor/flows/proto" ) // Suppress "imported and not used" errors @@ -739,7 +739,7 @@ var ( ) func request_API_VFSListDirectory_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSListRequest + var protoReq proto_5.VFSListRequest var metadata runtime.ServerMetadata var ( @@ -772,7 +772,7 @@ func request_API_VFSListDirectory_0(ctx context.Context, marshaler runtime.Marsh } func local_request_API_VFSListDirectory_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSListRequest + var protoReq proto_5.VFSListRequest var metadata runtime.ServerMetadata var ( @@ -843,7 +843,7 @@ var ( ) func request_API_VFSStatDirectory_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSListRequest + var protoReq proto_5.VFSListRequest var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -859,7 +859,7 @@ func request_API_VFSStatDirectory_0(ctx context.Context, marshaler runtime.Marsh } func local_request_API_VFSStatDirectory_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSListRequest + var protoReq proto_5.VFSListRequest var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -879,7 +879,7 @@ var ( ) func request_API_VFSStatDownload_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSStatDownloadRequest + var protoReq proto_5.VFSStatDownloadRequest var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -895,7 +895,7 @@ func request_API_VFSStatDownload_0(ctx context.Context, marshaler runtime.Marsha } func local_request_API_VFSStatDownload_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.VFSStatDownloadRequest + var protoReq proto_5.VFSStatDownloadRequest var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -947,7 +947,7 @@ func local_request_API_GetTable_0(ctx context.Context, marshaler runtime.Marshal } func request_API_CollectArtifact_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ArtifactCollectorArgs + var protoReq proto_5.ArtifactCollectorArgs var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -964,7 +964,7 @@ func request_API_CollectArtifact_0(ctx context.Context, marshaler runtime.Marsha } func local_request_API_CollectArtifact_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ArtifactCollectorArgs + var protoReq proto_5.ArtifactCollectorArgs var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1283,7 +1283,7 @@ var ( ) func request_API_GetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_1.Tool + var protoReq proto_0.Tool var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -1299,7 +1299,7 @@ func request_API_GetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, } func local_request_API_GetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_1.Tool + var protoReq proto_0.Tool var metadata runtime.ServerMetadata if err := req.ParseForm(); err != nil { @@ -1315,7 +1315,7 @@ func local_request_API_GetToolInfo_0(ctx context.Context, marshaler runtime.Mars } func request_API_SetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_1.Tool + var protoReq proto_0.Tool var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1332,7 +1332,7 @@ func request_API_SetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, } func local_request_API_SetToolInfo_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_1.Tool + var protoReq proto_0.Tool var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1401,7 +1401,7 @@ func local_request_API_GetServerMonitoringState_0(ctx context.Context, marshaler } func request_API_SetServerMonitoringState_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ArtifactCollectorArgs + var protoReq proto_5.ArtifactCollectorArgs var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1418,7 +1418,7 @@ func request_API_SetServerMonitoringState_0(ctx context.Context, marshaler runti } func local_request_API_SetServerMonitoringState_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ArtifactCollectorArgs + var protoReq proto_5.ArtifactCollectorArgs var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1453,7 +1453,7 @@ func local_request_API_GetClientMonitoringState_0(ctx context.Context, marshaler } func request_API_SetClientMonitoringState_0(ctx context.Context, marshaler runtime.Marshaler, client APIClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ClientEventTable + var protoReq proto_5.ClientEventTable var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) @@ -1470,7 +1470,7 @@ func request_API_SetClientMonitoringState_0(ctx context.Context, marshaler runti } func local_request_API_SetClientMonitoringState_0(ctx context.Context, marshaler runtime.Marshaler, server APIServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq proto_3.ClientEventTable + var protoReq proto_5.ClientEventTable var metadata runtime.ServerMetadata newReader, berr := utilities.IOReaderFactory(req.Body) diff --git a/api/proto/csv.pb.go b/api/proto/csv.pb.go index 4469cdacf1a..7652344655e 100644 --- a/api/proto/csv.pb.go +++ b/api/proto/csv.pb.go @@ -48,6 +48,10 @@ type GetTableRequest struct { NotebookId string `protobuf:"bytes,9,opt,name=notebook_id,json=notebookId,proto3" json:"notebook_id,omitempty"` CellId string `protobuf:"bytes,10,opt,name=cell_id,json=cellId,proto3" json:"cell_id,omitempty"` TableId int64 `protobuf:"varint,11,opt,name=table_id,json=tableId,proto3" json:"table_id,omitempty"` + // For timelines + Timeline string `protobuf:"bytes,16,opt,name=timeline,proto3" json:"timeline,omitempty"` + // Skip these timeline components. + SkipComponents []string `protobuf:"bytes,17,rep,name=skip_components,json=skipComponents,proto3" json:"skip_components,omitempty"` // For download handler when creating an export file - control // output format. Can be "csv", "jsonl" DownloadFormat string `protobuf:"bytes,12,opt,name=download_format,json=downloadFormat,proto3" json:"download_format,omitempty"` @@ -171,6 +175,20 @@ func (x *GetTableRequest) GetTableId() int64 { return 0 } +func (x *GetTableRequest) GetTimeline() string { + if x != nil { + return x.Timeline + } + return "" +} + +func (x *GetTableRequest) GetSkipComponents() []string { + if x != nil { + return x.SkipComponents + } + return nil +} + func (x *GetTableRequest) GetDownloadFormat() string { if x != nil { return x.DownloadFormat @@ -241,6 +259,8 @@ type GetTableResponse struct { Rows []*Row `protobuf:"bytes,2,rep,name=rows,proto3" json:"rows,omitempty"` TotalRows int64 `protobuf:"varint,3,opt,name=total_rows,json=totalRows,proto3" json:"total_rows,omitempty"` ColumnTypes []*proto.ColumnType `protobuf:"bytes,4,rep,name=column_types,json=columnTypes,proto3" json:"column_types,omitempty"` + StartTime int64 `protobuf:"varint,5,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime int64 `protobuf:"varint,6,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` } func (x *GetTableResponse) Reset() { @@ -303,6 +323,20 @@ func (x *GetTableResponse) GetColumnTypes() []*proto.ColumnType { return nil } +func (x *GetTableResponse) GetStartTime() int64 { + if x != nil { + return x.StartTime + } + return 0 +} + +func (x *GetTableResponse) GetEndTime() int64 { + if x != nil { + return x.EndTime + } + return 0 +} + var File_csv_proto protoreflect.FileDescriptor var file_csv_proto_rawDesc = []byte{ @@ -310,7 +344,7 @@ var file_csv_proto_rawDesc = []byte{ 0x74, 0x6f, 0x1a, 0x14, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x93, 0x03, 0x0a, 0x0f, 0x47, 0x65, 0x74, + 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd8, 0x03, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x72, 0x6f, 0x77, 0x18, 0x03, 0x20, @@ -331,28 +365,36 @@ var file_csv_proto_rawDesc = []byte{ 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x64, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x0c, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, - 0x0f, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0x19, - 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, 0xb6, 0x01, 0x0a, 0x10, 0x47, 0x65, - 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, - 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x42, - 0x13, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x0d, 0x12, 0x0b, 0x54, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x1e, 0x0a, - 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x1d, 0x0a, - 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x34, 0x0a, 0x0c, - 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x42, 0x31, 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, - 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, - 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x03, 0x52, 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x74, + 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, + 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x5f, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, + 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x66, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x73, 0x22, 0x19, 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, + 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, 0xf0, + 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x42, 0x13, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x0d, 0x12, 0x0b, 0x54, 0x68, + 0x65, 0x20, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, + 0x77, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x77, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x52, 0x6f, 0x77, + 0x73, 0x12, 0x34, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x42, 0x31, 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, + 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, + 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/csv.proto b/api/proto/csv.proto index cfa3009ca39..958cd67514e 100644 --- a/api/proto/csv.proto +++ b/api/proto/csv.proto @@ -38,10 +38,16 @@ message GetTableRequest { string cell_id = 10; int64 table_id = 11; + // For timelines + string timeline = 16; + // Skip these timeline components. + repeated string skip_components = 17; + // For download handler when creating an export file - control // output format. Can be "csv", "jsonl" string download_format = 12; + // If specified only emit these columns. repeated string columns = 15; } @@ -60,4 +66,7 @@ message GetTableResponse { int64 total_rows = 3; repeated ColumnType column_types = 4; + + int64 start_time = 5; + int64 end_time = 6; } \ No newline at end of file diff --git a/api/proto/notebooks.pb.go b/api/proto/notebooks.pb.go index 7699e02072c..06d25fa9265 100644 --- a/api/proto/notebooks.pb.go +++ b/api/proto/notebooks.pb.go @@ -329,7 +329,8 @@ type NotebookMetadata struct { AvailableDownloads *AvailableDownloads `protobuf:"bytes,10,opt,name=available_downloads,json=availableDownloads,proto3" json:"available_downloads,omitempty"` // These environment variables will be populated into each // notebook cell in this notebook. - Env []*Env `protobuf:"bytes,14,rep,name=env,proto3" json:"env,omitempty"` + Env []*Env `protobuf:"bytes,14,rep,name=env,proto3" json:"env,omitempty"` + Timelines []string `protobuf:"bytes,15,rep,name=timelines,proto3" json:"timelines,omitempty"` } func (x *NotebookMetadata) Reset() { @@ -469,6 +470,13 @@ func (x *NotebookMetadata) GetEnv() []*Env { return nil } +func (x *NotebookMetadata) GetTimelines() []string { + if x != nil { + return x.Timelines + } + return nil +} + type Notebooks struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -789,7 +797,7 @@ var file_notebooks_proto_rawDesc = []byte{ 0x06, 0x68, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0xb3, 0x04, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0xd1, 0x04, 0x0a, 0x10, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, @@ -825,44 +833,46 @@ var file_notebooks_proto_rawDesc = []byte{ 0x73, 0x52, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x1c, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x52, 0x03, - 0x65, 0x6e, 0x76, 0x22, 0x3a, 0x0a, 0x09, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x73, - 0x12, 0x2d, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, - 0xc0, 0x02, 0x0a, 0x0c, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x43, 0x65, 0x6c, 0x6c, - 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x6c, 0x79, 0x5f, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x10, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x45, 0x64, 0x69, 0x74, 0x69, - 0x6e, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6e, - 0x67, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, - 0x74, 0x69, 0x6e, 0x67, 0x12, 0x1c, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x0b, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x52, 0x03, 0x65, - 0x6e, 0x76, 0x22, 0x6c, 0x0a, 0x19, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x46, 0x69, - 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x1f, 0x0a, 0x0b, 0x6e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x49, 0x64, - 0x22, 0x2e, 0x0a, 0x1a, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x46, 0x69, 0x6c, 0x65, - 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, - 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, - 0x42, 0x31, 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, - 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, - 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x6e, 0x76, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, + 0x18, 0x0f, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x73, 0x22, 0x3a, 0x0a, 0x09, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x73, 0x12, 0x2d, + 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xc0, 0x02, + 0x0a, 0x0c, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x43, 0x65, 0x6c, 0x6c, 0x12, 0x14, + 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, + 0x5f, 0x65, 0x64, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x45, 0x64, 0x69, 0x74, 0x69, 0x6e, 0x67, + 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, + 0x6e, 0x67, 0x12, 0x1c, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x52, 0x03, 0x65, 0x6e, 0x76, + 0x22, 0x6c, 0x0a, 0x19, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x46, 0x69, 0x6c, 0x65, + 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, + 0x0b, 0x6e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x49, 0x64, 0x22, 0x2e, + 0x0a, 0x1a, 0x4e, 0x6f, 0x74, 0x65, 0x62, 0x6f, 0x6f, 0x6b, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, + 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x42, 0x31, + 0x5a, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, 0x63, + 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/notebooks.proto b/api/proto/notebooks.proto index 66ba3f58eaa..71dd399b406 100644 --- a/api/proto/notebooks.proto +++ b/api/proto/notebooks.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "flows.proto"; + package proto; option go_package = "www.velocidex.com/golang/velociraptor/api/proto"; @@ -70,6 +71,8 @@ message NotebookMetadata { // These environment variables will be populated into each // notebook cell in this notebook. repeated Env env = 14; + + repeated string timelines = 15; } message Notebooks { diff --git a/artifacts/definitions/Server/Utils/AddTimeline.yaml b/artifacts/definitions/Server/Utils/AddTimeline.yaml new file mode 100644 index 00000000000..dce9f5ea987 --- /dev/null +++ b/artifacts/definitions/Server/Utils/AddTimeline.yaml @@ -0,0 +1,35 @@ +name: Server.Utils.AddTimeline +description: | + Adds a new timeline to a super timeline. + +type: SERVER + +parameters: + - name: NotebookId + - name: Timeline + description: SuperTimeline name + - name: ChildName + description: Name of child timeline + - name: Query + description: A query that will be parsed and run. + - name: Key + description: Sort column for time + - name: RemoveLimit + description: If specified, we remove the limit clause before adding to the timeline. + type: bool + - name: Env + type: json + +sources: + - query: | + SELECT timeline_add( + notebook_id=NotebookId, + timeline=Timeline, + name=ChildName, + query={ + SELECT * FROM query(query=if(condition=RemoveLimit, + then=regex_replace(re="(?i)LIMIT [0-9]+", replace="", source=Query), + else=Query), env=Env) + }, + key=Key), RemoveLimit + FROM scope() diff --git a/bin/artifacts.go b/bin/artifacts.go index caa90e8a18d..57b10bb4f9a 100644 --- a/bin/artifacts.go +++ b/bin/artifacts.go @@ -30,7 +30,6 @@ import ( "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" - "www.velocidex.com/golang/velociraptor/json" logging "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/services" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -160,8 +159,6 @@ func doArtifactCollect() { spec.Set(name, collect_args) } - json.Dump(spec) - manager, err := services.GetRepositoryManager() kingpin.FatalIfError(err, "GetRepositoryManager") diff --git a/constants/constants.go b/constants/constants.go index 49949ff734a..f5fd022d3ab 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -39,6 +39,9 @@ const ( NOTEBOOK_INDEX = "/notebook_index/" USER_URN = "/users/" + // Timelines + TIMELINE_URN = "/timelines/" + // Well known flows - Request ID: LOG_SINK uint64 = 980 diff --git a/docs/references/vql.yaml b/docs/references/vql.yaml old mode 100755 new mode 100644 index 18ae285d426..da911d69d6c --- a/docs/references/vql.yaml +++ b/docs/references/vql.yaml @@ -1,19 +1,14 @@ # Autogenerated! Do not edit. - name: amsi - description: "" type: Function args: - name: string - description: A string to scan type: string - repeated: false + description: A string to scan required: true - category: "" - name: appcompatcache description: Parses the appcompatcache. type: Plugin - args: [] - category: windows - name: array description: | Create an array with all the args. @@ -30,68 +25,48 @@ Will return a single row with Users being an array of names. type: Function - args: [] - category: basic - name: artifact_definitions description: Dump artifact definitions. type: Plugin args: - name: names - description: Artifact definitions to dump type: string + description: Artifact definitions to dump repeated: true - required: false - name: deps - description: If true includes all dependencies as well. type: bool - repeated: false - required: false + description: If true includes all dependencies as well. - name: sanitize - description: If true we remove extra metadata. type: bool - repeated: false - required: false - category: server + description: If true we remove extra metadata. - name: artifact_delete description: Deletes an artifact from the global repository. type: Function args: - name: name - description: The Artifact to delete type: string - repeated: false - required: false - category: server + description: The Artifact to delete - name: artifact_set description: Sets an artifact into the global repository. type: Function args: - name: definition - description: Artifact definition in YAML type: string - repeated: false - required: false + description: Artifact definition in YAML - name: prefix - description: Required name prefix type: string - repeated: false - required: false - category: server + description: Required name prefix - name: atoi description: Convert a string to an int. type: Function args: - name: string - description: A string to convert to int type: Any - repeated: false + description: A string to convert to int required: true - category: basic - name: audit description: Register as an audit daemon in the kernel. type: Plugin - args: [] - category: linux - name: authenticode description: | This plugin uses the Windows API to extract authenticode signature @@ -102,56 +77,42 @@ type: Function args: - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: filename - description: The filename to parse. type: string - repeated: false + description: The filename to parse. required: true - name: verbose - description: Set to receive verbose information about all the certs. type: bool - repeated: false - required: false - category: windows + description: Set to receive verbose information about all the certs. - name: base64decode description: Decodes a base64 encoded string. type: Function args: - name: string - description: A string to decode type: string - repeated: false + description: A string to decode required: true - category: basic - name: base64encode description: Encodes a string into base64. type: Function args: - name: string - description: A string to decode type: string - repeated: false + description: A string to decode required: true - category: basic - name: basename description: Return the basename of the path. type: Function args: - name: path - description: Extract directory name of path type: string - repeated: false + description: Extract directory name of path required: true - name: sep - description: Separator to use (default /) type: string - repeated: false - required: false - category: basic + description: Separator to use (default /) - name: binary_parse description: | Parse a binary string with profile based parser. @@ -175,31 +136,21 @@ type: Function args: - name: offset - description: Start parsing from this offset. type: int64 - repeated: false - required: false + description: Start parsing from this offset. - name: string - description: The string to parse. type: string - repeated: false + description: The string to parse. required: true - name: profile - description: The profile to use. type: string - repeated: false - required: false + description: The profile to use. - name: iterator - description: An iterator to begin with. type: string - repeated: false - required: false + description: An iterator to begin with. - name: target - description: The target type to fetch. type: string - repeated: false - required: false - category: parsers + description: The target type to fetch. - name: binary_parse description: | Parse binary files using a profile. @@ -208,86 +159,58 @@ type: Plugin args: - name: offset - description: Start parsing from this offset type: int64 - repeated: false - required: false + description: Start parsing from this offset - name: file - description: Filename to parse type: string - repeated: false + description: Filename to parse required: true - name: accessor - description: Accessor to use (e.g. ntfs, data) type: string - repeated: false - required: false + description: Accessor to use (e.g. ntfs, data) - name: profile - description: Profile to use. type: string - repeated: false - required: false + description: Profile to use. - name: target - description: The target to fetch. type: string - repeated: false + description: The target to fetch. required: true - name: args - description: Args for the target class. type: vfilter.Any - repeated: false - required: false + description: Args for the target class. - name: start - description: The initial field in the target to fetch. type: string - repeated: false - required: false - category: parsers + description: The initial field in the target to fetch. - name: cache description: Creates a cache object type: Function args: - name: func - description: A function to evaluate type: LazyExpr - repeated: false + description: A function to evaluate required: true - name: name - description: The global name of this cache (needed when more than one) type: string - repeated: false - required: false + description: The global name of this cache (needed when more than one) - name: key - description: Cache key to use. type: string - repeated: false + description: Cache key to use. required: true - name: period - description: The latest age of the cache. type: int64 - repeated: false - required: false - category: basic + description: The latest age of the cache. - name: cancel_flow description: Cancels the flow. type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - name: flow_id - description: "" type: string - repeated: false - required: false - category: server - name: certificates description: Collect certificate from the system trust store. type: Plugin - args: [] - category: windows - name: chain description: | Chain the output of several queries into the same table. This plugin @@ -304,81 +227,54 @@ b={ SELECT ...}) ``` type: Plugin - args: [] - category: plugin - name: client_delete description: 'Delete all information related to a client. ' type: Plugin args: - name: client_id - description: "" type: string - repeated: false required: true - name: really_do_it - description: "" type: bool - repeated: false - required: false - category: server - name: client_info description: Returns client info (like the fqdn) from the datastore. type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: server - name: client_metadata description: Returns client metadata from the datastore. Client metadata is a set of free form key/value data type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: server - name: client_set_metadata description: Sets client metadata. Client metadata is a set of free form key/value data type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: server - name: clients description: Retrieve the list of clients. type: Plugin args: - name: search + type: string description: 'Client search string. Can have the following prefixes: ''lable:'', ''host:''' - type: string - repeated: false - required: false - name: start - description: First client to fetch (0)' type: uint64 - repeated: false - required: false + description: First client to fetch (0)' - name: count - description: Maximum number of clients to fetch (1000)' type: uint64 - repeated: false - required: false + description: Maximum number of clients to fetch (1000)' - name: client_id - description: "" type: string - repeated: false - required: false - category: server - name: clock description: | Generate a timestamp periodically. This is mostly useful for event @@ -403,72 +299,48 @@ type: Plugin args: - name: start - description: Start at this time. type: Any - repeated: false - required: false + description: Start at this time. - name: period - description: Wait this many seconds between events. type: int64 - repeated: false - required: false + description: Wait this many seconds between events. - name: ms - description: Wait this many ms between events. type: int64 - repeated: false - required: false - category: event + description: Wait this many ms between events. - name: collect description: Collect artifacts into a local file. type: Plugin args: - name: artifacts - description: A list of artifacts to collect. type: string + description: A list of artifacts to collect. repeated: true required: true - name: output - description: A path to write the output file on. type: string - repeated: false - required: false + description: A path to write the output file on. - name: report - description: A path to write the report on. type: string - repeated: false - required: false + description: A path to write the report on. - name: args - description: Optional parameters. type: Any - repeated: false - required: false + description: Optional parameters. - name: password - description: An optional password to encrypt the collection zip. type: string - repeated: false - required: false + description: An optional password to encrypt the collection zip. - name: format - description: Output format (csv, jsonl). type: string - repeated: false - required: false + description: Output format (csv, jsonl). - name: artifact_definitions - description: Optional additional custom artifacts. type: Any - repeated: false - required: false + description: Optional additional custom artifacts. - name: template + type: string description: The name of a template artifact (i.e. one which has report of type HTML). - type: string - repeated: false - required: false - name: level - description: Compression level between 0 (no compression) and 9. type: int64 - repeated: false - required: false - category: plugin + description: Compression level between 0 (no compression) and 9. - name: collect_client description: | Launch an artifact collection against a client. If the client_id @@ -477,66 +349,48 @@ type: Function args: - name: client_id - description: The client id to schedule a collection on type: string - repeated: false + description: The client id to schedule a collection on required: true - name: artifacts - description: A list of artifacts to collect type: string + description: A list of artifacts to collect repeated: true required: true - name: env - description: Parameters to apply to the artifact (an alternative to a full spec) type: Any - repeated: false - required: false + description: Parameters to apply to the artifact (an alternative to a full spec) - name: spec - description: Parameters to apply to the artifacts type: Any - repeated: false - required: false + description: Parameters to apply to the artifacts - name: timeout - description: Set query timeout (default 10 min) type: uint64 - repeated: false - required: false + description: Set query timeout (default 10 min) - name: ops_per_sec - description: Set query ops_per_sec value type: float64 - repeated: false - required: false + description: Set query ops_per_sec value - name: max_rows - description: Max number of rows to fetch type: uint64 - repeated: false - required: false + description: Max number of rows to fetch - name: max_bytes - description: Max number of bytes to upload type: uint64 - repeated: false - required: false - category: server + description: Max number of bytes to upload - name: column_filter description: Select columns from another query using regex. type: Plugin args: - name: query - description: This query will be run to produce the columns. type: StoredQuery - repeated: false + description: This query will be run to produce the columns. required: true - name: exclude - description: One of more regular expressions that will exclude columns. type: string + description: One of more regular expressions that will exclude columns. repeated: true - required: false - name: include - description: One of more regular expressions that will include columns. type: string + description: One of more regular expressions that will include columns. repeated: true - required: false - category: "" - name: compress description: | Compress a file in the server's FileStore. A compressed @@ -545,116 +399,82 @@ type: Function args: - name: path - description: A VFS path to compress type: string + description: A VFS path to compress repeated: true required: true - category: server - name: connections description: List all active connections type: Plugin - args: [] - category: client - name: copy description: Copy a file. type: Function args: - name: filename - description: The file to copy from. type: string - repeated: false + description: The file to copy from. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: dest - description: The destination file to write. type: string - repeated: false + description: The destination file to write. required: true - name: permissions - description: Required permissions (e.g. 'x'). type: string - repeated: false - required: false - category: basic + description: Required permissions (e.g. 'x'). - name: count description: Counts the items. type: Function args: - name: items - description: Not used anymore type: Any - repeated: false - required: false - category: basic + description: Not used anymore - name: create_flow_download description: Creates a download pack for the flow. type: Function args: - name: client_id - description: Client ID to export. type: string - repeated: false + description: Client ID to export. required: true - name: flow_id - description: The flow id to export. type: string - repeated: false + description: The flow id to export. required: true - name: wait - description: If set we wait for the download to complete before returning. type: bool - repeated: false - required: false + description: If set we wait for the download to complete before returning. - name: type - description: Type of download to create (e.g. 'report') default a full zip file. type: string - repeated: false - required: false + description: Type of download to create (e.g. 'report') default a full zip file. - name: template - description: Report template to use (defaults to Reporting.Default). type: string - repeated: false - required: false - category: server + description: Report template to use (defaults to Reporting.Default). - name: create_hunt_download description: Creates a download pack for a hunt. type: Function args: - name: hunt_id - description: Hunt ID to export. type: string - repeated: false + description: Hunt ID to export. required: true - name: only_combined - description: If set we only export combined results. type: bool - repeated: false - required: false + description: If set we only export combined results. - name: wait - description: If set we wait for the download to complete before returning. type: bool - repeated: false - required: false + description: If set we wait for the download to complete before returning. - name: format - description: Format to export (csv,json) defaults to both. type: string - repeated: false - required: false + description: Format to export (csv,json) defaults to both. - name: base - description: Base filename to write to. type: string - repeated: false - required: false - category: server + description: Base filename to write to. - name: dict description: Construct a dict from arbitrary keyword args. type: Function - args: [] - category: basic - name: diff description: | Executes 'query' periodically and emit differences from the last query. @@ -700,36 +520,27 @@ type: Plugin args: - name: query - description: Source for cached rows. type: StoredQuery - repeated: false + description: Source for cached rows. required: true - name: key - description: The column to use as key. type: string - repeated: false + description: The column to use as key. required: true - name: period - description: Number of seconds between evaluation of the query. type: int64 - repeated: false - required: false - category: event + description: Number of seconds between evaluation of the query. - name: dirname description: Return the directory path. type: Function args: - name: path - description: Extract directory name of path type: string - repeated: false + description: Extract directory name of path required: true - name: sep - description: Separator to use (default /) type: string - repeated: false - required: false - category: basic + description: Separator to use (default /) - name: dns description: | Monitor dns queries. @@ -750,126 +561,86 @@ {{% /notice %}} type: Plugin - args: [] - category: windows - name: elastic_upload description: Upload rows to elastic. type: Plugin args: - name: query - description: Source for rows to upload. type: StoredQuery - repeated: false + description: Source for rows to upload. required: true - name: threads - description: How many threads to use. type: int64 - repeated: false - required: false + description: How many threads to use. - name: index + type: string description: The name of the index to upload to. If not specified ensure a column is named '_index'. - type: string - repeated: false - required: false - name: type - description: The type of the index to upload to. type: string - repeated: false + description: The type of the index to upload to. required: true - name: chunk_size - description: The number of rows to send at the time. type: int64 - repeated: false - required: false + description: The number of rows to send at the time. - name: addresses - description: A list of Elasticsearch nodes to use. type: string + description: A list of Elasticsearch nodes to use. repeated: true - required: false - name: username - description: Username for HTTP Basic Authentication. type: string - repeated: false - required: false + description: Username for HTTP Basic Authentication. - name: password - description: Password for HTTP Basic Authentication. type: string - repeated: false - required: false + description: Password for HTTP Basic Authentication. - name: cloud_id - description: Endpoint for the Elastic Service (https://elastic.co/cloud). type: string - repeated: false - required: false + description: Endpoint for the Elastic Service (https://elastic.co/cloud). - name: api_key + type: string description: Base64-encoded token for authorization; if set, overrides username and password. - type: string - repeated: false - required: false - name: wait_time - description: Batch elastic upload this long (2 sec). type: int64 - repeated: false - required: false + description: Batch elastic upload this long (2 sec). - name: pipeline - description: Pipeline for uploads type: string - repeated: false - required: false - category: server + description: Pipeline for uploads - name: encode description: Encodes a string as as different type. Currently supported types include 'hex', 'base64'. type: Function args: - name: string - description: "" type: Any - repeated: false required: true - name: type - description: "" type: string - repeated: false required: true - category: basic - name: enumerate description: Collect all the items in each group by bin. type: Function args: - name: items - description: Not used anymore type: Any - repeated: false - required: false - category: basic + description: Not used anymore - name: enumerate_flow description: Enumerate all the files that make up a flow. type: Plugin args: - name: client_id - description: "" type: string - repeated: false required: true - name: flow_id - description: "" type: string - repeated: false - required: false - category: server - name: environ description: Get an environment variable. type: Function args: - name: var - description: Extract the var from the environment. type: string - repeated: false + description: Extract the var from the environment. required: true - category: basic - name: environ description: | The row returned will have all environment variables as @@ -878,12 +649,10 @@ type: Plugin args: - name: vars + type: string description: Extract these variables from the environment and return them one per row - type: string repeated: true - required: false - category: plugin - name: execve description: | This plugin launches an external command and captures its STDERR, @@ -904,21 +673,16 @@ type: Plugin args: - name: argv - description: Argv to run the command with. type: string + description: Argv to run the command with. repeated: true required: true - name: sep - description: The serparator that will be used to split the stdout into rows. type: string - repeated: false - required: false + description: The serparator that will be used to split the stdout into rows. - name: length - description: Size of buffer to capture output per row. type: int64 - repeated: false - required: false - category: plugin + description: Size of buffer to capture output per row. - name: expand description: | Expand the path using the environment. @@ -932,11 +696,9 @@ type: Function args: - name: path - description: A path with environment escapes type: string - repeated: false + description: A path with environment escapes required: true - category: basic - name: fifo description: | Executes 'query' and cache a number of rows from it. For each invocation @@ -991,26 +753,18 @@ type: Plugin args: - name: query - description: Source for cached rows. type: StoredQuery - repeated: false + description: Source for cached rows. required: true - name: max_age - description: Maximum number of seconds to hold rows in the fifo. type: int64 - repeated: false - required: false + description: Maximum number of seconds to hold rows in the fifo. - name: max_rows - description: Maximum number of rows to hold in the fifo. type: int64 - repeated: false - required: false + description: Maximum number of rows to hold in the fifo. - name: flush - description: If specified we flush all rows from cache after the call. type: bool - repeated: false - required: false - category: event + description: If specified we flush all rows from cache after the call. - name: file_store description: | Resolves file store paths into full filesystem paths. @@ -1035,158 +789,115 @@ type: Function args: - name: path - description: A VFS path to convert type: string + description: A VFS path to convert repeated: true required: true - category: server - name: file_store_delete description: 'Delete file store paths into full filesystem paths. ' type: Function args: - name: path - description: A VFS path to remove type: string - repeated: false + description: A VFS path to remove required: true - category: server - name: filesystems - description: "" type: Plugin - args: [] - category: plugin - name: filter description: | Filters a strings array by regex. type: Function args: - name: list - description: A list of items to filter type: string + description: A list of items to filter repeated: true required: true - name: regex - description: A regex to test each item type: string + description: A regex to test each item repeated: true required: true - category: basic - name: flatten description: Flatten the columns in query. If any column repeats then we repeat the entire row once for each item. type: Plugin args: - name: Name - description: "" type: string - repeated: false - required: false - category: plugin - name: flow_results description: Retrieve the results of a flow. type: Plugin args: - name: artifact - description: The artifact to retrieve type: string - repeated: false - required: false + description: The artifact to retrieve - name: source - description: An optional source within the artifact. type: string - repeated: false - required: false + description: An optional source within the artifact. - name: flow_id - description: The hunt id to read. type: string - repeated: false + description: The hunt id to read. required: true - name: client_id - description: The client id to extract type: string - repeated: false + description: The client id to extract required: true - category: server - name: flows description: Retrieve the flows launched on each client. type: Plugin args: - name: client_id - description: "" type: string - repeated: false required: true - name: flow_id - description: "" type: string - repeated: false - required: false - category: server - name: for description: Iterate over a list. type: Plugin args: - name: var - description: The variable to assign. type: string - repeated: false + description: The variable to assign. required: true - name: foreach - description: The variable to iterate over. type: StoredQuery - repeated: false + description: The variable to iterate over. required: true - name: query - description: Run this query over the item. type: StoredQuery - repeated: false - required: false - category: plugin + description: Run this query over the item. - name: foreach description: Executes 'query' once for each row in the 'row' query. type: Plugin args: - name: row - description: A query or slice which generates rows. type: LazyExpr - repeated: false + description: A query or slice which generates rows. required: true - name: query - description: Run this query for each row. type: StoredQuery - repeated: false - required: false + description: Run this query for each row. - name: async - description: If set we run all queries asyncronously (implies workers=1000). type: bool - repeated: false - required: false + description: If set we run all queries asyncronously (implies workers=1000). - name: workers - description: Total number of asyncronous workers. type: int64 - repeated: false - required: false + description: Total number of asyncronous workers. - name: column - description: If set we only extract the column from row. type: string - repeated: false - required: false - category: plugin + description: If set we only extract the column from row. - name: format description: Format one or more items according to a format string. type: Function args: - name: format - description: Format string to use type: string - repeated: false + description: Format string to use required: true - name: args - description: An array of elements to apply into the format string. type: Any - repeated: false - required: false - category: basic + description: An array of elements to apply into the format string. - name: get description: | Gets the member field from item. @@ -1207,41 +918,22 @@ type: Function args: - name: item - description: "" type: Any - repeated: false - required: false - name: member - description: "" type: string - repeated: false - required: false - name: field - description: "" type: Any - repeated: false - required: false - name: default - description: "" type: Any - repeated: false - required: false - category: basic - name: get_client_monitoring description: Retrieve the current client monitoring state. type: Function - args: [] - category: server - name: get_server_monitoring description: Retrieve the current client monitoring state. type: Function - args: [] - category: server - name: getpid description: Returns the current pid of the process. type: Function - args: [] - category: basic - name: glob description: | Retrieve files based on a list of glob expressions @@ -1296,91 +988,67 @@ type: Plugin args: - name: globs - description: One or more glob patterns to apply to the filesystem. type: string + description: One or more glob patterns to apply to the filesystem. repeated: true required: true - name: root - description: The root directory to glob from (default ''). type: string - repeated: false - required: false + description: The root directory to glob from (default ''). - name: accessor - description: An accessor to use. type: string - repeated: false - required: false + description: An accessor to use. - name: nosymlink - description: If set we do not follow symlinks. type: bool - repeated: false - required: false - category: plugin + description: If set we do not follow symlinks. - name: grep description: Search a file for keywords. type: Function args: - name: path - description: path to open. type: string - repeated: false + description: path to open. required: true - name: accessor - description: An accessor to use. type: string - repeated: false - required: false + description: An accessor to use. - name: keywords - description: Keywords to search for. type: string + description: Keywords to search for. repeated: true required: true - name: context - description: Extract this many bytes as context around hits. type: int - repeated: false - required: false - category: plugin + description: Extract this many bytes as context around hits. - name: grok description: Parse a string using a Grok expression. type: Function args: - name: grok - description: Grok pattern. type: string - repeated: false + description: Grok pattern. required: true - name: data - description: String to parse. type: string - repeated: false + description: String to parse. required: true - name: patterns - description: Additional patterns. type: Any - repeated: false - required: false + description: Additional patterns. - name: all_captures - description: Extract all captures. type: bool - repeated: false - required: false - category: parsers + description: Extract all captures. - name: gui_users description: Retrieve the list of users on the server. type: Plugin - args: [] - category: server - name: handles description: Enumerate process handles. type: Plugin args: - name: pid - description: The PID to dump out. type: int64 - repeated: false + description: The PID to dump out. required: true - category: windows - name: hash description: | Calculate the hash of a file. @@ -1389,16 +1057,12 @@ type: Function args: - name: path - description: Path to open and hash. type: string - repeated: false + description: Path to open and hash. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: plugin + description: The accessor to use - name: http_client description: | Make a http request. @@ -1452,53 +1116,35 @@ type: Plugin args: - name: url - description: The URL to fetch type: string - repeated: false + description: The URL to fetch required: true - name: params - description: Parameters to encode as POST or GET query strings type: Any - repeated: false - required: false + description: Parameters to encode as POST or GET query strings - name: headers - description: A dict of headers to send. type: Any - repeated: false - required: false + description: A dict of headers to send. - name: method - description: HTTP method to use (GET, POST) type: string - repeated: false - required: false + description: HTTP method to use (GET, POST) - name: data + type: string description: If specified we write this raw data into a POST request instead of encoding the params above. - type: string - repeated: false - required: false - name: chunk_size - description: Read input with this chunk size and send each chunk as a row type: int - repeated: false - required: false + description: Read input with this chunk size and send each chunk as a row - name: disable_ssl_security - description: Disable ssl certificate verifications. type: bool - repeated: false - required: false + description: Disable ssl certificate verifications. - name: tempfile_extension + type: string description: If specified we write to a tempfile. The content field will contain the full path to the tempfile. - type: string - repeated: false - required: false - name: remove_last - description: If set we delay removal as much as possible. type: bool - repeated: false - required: false - category: plugin + description: If set we delay removal as much as possible. - name: humanize description: | Format items in human readable way. @@ -1507,131 +1153,90 @@ type: Function args: - name: bytes - description: Format bytes with units type: int64 - repeated: false - required: false - category: basic + description: Format bytes with units - name: hunt description: Launch an artifact collection against a client. type: Function args: - name: description - description: Description of the hunt type: string - repeated: false + description: Description of the hunt required: true - name: artifacts - description: A list of artifacts to collect type: string + description: A list of artifacts to collect repeated: true required: true - name: expires - description: Number of milliseconds since epoch for expiry type: uint64 - repeated: false - required: false + description: Number of milliseconds since epoch for expiry - name: spec - description: Parameters to apply to the artifacts type: Any - repeated: false - required: false + description: Parameters to apply to the artifacts - name: timeout - description: Set query timeout (default 10 min) type: uint64 - repeated: false - required: false + description: Set query timeout (default 10 min) - name: ops_per_sec - description: Set query ops_per_sec value type: float64 - repeated: false - required: false + description: Set query ops_per_sec value - name: max_rows - description: Max number of rows to fetch type: uint64 - repeated: false - required: false + description: Max number of rows to fetch - name: max_bytes - description: Max number of bytes to upload type: uint64 - repeated: false - required: false + description: Max number of bytes to upload - name: pause - description: If specified the new hunt will be in the paused state type: bool - repeated: false - required: false - category: server + description: If specified the new hunt will be in the paused state - name: hunt_add description: Assign a client to a hunt. type: Function args: - name: ClientId - description: "" type: string - repeated: false required: true - name: HuntId - description: "" type: string - repeated: false required: true - category: server - name: hunt_flows description: Retrieve the flows launched by a hunt. type: Plugin args: - name: hunt_id - description: The hunt id to inspect. type: string - repeated: false + description: The hunt id to inspect. required: true - name: start_row - description: The first row to show (used for paging). type: int64 - repeated: false - required: false + description: The first row to show (used for paging). - name: limit - description: Number of rows to show (used for paging). type: int64 - repeated: false - required: false - category: server + description: Number of rows to show (used for paging). - name: hunt_results description: Retrieve the results of a hunt. type: Plugin args: - name: artifact - description: The artifact to retrieve type: string - repeated: false - required: false + description: The artifact to retrieve - name: source - description: An optional source within the artifact. type: string - repeated: false - required: false + description: An optional source within the artifact. - name: hunt_id - description: The hunt id to read. type: string - repeated: false + description: The hunt id to read. required: true - name: brief - description: If set we return less columns. type: bool - repeated: false - required: false - category: server + description: If set we return less columns. - name: hunts description: Retrieve the list of hunts. type: Plugin args: - name: hunt_id - description: A hunt id to read, if not specified we list all of them. type: string - repeated: false - required: false - category: server + description: A hunt id to read, if not specified we list all of them. - name: if description: | Conditional execution of query @@ -1646,21 +1251,12 @@ type: Function args: - name: condition - description: "" type: Any - repeated: false required: true - name: then - description: "" type: LazyExpr - repeated: false - required: false - name: else - description: "" type: LazyExpr - repeated: false - required: false - category: basic - name: if description: | Conditional execution of query @@ -1676,46 +1272,31 @@ type: Plugin args: - name: condition - description: "" type: Any - repeated: false required: true - name: then - description: "" type: StoredQuery - repeated: false required: true - name: else - description: "" type: StoredQuery - repeated: false - required: false - category: plugin - name: import_collection description: Imports an offline collection zip file (experimental). type: Function args: - name: client_id - description: The client id to import to. Use 'auto' to generate a new client id. type: string - repeated: false + description: The client id to import to. Use 'auto' to generate a new client id. required: true - name: hostname - description: When creating a new client, set this as the hostname. type: string - repeated: false - required: false + description: When creating a new client, set this as the hostname. - name: filename - description: Path on server to the collector zip. type: string - repeated: false + description: Path on server to the collector zip. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: "" + description: The accessor to use - name: info description: | Get information about the running host. @@ -1727,78 +1308,48 @@ This plugin is very useful in preconditions as it restricts a query to certain OS or versions. type: Plugin - args: [] - category: plugin - name: int description: Truncate to an integer. type: Function args: - name: int - description: The integer to round type: Any - repeated: false - required: false - category: plugin + description: The integer to round - name: interfaces description: List all active interfaces. type: Plugin - args: [] - category: windows - name: inventory description: Retrieve the tools inventory. type: Plugin - args: [] - category: server - name: inventory_add description: Add tool to ThirdParty inventory. type: Function args: - name: tool - description: "" type: string - repeated: false required: true - name: serve_locally - description: "" type: bool - repeated: false - required: false - name: url - description: "" type: string - repeated: false - required: false - name: hash - description: "" type: string - repeated: false - required: false - name: filename - description: The name of the file on the endpoint type: string - repeated: false - required: false + description: The name of the file on the endpoint - name: file - description: An optional file to upload type: string - repeated: false - required: false + description: An optional file to upload - name: accessor - description: The accessor to use to read the file. type: string - repeated: false - required: false - category: server + description: The accessor to use to read the file. - name: inventory_get description: Get tool info from inventory service. type: Function args: - name: tool - description: "" type: string - repeated: false required: true - category: server - name: ip description: | Format an IP address. @@ -1817,36 +1368,24 @@ type: Function args: - name: netaddr4_le - description: A network order IPv4 address (as little endian). type: int64 - repeated: false - required: false + description: A network order IPv4 address (as little endian). - name: netaddr4_be - description: A network order IPv4 address (as big endian). type: int64 - repeated: false - required: false - category: plugin + description: A network order IPv4 address (as big endian). - name: items description: Iterate over dict members producing _key and _value columns type: Function args: - name: item - description: "" type: Any - repeated: false - required: false - category: basic - name: items description: Enumerate all members of the item (similar to Pythons items() method. type: Plugin args: - name: item - description: The item to enumerate. type: Any - repeated: false - required: false - category: basic + description: The item to enumerate. - name: join description: | Join all the args on a separator. @@ -1855,188 +1394,144 @@ type: Function args: - name: array - description: The array to join type: string + description: The array to join repeated: true required: true - name: sep - description: The separator type: string - repeated: false - required: false - category: basic + description: The separator - name: js description: Compile and run javascript code. type: Function args: - name: js - description: The body of the javascript code. type: string - repeated: false + description: The body of the javascript code. required: true - name: key - description: If set use this key to cache the JS VM. type: string - repeated: false - required: false - category: experimental + description: If set use this key to cache the JS VM. - name: js_call description: Compile and run javascript code. type: Function args: - name: func - description: JS function to call. type: string - repeated: false + description: JS function to call. required: true - name: args - description: Positional args for the function. type: Any - repeated: false - required: false + description: Positional args for the function. - name: key - description: If set use this key to cache the JS VM. type: string - repeated: false - required: false - category: experimental + description: If set use this key to cache the JS VM. - name: js_get description: Get a variable's value from the JS VM. type: Function args: - name: var - description: The variable to get from the JS VM. type: string - repeated: false + description: The variable to get from the JS VM. required: true - name: key - description: If set use this key to cache the JS VM. type: string - repeated: false - required: false - category: plugin + description: If set use this key to cache the JS VM. - name: js_set description: Set a variables value in the JS VM. type: Function args: - name: var - description: The variable to set inside the JS VM. type: string - repeated: false + description: The variable to set inside the JS VM. required: true - name: value - description: The value to set inside the VM. type: Any - repeated: false + description: The value to set inside the VM. required: true - name: key - description: If set use this key to cache the JS VM. type: string - repeated: false - required: false - category: plugin + description: If set use this key to cache the JS VM. - name: killkillkill description: Kills the client and forces a restart - this is very aggresive! type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: basic - name: label description: | Add the labels to the client. If op is 'remove' then remove these labels. type: Function args: - name: client_id - description: Client ID to label. type: string - repeated: false + description: Client ID to label. required: true - name: labels - description: A list of labels to apply type: string + description: A list of labels to apply repeated: true required: true - name: op - description: An operation on the labels (set, check, remove) type: string - repeated: false - required: false - category: server + description: An operation on the labels (set, check, remove) - name: len description: Returns the length of an object. type: Function args: - name: list - description: A list of items too filter type: Any - repeated: false + description: A list of items too filter required: true - category: basic - name: log description: Log the message. type: Function args: - name: message - description: Message to log. type: string - repeated: false + description: Message to log. required: true - category: basic - name: lookupSID description: Get information about the SID. type: Function args: - name: sid - description: 'A SID to lookup using LookupAccountSid ' type: string - repeated: false + description: 'A SID to lookup using LookupAccountSid ' required: true - category: windows - name: lowcase - description: "" type: Function args: - name: string - description: A string to lower type: string - repeated: false + description: A string to lower required: true - category: basic - name: mail description: Send Email to a remote server. type: Plugin args: - name: to - description: Receipient of the mail type: string + description: Receipient of the mail repeated: true required: true - name: cc - description: A cc for the mail type: string + description: A cc for the mail repeated: true - required: false - name: subject - description: The subject. type: string - repeated: false - required: false + description: The subject. - name: body - description: The body of the mail. type: string - repeated: false + description: The body of the mail. required: true - name: period + type: int64 description: How long to wait before sending the next mail - help to throttle mails. - type: int64 - repeated: false required: true - category: server - name: max description: | Finds the largest item in the aggregate. @@ -2054,31 +1549,23 @@ type: Function args: - name: item - description: "" type: int64 - repeated: false required: true - category: basic - name: memoize description: Memoize a query into memory. type: Function args: - name: query - description: Query to expand into memory type: LazyExpr - repeated: false + description: Query to expand into memory required: true - name: key - description: The name of the column to use as a key. type: string - repeated: false + description: The name of the column to use as a key. required: true - name: period - description: The latest age of the cache. type: int64 - repeated: false - required: false - category: basic + description: The latest age of the cache. - name: min description: | Finds the smallest item in the aggregate. @@ -2096,112 +1583,78 @@ type: Function args: - name: item - description: "" type: int64 - repeated: false required: true - category: basic - name: mock description: Mock a plugin. type: Function args: - name: plugin - description: The plugin to mock type: string - repeated: false - required: false + description: The plugin to mock - name: function - description: The function to mock type: string - repeated: false - required: false + description: The function to mock - name: artifact - description: The artifact to mock type: Any - repeated: false - required: false + description: The artifact to mock - name: results - description: The result to return type: Any - repeated: false + description: The result to return required: true - category: utils - name: mock_check description: Check expectations on a mock. type: Function args: - name: plugin - description: The plugin to mock type: string - repeated: false - required: false + description: The plugin to mock - name: function - description: The function to mock type: string - repeated: false - required: false + description: The function to mock - name: expected_calls - description: How many times plugin should be called type: int - repeated: false - required: false + description: How many times plugin should be called - name: clear - description: This call will clear previous mocks for this plugin type: bool - repeated: false - required: false - category: utils + description: This call will clear previous mocks for this plugin - name: modules description: Enumerate Loaded DLLs. type: Plugin args: - name: pid - description: The PID to dump out. type: int64 - repeated: false + description: The PID to dump out. required: true - category: windows - name: monitoring description: Extract monitoring log from a client. If client_id is not specified we watch the global journal which contains event logs from all clients. type: Plugin args: - name: artifact - description: The event artifact name to watch type: string - repeated: false + description: The event artifact name to watch required: true - category: server - name: netstat description: Collect network information. type: Plugin - args: [] - category: windows - name: notebook_delete description: 'Delete a notebook with all its cells. ' type: Plugin args: - name: notebook_id - description: "" type: string - repeated: false required: true - name: really_do_it - description: "" type: bool - repeated: false - required: false - category: server - name: now description: Returns current time in seconds since epoch. type: Function args: - name: string - description: A string to convert to int type: Any - repeated: false + description: A string to convert to int required: true - category: basic - name: olevba description: | Extracts VBA Macros from Office documents. @@ -2212,136 +1665,93 @@ type: Plugin args: - name: file - description: A list of filenames to open as OLE files. type: string + description: A list of filenames to open as OLE files. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: max_size - description: Maximum size of file we load into memory. type: int64 - repeated: false - required: false - category: parsers + description: Maximum size of file we load into memory. - name: parallelize description: Runs query on result batches in parallel. type: Plugin args: - name: query - description: The query will be run in parallel over batches. type: StoredQuery - repeated: false + description: The query will be run in parallel over batches. required: true - name: client_id - description: The client id to extract type: string - repeated: false - required: false + description: The client id to extract - name: flow_id - description: A flow ID (client or server artifacts) type: string - repeated: false - required: false + description: A flow ID (client or server artifacts) - name: hunt_id - description: Retrieve sources from this hunt (combines all results from all clients) type: string - repeated: false - required: false + description: Retrieve sources from this hunt (combines all results from all clients) - name: artifact - description: The name of the artifact collection to fetch type: string - repeated: false - required: false + description: The name of the artifact collection to fetch - name: source - description: An optional named source within the artifact type: string - repeated: false - required: false + description: An optional named source within the artifact - name: start_time - description: Start return events from this date (for event sources) type: int64 - repeated: false - required: false + description: Start return events from this date (for event sources) - name: end_time - description: Stop end events reach this time (event sources). type: int64 - repeated: false - required: false + description: Stop end events reach this time (event sources). - name: notebook_id - description: The notebook to read from (shoud also include cell id) type: string - repeated: false - required: false + description: The notebook to read from (shoud also include cell id) - name: notebook_cell_id - description: The notebook cell read from (shoud also include notebook id) type: string - repeated: false - required: false + description: The notebook cell read from (shoud also include notebook id) - name: notebook_cell_table - description: A notebook cell can have multiple tables.) type: int64 - repeated: false - required: false + description: A notebook cell can have multiple tables.) - name: workers - description: Number of workers to spawn.) type: int64 - repeated: false - required: false + description: Number of workers to spawn.) - name: batch - description: Number of rows in each batch.) type: int64 - repeated: false - required: false - category: server + description: Number of rows in each batch.) - name: parse_auditd description: Parse log files generated by auditd. type: Plugin args: - name: filename - description: A list of log files to parse. type: string + description: A list of log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_binary description: Parse a binary file into a datastructure using a profile. type: Function args: - name: filename - description: Binary file to open. type: string - repeated: false + description: Binary file to open. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: profile - description: Profile to use (see https://github.com/Velocidex/vtypes). type: string - repeated: false - required: false + description: Profile to use (see https://github.com/Velocidex/vtypes). - name: struct - description: Name of the struct in the profile to instantiate. type: string - repeated: false + description: Name of the struct in the profile to instantiate. required: true - name: offset - description: Start parsing from this offset type: int64 - repeated: false - required: false - category: parsers + description: Start parsing from this offset - name: parse_csv description: | Parses events from a CSV file. @@ -2374,46 +1784,33 @@ type: Plugin args: - name: filename - description: CSV files to open type: string + description: CSV files to open repeated: true required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: auto_headers - description: If unset the first row is headers type: bool - repeated: false - required: false + description: If unset the first row is headers - name: separator - description: Comma separator type: string - repeated: false - required: false - category: parsers + description: Comma separator - name: parse_ese description: Opens an ESE file and dump a table. type: Plugin args: - name: file - description: "" type: string - repeated: false required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: table - description: A table name to dump type: string - repeated: false + description: A table name to dump required: true - category: parsers - name: parse_evtx description: | Parses events from an EVTX file. @@ -2443,31 +1840,24 @@ type: Plugin args: - name: filename - description: A list of event log files to parse. type: string + description: A list of event log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: messagedb - description: A Message database from https://github.com/Velocidex/evtx-data. type: string - repeated: false - required: false - category: parsers + description: A Message database from https://github.com/Velocidex/evtx-data. - name: parse_float description: Convert a string to a float. type: Function args: - name: string - description: A string to convert to int type: Any - repeated: false + description: A string to convert to int required: true - category: parsers - name: parse_json description: | Parse a JSON string into an object. @@ -2478,11 +1868,9 @@ type: Function args: - name: data - description: Json encoded string. type: string - repeated: false + description: Json encoded string. required: true - category: parsers - name: parse_json_array description: | Parse a JSON string into an array. @@ -2492,169 +1880,124 @@ type: Function args: - name: data - description: Json encoded string. type: string - repeated: false + description: Json encoded string. required: true - category: parsers - name: parse_json_array description: Parses events from a line oriented json file. type: Plugin args: - name: data - description: Json encoded string. type: string - repeated: false + description: Json encoded string. required: true - category: parsers - name: parse_jsonl description: Parses a line oriented json file. type: Plugin args: - name: filename - description: JSON file to open type: string - repeated: false + description: JSON file to open required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: parsers + description: The accessor to use - name: parse_lines description: Parse a file separated into lines. type: Plugin args: - name: filename - description: A list of log files to parse. type: string + description: A list of log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_mft description: Scan the $MFT from an NTFS volume. type: Plugin args: - name: filename - description: A list of event log files to parse. type: string - repeated: false + description: A list of event log files to parse. required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_ntfs description: Parse an NTFS image file. type: Function args: - name: device + type: string description: The device file to open. This may be a full path - we will figure out the device automatically. - type: string - repeated: false required: true - name: inode - description: The MFT entry to parse in inode notation (5-144-1). type: string - repeated: false - required: false + description: The MFT entry to parse in inode notation (5-144-1). - name: mft - description: The MFT entry to parse. type: int64 - repeated: false - required: false + description: The MFT entry to parse. - name: mft_offset - description: The offset to the MFT entry to parse. type: int64 - repeated: false - required: false - category: parsers + description: The offset to the MFT entry to parse. - name: parse_ntfs_i30 description: Scan the $I30 stream from an NTFS MFT entry. type: Plugin args: - name: device + type: string description: The device file to open. This may be a full path - we will figure out the device automatically. - type: string - repeated: false required: true - name: inode - description: The MFT entry to parse in inode notation (5-144-1). type: string - repeated: false - required: false + description: The MFT entry to parse in inode notation (5-144-1). - name: mft - description: The MFT entry to parse. type: int64 - repeated: false - required: false + description: The MFT entry to parse. - name: mft_offset - description: The offset to the MFT entry to parse. type: int64 - repeated: false - required: false - category: parsers + description: The offset to the MFT entry to parse. - name: parse_ntfs_ranges description: Show the run ranges for an NTFS stream. type: Plugin args: - name: device + type: string description: The device file to open. This may be a full path - we will figure out the device automatically. - type: string - repeated: false required: true - name: inode - description: The MFT entry to parse in inode notation (5-144-1). type: string - repeated: false - required: false + description: The MFT entry to parse in inode notation (5-144-1). - name: mft - description: The MFT entry to parse. type: int64 - repeated: false - required: false + description: The MFT entry to parse. - name: mft_offset - description: The offset to the MFT entry to parse. type: int64 - repeated: false - required: false - category: parsers + description: The offset to the MFT entry to parse. - name: parse_pe description: Parse a PE file. type: Function args: - name: file - description: The PE file to open. type: string - repeated: false + description: The PE file to open. required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_pkcs7 description: Parse a DER encoded pkcs7 string into an object. type: Function args: - name: data - description: PKCS7 DER encoded string. type: string - repeated: false + description: PKCS7 DER encoded string. required: true - category: "" - name: parse_records_with_regex description: | Parses a file with a set of regexp and yields matches as records. The @@ -2730,118 +2073,93 @@ type: Plugin args: - name: file - description: A list of files to parse. type: string + description: A list of files to parse. repeated: true required: true - name: regex - description: A list of regex to apply to the file data. type: string + description: A list of regex to apply to the file data. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_recyclebin description: Parses a $I file found in the $Recycle.Bin type: Plugin args: - name: filename - description: Files to be parsed. type: string + description: Files to be parsed. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: parse_string_with_regex description: Parse a string with a set of regex and extract fields. Returns a dict with fields populated from all regex capture variables. type: Function args: - name: string - description: A string to parse. type: string - repeated: false + description: A string to parse. required: true - name: regex - description: The regex to apply. type: string + description: The regex to apply. repeated: true required: true - category: parsers - name: parse_usn description: Parse the USN journal from a device. type: Plugin args: - name: device - description: The device file to open. type: string - repeated: false + description: The device file to open. required: true - name: start_offset - description: The starting offset of the first USN record to parse. type: int64 - repeated: false - required: false - category: parsers + description: The starting offset of the first USN record to parse. - name: parse_x509 description: Parse a DER encoded x509 string into an object. type: Function args: - name: data - description: X509 DER encoded string. type: string - repeated: false + description: X509 DER encoded string. required: true - category: "" - name: parse_xml description: | Parse an XML document into a dict like object. type: Function args: - name: file - description: XML file to open. type: string - repeated: false + description: XML file to open. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: parsers + description: The accessor to use - name: parse_yaml description: Parse yaml into an object. type: Function args: - name: filename - description: Yaml Filename type: string - repeated: false + description: Yaml Filename required: true - name: accessor - description: File accessor type: string - repeated: false - required: false - category: "" + description: File accessor - name: partitions description: List all partititions type: Plugin args: - name: all - description: If specified list all Partitions type: bool - repeated: false - required: false - category: windows + description: If specified list all Partitions - name: patch description: | Patch a JSON object with a json patch or merge. @@ -2884,92 +2202,71 @@ type: Function args: - name: item - description: The item to path type: Any - repeated: false + description: The item to path required: true - name: patch - description: A JSON Patch to apply type: Any - repeated: false - required: false + description: A JSON Patch to apply - name: merge - description: A merge-patch to apply type: Any - repeated: false - required: false - category: server + description: A merge-patch to apply - name: path_join description: Build a path by joining all components. type: Function args: - name: components - description: Path components to join. type: string + description: Path components to join. repeated: true required: true - name: sep - description: Separator to use (default /) type: string - repeated: false - required: false - category: basic + description: Separator to use (default /) - name: path_split description: Split a path into components. Note this is more complex than just split() because it takes into account path escaping. type: Function args: - name: path - description: Path to split into components. type: string - repeated: false + description: Path to split into components. required: true - category: basic - name: plist description: Parse plist file type: Function args: - name: file - description: A list of files to parse. type: string - repeated: false + description: A list of files to parse. required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: plist description: Parses a plist file. type: Plugin args: - name: file - description: A list of files to parse. type: string + description: A list of files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: prefetch description: Parses a prefetch file. type: Plugin args: - name: filename - description: A list of event log files to parse. type: string + description: A list of event log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: parsers + description: The accessor to use. - name: proc_dump description: | Dumps process memory. @@ -2983,11 +2280,9 @@ type: Plugin args: - name: pid - description: The PID to dump out. type: int64 - repeated: false + description: The PID to dump out. required: true - category: windows - name: proc_yara description: | Scan processes using yara rules. @@ -3002,91 +2297,59 @@ type: Plugin args: - name: rules - description: Yara rules type: string - repeated: false + description: Yara rules required: true - name: pid - description: The pid to scan type: int - repeated: false + description: The pid to scan required: true - name: context - description: Return this many bytes either side of a hit type: int - repeated: false - required: false + description: Return this many bytes either side of a hit - name: key - description: If set use this key to cache the yara rules. type: string - repeated: false - required: false - category: windows + description: If set use this key to cache the yara rules. - name: profile description: Returns a profile dump from the running process. type: Plugin args: - name: allocs - description: A sampling of all past memory allocations type: bool - repeated: false - required: false + description: A sampling of all past memory allocations - name: block - description: Stack traces that led to blocking on synchronization primitives type: bool - repeated: false - required: false + description: Stack traces that led to blocking on synchronization primitives - name: goroutine - description: Stack traces of all current goroutines type: bool - repeated: false - required: false + description: Stack traces of all current goroutines - name: heap - description: A sampling of memory allocations of live objects. type: bool - repeated: false - required: false + description: A sampling of memory allocations of live objects. - name: mutex - description: Stack traces of holders of contended mutexes type: bool - repeated: false - required: false + description: Stack traces of holders of contended mutexes - name: profile - description: CPU profile. type: bool - repeated: false - required: false + description: CPU profile. - name: trace - description: CPU trace. type: bool - repeated: false - required: false + description: CPU trace. - name: debug - description: Debug level type: int64 - repeated: false - required: false + description: Debug level - name: logs - description: Recent logs type: bool - repeated: false - required: false + description: Recent logs - name: queries - description: Recent Queries run type: bool - repeated: false - required: false + description: Recent Queries run - name: metrics - description: Collect metrics type: bool - repeated: false - required: false + description: Collect metrics - name: duration - description: Duration of samples (default 30 sec) type: int64 - repeated: false - required: false - category: plugins + description: Duration of samples (default 30 sec) - name: pslist description: | Enumerate running processes. @@ -3098,66 +2361,49 @@ type: Plugin args: - name: pid - description: A process ID to list. If not provided list all processes. type: int64 - repeated: false - required: false - category: plugin + description: A process ID to list. If not provided list all processes. - name: query description: Launch a subquery and materialize it into a list of rows. type: Function args: - name: vql - description: "" type: vfilter.StoredQuery - repeated: false required: true - category: basic - name: query description: Evaluate a VQL query. type: Plugin args: - name: query - description: A VQL Query to parse and execute. type: string - repeated: false + description: A VQL Query to parse and execute. required: true - name: env - description: A dict of args to insert into the scope. type: Any - repeated: false - required: false - category: "" + description: A dict of args to insert into the scope. - name: rand description: Selects a random number. type: Function args: - name: range - description: Selects a random number up to this range. type: int64 - repeated: false - required: false - category: basic + description: Selects a random number up to this range. - name: range description: Iterate over range. type: Plugin args: - name: start - description: Start index (0 based) type: int64 - repeated: false + description: Start index (0 based) required: true - name: end - description: End index (0 based) type: int64 - repeated: false + description: End index (0 based) required: true - name: step - description: End index (0 based) type: int64 - repeated: false + description: End index (0 based) required: true - category: "" - name: rate description: | Calculates the rate (derivative) between two quantities. @@ -3171,41 +2417,30 @@ type: Function args: - name: x - description: The X float type: float64 - repeated: false + description: The X float required: true - name: "y" - description: The Y float type: float64 - repeated: false + description: The Y float required: true - category: server - name: read_file description: Read a file into a string. type: Function args: - name: length - description: Max length of the file to read. type: int - repeated: false - required: false + description: Max length of the file to read. - name: offset - description: Where to read from the file. type: int64 - repeated: false - required: false + description: Where to read from the file. - name: filename - description: One or more files to open. type: string - repeated: false + description: One or more files to open. required: true - name: accessor - description: An accessor to use. type: string - repeated: false - required: false - category: basic + description: An accessor to use. - name: read_file description: | Read files in chunks. @@ -3219,26 +2454,19 @@ type: Plugin args: - name: chunk - description: length of each chunk to read from the file. type: int - repeated: false - required: false + description: length of each chunk to read from the file. - name: max_length - description: Max length of the file to read. type: int - repeated: false - required: false + description: Max length of the file to read. - name: filenames - description: One or more files to open. type: string + description: One or more files to open. repeated: true required: true - name: accessor - description: An accessor to use. type: string - repeated: false - required: false - category: plugin + description: An accessor to use. - name: read_reg_key description: | This is a convenience plugin which applies the globs to the registry @@ -3251,67 +2479,51 @@ type: Plugin args: - name: globs - description: Glob expressions to apply. type: string + description: Glob expressions to apply. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: windows + description: The accessor to use. - name: regex_replace description: Search and replace a string with a regexp. Note you can use $1 to replace the capture string. type: Function args: - name: source - description: The source string to replace. type: string - repeated: false + description: The source string to replace. required: true - name: replace - description: The substitute string. type: string - repeated: false + description: The substitute string. required: true - name: re - description: A regex to apply type: string - repeated: false + description: A regex to apply required: true - category: parsers - name: relpath description: Return the relative path of . type: Function args: - name: path - description: Extract directory name of path type: string - repeated: false + description: Extract directory name of path required: true - name: base - description: The base of the path type: string - repeated: false + description: The base of the path required: true - name: sep - description: Separator to use (default native) type: string - repeated: false - required: false - category: "" + description: Separator to use (default native) - name: rot13 description: Apply rot13 deobfuscation to the string. type: Function args: - name: string - description: "" type: string - repeated: false - required: false - category: parsers - name: sample description: | Executes 'query' and samples every n'th row. @@ -3321,21 +2533,16 @@ type: Plugin args: - name: query - description: Source query. type: StoredQuery - repeated: false + description: Source query. required: true - name: "n" - description: Pick every n row from query. type: int64 - repeated: false + description: Pick every n row from query. required: true - category: server - name: scope description: return the scope. type: Function - args: [] - category: basic - name: scope description: | The scope plugin returns the current scope as a single row. @@ -3349,120 +2556,88 @@ SELECT 1+1 As Two FROM scop() ``` type: Plugin - args: [] - category: plugin - name: search description: Search the server client's index. type: Plugin args: - name: query - description: The query string. type: string - repeated: false - required: false + description: The query string. - name: offset - description: Skip this many results. type: uint64 - repeated: false - required: false + description: Skip this many results. - name: limit - description: Only return limited results type: uint64 - repeated: false - required: false + description: Only return limited results - name: type - description: The type of search (e.g. 'key') type: string - repeated: false - required: false - category: server + description: The type of search (e.g. 'key') - name: serialize description: Encode an object as a string (csv or json). type: Function args: - name: item - description: The item to encode type: Any - repeated: false + description: The item to encode required: true - name: format - description: Encoding format (csv,json) type: string - repeated: false - required: false - category: basic + description: Encoding format (csv,json) - name: server_metadata description: Returns client metadata from the datastore. Client metadata is a set of free form key/value data type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: server - name: server_set_metadata description: Sets client metadata. Client metadata is a set of free form key/value data type: Function args: - name: client_id - description: "" type: string - repeated: false required: true - category: server - name: set_client_monitoring description: Sets the current client monitoring state. type: Function args: - name: value - description: The Value to set type: Any - repeated: false + description: The Value to set required: true - category: server - name: set_server_monitoring description: Sets the current server monitoring state. type: Function args: - name: value - description: The Value to set type: Any - repeated: false + description: The Value to set required: true - category: server - name: sleep description: Sleep for the specified number of seconds. Always returns true. type: Function args: - name: time - description: The number of seconds to sleep type: int64 - repeated: false - required: false - category: basic + description: The number of seconds to sleep - name: slice description: Slice an array. type: Function args: - name: list - description: A list of items to slice type: Any - repeated: false + description: A list of items to slice required: true - name: start - description: Start index (0 based) type: uint64 - repeated: false + description: Start index (0 based) required: true - name: end - description: End index (0 based) type: uint64 - repeated: false + description: End index (0 based) required: true - category: "" - name: source description: | Retrieve rows from an artifact's source. @@ -3478,338 +2653,243 @@ type: Plugin args: - name: client_id - description: The client id to extract type: string - repeated: false - required: false + description: The client id to extract - name: flow_id - description: A flow ID (client or server artifacts) type: string - repeated: false - required: false + description: A flow ID (client or server artifacts) - name: hunt_id - description: Retrieve sources from this hunt (combines all results from all clients) type: string - repeated: false - required: false + description: Retrieve sources from this hunt (combines all results from all clients) - name: artifact - description: The name of the artifact collection to fetch type: string - repeated: false - required: false + description: The name of the artifact collection to fetch - name: source - description: An optional named source within the artifact type: string - repeated: false - required: false + description: An optional named source within the artifact - name: start_time - description: Start return events from this date (for event sources) type: int64 - repeated: false - required: false + description: Start return events from this date (for event sources) - name: end_time - description: Stop end events reach this time (event sources). type: int64 - repeated: false - required: false + description: Stop end events reach this time (event sources). - name: notebook_id - description: The notebook to read from (shoud also include cell id) type: string - repeated: false - required: false + description: The notebook to read from (shoud also include cell id) - name: notebook_cell_id - description: The notebook cell read from (shoud also include notebook id) type: string - repeated: false - required: false + description: The notebook cell read from (shoud also include notebook id) - name: notebook_cell_table - description: A notebook cell can have multiple tables.) type: int64 - repeated: false - required: false + description: A notebook cell can have multiple tables.) - name: start_row - description: Start reading the result set from this row type: int64 - repeated: false - required: false + description: Start reading the result set from this row - name: count - description: Maximum number of clients to fetch (default unlimited)' type: int64 - repeated: false - required: false - category: server + description: Maximum number of clients to fetch (default unlimited)' - name: split description: Splits a string into an array based on a regexp separator. type: Function args: - name: string - description: The value to split type: string - repeated: false + description: The value to split required: true - name: sep - description: The serparator that will be used to split type: string - repeated: false + description: The serparator that will be used to split required: true - category: basic - name: split_records description: Parses files by splitting lines into records. type: Plugin args: - name: filenames - description: Files to parse. type: string + description: Files to parse. repeated: true required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: regex - description: The split regular expression (e.g. a comma) type: string - repeated: false + description: The split regular expression (e.g. a comma) required: true - name: columns + type: string description: If the first row is not the headers, this arg must provide a list of column names for each value. - type: string repeated: true - required: false - name: first_row_is_headers - description: A bool indicating if we should get column names from the first row. type: bool - repeated: false - required: false + description: A bool indicating if we should get column names from the first row. - name: count - description: Only split into this many columns if possible. type: int - repeated: false - required: false - category: parsers + description: Only split into this many columns if possible. - name: splunk_upload description: Upload rows to splunk. type: Plugin args: - name: query - description: Source for rows to upload. type: StoredQuery - repeated: false + description: Source for rows to upload. required: true - name: threads - description: How many threads to use. type: int64 - repeated: false - required: false + description: How many threads to use. - name: url - description: The Splunk Event Collector URL. type: string - repeated: false - required: false + description: The Splunk Event Collector URL. - name: token - description: Splunk HEC Token. type: string - repeated: false - required: false + description: Splunk HEC Token. - name: index - description: The name of the index to upload to. type: string - repeated: false + description: The name of the index to upload to. required: true - name: source - description: The source field for splunk. If not specified this will be 'velociraptor'. type: string - repeated: false - required: false + description: The source field for splunk. If not specified this will be 'velociraptor'. - name: sourcetype - description: The sourcetype field for splunk. If not specified this will 'vql' type: string - repeated: false - required: false + description: The sourcetype field for splunk. If not specified this will 'vql' - name: chunk_size - description: The number of rows to send at the time. type: int64 - repeated: false - required: false + description: The number of rows to send at the time. - name: skip_verify - description: 'Skip SSL verification(default: False).' type: bool - repeated: false - required: false + description: 'Skip SSL verification(default: False).' - name: wait_time - description: Batch splunk upload this long (2 sec). type: int64 - repeated: false - required: false - category: server + description: Batch splunk upload this long (2 sec). - name: sql description: Run queries against sqlite, mysql, and postgres databases type: Plugin args: - name: driver - description: sqlite, mysql,or postgres type: string - repeated: false + description: sqlite, mysql,or postgres required: true - name: connstring - description: SQL Connection String type: string - repeated: false - required: false + description: SQL Connection String - name: file - description: Required if using sqlite driver type: string - repeated: false - required: false + description: Required if using sqlite driver - name: accessor - description: The accessor to use if using sqlite type: string - repeated: false - required: false + description: The accessor to use if using sqlite - name: query - description: "" type: string - repeated: false required: true - name: args - description: "" type: Any - repeated: false - required: false - category: "" - name: sqlite description: Opens an SQLite file and run a query against it. type: Plugin args: - name: file - description: "" type: string - repeated: false required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: query - description: "" type: string - repeated: false required: true - name: args - description: "" type: Any - repeated: false - required: false - category: parsers - name: srum_lookup_id description: Lookup a SRUM id. type: Function args: - name: file - description: "" type: string - repeated: false required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: id - description: "" type: int64 - repeated: false required: true - category: windows +- name: starl + description: Compile a starlark code block - returns a module usable in VQL + type: Function + args: + - name: code + type: string + description: The body of the starlark code. + required: true + - name: key + type: string + description: If set use this key to cache the Starlark code block. + - name: globals + type: Any + description: Dictionary of values to feed into Starlark environment - name: stat description: Get file information. Unlike glob() this does not support wildcards. type: Plugin args: - name: filename - description: One or more files to open. type: string + description: One or more files to open. repeated: true required: true - name: accessor - description: An accessor to use. type: string - repeated: false - required: false - category: plugin + description: An accessor to use. - name: str description: Normalize a String. type: Function args: - name: str - description: The string to normalize type: Any - repeated: false + description: The string to normalize required: true - category: basic - name: strip description: Strip a prefix from a string. type: Function args: - name: string - description: The string to strip type: string - repeated: false + description: The string to strip required: true - name: prefix - description: The prefix to strip type: string - repeated: false - required: false - category: basic + description: The prefix to strip - name: substr description: Create a substring from a string type: Function args: - name: str - description: The string to shorten type: string - repeated: false + description: The string to shorten required: true - name: start - description: Beginning index of substring type: int - repeated: false - required: false + description: Beginning index of substring - name: end - description: End index of substring type: int - repeated: false - required: false - category: "" + description: End index of substring - name: sum description: Sums the items. type: Function args: - name: item - description: "" type: int64 - repeated: false required: true - category: "" - name: switch description: Executes each query. The first query to return any rows will be emitted. type: Plugin - args: [] - category: plugin - name: tempdir description: Create a temporary directory. The directory will be removed when the query ends. type: Function args: - name: remove_last - description: If set we delay removal as much as possible. type: bool - repeated: false - required: false - category: basic + description: If set we delay removal as much as possible. - name: tempfile description: | Create a temporary file and write some data into it. @@ -3818,126 +2898,98 @@ type: Function args: - name: data - description: Data to write in the tempfile. type: string + description: Data to write in the tempfile. repeated: true - required: false - name: extension - description: An extension to place in the tempfile. type: string - repeated: false - required: false + description: An extension to place in the tempfile. - name: permissions - description: Required permissions (e.g. 'x'). type: string - repeated: false - required: false + description: Required permissions (e.g. 'x'). - name: remove_last - description: If set we delay removal as much as possible. type: bool - repeated: false - required: false - category: plugin + description: If set we delay removal as much as possible. +- name: timeline_add + description: Add a new query to a timeline. + type: Function + args: + - name: timeline + type: string + required: true + - name: query + type: StoredQuery + description: Run this query to generate the timeline. + required: true + - name: key + type: string + description: The column representing the time. + required: true - name: timestamp description: Convert from different types to a time.Time. type: Function args: - name: epoch - description: "" type: Any - repeated: false - required: false - name: cocoatime - description: "" type: int64 - repeated: false - required: false - name: mactime - description: HFS+ type: int64 - repeated: false - required: false + description: HFS+ - name: winfiletime - description: "" type: int64 - repeated: false - required: false - name: string - description: Guess a timestamp from a string type: string - repeated: false - required: false + description: Guess a timestamp from a string - name: us_style - description: US Style Month/Day/Year type: bool - repeated: false - required: false - category: basic + description: US Style Month/Day/Year - name: to_dict description: Construct a dict from another object. type: Function args: - name: item - description: "" type: Any - repeated: false - required: false - category: basic - name: token description: Extract process token. type: Function args: - name: pid - description: The PID to get the token for. type: int64 - repeated: false + description: The PID to get the token for. required: true - category: windows - name: unhex description: Apply hex decoding to the string. type: Function args: - name: string - description: Hex string to decode type: string - repeated: false - required: false - category: "" + description: Hex string to decode - name: unzip description: Unzips a file into a directory type: Plugin args: - name: filename - description: File to unzip. type: string - repeated: false + description: File to unzip. required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: filename_filter - description: Only extract members matching this filter. type: string - repeated: false - required: false + description: Only extract members matching this filter. - name: output_directory - description: Where to unzip to type: string - repeated: false + description: Where to unzip to required: true - category: "" - name: upcase - description: "" type: Function args: - name: string - description: A string to lower type: string - repeated: false + description: A string to lower required: true - category: basic - name: upload description: | Upload a file to the upload service. For a Velociraptor client this @@ -3948,21 +3000,15 @@ type: Function args: - name: file - description: The file to upload type: string - repeated: false + description: The file to upload required: true - name: name - description: The name of the file that should be stored on the server type: string - repeated: false - required: false + description: The name of the file that should be stored on the server - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: plugin + description: The accessor to use - name: upload description: | Upload files to the server. @@ -3976,201 +3022,145 @@ type: Plugin args: - name: files - description: A list of files to upload type: string + description: A list of files to upload repeated: true required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false - category: plugin + description: The accessor to use - name: upload_gcs description: Upload files to GCS. type: Function args: - name: file - description: The file to upload type: string - repeated: false + description: The file to upload required: true - name: name - description: The name of the file that should be stored on the server type: string - repeated: false - required: false + description: The name of the file that should be stored on the server - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: bucket - description: The bucket to upload to type: string - repeated: false + description: The bucket to upload to required: true - name: project - description: The project to upload to type: string - repeated: false + description: The project to upload to required: true - name: credentials - description: The credentials to use type: string - repeated: false + description: The credentials to use required: true - category: plugin - name: upload_s3 description: Upload files to S3. type: Function args: - name: file - description: The file to upload type: string - repeated: false + description: The file to upload required: true - name: name - description: The name of the file that should be stored on the server type: string - repeated: false - required: false + description: The name of the file that should be stored on the server - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: bucket - description: The bucket to upload to type: string - repeated: false + description: The bucket to upload to required: true - name: region - description: The region the bucket is in type: string - repeated: false + description: The region the bucket is in required: true - name: credentialskey - description: The AWS key credentials to use type: string - repeated: false + description: The AWS key credentials to use required: true - name: credentialssecret - description: The AWS secret credentials to use type: string - repeated: false + description: The AWS secret credentials to use required: true - name: endpoint - description: The Endpoint to use type: string - repeated: false - required: false + description: The Endpoint to use - name: serversideencryption - description: The server side encryption method to use type: string - repeated: false - required: false + description: The server side encryption method to use - name: noverifycert - description: Skip TLS Verification type: bool - repeated: false - required: false - category: plugin + description: Skip TLS Verification - name: upload_sftp description: Upload files to SFTP. type: Function args: - name: file - description: The file to upload type: string - repeated: false + description: The file to upload required: true - name: name - description: The name of the file that should be stored on the server type: string - repeated: false - required: false + description: The name of the file that should be stored on the server - name: user - description: The username to connect to the endpoint with type: string - repeated: false + description: The username to connect to the endpoint with required: true - name: path - description: Path on server to upload file to type: string - repeated: false + description: Path on server to upload file to required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: privatekey - description: The private key to use type: string - repeated: false + description: The private key to use required: true - name: endpoint - description: The Endpoint to use type: string - repeated: false + description: The Endpoint to use required: true - name: hostkey - description: Host key to verify. Blank to disable type: string - repeated: false - required: false - category: basic + description: Host key to verify. Blank to disable - name: upload_webdav description: Upload files to a WebDAV server. type: Function args: - name: file - description: The file to upload type: string - repeated: false + description: The file to upload required: true - name: name - description: The name that the file should have on the server type: string - repeated: false - required: false + description: The name that the file should have on the server - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: url - description: The WebDAV url type: string - repeated: false + description: The WebDAV url required: true - name: basic_auth_user - description: The username to use in HTTP basic auth type: string - repeated: false - required: false + description: The username to use in HTTP basic auth - name: basic_auth_password - description: The password to use in HTTP basic auth type: string - repeated: false - required: false - category: basic + description: The password to use in HTTP basic auth - name: uploads description: Retrieve information about a flow's uploads. type: Plugin args: - name: client_id - description: The client id to extract type: string - repeated: false - required: false + description: The client id to extract - name: flow_id - description: A flow ID (client or server artifacts) type: string - repeated: false - required: false - category: server + description: A flow ID (client or server artifacts) - name: url description: | Construct a URL or parse one. @@ -4198,72 +3188,51 @@ type: Function args: - name: scheme - description: The scheme to use type: string - repeated: false - required: false + description: The scheme to use - name: host - description: The host component type: string - repeated: false - required: false + description: The host component - name: path - description: The path component type: string - repeated: false - required: false + description: The path component - name: fragment - description: The fragment type: string - repeated: false - required: false + description: The fragment - name: parse - description: A url to parse type: string - repeated: false - required: false - category: basic + description: A url to parse - name: users description: Display information about workstation local users. This is obtained through the NetUserEnum() API. type: Plugin - args: [] - category: windows - name: utf16 description: Parse input from utf16. type: Function args: - name: string - description: A string to decode type: string - repeated: false + description: A string to decode required: true - category: basic - name: utf16_encode description: Encode a string to utf16 bytes. type: Function args: - name: string - description: A string to decode type: string - repeated: false + description: A string to decode required: true - category: basic - name: uuid description: Generate a UUID. type: Function - args: [] - category: basic - name: vad description: Enumerate process memory regions. type: Plugin args: - name: pid - description: The PID to dump out. type: int64 - repeated: false + description: The PID to dump out. required: true - category: windows - name: version description: |2 @@ -4286,31 +3255,21 @@ type: Function args: - name: function - description: "" type: string - repeated: false - required: false - name: plugin - description: "" type: string - repeated: false - required: false - category: basic - name: watch_auditd description: Watch log files generated by auditd. type: Plugin args: - name: filename - description: A list of log files to parse. type: string + description: A list of log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: event + description: The accessor to use. - name: watch_csv description: | Watch a CSV file and stream events from it. Note: This is an event @@ -4321,51 +3280,39 @@ type: Plugin args: - name: filename - description: CSV files to open type: string + description: CSV files to open repeated: true required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: auto_headers - description: If unset the first row is headers type: bool - repeated: false - required: false + description: If unset the first row is headers - name: separator - description: Comma separator type: string - repeated: false - required: false - category: event + description: Comma separator - name: watch_etw description: Watch for events from an ETW provider. type: Plugin args: + - name: name + type: string + description: 'A session name ' - name: guid - description: 'A Provider GUID to watch ' type: string - repeated: false + description: 'A Provider GUID to watch ' required: true - name: any - description: 'Any Keywords ' type: uint64 - repeated: false - required: false + description: 'Any Keywords ' - name: all - description: 'All Keywords ' type: uint64 - repeated: false - required: false + description: 'All Keywords ' - name: level - description: Log level (0-5) type: int64 - repeated: false - required: false - category: event + description: Log level (0-5) - name: watch_evtx description: | Watch an EVTX file and stream events from it. @@ -4384,21 +3331,16 @@ type: Plugin args: - name: filename - description: A list of event log files to parse. type: string + description: A list of event log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false + description: The accessor to use. - name: messagedb - description: A Message database from https://github.com/Velocidex/evtx-data. type: string - repeated: false - required: false - category: event + description: A Message database from https://github.com/Velocidex/evtx-data. - name: watch_monitoring description: | Watch clients' monitoring log. This is an event plugin. This @@ -4406,51 +3348,39 @@ type: Plugin args: - name: artifact - description: The event artifact name to watch type: string - repeated: false + description: The event artifact name to watch required: true - category: event - name: watch_syslog description: 'Watch a syslog file and stream events from it. ' type: Plugin args: - name: filename - description: A list of log files to parse. type: string + description: A list of log files to parse. repeated: true required: true - name: accessor - description: The accessor to use. type: string - repeated: false - required: false - category: event + description: The accessor to use. - name: watch_usn description: Watch the USN journal from a device. type: Plugin args: - name: device - description: The device file to open. type: string - repeated: false + description: The device file to open. required: true - category: event - name: whoami description: Returns the username that is running the query. type: Function - args: [] - category: "" - name: winobj description: Enumerate The Windows Object Manager namespace. type: Plugin args: - name: path - description: Object namespace path. type: string - repeated: false - required: false - category: windows + description: Object namespace path. - name: wmi description: | Execute simple WMI queries synchronously. @@ -4463,16 +3393,12 @@ type: Plugin args: - name: query - description: The WMI query to issue. type: string - repeated: false + description: The WMI query to issue. required: true - name: namespace - description: The WMI namespace to use (ROOT/CIMV2) type: string - repeated: false - required: false - category: windows + description: The WMI namespace to use (ROOT/CIMV2) - name: wmi_events description: | Executes an evented WMI queries asynchronously. @@ -4481,41 +3407,32 @@ type: Plugin args: - name: query - description: WMI query to run. type: string - repeated: false + description: WMI query to run. required: true - name: namespace - description: WMI namespace type: string - repeated: false + description: WMI namespace required: true - name: wait - description: Wait this many seconds for events and then quit. type: int64 - repeated: false + description: Wait this many seconds for events and then quit. required: true - category: event - name: write_csv description: Write a query into a CSV file. type: Plugin args: - name: filename - description: CSV files to open type: string - repeated: false + description: CSV files to open required: true - name: accessor - description: The accessor to use type: string - repeated: false - required: false + description: The accessor to use - name: query - description: query to write into the file. type: StoredQuery - repeated: false + description: query to write into the file. required: true - category: plugin - name: yara description: | Scan files using yara rules. @@ -4568,49 +3485,33 @@ type: Plugin args: - name: rules - description: Yara rules in the yara DSL. type: string - repeated: false + description: Yara rules in the yara DSL. required: true - name: files - description: The list of files to scan. type: string + description: The list of files to scan. repeated: true required: true - name: accessor - description: Accessor (e.g. NTFS) type: string - repeated: false - required: false + description: Accessor (e.g. NTFS) - name: context - description: How many bytes to include around each hit type: int - repeated: false - required: false + description: How many bytes to include around each hit - name: start - description: The start offset to scan type: uint64 - repeated: false - required: false + description: The start offset to scan - name: end - description: End scanning at this offset (100mb) type: uint64 - repeated: false - required: false + description: End scanning at this offset (100mb) - name: number - description: Stop after this many hits (1). type: int64 - repeated: false - required: false + description: Stop after this many hits (1). - name: blocksize - description: Blocksize for scanning (1mb). type: uint64 - repeated: false - required: false + description: Blocksize for scanning (1mb). - name: key - description: If set use this key to cache the yara rules. type: string - repeated: false - required: false - category: plugin + description: If set use this key to cache the yara rules. diff --git a/gui/velociraptor/package-lock.json b/gui/velociraptor/package-lock.json index d53fc0508fe..526acccc48e 100644 --- a/gui/velociraptor/package-lock.json +++ b/gui/velociraptor/package-lock.json @@ -42,6 +42,7 @@ "react-bootstrap-table2-paginator": "^2.1.2", "react-bootstrap-table2-toolkit": "^2.1.3", "react-bootstrap-typeahead": "^5.1.1", + "react-calendar-timeline": "^0.27.0", "react-datepicker": "^3.3.0", "react-datetime-picker": "^3.0.4", "react-dom": "^16.13.1", @@ -2184,6 +2185,12 @@ "@hapi/hoek": "^8.3.0" } }, + "node_modules/@interactjs/types": { + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@interactjs/types/-/types-1.10.11.tgz", + "integrity": "sha512-YRsVFWjL8Gkkvlx3qnjeaxW4fnibSJ9791g8BA7Pv5ANByI64WmtR1vU7A2rXcrOn8XvyCEfY0ss1s8NhZP+MA==", + "peer": true + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -5270,6 +5277,11 @@ "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=" }, + "node_modules/batch-processor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/batch-processor/-/batch-processor-1.0.0.tgz", + "integrity": "sha1-dclcMrdI4IUNEMKxaPa9vpiRrOg=" + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -7767,6 +7779,14 @@ "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.749.tgz", "integrity": "sha512-F+v2zxZgw/fMwPz/VUGIggG4ZndDsYy0vlpthi3tjmDZlcfbhN5mYW0evXUsBr2sUtuDANFtle410A9u/sd/4A==" }, + "node_modules/element-resize-detector": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/element-resize-detector/-/element-resize-detector-1.2.3.tgz", + "integrity": "sha512-+dhNzUgLpq9ol5tyhoG7YLoXL3ssjfFW+0gpszXPwRU6NjGr1fVHMEAF8fVzIiRJq57Nre0RFeIjJwI8Nh2NmQ==", + "dependencies": { + "batch-processor": "1.0.0" + } + }, "node_modules/elliptic": { "version": "6.5.4", "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", @@ -10707,6 +10727,15 @@ "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==" }, + "node_modules/interactjs": { + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/interactjs/-/interactjs-1.10.11.tgz", + "integrity": "sha512-VPUWsGAOPmrZe1YF7Fq/4AIBBZ+3FikZRS8bpzT6VsAfUuhxl/CKJY73IAiZHd3fz9p174CXErn0Qs81XEFICA==", + "peer": true, + "dependencies": { + "@interactjs/types": "1.10.11" + } + }, "node_modules/internal-ip": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-4.3.0.tgz", @@ -17264,6 +17293,25 @@ "prop-types": "^15.6.0" } }, + "node_modules/react-calendar-timeline": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/react-calendar-timeline/-/react-calendar-timeline-0.27.0.tgz", + "integrity": "sha512-LOC7yrLT5bkfZAhfUKDhDkxyYd14dMnb3jXJzkpOyciPJyUwYWhxCDhjxwddfXjcB317NJVpszcqpk7xJw0mQg==", + "dependencies": { + "classnames": "^2.2.6", + "create-react-context": "^0.3.0", + "element-resize-detector": "^1.1.12", + "lodash.isequal": "^4.5.0", + "memoize-one": "^5.1.1" + }, + "peerDependencies": { + "interactjs": "^1.3.4", + "moment": "*", + "prop-types": "^15.6.2", + "react": ">=16.3", + "react-dom": ">=16.3" + } + }, "node_modules/react-clock": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/react-clock/-/react-clock-2.4.0.tgz", @@ -25045,6 +25093,12 @@ "@hapi/hoek": "^8.3.0" } }, + "@interactjs/types": { + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/@interactjs/types/-/types-1.10.11.tgz", + "integrity": "sha512-YRsVFWjL8Gkkvlx3qnjeaxW4fnibSJ9791g8BA7Pv5ANByI64WmtR1vU7A2rXcrOn8XvyCEfY0ss1s8NhZP+MA==", + "peer": true + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -27612,6 +27666,11 @@ "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=" }, + "batch-processor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/batch-processor/-/batch-processor-1.0.0.tgz", + "integrity": "sha1-dclcMrdI4IUNEMKxaPa9vpiRrOg=" + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -29714,6 +29773,14 @@ "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.749.tgz", "integrity": "sha512-F+v2zxZgw/fMwPz/VUGIggG4ZndDsYy0vlpthi3tjmDZlcfbhN5mYW0evXUsBr2sUtuDANFtle410A9u/sd/4A==" }, + "element-resize-detector": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/element-resize-detector/-/element-resize-detector-1.2.3.tgz", + "integrity": "sha512-+dhNzUgLpq9ol5tyhoG7YLoXL3ssjfFW+0gpszXPwRU6NjGr1fVHMEAF8fVzIiRJq57Nre0RFeIjJwI8Nh2NmQ==", + "requires": { + "batch-processor": "1.0.0" + } + }, "elliptic": { "version": "6.5.4", "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", @@ -32071,6 +32138,15 @@ "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==" }, + "interactjs": { + "version": "1.10.11", + "resolved": "https://registry.npmjs.org/interactjs/-/interactjs-1.10.11.tgz", + "integrity": "sha512-VPUWsGAOPmrZe1YF7Fq/4AIBBZ+3FikZRS8bpzT6VsAfUuhxl/CKJY73IAiZHd3fz9p174CXErn0Qs81XEFICA==", + "peer": true, + "requires": { + "@interactjs/types": "1.10.11" + } + }, "internal-ip": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-4.3.0.tgz", @@ -37410,6 +37486,18 @@ "prop-types": "^15.6.0" } }, + "react-calendar-timeline": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/react-calendar-timeline/-/react-calendar-timeline-0.27.0.tgz", + "integrity": "sha512-LOC7yrLT5bkfZAhfUKDhDkxyYd14dMnb3jXJzkpOyciPJyUwYWhxCDhjxwddfXjcB317NJVpszcqpk7xJw0mQg==", + "requires": { + "classnames": "^2.2.6", + "create-react-context": "^0.3.0", + "element-resize-detector": "^1.1.12", + "lodash.isequal": "^4.5.0", + "memoize-one": "^5.1.1" + } + }, "react-clock": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/react-clock/-/react-clock-2.4.0.tgz", diff --git a/gui/velociraptor/package.json b/gui/velociraptor/package.json index cfb2eda49ea..f1a11de9cd2 100644 --- a/gui/velociraptor/package.json +++ b/gui/velociraptor/package.json @@ -38,6 +38,7 @@ "react-bootstrap-table2-paginator": "^2.1.2", "react-bootstrap-table2-toolkit": "^2.1.3", "react-bootstrap-typeahead": "^5.1.1", + "react-calendar-timeline": "^0.27.0", "react-datepicker": "^3.3.0", "react-datetime-picker": "^3.0.4", "react-dom": "^16.13.1", diff --git a/gui/velociraptor/src/_variables.css b/gui/velociraptor/src/_variables.css index 0f64783cf8e..7cc18424414 100644 --- a/gui/velociraptor/src/_variables.css +++ b/gui/velociraptor/src/_variables.css @@ -62,4 +62,14 @@ --color-diff-added: #dff0d8; --color-diff-removed: #f2dede; --color-troggle-highlight-color: #cdcdcd; + + --color-timeline-header: #cdcdcd60; + --color-timeline-table-shown: #dd4b3930; + --color-timeline-1: #dff0d860; + --color-timeline-2: #a9444220; + --color-timeline-3: #2aabd2; + --color-timeline-4: #f0c36d; + --color-timeline-5: #bbb; + --color-timeline-6: #008000; + --color-timeline-7: #471ecb; } diff --git a/gui/velociraptor/src/components/artifacts/reporting.js b/gui/velociraptor/src/components/artifacts/reporting.js index d798a994166..5bd65148132 100644 --- a/gui/velociraptor/src/components/artifacts/reporting.js +++ b/gui/velociraptor/src/components/artifacts/reporting.js @@ -11,6 +11,7 @@ import VeloTable from '../core/table.js'; import VeloLineChart from './line-charts.js'; import Spinner from '../utils/spinner.js'; import ToolViewer from "../tools/tool-viewer.js"; +import Timeline from "../timeline/timeline.js"; // Renders a report in the DOM. @@ -124,6 +125,7 @@ export default class VeloReportViewer extends React.Component { render() { let template = parse(this.cleanupHTML(this.state.template), { replace: (domNode) => { + console.log(domNode.name); if (domNode.name === "inline-table-viewer") { console.log(this.state.template); try { @@ -152,11 +154,19 @@ export default class VeloReportViewer extends React.Component { ); }; + if (domNode.name === "grr-tool-viewer") { return ( ); }; + + if (domNode.name === "grr-timeline") { + return ( + + ); + }; + if (domNode.name === "grr-line-chart") { // Figure out where the data is: attribs.value is // something like data['table2'] diff --git a/gui/velociraptor/src/components/notebooks/notebook-cell-renderer.js b/gui/velociraptor/src/components/notebooks/notebook-cell-renderer.js index 40954d329b7..ef985d53125 100644 --- a/gui/velociraptor/src/components/notebooks/notebook-cell-renderer.js +++ b/gui/velociraptor/src/components/notebooks/notebook-cell-renderer.js @@ -23,6 +23,7 @@ import AddCellFromFlowDialog from './add-cell-from-flow.js'; import Completer from '../artifacts/syntax.js'; import { getHuntColumns } from '../hunts/hunt-list.js'; import VeloTimestamp from "../utils/time.js"; +import { AddTimelineDialog, AddVQLCellToTimeline } from "./timelines.js"; import axios from 'axios'; import api from '../core/api-service.js'; @@ -131,6 +132,7 @@ export default class NotebookCellRenderer extends React.Component { static propTypes = { cell_metadata: PropTypes.object, notebook_id: PropTypes.string, + notebook_metadata: PropTypes.object.isRequired, selected_cell_id: PropTypes.string, setSelectedCellId: PropTypes.func, @@ -156,6 +158,8 @@ export default class NotebookCellRenderer extends React.Component { currently_editing: false, input: "", + showAddTimeline: false, + showAddCellToTimeline: false, showAddCellFromHunt: false, showAddCellFromFlow: false, showCreateArtifactFromCell: false, @@ -433,6 +437,13 @@ export default class NotebookCellRenderer extends React.Component { + {this.state.cell && this.state.cell.type === "VQL" && + } + @@ -454,6 +465,11 @@ export default class NotebookCellRenderer extends React.Component { VQL
+ this.setState({showAddTimeline: true})}> + Add Timeline + @@ -558,6 +574,20 @@ export default class NotebookCellRenderer extends React.Component { vql={this.state.input} onClose={()=>this.setState({showCreateArtifactFromCell: false})} /> } + { this.state.showAddTimeline && + { + this.props.addCell(this.state.cell.cell_id, type, text); + }} + closeDialog={()=>this.setState({showAddTimeline: false})} /> + } + { this.state.showAddCellToTimeline && + this.setState({showAddCellToTimeline: false})} /> + }
@@ -595,6 +625,7 @@ export default class NotebookCellRenderer extends React.Component { > { selected && _.map(this.state.cell.messages, (msg, idx) => { diff --git a/gui/velociraptor/src/components/notebooks/notebook-renderer.js b/gui/velociraptor/src/components/notebooks/notebook-renderer.js index 1dbc015e2f6..b8eca282ae9 100644 --- a/gui/velociraptor/src/components/notebooks/notebook-renderer.js +++ b/gui/velociraptor/src/components/notebooks/notebook-renderer.js @@ -171,6 +171,7 @@ export default class NotebookRenderer extends React.Component { selected_cell_id={this.state.selected_cell_id} setSelectedCellId={this.setSelectedCellId} notebook_id={this.props.notebook.notebook_id} + notebook_metadata={this.props.notebook} cell_metadata={cell_md} key={idx} upCell={this.upCell} downCell={this.downCell} diff --git a/gui/velociraptor/src/components/notebooks/notebook-report-renderer.js b/gui/velociraptor/src/components/notebooks/notebook-report-renderer.js index a1c884a146b..d3e4c38cf50 100644 --- a/gui/velociraptor/src/components/notebooks/notebook-report-renderer.js +++ b/gui/velociraptor/src/components/notebooks/notebook-report-renderer.js @@ -3,6 +3,8 @@ import PropTypes from 'prop-types'; import parse from 'html-react-parser'; import VeloTable from '../core/table.js'; +import TimelineRenderer from "../timeline/timeline.js"; +import VeloLineChart from '../artifacts/line-charts.js'; import NotebookTableRenderer from './notebook-table-renderer.js'; @@ -10,6 +12,7 @@ export default class NotebookReportRenderer extends React.Component { static propTypes = { refresh: PropTypes.func, cell: PropTypes.object, + notebook_id: PropTypes.string, }; render() { @@ -37,6 +40,15 @@ export default class NotebookReportRenderer extends React.Component { }; } + if (domNode.name === "grr-timeline") { + return ( + + ); + }; + if (domNode.name === "grr-csv-viewer") { try { let params = JSON.parse(domNode.attribs.params); @@ -50,6 +62,23 @@ export default class NotebookReportRenderer extends React.Component { return domNode; } }; + + if (domNode.name === "grr-line-chart") { + // Figure out where the data is: attribs.value is + // something like data['table2'] + let re = /'([^']+)'/; + let match = re.exec(domNode.attribs.value); + let data = this.state.data[match[1]]; + let rows = JSON.parse(data.Response); + let params = JSON.parse(domNode.attribs.params); + + return ( + + ); + }; + return domNode; } }); diff --git a/gui/velociraptor/src/components/notebooks/timelines.js b/gui/velociraptor/src/components/notebooks/timelines.js new file mode 100644 index 00000000000..30180d2e7a1 --- /dev/null +++ b/gui/velociraptor/src/components/notebooks/timelines.js @@ -0,0 +1,267 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import _ from 'lodash'; +import axios from 'axios'; +import api from '../core/api-service.js'; + +import parse from 'html-react-parser'; +import Modal from 'react-bootstrap/Modal'; +import CreatableSelect from 'react-select/creatable'; +import Select from 'react-select'; +import Button from 'react-bootstrap/Button'; +import Form from 'react-bootstrap/Form'; +import Col from 'react-bootstrap/Col'; +import Row from 'react-bootstrap/Row'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; + + +const POLL_TIME = 2000; + +// Adds a new timeline cell below this one. +export class AddTimelineDialog extends React.Component { + static propTypes = { + notebook_metadata: PropTypes.object.isRequired, + addCell: PropTypes.func, + closeDialog: PropTypes.func.isRequired, + } + + state = { + timeline: "", + } + + addTimeline = ()=>{ + if (this.state.timeline) { + this.props.addCell('{{ Timeline "' + this.state.timeline + '" }}', "Markdown"); + } + this.props.closeDialog(); + } + + render() { + let options = _.map(this.props.notebook_metadata.timelines, x=>{ + return {value: x, label: x, isFixed: true, color: "#00B8D9"}; + }); + return ( + + + Add Timeline + + + this.setState({time_column: e && e.value})} + placeholder="Time Column" + /> + + + + + + + + + + ); + } +} diff --git a/gui/velociraptor/src/components/timeline/timeline.css b/gui/velociraptor/src/components/timeline/timeline.css new file mode 100644 index 00000000000..12758ef330a --- /dev/null +++ b/gui/velociraptor/src/components/timeline/timeline.css @@ -0,0 +1,84 @@ +.timeline-value-item { + margin-right: 2ex; +} + +.timeline-value { + display: inline-flex; +} + +.timeline-table-item { + color: var(--color-timeline-table-shown); + background: var(--color-timeline-table-shown); +} + +.super-timeline .form-check { + line-height: normal; +} + +.super-timeline .react-calendar-timeline .rct-header-root { + background: var(--color-timeline-header); +} + +.super-timeline .react-calendar-timeline .rct-dateHeader { + background: var(--color-timeline-header); + color: var(--color-default-font); +} + +.super-timeline .react-calendar-timeline .rct-dateHeader-primary { + color: var(--color-default-font); +} + +.super-timeline .hidden-header { + display: none; +} + +.super-timeline td { + padding-top: 0; + padding-bottom: 0; +} + +.super-timeline .velo-table { + max-height: 50vh; + overflow-y: auto; +} + +td.timeline-time { + width: 15em; + overflow-x: hidden; +} + +.timeline-item-1 { + background: var(--color-timeline-1); + color: var(--color-default-font); + } + +.timeline-item-2 { + background: var(--color-timeline-2); + color: var(--color-default-font); +} + +.timeline-item-3 { + background: var(--color-timeline-3); + color: var(--color-default-font); +} + +.timeline-item-4 { + background: var(--color-timeline-4); +} + +.timeline-item-5 { + background: var(--color-timeline-5); +} + +.timeline-item-6 { + background: var(--color-timeline-6); +} + +.timeline-item-7 { + background: var(--color-timeline-7); +} + +.timeline-marker { + background-color: var(--color-default-font); + width: 5px; +} diff --git a/gui/velociraptor/src/components/timeline/timeline.js b/gui/velociraptor/src/components/timeline/timeline.js new file mode 100644 index 00000000000..bec065d58fa --- /dev/null +++ b/gui/velociraptor/src/components/timeline/timeline.js @@ -0,0 +1,379 @@ +import "./timeline.css"; + +import React, { Component } from 'react'; +import PropTypes from 'prop-types'; +import _ from 'lodash'; +import Timeline, { + TimelineMarkers, + CustomMarker, +} from 'react-calendar-timeline'; +import api from '../core/api-service.js'; +import axios from 'axios'; +import { PrepareData } from '../core/table.js'; +import VeloTimestamp from "../utils/time.js"; +import VeloValueRenderer from '../utils/value.js'; +import Form from 'react-bootstrap/Form'; + +// make sure you include the timeline stylesheet or the timeline will not be styled +import 'react-calendar-timeline/lib/Timeline.css'; +import moment from 'moment'; +import Button from 'react-bootstrap/Button'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import BootstrapTable from 'react-bootstrap-table-next'; +import ButtonGroup from 'react-bootstrap/ButtonGroup'; +import Navbar from 'react-bootstrap/Navbar'; + +class TimelineValueRenderer extends Component { + static propTypes = { + value: PropTypes.object, + } + state = { + expanded: false, + } + render() { + return ( +
+ { this.state.expanded ? + + +
+ +
+
+ : + + + { _.map(this.props.value, (v, k) => { + return {k}: {v}; + })} + + } +
+ ); + } +} + +class TimelineTableRenderer extends Component { + static propTypes = { + rows: PropTypes.array, + timelines: PropTypes.object, + } + + getTimelineClass = (name) => { + let timelines = this.props.timelines.timelines; + for(let i=0;i
; + } + + let rows = this.props.rows; + let columns = [ + {dataField: '_id', hidden: true}, + {dataField: 'Time', + text: "Time", + classes: "timeline-time", + formatter: (cell, row, rowIndex) => { + return
+ +
; + }}, + {dataField: 'Data', + text: "Data", + formatter: (cell, row, rowIndex) => { + return ; + }}, + ]; + + // Add an id field for react ordering. + for (var j=0; j + +
+ ); + } +} + + +export default class TimelineRenderer extends React.Component { + static propTypes = { + name: PropTypes.string, + notebook_id: PropTypes.string, + params: PropTypes.string, + } + + componentDidMount = () => { + this.source = axios.CancelToken.source(); + this.fetchRows(); + } + + componentWillUnmount() { + this.source.cancel(); + } + + componentDidUpdate(prevProps, prevState, snapshot) { + if (!_.isEqual(prevState.version, this.state.version)) { + return true; + } + + if (!_.isEqual(prevState.start_time, this.state.start_time)) { + this.fetchRows(); + return true; + }; + + if (!_.isEqual(prevState.row_count, this.state.row_count)) { + this.fetchRows(); + return true; + }; + + return false; + } + + state = { + start_time: 0, + table_start: 0, + table_end: 0, + loading: true, + disabled: {}, + version: 0, + row_count: 10, + }; + + fetchRows = () => { + let skip_components = []; + _.map(this.state.disabled, (v,k)=>{ + if(v) { + skip_components.push(k); + }; + }); + + let params = { + type: "TIMELINE", + timeline: this.props.name, + start_time: this.state.start_time * 1000000, + rows: this.state.row_count, + skip_components: skip_components, + notebook_id: this.props.notebook_id, + }; + + let url = this.props.url || "v1/GetTable"; + + this.source.cancel(); + this.source = axios.CancelToken.source(); + + this.setState({loading: true}); + + api.get(url, params, this.source.token).then((response) => { + if (response.cancel) { + return; + } + + let pageData = PrepareData(response.data); + this.setState({ + table_start: response.data.start_time / 1000000, + table_end: response.data.end_time / 1000000, + columns: pageData.columns, + rows: pageData.rows, + version: Date(), + }); + }); + }; + + groupRenderer = ({ group }) => { + if (group.id < 0) { + return
{group.title}
; + } + + return ( +
+ { + let disabled = this.state.disabled; + disabled[group.id] = !disabled[group.id]; + this.setState({disabled: disabled}); + this.fetchRows(); + }} + /> + + ); + }; + + pageSizeSelector = () => { + let options = [10, 20, 50, 100]; + + return
+ { _.map(options, option=>{ + return ; + }) } +
; + } + + nextPage = ()=>{ + if (this.state.table_end > 0) { + this.setState({start_time: this.state.table_end + 1}); + } + } + + render() { + let super_timeline = {timelines:[]}; + try { + super_timeline = JSON.parse(this.props.params); + } catch(e) { + return <>; + } + + let groups = [{id: -1, title: "Table View"}]; + let items = [{ + id:-1, group: -1, + start_time: this.state.table_start, + end_time: this.state.table_end, + canMove: false, + canResize: false, + canChangeGroup: false, + itemProps: { + className: 'timeline-table-item', + style: { + background: undefined, + color: undefined, + }, + }, + }]; + let smallest = 10000000000000000; + let largest = 0; + let timelines = super_timeline.timelines || []; + + for (let i=0;i largest) { + largest = end; + } + + groups.push({ + id: timeline.id, + disabled: this.state.disabled[timeline.id], + title: timeline.id, + }); + items.push({ + id: i+1, group: timeline.id, + start_time: start, + end_time: end, + canMove: false, + canResize: false, + canChangeGroup: false, + itemProps: { + className: 'timeline-item-' + ((i + 1) % 8), + style: { + background: undefined, + color: undefined, + } + }, + }); + } + + if (smallest > largest) { + smallest = largest; + } + + if (_.isNaN(smallest)) { + smallest = 0; + largest =0; + } + + return
Super-timeline {this.props.name} + + + { this.pageSizeSelector() } + + + + { + console.log(time); + this.setState({start_time: time}); + }} + onItemSelect={(itemId, e, time) => { + this.setState({start_time: time}); + return false; + }} + onItemClick={(itemId, e, time) => { + console.log(time); + this.setState({start_time: time}); + return false; + }} + groupRenderer={this.groupRenderer} + > + + + { ({ styles, date }) => { + styles.backgroundColor = undefined; + styles.width = undefined; + return
; + }} + + + + { this.state.columns && + + } +
; + } +} diff --git a/gui/velociraptor/src/components/vfs/file-details.css b/gui/velociraptor/src/components/vfs/file-details.css index 6c873d767a3..29d8962ad51 100644 --- a/gui/velociraptor/src/components/vfs/file-details.css +++ b/gui/velociraptor/src/components/vfs/file-details.css @@ -3,6 +3,10 @@ background-color: var(--color-default-background); } +.padded .velo-table { + overflow-x: auto; +} + .tab-content { background-color: var(--color-default-background); } diff --git a/gui/velociraptor/src/dark-mode.css b/gui/velociraptor/src/dark-mode.css index 8066b482cd7..a59aa259ee9 100644 --- a/gui/velociraptor/src/dark-mode.css +++ b/gui/velociraptor/src/dark-mode.css @@ -56,6 +56,17 @@ --color-monospace-color: #eeeeee; --scrollbar-track: #121212; --app-footer-background-color: #444444; + + --color-default-font: #eeeeee; + --color-timeline-header: #cdcdcd20; + --color-timeline-table-shown: #dd4b3920; + --color-timeline-1: #dff0d820; + --color-timeline-2: #a9444220; + --color-timeline-3: #2aabd2; + --color-timeline-4: #f0c36d; + --color-timeline-5: #bbb; + --color-timeline-6: #008000; + --color-timeline-7: #471ecb; } .dark-mode .btn { diff --git a/gui/velociraptor/src/index.js b/gui/velociraptor/src/index.js index 5169723b0bb..afc8ded7810 100644 --- a/gui/velociraptor/src/index.js +++ b/gui/velociraptor/src/index.js @@ -19,7 +19,7 @@ import { faHome, faCrosshairs, faWrench, faEye, faServer, faBook, faLaptop, faWindowClose, faPencilAlt, faArrowUp, faArrowDown, faPlus, faSave, faTrash, faBinoculars, faUpload, faExternalLinkAlt, faTags, faTimes, faFolder, faSignOutAlt, faBroom, faPaperPlane, faEdit, faChevronDown, faFileDownload, - faEraser, faFileCsv, faFileImport, + faEraser, faFileCsv, faFileImport, faMinus, faForward, faCalendarAlt, } from '@fortawesome/free-solid-svg-icons'; library.add(faHome, faCrosshairs, faWrench, faEye, faServer, faBook, faLaptop, @@ -30,7 +30,8 @@ library.add(faHome, faCrosshairs, faWrench, faEye, faServer, faBook, faLaptop, faWindowClose, faPencilAlt, faArrowUp, faArrowDown, faPlus, faSave, faTrash, faFolderOpen , faHistory, faBinoculars, faUpload, faExternalLinkAlt, faTags, faTimes, faFolder, faSignOutAlt, faBroom, faPaperPlane, faEdit, - faChevronDown, faFileDownload, faEraser, faFileCsv, faFileImport, + faChevronDown, faFileDownload, faEraser, faFileCsv, faFileImport, faMinus, + faForward, faCalendarAlt, ); ReactDOM.render( diff --git a/gui/velociraptor/yarn.lock b/gui/velociraptor/yarn.lock index 9f104eafe7d..b8000cd593a 100644 --- a/gui/velociraptor/yarn.lock +++ b/gui/velociraptor/yarn.lock @@ -1418,6 +1418,11 @@ dependencies: "@hapi/hoek" "^8.3.0" +"@interactjs/types@1.10.11": + "integrity" "sha512-YRsVFWjL8Gkkvlx3qnjeaxW4fnibSJ9791g8BA7Pv5ANByI64WmtR1vU7A2rXcrOn8XvyCEfY0ss1s8NhZP+MA==" + "resolved" "https://registry.npmjs.org/@interactjs/types/-/types-1.10.11.tgz" + "version" "1.10.11" + "@istanbuljs/load-nyc-config@^1.0.0": "integrity" "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==" "resolved" "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz" @@ -3041,6 +3046,11 @@ "resolved" "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz" "version" "1.3.1" +"batch-processor@1.0.0": + "integrity" "sha1-dclcMrdI4IUNEMKxaPa9vpiRrOg=" + "resolved" "https://registry.npmjs.org/batch-processor/-/batch-processor-1.0.0.tgz" + "version" "1.0.0" + "batch@0.6.1": "integrity" "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=" "resolved" "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz" @@ -4829,6 +4839,13 @@ "resolved" "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.749.tgz" "version" "1.3.749" +"element-resize-detector@^1.1.12": + "integrity" "sha512-+dhNzUgLpq9ol5tyhoG7YLoXL3ssjfFW+0gpszXPwRU6NjGr1fVHMEAF8fVzIiRJq57Nre0RFeIjJwI8Nh2NmQ==" + "resolved" "https://registry.npmjs.org/element-resize-detector/-/element-resize-detector-1.2.3.tgz" + "version" "1.2.3" + dependencies: + "batch-processor" "1.0.0" + "elliptic@^6.5.3": "integrity" "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==" "resolved" "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz" @@ -6581,6 +6598,13 @@ "resolved" "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz" "version" "0.1.1" +"interactjs@^1.3.4": + "integrity" "sha512-VPUWsGAOPmrZe1YF7Fq/4AIBBZ+3FikZRS8bpzT6VsAfUuhxl/CKJY73IAiZHd3fz9p174CXErn0Qs81XEFICA==" + "resolved" "https://registry.npmjs.org/interactjs/-/interactjs-1.10.11.tgz" + "version" "1.10.11" + dependencies: + "@interactjs/types" "1.10.11" + "internal-ip@^4.3.0": "integrity" "sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==" "resolved" "https://registry.npmjs.org/internal-ip/-/internal-ip-4.3.0.tgz" @@ -8044,7 +8068,7 @@ "resolved" "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" "version" "0.3.0" -"memoize-one@^5.0.0": +"memoize-one@^5.0.0", "memoize-one@^5.1.1": "integrity" "sha512-HKeeBpWvqiVJD57ZUAsJNm71eHTykffzcLZVYWiVfQeI1rJtuEaS7hQiEpWfVVk18donPwJEcFKIkCmPJNOhHA==" "resolved" "https://registry.npmjs.org/memoize-one/-/memoize-one-5.1.1.tgz" "version" "5.1.1" @@ -8279,7 +8303,7 @@ "resolved" "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz" "version" "1.0.4" -"moment@^2.28.0": +"moment@*", "moment@^2.28.0": "integrity" "sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw==" "resolved" "https://registry.npmjs.org/moment/-/moment-2.28.0.tgz" "version" "2.28.0" @@ -10250,6 +10274,17 @@ "uncontrollable" "^7.0.0" "warning" "^4.0.3" +"react-calendar-timeline@^0.27.0": + "integrity" "sha512-LOC7yrLT5bkfZAhfUKDhDkxyYd14dMnb3jXJzkpOyciPJyUwYWhxCDhjxwddfXjcB317NJVpszcqpk7xJw0mQg==" + "resolved" "https://registry.npmjs.org/react-calendar-timeline/-/react-calendar-timeline-0.27.0.tgz" + "version" "0.27.0" + dependencies: + "classnames" "^2.2.6" + "create-react-context" "^0.3.0" + "element-resize-detector" "^1.1.12" + "lodash.isequal" "^4.5.0" + "memoize-one" "^5.1.1" + "react-calendar@^3.0.0": "integrity" "sha512-xoKdRe6FrnZ30LD9pyr20fhet1uwSbc6srLGm1ib7G4b7tAXniZrwzrJ4YV/Hbmmwf/zAFGyXtBzLAIV1KNvuA==" "resolved" "https://registry.npmjs.org/react-calendar/-/react-calendar-3.1.0.tgz" @@ -10340,7 +10375,7 @@ "strip-ansi" "6.0.0" "text-table" "0.2.0" -"react-dom@^15.0.0 || ^16.0.0 || ^17.0.0", "react-dom@^16.0.0 || ^17.0.0", "react-dom@^16.13.1", "react-dom@^16.3.0", "react-dom@^16.8.0 || ^17.0.0", "react-dom@^17.0.0 || ^16.3.0 || ^15.5.4", "react-dom@>=15.0.0": +"react-dom@^15.0.0 || ^16.0.0 || ^17.0.0", "react-dom@^16.0.0 || ^17.0.0", "react-dom@^16.13.1", "react-dom@^16.3.0", "react-dom@^16.8.0 || ^17.0.0", "react-dom@^17.0.0 || ^16.3.0 || ^15.5.4", "react-dom@>=15.0.0", "react-dom@>=16.3": "integrity" "sha512-81PIMmVLnCNLO/fFOQxdQkvEq/+Hfpv24XNJfpyZhTRfO0QcmQIF/PgCa1zCOj2w1hrn12MFLyaJ/G0+Mxtfag==" "resolved" "https://registry.npmjs.org/react-dom/-/react-dom-16.13.1.tgz" "version" "16.13.1" @@ -10698,7 +10733,7 @@ "shallowequal" "^1.1.0" "velocity-react" "^1.4.1" -"react@^15.0.0 || ^16.0.0 || ^17.0.0", "react@^15.0.2 || ^16.0.0 || ^17.0.0", "react@^15.5.4", "react@^16.0.0 || ^17.0.0", "react@^16.14.0", "react@^16.3.0", "react@^16.3.0 || ^17.0.0", "react@^16.8.0 || ^17.0.0", "react@^17.0.0 || ^16.3.0 || ^15.5.4", "react@>= 16", "react@>=15.0.0": +"react@^15.0.0 || ^16.0.0 || ^17.0.0", "react@^15.0.2 || ^16.0.0 || ^17.0.0", "react@^15.5.4", "react@^16.0.0 || ^17.0.0", "react@^16.14.0", "react@^16.3.0", "react@^16.3.0 || ^17.0.0", "react@^16.8.0 || ^17.0.0", "react@^17.0.0 || ^16.3.0 || ^15.5.4", "react@>= 16", "react@>=15.0.0", "react@>=16.3": "integrity" "sha512-0X2CImDkJGApiAlcf0ODKIneSwBPhqJawOa5wCtKbu7ZECrmS26NvtSILynQ66cgkT/RJ4LidJOc3bUESwmU8g==" "resolved" "https://registry.npmjs.org/react/-/react-16.14.0.tgz" "version" "16.14.0" diff --git a/make_proto.sh b/make_proto.sh index 8b8afef39c2..4ef21f9f269 100755 --- a/make_proto.sh +++ b/make_proto.sh @@ -21,6 +21,7 @@ for i in $CWD/proto/ $CWD/crypto/proto/ \ $CWD/actions/proto/ \ $CWD/services/frontend/proto/ \ $CWD/config/proto/ \ + $CWD/timelines/proto/ \ $CWD/acls/proto/ \ $CWD/flows/proto/ ; do echo Building protos in $i diff --git a/reporting/gui.go b/reporting/gui.go index a79b88d19e1..524670cdfb9 100644 --- a/reporting/gui.go +++ b/reporting/gui.go @@ -16,7 +16,6 @@ import ( "github.com/Depado/bfchroma" "github.com/Masterminds/sprig" - "github.com/Velocidex/json" "github.com/Velocidex/ordereddict" "github.com/pkg/errors" @@ -29,7 +28,9 @@ import ( "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/result_sets" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/timelines" "www.velocidex.com/golang/velociraptor/utils" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" @@ -258,6 +259,18 @@ func (self *GuiTemplateEngine) Timeline(values ...interface{}) string { default: return "" + case string: + timeline_path_manager := self.path_manager.Notebook().Timeline(t) + parameters := "{}" + reader, err := timelines.NewSuperTimelineReader(self.config_obj, timeline_path_manager, nil) + if err == nil { + parameters = json.MustMarshalString(reader.Stat()) + } + + return fmt.Sprintf( + `
`, t, parameters) + case []*NotebookCellQuery: result := "" for _, item := range t { @@ -583,7 +596,7 @@ func NewBlueMondayPolicy() *bluemonday.Policy { p.AllowAttrs("value", "params").OnElements("grr-csv-viewer") p.AllowAttrs("value", "params").OnElements("inline-table-viewer") p.AllowAttrs("value", "params").OnElements("grr-line-chart") - p.AllowAttrs("value", "params").OnElements("grr-timeline") + p.AllowAttrs("name", "params").OnElements("grr-timeline") p.AllowAttrs("name").OnElements("grr-tool-viewer") // Required for syntax highlighting. diff --git a/reporting/paths.go b/reporting/paths.go index d86b61193c4..af257241c78 100644 --- a/reporting/paths.go +++ b/reporting/paths.go @@ -10,6 +10,7 @@ import ( "github.com/Velocidex/ordereddict" "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/timelines" ) type NotebookPathManager struct { @@ -53,6 +54,13 @@ func (self *NotebookPathManager) ZipExport() string { time.Now().Format("20060102150405Z"))) } +func (self *NotebookPathManager) Timeline(name string) *timelines.SuperTimelinePathManager { + return &timelines.SuperTimelinePathManager{ + Root: self.Directory(), + Name: name, + } +} + var notebook_regex = regexp.MustCompile(`N\.(F\.[^-]+?)-(C\..+|server)`) func NewNotebookPathManager(notebook_id string) *NotebookPathManager { @@ -67,7 +75,7 @@ func NewNotebookPathManager(notebook_id string) *NotebookPathManager { matches := notebook_regex.FindStringSubmatch(notebook_id) if len(matches) == 3 { - // For hunt notebooks store them in the hunt itself. + // For collections notebooks store them in the hunt itself. return &NotebookPathManager{ notebook_id: notebook_id, root: path.Join("/clients/", matches[2], @@ -91,6 +99,13 @@ func (self *NotebookCellPathManager) Path() string { return path.Join(self.root, self.notebook_id, self.cell_id+".json") } +func (self *NotebookCellPathManager) Notebook() *NotebookPathManager { + return &NotebookPathManager{ + notebook_id: self.notebook_id, + root: self.root, + } +} + func (self *NotebookCellPathManager) Item(name string) string { return path.Join(self.root, self.notebook_id, self.cell_id, name) } diff --git a/reporting/timelines.go b/reporting/timelines.go new file mode 100644 index 00000000000..507283a8be2 --- /dev/null +++ b/reporting/timelines.go @@ -0,0 +1 @@ +package reporting diff --git a/timelines/paths.go b/timelines/paths.go new file mode 100644 index 00000000000..962631a1503 --- /dev/null +++ b/timelines/paths.go @@ -0,0 +1,37 @@ +package timelines + +import ( + "path" +) + +type TimelinePathManager struct { + Name string + root string +} + +func (self TimelinePathManager) Path() string { + return self.root + "/" + self.Name + ".json" +} + +func (self TimelinePathManager) Index() string { + return self.root + "/" + self.Name + ".idx" +} + +// A Supertimeline is a collection of individual timelines. Create +// this path manager using a notebook path manager. +type SuperTimelinePathManager struct { + Name string + Root string // Base directory where we store the timeline. +} + +func (self *SuperTimelinePathManager) Path() string { + return path.Join("/", self.Root, "timelines", self.Name+".json") +} + +// Add a child timeline to the super timeline. +func (self *SuperTimelinePathManager) GetChild(child_name string) *TimelinePathManager { + return &TimelinePathManager{ + Name: child_name, + root: path.Join("/", self.Root, "timelines", self.Name), + } +} diff --git a/timelines/proto/timelines.pb.go b/timelines/proto/timelines.pb.go new file mode 100644 index 00000000000..13eb053dc60 --- /dev/null +++ b/timelines/proto/timelines.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0-devel +// protoc v3.12.3 +// source: timelines.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + _ "www.velocidex.com/golang/velociraptor/proto" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Timeline struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StartTime int64 `protobuf:"varint,1,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime int64 `protobuf:"varint,2,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Id string `protobuf:"bytes,4,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *Timeline) Reset() { + *x = Timeline{} + if protoimpl.UnsafeEnabled { + mi := &file_timelines_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Timeline) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Timeline) ProtoMessage() {} + +func (x *Timeline) ProtoReflect() protoreflect.Message { + mi := &file_timelines_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Timeline.ProtoReflect.Descriptor instead. +func (*Timeline) Descriptor() ([]byte, []int) { + return file_timelines_proto_rawDescGZIP(), []int{0} +} + +func (x *Timeline) GetStartTime() int64 { + if x != nil { + return x.StartTime + } + return 0 +} + +func (x *Timeline) GetEndTime() int64 { + if x != nil { + return x.EndTime + } + return 0 +} + +func (x *Timeline) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Timeline) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type SuperTimeline struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Timelines []*Timeline `protobuf:"bytes,2,rep,name=timelines,proto3" json:"timelines,omitempty"` +} + +func (x *SuperTimeline) Reset() { + *x = SuperTimeline{} + if protoimpl.UnsafeEnabled { + mi := &file_timelines_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SuperTimeline) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuperTimeline) ProtoMessage() {} + +func (x *SuperTimeline) ProtoReflect() protoreflect.Message { + mi := &file_timelines_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuperTimeline.ProtoReflect.Descriptor instead. +func (*SuperTimeline) Descriptor() ([]byte, []int) { + return file_timelines_proto_rawDescGZIP(), []int{1} +} + +func (x *SuperTimeline) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SuperTimeline) GetTimelines() []*Timeline { + if x != nil { + return x.Timelines + } + return nil +} + +var File_timelines_proto protoreflect.FileDescriptor + +var file_timelines_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x14, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x73, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x68, + 0x0a, 0x08, 0x54, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x52, 0x0a, 0x0d, 0x53, 0x75, 0x70, 0x65, + 0x72, 0x54, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x42, 0x37, 0x5a, 0x35, + 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, + 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_timelines_proto_rawDescOnce sync.Once + file_timelines_proto_rawDescData = file_timelines_proto_rawDesc +) + +func file_timelines_proto_rawDescGZIP() []byte { + file_timelines_proto_rawDescOnce.Do(func() { + file_timelines_proto_rawDescData = protoimpl.X.CompressGZIP(file_timelines_proto_rawDescData) + }) + return file_timelines_proto_rawDescData +} + +var file_timelines_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_timelines_proto_goTypes = []interface{}{ + (*Timeline)(nil), // 0: proto.Timeline + (*SuperTimeline)(nil), // 1: proto.SuperTimeline +} +var file_timelines_proto_depIdxs = []int32{ + 0, // 0: proto.SuperTimeline.timelines:type_name -> proto.Timeline + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_timelines_proto_init() } +func file_timelines_proto_init() { + if File_timelines_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_timelines_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Timeline); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_timelines_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SuperTimeline); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_timelines_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_timelines_proto_goTypes, + DependencyIndexes: file_timelines_proto_depIdxs, + MessageInfos: file_timelines_proto_msgTypes, + }.Build() + File_timelines_proto = out.File + file_timelines_proto_rawDesc = nil + file_timelines_proto_goTypes = nil + file_timelines_proto_depIdxs = nil +} diff --git a/timelines/proto/timelines.proto b/timelines/proto/timelines.proto new file mode 100644 index 00000000000..a421db563df --- /dev/null +++ b/timelines/proto/timelines.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +import "proto/semantic.proto"; + +package proto; + +option go_package = "www.velocidex.com/golang/velociraptor/timelines/proto"; + +message Timeline { + int64 start_time = 1; + int64 end_time = 2; + string name = 3; + string id = 4; +} + +message SuperTimeline { + string name = 1; + repeated Timeline timelines = 2; +} diff --git a/timelines/reader.go b/timelines/reader.go new file mode 100644 index 00000000000..93c35f20a1c --- /dev/null +++ b/timelines/reader.go @@ -0,0 +1,163 @@ +package timelines + +import ( + "bufio" + "context" + "encoding/binary" + "fmt" + "os" + "sort" + "time" + + "github.com/Velocidex/ordereddict" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/glob" + timelines_proto "www.velocidex.com/golang/velociraptor/timelines/proto" +) + +type TimelineItem struct { + Row *ordereddict.Dict + Time int64 + Source string +} + +type TimelineReader struct { + id string + current_idx int + offset int64 + fd api.FileReader + index_fd api.FileReader + index_stat glob.FileInfo +} + +func (self *TimelineReader) getIndex(i int) (*IndexRecord, error) { + idx_record := &IndexRecord{} + _, err := self.index_fd.Seek(int64(i)*IndexRecordSize, 0) + if err != nil { + return nil, err + } + + err = binary.Read(self.index_fd, binary.LittleEndian, idx_record) + return idx_record, err +} + +func (self *TimelineReader) Stat() *timelines_proto.Timeline { + first_record, _ := self.getIndex(0) + last_record, _ := self.getIndex(int(self.index_stat.Size()/IndexRecordSize - 1)) + + if first_record == nil || last_record == nil { + return &timelines_proto.Timeline{Id: self.id} + } + + return &timelines_proto.Timeline{ + Id: self.id, + StartTime: first_record.Timestamp, + EndTime: last_record.Timestamp, + } +} + +func (self *TimelineReader) SeekToTime(timestamp time.Time) error { + timestamp_int := timestamp.UnixNano() + number_of_points := self.index_stat.Size() / IndexRecordSize + + self.current_idx = sort.Search(int(number_of_points), func(i int) bool { + // Read the index record at offset i + idx_record, _ := self.getIndex(i) + return idx_record.Timestamp >= timestamp_int + }) + idx_record, err := self.getIndex(self.current_idx) + self.offset = idx_record.Offset + return err +} + +func (self *TimelineReader) Read(ctx context.Context) <-chan TimelineItem { + output_chan := make(chan TimelineItem) + + go func() { + defer close(output_chan) + + self.fd.Seek(self.offset, os.SEEK_SET) + fmt.Printf("Seekin g to %v\n", self.offset) + + reader := bufio.NewReader(self.fd) + for { + select { + case <-ctx.Done(): + return + + default: + row_data, err := reader.ReadBytes('\n') + if err != nil { + return + } + + // We have reached the end. + if len(row_data) == 0 { + return + } + + idx_record, err := self.getIndex(self.current_idx) + if err != nil { + return + } + self.current_idx++ + + item := ordereddict.NewDict() + + // We failed to unmarshal one line of + // JSON - it may be corrupted, go to + // the next one. + err = item.UnmarshalJSON(row_data) + if err != nil { + continue + } + + output_chan <- TimelineItem{ + Source: self.id, + Row: item, + Time: idx_record.Timestamp, + } + } + } + }() + + return output_chan + +} + +func (self *TimelineReader) Close() { + self.fd.Close() + self.index_fd.Close() +} + +func NewTimelineReader(config_obj *config_proto.Config, + path_manager *TimelinePathManager) (*TimelineReader, error) { + file_store_factory := file_store.GetFileStore(config_obj) + fd, err := file_store_factory.ReadFile(path_manager.Path()) + if err != nil { + return nil, err + } + + index_fd, err := file_store_factory.ReadFile(path_manager.Index()) + if err != nil { + fd.Close() + return nil, err + } + + stats, err := index_fd.Stat() + if err != nil { + fd.Close() + index_fd.Close() + return nil, err + } + + return &TimelineReader{ + id: path_manager.Name, + fd: fd, + index_fd: index_fd, + index_stat: stats, + }, nil + +} diff --git a/timelines/supertimeline.go b/timelines/supertimeline.go new file mode 100644 index 00000000000..d726129bad9 --- /dev/null +++ b/timelines/supertimeline.go @@ -0,0 +1,200 @@ +package timelines + +import ( + "context" + "fmt" + "time" + + "google.golang.org/protobuf/proto" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/datastore" + timelines_proto "www.velocidex.com/golang/velociraptor/timelines/proto" + "www.velocidex.com/golang/velociraptor/utils" +) + +type SuperTimelineReader struct { + *timelines_proto.SuperTimeline + + readers []*TimelineReader +} + +func (self *SuperTimelineReader) Stat() *timelines_proto.SuperTimeline { + result := proto.Clone(self.SuperTimeline).(*timelines_proto.SuperTimeline) + result.Timelines = nil + for _, reader := range self.readers { + result.Timelines = append(result.Timelines, reader.Stat()) + } + + return result +} + +func (self *SuperTimelineReader) Close() { + for _, reader := range self.readers { + reader.Close() + } +} + +func (self *SuperTimelineReader) SeekToTime(timestamp time.Time) { + for _, reader := range self.readers { + reader.SeekToTime(timestamp) + } +} + +// Gets the smallest item or null if no items available. +func (self *SuperTimelineReader) getSmallest( + ctx context.Context, + slots []*TimelineItem, chans []<-chan TimelineItem) *TimelineItem { + + var smallest *TimelineItem + var smallest_idx int + + for idx := 0; idx < len(slots); idx++ { + // Backfill slot if needed + if slots[idx] == nil { + select { + case <-ctx.Done(): + return nil + + case item, ok := <-chans[idx]: + if !ok { + // Channel is closed, try the + // next slot. + continue + } + + // Store the item in the slot. + slots[idx] = &item + } + } + + // Check if the item is smallest than the result. + if smallest == nil || slots[idx].Time < smallest.Time { + smallest = slots[idx] + smallest_idx = idx + } + } + + // No smallest found + if smallest == nil { + return nil + } + + // Take the smallest and backfill + slots[smallest_idx] = nil + return smallest +} + +func (self *SuperTimelineReader) Read(ctx context.Context) <-chan TimelineItem { + output_chan := make(chan TimelineItem) + + go func() { + defer close(output_chan) + + slots := make([]*TimelineItem, len(self.readers)) + chans := make([]<-chan TimelineItem, len(self.readers)) + + // Fill in the initial set + for idx, reader := range self.readers { + chans[idx] = reader.Read(ctx) + } + + for { + smallest := self.getSmallest(ctx, slots, chans) + if smallest == nil { + return + } + + output_chan <- *smallest + } + }() + + return output_chan +} + +func NewSuperTimelineReader( + config_obj *config_proto.Config, + path_manager *SuperTimelinePathManager, + skip_components []string) (*SuperTimelineReader, error) { + db, err := datastore.GetDB(config_obj) + if err != nil { + return nil, err + } + + result := &SuperTimelineReader{SuperTimeline: &timelines_proto.SuperTimeline{}} + err = db.GetSubject(config_obj, path_manager.Path(), result.SuperTimeline) + if err != nil { + return nil, err + } + + // Open all the readers. + for _, timeline := range result.Timelines { + if utils.InString(skip_components, timeline.Id) { + continue + } + reader, err := NewTimelineReader(config_obj, path_manager.GetChild(timeline.Id)) + if err != nil { + fmt.Printf("NewSuperTimelineReader err: %v\n", err) + result.Close() + return nil, err + } + result.readers = append(result.readers, reader) + } + return result, nil +} + +type SuperTimelineWriter struct { + *timelines_proto.SuperTimeline + config_obj *config_proto.Config + path_manager *SuperTimelinePathManager +} + +func (self *SuperTimelineWriter) Close() { + db, err := datastore.GetDB(self.config_obj) + if err != nil { + return + } + db.SetSubject(self.config_obj, self.path_manager.Path(), self.SuperTimeline) +} + +func (self *SuperTimelineWriter) AddChild(name string) (*TimelineWriter, error) { + new_timeline_path_manager := self.path_manager.GetChild(name) + writer, err := NewTimelineWriter(self.config_obj, new_timeline_path_manager) + if err != nil { + return nil, err + } + + // Only add a new child if it is not already in there. + for _, item := range self.Timelines { + if item.Id == new_timeline_path_manager.Name { + return writer, err + } + } + + self.Timelines = append(self.Timelines, &timelines_proto.Timeline{ + Id: new_timeline_path_manager.Name, + }) + return writer, err +} + +func NewSuperTimelineWriter( + config_obj *config_proto.Config, + path_manager *SuperTimelinePathManager) (*SuperTimelineWriter, error) { + + self := &SuperTimelineWriter{ + SuperTimeline: &timelines_proto.SuperTimeline{}, + config_obj: config_obj, + path_manager: path_manager, + } + + db, err := datastore.GetDB(config_obj) + if err != nil { + return nil, err + } + + err = db.GetSubject(config_obj, self.path_manager.Path(), self.SuperTimeline) + if err != nil { + self.SuperTimeline.Name = path_manager.Name + } + + return self, nil +} diff --git a/timelines/timelines.go b/timelines/timelines.go new file mode 100644 index 00000000000..0b2d3f28524 --- /dev/null +++ b/timelines/timelines.go @@ -0,0 +1 @@ +package timelines diff --git a/timelines/timelines_test.go b/timelines/timelines_test.go new file mode 100644 index 00000000000..20466c75347 --- /dev/null +++ b/timelines/timelines_test.go @@ -0,0 +1,119 @@ +package timelines + +import ( + "context" + "testing" + "time" + + "github.com/Velocidex/ordereddict" + "github.com/alecthomas/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "www.velocidex.com/golang/velociraptor/config" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/test_utils" +) + +type TimelineTestSuite struct { + suite.Suite + + config_obj *config_proto.Config + file_store api.FileStore +} + +func (self *TimelineTestSuite) SetupTest() { + var err error + self.config_obj, err = new(config.Loader).WithFileLoader( + "../http_comms/test_data/server.config.yaml"). + WithRequiredFrontend().WithWriteback(). + LoadAndValidate() + require.NoError(self.T(), err) + + self.file_store = file_store.GetFileStore(self.config_obj) +} + +func (self *TimelineTestSuite) TearDownTest() { + test_utils.GetMemoryFileStore(self.T(), self.config_obj).Clear() +} + +func (self *TimelineTestSuite) TestSuperTimelineWriter() { + path_manager := &SuperTimelinePathManager{"Test", "notebooks/N.1234/"} + super, err := NewSuperTimelineWriter(self.config_obj, path_manager) + assert.NoError(self.T(), err) + + timeline, err := super.AddChild("1") + assert.NoError(self.T(), err) + + timeline2, err := super.AddChild("2") + assert.NoError(self.T(), err) + + for i := int64(0); i <= 10; i++ { + // This timeline contains evens + timeline.Write(time.Unix(i*2, 0), ordereddict.NewDict().Set("Item", i*2)) + + // This timeline contains odds + timeline2.Write(time.Unix(i*2+1, 0), ordereddict.NewDict().Set("Item", i*2+1)) + } + timeline.Close() + timeline2.Close() + super.Close() + + // test_utils.GetMemoryFileStore(self.T(), self.config_obj).Debug() + reader, err := NewSuperTimelineReader(self.config_obj, path_manager, nil) + assert.NoError(self.T(), err) + defer reader.Close() + + for _, ts := range []int64{3, 4, 7} { + reader.SeekToTime(time.Unix(ts, 0)) + + ctx := context.Background() + var last_id int64 + + for item := range reader.Read(ctx) { + value, ok := item.Row.GetInt64("Item") + assert.True(self.T(), ok) + assert.True(self.T(), value >= ts) + + // Items should be sequential - odds come from + // one timeline and events from the other. + if last_id > 0 { + assert.Equal(self.T(), last_id+1, value) + } + last_id = value + } + } +} + +func (self *TimelineTestSuite) TestTimelineWriter() { + path_manager := &TimelinePathManager{"T.1234", "Test"} + timeline, err := NewTimelineWriter(self.config_obj, path_manager) + assert.NoError(self.T(), err) + + for i := int64(0); i <= 10; i++ { + timeline.Write(time.Unix(i*2, 0), ordereddict.NewDict().Set("Item", i*2)) + } + timeline.Close() + + // test_utils.GetMemoryFileStore(self.T(), self.config_obj).Debug() + + reader, err := NewTimelineReader(self.config_obj, path_manager) + assert.NoError(self.T(), err) + defer reader.Close() + + ctx := context.Background() + + for _, ts := range []int64{3, 4, 7} { + reader.SeekToTime(time.Unix(ts, 0)) + for row := range reader.Read(ctx) { + value, ok := row.Row.GetInt64("Item") + assert.True(self.T(), ok) + assert.True(self.T(), value >= ts) + } + } +} + +func TestTimelineWriter(t *testing.T) { + suite.Run(t, &TimelineTestSuite{}) +} diff --git a/timelines/writer.go b/timelines/writer.go new file mode 100644 index 00000000000..838212df7d3 --- /dev/null +++ b/timelines/writer.go @@ -0,0 +1,106 @@ +package timelines + +import ( + "bytes" + "encoding/binary" + "time" + + "github.com/Velocidex/ordereddict" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/json" + vjson "www.velocidex.com/golang/velociraptor/json" +) + +const ( + IndexRecordSize = 24 +) + +type IndexRecord struct { + Timestamp int64 + + // Offset to the row data + Offset int64 + + // Annotation Offset + Annotation int64 +} + +type TimelineWriter struct { + last_time time.Time + opts *json.EncOpts + fd api.FileWriter + index_fd api.FileWriter +} + +func (self *TimelineWriter) Write( + timestamp time.Time, row *ordereddict.Dict) error { + offset, err := self.fd.Size() + if err != nil { + return err + } + + out := &bytes.Buffer{} + offsets := &bytes.Buffer{} + serialized, err := vjson.MarshalWithOptions(row, self.opts) + if err != nil { + return err + } + + // Write line delimited JSON + out.Write(serialized) + out.Write([]byte{'\n'}) + idx_record := &IndexRecord{ + Timestamp: timestamp.UnixNano(), + Offset: offset, + } + + err = binary.Write(offsets, binary.LittleEndian, idx_record) + if err != nil { + return err + } + + // Include the line feed in the count. + offset += int64(len(serialized) + 1) + + _, err = self.fd.Write(out.Bytes()) + if err != nil { + return err + } + _, err = self.index_fd.Write(offsets.Bytes()) + return err +} + +func (self *TimelineWriter) Truncate() { + self.fd.Truncate() + self.index_fd.Truncate() +} + +func (self *TimelineWriter) Close() { + self.fd.Close() + self.index_fd.Close() +} + +func NewTimelineWriter( + config_obj *config_proto.Config, + path_manager *TimelinePathManager) (*TimelineWriter, error) { + file_store_factory := file_store.GetFileStore(config_obj) + fd, err := file_store_factory.WriteFile( + path_manager.Path()) + if err != nil { + return nil, err + } + fd.Truncate() + + index_fd, err := file_store_factory.WriteFile( + path_manager.Index()) + if err != nil { + fd.Close() + return nil, err + } + index_fd.Truncate() + + return &TimelineWriter{fd: fd, index_fd: index_fd}, nil + +} diff --git a/vql/server/timelines/create.go b/vql/server/timelines/create.go new file mode 100644 index 00000000000..956e15424c8 --- /dev/null +++ b/vql/server/timelines/create.go @@ -0,0 +1,141 @@ +package timelines + +import ( + "context" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/acls" + api_proto "www.velocidex.com/golang/velociraptor/api/proto" + "www.velocidex.com/golang/velociraptor/datastore" + "www.velocidex.com/golang/velociraptor/reporting" + "www.velocidex.com/golang/velociraptor/timelines" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/velociraptor/vql/functions" + "www.velocidex.com/golang/velociraptor/vql/sorter" + "www.velocidex.com/golang/vfilter" + "www.velocidex.com/golang/vfilter/arg_parser" + "www.velocidex.com/golang/vfilter/types" +) + +type AddTimelineFunctionArgs struct { + Timeline string `vfilter:"required,field=timeline,doc=Supertimeline to add to"` + Name string `vfilter:"required,field=name,doc=Name of child timeline"` + Query types.StoredQuery `vfilter:"required,field=query,doc=Run this query to generate the timeline."` + Key string `vfilter:"required,field=key,doc=The column representing the time."` + NotebookId string `vfilter:"optional,field=notebook_id,doc=The notebook ID the timeline is stored in."` +} + +type AddTimelineFunction struct{} + +func (self *AddTimelineFunction) Call(ctx context.Context, + scope vfilter.Scope, + args *ordereddict.Dict) vfilter.Any { + + err := vql_subsystem.CheckAccess(scope, acls.READ_RESULTS) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + + arg := &AddTimelineFunctionArgs{} + err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + + config_obj, ok := vql_subsystem.GetServerConfig(scope) + if !ok { + scope.Log("Command can only run on the server") + return vfilter.Null{} + } + + notebook_id := arg.NotebookId + if notebook_id == "" { + notebook_id = vql_subsystem.GetStringFromRow(scope, scope, "NotebookId") + } + + if notebook_id == "" { + scope.Log("timeline_add: Notebook ID must be specified") + return vfilter.Null{} + } + + notebook_path_manager := reporting.NewNotebookPathManager(notebook_id) + super, err := timelines.NewSuperTimelineWriter( + config_obj, notebook_path_manager.Timeline(arg.Timeline)) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + defer super.Close() + + // make a new timeline to store in the super timeline. + writer, err := super.AddChild(arg.Name) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + defer writer.Close() + + writer.Truncate() + + subscope := scope.Copy() + sub_ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Timelines have to be sorted, so we force them to be sorted + // by the key. + sorter := sorter.MergeSorter{10000} + sorted_chan := sorter.Sort(sub_ctx, subscope, arg.Query.Eval(sub_ctx, subscope), + arg.Key, false /* desc */) + + for row := range sorted_chan { + key, pres := scope.Associative(row, arg.Key) + if !pres { + scope.Log("timeline_add: Key %v is not found in query", arg.Key) + return vfilter.Null{} + } + + ts, err := functions.TimeFromAny(scope, key) + if err == nil { + writer.Write(ts, vfilter.RowToDict(sub_ctx, subscope, row)) + } + } + + // Now record the new timeline in the notebook if needed. + db, _ := datastore.GetDB(config_obj) + notebook_metadata := &api_proto.NotebookMetadata{} + err = db.GetSubject(config_obj, notebook_path_manager.Path(), notebook_metadata) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + + for _, item := range notebook_metadata.Timelines { + if item == arg.Timeline { + return super.SuperTimeline + } + } + + notebook_metadata.Timelines = append(notebook_metadata.Timelines, arg.Timeline) + err = db.SetSubject(config_obj, notebook_path_manager.Path(), notebook_metadata) + if err != nil { + scope.Log("timeline_add: %v", err) + return vfilter.Null{} + } + + return super.SuperTimeline +} + +func (self AddTimelineFunction) Info( + scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo { + return &vfilter.FunctionInfo{ + Name: "timeline_add", + Doc: "Add a new query to a timeline.", + ArgType: type_map.AddType(scope, &AddTimelineFunctionArgs{}), + } +} + +func init() { + vql_subsystem.RegisterFunction(&AddTimelineFunction{}) +} diff --git a/vql/server/timelines/reader.go b/vql/server/timelines/reader.go new file mode 100644 index 00000000000..0110d68c113 --- /dev/null +++ b/vql/server/timelines/reader.go @@ -0,0 +1,101 @@ +package timelines + +import ( + "context" + "time" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/acls" + "www.velocidex.com/golang/velociraptor/reporting" + "www.velocidex.com/golang/velociraptor/timelines" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/velociraptor/vql/functions" + "www.velocidex.com/golang/vfilter" + "www.velocidex.com/golang/vfilter/arg_parser" +) + +type TimelinePluginArgs struct { + Timeline string `vfilter:"required,field=timeline,doc=Name of the timeline to read"` + SkipComponents []string `vfilter:"optional,field=skip,doc=List of child components to skip"` + StartTime vfilter.Any `vfilter:"optional,field=start,doc=First timestamp to fetch"` + NotebookId string `vfilter:"optional,field=notebook_id,doc=The notebook ID the timeline is stored in."` +} + +type TimelinePlugin struct{} + +func (self TimelinePlugin) Call( + ctx context.Context, + scope vfilter.Scope, + args *ordereddict.Dict) <-chan vfilter.Row { + output_chan := make(chan vfilter.Row) + + go func() { + defer close(output_chan) + + err := vql_subsystem.CheckAccess(scope, acls.READ_RESULTS) + if err != nil { + scope.Log("timeline: %v", err) + return + } + + arg := &TimelinePluginArgs{} + err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) + if err != nil { + scope.Log("timeline: %v", err) + return + } + + config_obj, ok := vql_subsystem.GetServerConfig(scope) + if !ok { + scope.Log("Command can only run on the server") + return + } + + notebook_id := arg.NotebookId + if notebook_id == "" { + notebook_id = vql_subsystem.GetStringFromRow(scope, scope, "NotebookId") + } + + if notebook_id == "" { + scope.Log("timeline_add: Notebook ID must be specified") + return + } + + super_path_manager := reporting.NewNotebookPathManager(notebook_id) + reader, err := timelines.NewSuperTimelineReader(config_obj, + super_path_manager.Timeline(arg.Timeline), arg.SkipComponents) + if err != nil { + scope.Log("timeline: %v", err) + return + } + defer reader.Close() + + if arg.StartTime != nil { + start, err := functions.TimeFromAny(scope, arg.StartTime) + if err != nil { + scope.Log("timeline: %v", err) + return + } + + reader.SeekToTime(start) + } + + for item := range reader.Read(ctx) { + output_chan <- item.Row.Set("_ts", time.Unix(0, item.Time)) + } + }() + + return output_chan +} + +func (self TimelinePlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo { + return &vfilter.PluginInfo{ + Name: "timeline", + Doc: "Read a timeline. You can create a timeline with the timeline_add() function", + ArgType: type_map.AddType(scope, &TimelinePluginArgs{}), + } +} + +func init() { + vql_subsystem.RegisterPlugin(&TimelinePlugin{}) +} diff --git a/vql/tools/query.go b/vql/tools/query.go index c736cfb2e53..74cd8776766 100644 --- a/vql/tools/query.go +++ b/vql/tools/query.go @@ -4,6 +4,7 @@ import ( "context" "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/services" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" "www.velocidex.com/golang/vfilter/arg_parser" @@ -49,10 +50,22 @@ func (self QueryPlugin) Call( env.Set(member, v) } - // Parse and compile the query - subscope := scope.Copy() + // Build a completely new scope to evaluate the query + // in. + builder := services.ScopeBuilderFromScope(scope) + + // Make a new scope for each artifact. + manager, err := services.GetRepositoryManager() + if err != nil { + scope.Log("query: %v", err) + return + } + + subscope := manager.BuildScope(builder).AppendVars(env) defer subscope.Close() + // Parse and compile the query + scope.Log("query: running query %v", arg.Query) statements, err := vfilter.MultiParse(arg.Query) if err != nil { scope.Log("query: %v", err) @@ -60,7 +73,7 @@ func (self QueryPlugin) Call( } for _, vql := range statements { - row_chan := vql.Eval(ctx, scope) + row_chan := vql.Eval(ctx, subscope) for { select { case <-ctx.Done(): diff --git a/vql_plugins/server.go b/vql_plugins/server.go index 00e1990f870..e42f8aa2e91 100644 --- a/vql_plugins/server.go +++ b/vql_plugins/server.go @@ -29,4 +29,5 @@ import ( _ "www.velocidex.com/golang/velociraptor/vql/server/downloads" _ "www.velocidex.com/golang/velociraptor/vql/server/hunts" _ "www.velocidex.com/golang/velociraptor/vql/server/notebooks" + _ "www.velocidex.com/golang/velociraptor/vql/server/timelines" )