From e4f59710e6293445322ee0e8a558abb0429eeb65 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 28 Apr 2020 17:41:09 +1000 Subject: [PATCH] Large refactor to use a path manager (#331) In the process of adding support to the mysql data store it is becoming clear that in order to utilize the best intrinsic features of each data store technology we can not rely on a pure filesystem like abstraction. It is useful to hint to different implementations the likely access pattern so better optimizations can be undertaken. Previously the layout of the different objects stored in the filestore was hard coded throughout the code in various path construction statements. This change introduces a path manager - an object responsible for mapping a particular entity into the filestore. The path manager is the perfect way we can hint to the data store how to treat different objects. For example, the FlowPathManager is responsible for store various flow related items. Therefore we can use it to build paths like flow_path_manager.Log() for the result set storing flow logs, etc. A path manager is responsible for accessing a result set in the filestore (similar to a path but smarter). In the future all file store operations will be made using path managers. The filestore is now also responsible for storing and reading result set (i.e. rows) as well as just bulk files. This makes it easier for other code to use because we dont need to csv serialization/deserialization by hand now. Filestore's interface is: * PushRows() -> pushes rows into the result set specified by the path manager. We dont actually care exactly where they are stored. * GetTimeRange() -> Gets those rows that were inserted in the time range specified (or all times) Therefore callers do not need to worry about managing the data themselves - and the specific datastore can now use e.g. timestamp indexes to read data more efficiently. Queuing is also moved to the file store: * Watch() -> registers interest in a queue name (usually an artifact name) and watches for new events. Filestores may implement any message passing technique to make it work. Directory based file store is designed to work in a single process. Therefore for directory based file store we use internal thread based notification mechanism (extremely low latency). This is a somewhat breaking change is that some of the layout of objects in the file store has changed. Velociraptor now uses line delimited json internally instead of CSV although we try to read csv files as a fallback for backward compatibility. If you upgrade you might need to reset the file store. --- api/api.go | 21 +- api/artifacts.go | 59 ++-- api/clients.go | 9 +- api/csv.go | 71 +++-- api/download.go | 39 +-- api/proto/csv.pb.go | 84 ++++-- api/proto/csv.proto | 7 + artifacts/assets/ab0x.go | 76 ++++- artifacts/builder.go | 8 +- .../Server/Internal/Notification.yaml | 10 + .../definitions/System/Flow/Archive.yaml | 6 + .../System/Hunt/Participation.yaml | 10 +- bin/inspect.go | 3 +- constants/paths.go | 13 - crypto/resolver.go | 12 +- datastore/filebased.go | 19 +- file_store/api/accessor.go | 153 ++++++++++ file_store/api/queues.go | 25 +- file_store/api/testsuite.go | 69 ++++- file_store/api/uploader.go | 25 +- file_store/directory/csv.go | 84 ++++++ file_store/directory/directory.go | 7 +- file_store/directory/directory_accessor.go | 155 ---------- file_store/directory/json.go | 65 +++++ file_store/directory/queue.go | 70 +++++ file_store/directory/queue_test.go | 28 ++ file_store/directory/result_sets.go | 71 +++++ file_store/file_store.go | 16 +- file_store/memory/memory.go | 14 +- file_store/memory/queue.go | 57 ++-- file_store/memory/queue_test.go | 15 +- file_store/mysql/mysql.go | 16 +- file_store/mysql/queue.go | 166 +++++++++++ file_store/queue.go | 18 +- file_store/result_set.go | 33 +++ flows/api.go | 13 +- flows/artifacts.go | 135 ++++----- flows/foreman.go | 2 +- flows/hunts.go | 20 +- flows/monitoring.go | 23 +- flows/proto/artifact_collector.pb.go | 200 +++++++------ flows/proto/artifact_collector.proto | 4 +- go.mod | 238 +++++++-------- go.sum | 2 + .../artifact/server-events-directive.js | 18 +- .../core/csv-viewer-directive.js | 3 +- .../flow/flow-inspector-directive.js | 6 +- .../flow/flow-log-directive.js | 8 +- .../flow/flow-results-directive.js | 28 +- .../hunt/hunt-clients-directive.js | 6 +- http_comms/comms.go | 1 + paths/client.go | 67 +++++ paths/clients.go | 27 -- paths/flow_metadata.go | 134 +++++++++ paths/hunt_metadata.go | 61 ++++ paths/paths.go | 183 +----------- reporting/container.go | 8 +- result_sets/paths.go | 270 ++++++++++++++++++ result_sets/paths_test.go | 165 +++++++++++ result_sets/result_sets.go | 83 ++++++ server/comms.go | 1 + server/server.go | 4 +- server/server_test.go | 30 +- services/hunt_dispatcher.go | 22 +- services/hunt_manager.go | 47 +-- services/hunt_manager_test.go | 50 ++-- services/interrogation.go | 43 ++- services/journal.go | 71 +---- services/notfications.go | 65 +++-- services/server_artifacts.go | 60 ++-- services/server_monitoring.go | 110 ++----- services/services.go | 2 +- services/vfs_service.go | 6 +- uploads/api.go | 27 -- uploads/client_uploader.go | 5 +- uploads/file_based.go | 5 +- utils/clock.go | 21 ++ utils/json.go | 75 ++++- vql/networking/upload.go | 8 +- vql/server/flows.go | 65 ++--- vql/server/hunts.go | 66 ++--- vql/server/monitoring.go | 116 +------- vql/server/results.go | 224 ++------------- vql/tools/gcs_upload.go | 8 +- vql/tools/s3_upload.go | 10 +- 85 files changed, 2700 insertions(+), 1680 deletions(-) create mode 100644 artifacts/definitions/Server/Internal/Notification.yaml create mode 100644 artifacts/definitions/System/Flow/Archive.yaml delete mode 100644 constants/paths.go create mode 100644 file_store/api/accessor.go create mode 100644 file_store/directory/csv.go delete mode 100644 file_store/directory/directory_accessor.go create mode 100644 file_store/directory/json.go create mode 100644 file_store/directory/queue.go create mode 100644 file_store/directory/queue_test.go create mode 100644 file_store/directory/result_sets.go create mode 100644 file_store/mysql/queue.go create mode 100644 file_store/result_set.go create mode 100644 paths/client.go delete mode 100644 paths/clients.go create mode 100644 paths/flow_metadata.go create mode 100644 paths/hunt_metadata.go create mode 100644 result_sets/paths.go create mode 100644 result_sets/paths_test.go create mode 100644 result_sets/result_sets.go create mode 100644 utils/clock.go diff --git a/api/api.go b/api/api.go index 5539144e107..38f1be60cad 100644 --- a/api/api.go +++ b/api/api.go @@ -52,6 +52,7 @@ import ( "www.velocidex.com/golang/velociraptor/grpc_client" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/server" "www.velocidex.com/golang/velociraptor/services" users "www.velocidex.com/golang/velociraptor/users" @@ -180,7 +181,7 @@ func (self *ApiServer) CollectArtifact( result.FlowId = flow_id - err = services.NotifyClient(in.ClientId) + err = services.NotifyClient(self.config, in.ClientId) if err != nil { return nil, err } @@ -395,10 +396,10 @@ func (self *ApiServer) NotifyClients( if in.NotifyAll { self.server_obj.Info("sending notification to everyone") - services.NotifyAll() + services.NotifyAll(self.config) } else if in.ClientId != "" { self.server_obj.Info("sending notification to %s", in.ClientId) - services.NotifyClient(in.ClientId) + services.NotifyClient(self.config, in.ClientId) } else { return nil, status.Error(codes.InvalidArgument, "client id should be specified") @@ -638,7 +639,7 @@ func (self *ApiServer) GetTable( "User is not allowed to view results.") } - result, err := getTable(self.config, in) + result, err := getTable(ctx, self.config, in) if err != nil { return &api_proto.GetTableResponse{}, nil } @@ -793,8 +794,16 @@ func (self *ApiServer) WriteEvent( "Permission denied: PUBLISH "+peer_name+" to "+in.Query.Name) } - return &empty.Empty{}, services.GetJournal().Push( - in.Query.Name, peer_name, 0, []byte(in.Response)) + rows, err := utils.ParseJsonToDicts([]byte(in.Response)) + if err != nil { + return nil, err + } + + path_manager := result_sets.NewArtifactPathManager(self.config, + peer_name, "", in.Query.Name) + + return &empty.Empty{}, services.GetJournal().PushRows( + path_manager, rows) } return nil, status.Error(codes.InvalidArgument, "no peer certs?") diff --git a/api/artifacts.go b/api/artifacts.go index c0ebbb8f632..28b4d236c30 100644 --- a/api/artifacts.go +++ b/api/artifacts.go @@ -19,6 +19,7 @@ package api import ( "errors" + "os" "path" "regexp" "strings" @@ -35,6 +36,7 @@ import ( file_store "www.velocidex.com/golang/velociraptor/file_store" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" users "www.velocidex.com/golang/velociraptor/users" ) @@ -246,39 +248,40 @@ func (self *ApiServer) ListAvailableEventResults( "User is not allowed to view results.") } - result := &api_proto.ListAvailableEventResultsResponse{} - - root_path := "server_artifacts" - - if in.ClientId != "" { - root_path = "clients/" + in.ClientId + "/monitoring" - } - + path_manager := result_sets.NewMonitoringArtifactPathManager(in.ClientId) file_store_factory := file_store.GetFileStore(self.config) - dir_list, err := file_store_factory.ListDirectory(root_path) - if err != nil { - return nil, err - } - for _, dirname := range dir_list { - available_event := &api_proto.AvailableEvent{ - Artifact: dirname.Name(), - } - result.Logs = append(result.Logs, available_event) - - timestamps, err := file_store_factory.ListDirectory( - path.Join(root_path, dirname.Name())) - if err == nil { - for _, filename := range timestamps { - timestamp := paths.DayNameToTimestamp( - filename.Name()) - if timestamp > 0 { - available_event.Timestamps = append( - available_event.Timestamps, + seen := make(map[string]*api_proto.AvailableEvent) + err = file_store_factory.Walk(path_manager.Path(), + func(full_path string, info os.FileInfo, err error) error { + if !info.IsDir() && info.Size() > 0 { + relative_path := strings.TrimPrefix(full_path, path_manager.Path()) + artifact_name := strings.TrimLeft(path.Dir(relative_path), "/") + date_name := path.Base(relative_path) + timestamp := paths.DayNameToTimestamp(date_name) + + if timestamp != 0 { + event, pres := seen[artifact_name] + if !pres { + event = &api_proto.AvailableEvent{ + Artifact: artifact_name, + } + } + + event.Timestamps = append(event.Timestamps, int32(timestamp)) + seen[artifact_name] = event } } - } + return nil + }) + if err != nil { + return nil, err + } + + result := &api_proto.ListAvailableEventResultsResponse{} + for _, item := range seen { + result.Logs = append(result.Logs, item) } return result, nil diff --git a/api/clients.go b/api/clients.go index 3de9d42fc31..d3fbb1cd0b2 100644 --- a/api/clients.go +++ b/api/clients.go @@ -52,7 +52,7 @@ func GetApiClient( return nil, errors.New("client_id must start with C") } - client_urn := paths.GetClientMetadataPath(client_id) + client_path_manager := paths.NewClientPathManager(client_id) db, err := datastore.GetDB(config_obj) if err != nil { return nil, err @@ -68,7 +68,8 @@ func GetApiClient( } client_info := &actions_proto.ClientInfo{} - err = db.GetSubject(config_obj, client_urn, client_info) + err = db.GetSubject(config_obj, + client_path_manager.Path(), client_info) if err != nil { return nil, err } @@ -87,7 +88,7 @@ func GetApiClient( } public_key_info := &crypto_proto.PublicKey{} - err = db.GetSubject(config_obj, paths.GetClientKeyPath(client_id), + err = db.GetSubject(config_obj, client_path_manager.Key().Path(), public_key_info) if err != nil { return nil, err @@ -95,7 +96,7 @@ func GetApiClient( result.FirstSeenAt = public_key_info.EnrollTime - err = db.GetSubject(config_obj, paths.GetClientPingPath(client_id), + err = db.GetSubject(config_obj, client_path_manager.Ping().Path(), client_info) if err != nil { return nil, err diff --git a/api/csv.go b/api/csv.go index 11b8b9c1245..66912566b11 100644 --- a/api/csv.go +++ b/api/csv.go @@ -18,16 +18,21 @@ package api import ( - "io" - + context "golang.org/x/net/context" file_store "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/csv" + "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" ) -func getTable(config_obj *config_proto.Config, in *api_proto.GetTableRequest) ( +func getTable( + ctx context.Context, + config_obj *config_proto.Config, + in *api_proto.GetTableRequest) ( *api_proto.GetTableResponse, error) { rows := uint64(0) @@ -35,36 +40,52 @@ func getTable(config_obj *config_proto.Config, in *api_proto.GetTableRequest) ( in.Rows = 500 } - fd, err := file_store.GetFileStore(config_obj).ReadFile(in.Path) - if err != nil { - return nil, err - } + var path_manager api.PathManager + + if in.FlowId != "" && in.Artifact != "" { + path_manager = result_sets.NewArtifactPathManager( + config_obj, in.ClientId, in.FlowId, in.Artifact) - csv_reader := csv.NewReader(fd) - headers, err := csv_reader.Read() - if err != nil { - if err == io.EOF { - return &api_proto.GetTableResponse{}, nil + } else if in.FlowId != "" && in.Type != "" { + flow_path_manager := paths.NewFlowPathManager( + in.ClientId, in.FlowId) + switch in.Type { + case "log": + path_manager = flow_path_manager.Log() + case "uploads": + path_manager = flow_path_manager.UploadMetadata() } - return nil, err + } else if in.HuntId != "" && in.Type == "clients" { + path_manager = paths.NewHuntPathManager(in.HuntId).Clients() } - result := &api_proto.GetTableResponse{ - Columns: headers, - } + result := &api_proto.GetTableResponse{} - for { - row_data, err := csv_reader.Read() + if path_manager != nil { + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + path_manager, 0, 0) if err != nil { - break + return nil, err } - result.Rows = append(result.Rows, &api_proto.Row{ - Cell: row_data, - }) - rows += 1 - if rows > in.Rows { - break + for row := range row_chan { + if result.Columns == nil { + result.Columns = row.Keys() + } + + row_data := make([]string, 0, len(result.Columns)) + for _, key := range row.Keys() { + value, _ := row.Get(key) + row_data = append(row_data, csv.AnyToString(value)) + } + result.Rows = append(result.Rows, &api_proto.Row{ + Cell: row_data, + }) + + rows += 1 + if rows > in.Rows { + break + } } } diff --git a/api/download.go b/api/download.go index 853b5172f88..1fb7a310507 100644 --- a/api/download.go +++ b/api/download.go @@ -15,8 +15,9 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -// Implement downloads. For now we do not use gRPC for this but -// implement it directly in the API. + +// Implement downloads. We do not use gRPC for this but implement it +// directly in the API. package api import ( @@ -42,6 +43,7 @@ import ( "www.velocidex.com/golang/velociraptor/flows" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/utils" ) @@ -93,17 +95,20 @@ func downloadFlowToZip( return err } + flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) + // Copy the flow's logs. - copier(path.Join(flow_details.Context.Urn, "logs")) - - // Copy CSV files - for _, artifacts_with_results := range flow_details.Context.ArtifactsWithResults { - artifact_name, source := paths.SplitFullSourceName(artifacts_with_results) - csv_path := paths.GetCSVPath( - flow_details.Context.Request.ClientId, "*", - flow_details.Context.SessionId, - artifact_name, source, paths.MODE_CLIENT) - copier(csv_path) + copier(flow_path_manager.Log().Path()) + + // Copy result sets + for _, artifact_with_results := range flow_details.Context.ArtifactsWithResults { + path_manager := result_sets.NewArtifactPathManager(config_obj, + flow_details.Context.Request.ClientId, + flow_details.Context.SessionId, artifact_with_results) + rs_path, err := path_manager.GetPathForWriting() + if err == nil { + copier(rs_path) + } } // Get all file uploads @@ -117,14 +122,13 @@ func downloadFlowToZip( // are. Users can do their own post processing. // File uploads are stored in their own CSV file. - file_path := paths.GetUploadsMetadata(client_id, flow_id) - fd, err := file_store_factory.ReadFile(file_path) + row_chan, err := file_store.GetTimeRange( + ctx, config_obj, flow_path_manager.UploadMetadata(), 0, 0) if err != nil { return err } - defer fd.Close() - for row := range csv.GetCSVReader(fd) { + for row := range row_chan { vfs_path_any, pres := row.Get("vfs_path") if pres { err = copier(vfs_path_any.(string)) @@ -140,7 +144,8 @@ func createDownloadFile(config_obj *config_proto.Config, return errors.New("Client Id and Flow Id should be specified.") } - download_file := paths.GetDownloadsFile(client_id, flow_id) + flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) + download_file := flow_path_manager.GetDownloadsFile().Path() logger := logging.GetLogger(config_obj, &logging.GUIComponent) logger.WithFields(logrus.Fields{ diff --git a/api/proto/csv.pb.go b/api/proto/csv.pb.go index 690f6d02002..08ed0ae76b1 100644 --- a/api/proto/csv.pb.go +++ b/api/proto/csv.pb.go @@ -22,10 +22,15 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type GetTableRequest struct { + // Deprecated! Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` Rows uint64 `protobuf:"varint,2,opt,name=rows,proto3" json:"rows,omitempty"` StartRow uint64 `protobuf:"varint,3,opt,name=start_row,json=startRow,proto3" json:"start_row,omitempty"` ClientId string `protobuf:"bytes,4,opt,name=client_id,json=clientId,proto3" json:"client_id,omitempty"` + FlowId string `protobuf:"bytes,5,opt,name=flow_id,json=flowId,proto3" json:"flow_id,omitempty"` + Artifact string `protobuf:"bytes,6,opt,name=artifact,proto3" json:"artifact,omitempty"` + Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` + HuntId string `protobuf:"bytes,8,opt,name=hunt_id,json=huntId,proto3" json:"hunt_id,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -84,6 +89,34 @@ func (m *GetTableRequest) GetClientId() string { return "" } +func (m *GetTableRequest) GetFlowId() string { + if m != nil { + return m.FlowId + } + return "" +} + +func (m *GetTableRequest) GetArtifact() string { + if m != nil { + return m.Artifact + } + return "" +} + +func (m *GetTableRequest) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *GetTableRequest) GetHuntId() string { + if m != nil { + return m.HuntId + } + return "" +} + type Row struct { Cell []string `protobuf:"bytes,1,rep,name=cell,proto3" json:"cell,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -181,28 +214,31 @@ func init() { } var fileDescriptor_3e185232356a2266 = []byte{ - // 359 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0xcd, 0xae, 0xd3, 0x30, - 0x10, 0x85, 0x15, 0x12, 0xe0, 0xc6, 0x57, 0x08, 0xe4, 0x0b, 0x22, 0xa5, 0x12, 0x8c, 0xba, 0x21, - 0x1b, 0x12, 0x41, 0x37, 0x2c, 0x51, 0x90, 0x4a, 0xd9, 0xba, 0x55, 0x17, 0x08, 0xa9, 0x72, 0x1d, - 0xb7, 0xb1, 0xe4, 0xd8, 0xc1, 0x9e, 0xd6, 0xf0, 0x7e, 0x3c, 0x09, 0x3c, 0x06, 0x2c, 0x50, 0x1c, - 0xfe, 0x56, 0x63, 0xcd, 0x39, 0x9f, 0x35, 0xe7, 0x90, 0x5c, 0xf8, 0x4b, 0x35, 0x38, 0x8b, 0x96, - 0xde, 0x8e, 0xe3, 0xc9, 0xc3, 0x38, 0x6a, 0x2f, 0x7b, 0x6e, 0x50, 0x89, 0x49, 0x5c, 0xfc, 0x48, - 0xc8, 0xfd, 0x77, 0x12, 0xb7, 0xfc, 0xa0, 0x25, 0x93, 0x9f, 0xce, 0xd2, 0x23, 0xfd, 0x48, 0xb2, - 0x81, 0x63, 0x57, 0x24, 0x90, 0x94, 0x79, 0xb3, 0xfe, 0xf6, 0xf3, 0xfb, 0xd7, 0xa4, 0xa1, 0x6f, - 0xb6, 0x9d, 0x84, 0x71, 0x0f, 0x41, 0x61, 0xa7, 0x0c, 0x60, 0x27, 0x41, 0x68, 0x25, 0x0d, 0x3e, - 0xf7, 0xb0, 0x5b, 0x6d, 0x00, 0x3b, 0x8e, 0x20, 0xac, 0x41, 0xae, 0x8c, 0x87, 0xb7, 0x9b, 0x1d, - 0x1c, 0x95, 0x96, 0x80, 0x16, 0x9c, 0xe4, 0x6d, 0xc5, 0xe2, 0xaf, 0x74, 0x49, 0x32, 0x67, 0x83, - 0x2f, 0x6e, 0x41, 0x52, 0x66, 0xcd, 0xb3, 0xf8, 0xfb, 0x8c, 0x3e, 0x5e, 0xdb, 0x00, 0x3d, 0x37, - 0x5f, 0x60, 0x14, 0x47, 0xe6, 0x28, 0x51, 0x74, 0x15, 0x8b, 0x66, 0xfa, 0x9a, 0xe4, 0x1e, 0xb9, - 0xc3, 0xbd, 0xb3, 0xa1, 0x48, 0x23, 0x39, 0x8f, 0xe4, 0x23, 0x7a, 0xb3, 0x52, 0xce, 0xe3, 0x88, - 0xfd, 0x47, 0x5d, 0x45, 0x37, 0xb3, 0x81, 0xce, 0x49, 0x3e, 0x1d, 0xb9, 0x57, 0x6d, 0x91, 0x8d, - 0x89, 0xd8, 0xd5, 0xb4, 0x78, 0xdf, 0x2e, 0x66, 0x24, 0x1d, 0x3d, 0x94, 0x64, 0x42, 0x6a, 0x5d, - 0x24, 0x90, 0x96, 0x39, 0x8b, 0xef, 0x05, 0x27, 0x0f, 0xfe, 0xf5, 0xe2, 0x07, 0x6b, 0xbc, 0xa4, - 0x2f, 0xc8, 0x5d, 0x61, 0xf5, 0xb9, 0x37, 0x7e, 0xb2, 0x36, 0x37, 0xf1, 0x86, 0x7b, 0xf4, 0x7a, - 0xec, 0xe6, 0xb7, 0xc4, 0xfe, 0x78, 0xe8, 0xd3, 0xbf, 0x49, 0xd3, 0xf2, 0xfa, 0x15, 0x99, 0x1a, - 0xaf, 0x98, 0x0d, 0x53, 0xa8, 0xe6, 0xe5, 0x87, 0x3a, 0x84, 0x50, 0x5d, 0xa4, 0xb6, 0x42, 0xb5, - 0xf2, 0x73, 0x25, 0x6c, 0x5f, 0x9f, 0xac, 0xe6, 0xe6, 0x54, 0x4f, 0x4b, 0xc7, 0x07, 0xb4, 0xae, - 0xe6, 0x83, 0xaa, 0x23, 0x7c, 0xb8, 0x13, 0xc7, 0xf2, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x34, - 0xc8, 0x33, 0x4e, 0xdf, 0x01, 0x00, 0x00, + // 410 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x91, 0xc1, 0x6e, 0xd4, 0x30, + 0x10, 0x86, 0x95, 0x6e, 0xba, 0xbb, 0x71, 0x85, 0x40, 0x2e, 0xa8, 0x69, 0x2b, 0x81, 0xb5, 0x17, + 0xf6, 0x42, 0x22, 0xe8, 0x85, 0x23, 0x0a, 0x52, 0xe9, 0x5e, 0xd3, 0xaa, 0x07, 0x84, 0x54, 0xb9, + 0x8e, 0xd3, 0x58, 0x72, 0x3c, 0xc1, 0x9e, 0xad, 0xe9, 0xfb, 0xf1, 0x1a, 0x5c, 0xe0, 0x35, 0x38, + 0x20, 0xdb, 0x2c, 0xf4, 0x34, 0x9e, 0xf9, 0xe7, 0xfb, 0x63, 0xff, 0x21, 0x85, 0x70, 0xf7, 0xd5, + 0x64, 0x01, 0x81, 0xee, 0xc7, 0x72, 0xf2, 0x3c, 0x96, 0xda, 0xc9, 0x91, 0x1b, 0x54, 0x22, 0x89, + 0xab, 0x1f, 0x7b, 0xe4, 0xe9, 0x27, 0x89, 0x57, 0xfc, 0x56, 0xcb, 0x56, 0x7e, 0xdd, 0x4a, 0x87, + 0xf4, 0x0b, 0xc9, 0x27, 0x8e, 0x43, 0x99, 0xb1, 0x6c, 0x5d, 0x34, 0x17, 0x3f, 0x7f, 0xff, 0xfa, + 0x9e, 0x35, 0xf4, 0xc3, 0xd5, 0x20, 0x59, 0x98, 0x33, 0xaf, 0x70, 0x50, 0x86, 0xe1, 0x20, 0x99, + 0xd0, 0x4a, 0x1a, 0x7c, 0xed, 0xd8, 0xf5, 0xf9, 0x25, 0xc3, 0x81, 0x23, 0x13, 0x60, 0x90, 0x2b, + 0xe3, 0xd8, 0xc7, 0xcb, 0x6b, 0xd6, 0x2b, 0x2d, 0x19, 0x02, 0xb3, 0x92, 0x77, 0x55, 0x1b, 0x5d, + 0xe9, 0x19, 0xc9, 0x2d, 0x78, 0x57, 0xee, 0xb1, 0x6c, 0x9d, 0x37, 0xaf, 0xa2, 0xfb, 0x31, 0x3d, + 0xba, 0x00, 0xcf, 0x46, 0x6e, 0x1e, 0x58, 0x10, 0x03, 0xd3, 0x4b, 0x14, 0x43, 0xd5, 0xc6, 0x65, + 0xfa, 0x9e, 0x14, 0x0e, 0xb9, 0xc5, 0x1b, 0x0b, 0xbe, 0x9c, 0x45, 0xf2, 0x34, 0x92, 0x2f, 0xe8, + 0xe1, 0xb9, 0xb2, 0x0e, 0x03, 0xf6, 0x88, 0x5a, 0xc6, 0xed, 0x16, 0x3c, 0x3d, 0x25, 0x45, 0xba, + 0xe4, 0x8d, 0xea, 0xca, 0x3c, 0xbc, 0xa8, 0x5d, 0xa6, 0xc1, 0xa6, 0xa3, 0x47, 0x64, 0xd1, 0x6b, + 0xf0, 0x41, 0xda, 0x8f, 0xd2, 0x3c, 0xb4, 0x9b, 0x8e, 0x9e, 0x90, 0x25, 0xb7, 0xa8, 0x7a, 0x2e, + 0xb0, 0x9c, 0x27, 0x68, 0xd7, 0x53, 0x4a, 0x72, 0x7c, 0x98, 0x64, 0xb9, 0x88, 0xf3, 0x78, 0x0e, + 0x46, 0xc3, 0x36, 0x7d, 0x63, 0x99, 0x8c, 0x42, 0xbb, 0xe9, 0x56, 0xc7, 0x64, 0x16, 0x6e, 0x41, + 0x49, 0x2e, 0xa4, 0xd6, 0x65, 0xc6, 0x66, 0x81, 0x09, 0xe7, 0x15, 0x27, 0xcf, 0xfe, 0x27, 0xef, + 0x26, 0x30, 0x4e, 0xd2, 0x37, 0x64, 0x21, 0x40, 0x6f, 0x47, 0xe3, 0xd2, 0x6a, 0x73, 0x18, 0x5f, + 0xf9, 0x84, 0x1e, 0x84, 0xf4, 0xff, 0x4a, 0xed, 0x6e, 0x87, 0xbe, 0xfc, 0x97, 0xe5, 0x6c, 0x7d, + 0xf0, 0x8e, 0xa4, 0x7f, 0x5a, 0xb5, 0xe0, 0x53, 0x6c, 0xcd, 0xdb, 0xcf, 0xb5, 0xf7, 0xbe, 0xba, + 0x97, 0x1a, 0x84, 0xea, 0xe4, 0xb7, 0x4a, 0xc0, 0x58, 0xdf, 0x81, 0xe6, 0xe6, 0xae, 0x4e, 0x43, + 0xcb, 0x27, 0x04, 0x5b, 0xf3, 0x49, 0xd5, 0x11, 0xbe, 0x9d, 0xc7, 0x72, 0xf6, 0x27, 0x00, 0x00, + 0xff, 0xff, 0x7c, 0x01, 0x8a, 0xe3, 0x41, 0x02, 0x00, 0x00, } diff --git a/api/proto/csv.proto b/api/proto/csv.proto index d6e93ea2121..be065149da9 100644 --- a/api/proto/csv.proto +++ b/api/proto/csv.proto @@ -7,6 +7,7 @@ package proto; option go_package = "www.velocidex.com/golang/velociraptor/api/proto"; message GetTableRequest { + // Deprecated! string path = 1 [(sem_type) = { description: "The path within the client's VFS " "that contains CSV file to read." @@ -21,6 +22,12 @@ message GetTableRequest { }]; string client_id = 4; + string flow_id = 5; + string artifact = 6; + + string type = 7; + + string hunt_id = 8; } message Row { diff --git a/artifacts/assets/ab0x.go b/artifacts/assets/ab0x.go index 6edc3932d91..bf3da27bc2f 100644 --- a/artifacts/assets/ab0x.go +++ b/artifacts/assets/ab0x.go @@ -189,6 +189,9 @@ var FileArtifactsDefinitionsServerInternalArtifactDescriptionYaml = []byte("\x1f // FileArtifactsDefinitionsServerInternalInterrogateYaml is "artifacts/definitions/Server/Internal/Interrogate.yaml" var FileArtifactsDefinitionsServerInternalInterrogateYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x93\xcd\x6e\xda\x40\x10\xc7\xef\x7e\x8a\x11\x17\x42\x95\xf8\x01\x90\x7c\x48\x90\x69\xac\x52\x52\x01\xa2\x47\xb4\xac\x87\xb0\xaa\xbd\x4b\x66\xc7\xb1\x50\x4b\x9f\xbd\x5a\x1b\x5a\xa0\x6b\x9b\xcc\x01\x31\x9e\xff\x8c\xe7\xe3\x67\x2d\x72\x1c\xc2\x1c\xe9\x1d\x29\x4c\x34\x23\x69\x91\xd5\x7f\xc8\xbc\x0a\xc6\x20\x45\x2b\x49\xed\x58\x19\x3d\x84\x5f\x01\xc0\xa3\x06\x75\x14\x82\x20\x56\x1b\x21\x19\x0a\x8b\x29\x30\x09\xf9\x03\x34\x96\x20\x33\x85\x9a\x6b\x5d\x55\x47\x19\x6d\x61\xbd\x07\xde\x62\x00\x90\x9c\x3f\x07\x8b\xf4\xae\x24\x86\x41\xc0\xfb\x9d\xeb\x26\x9e\x2d\xe3\xd9\x2a\x5e\xc6\xd3\x45\x10\x58\x53\x90\x44\x3b\x0c\x00\x1e\xe0\xad\x40\x52\xb5\xe3\xec\x01\xe6\xf1\x24\x1e\x2d\xe0\x13\x8c\x67\x2f\x5f\x61\x63\x08\x85\xdc\xde\x1d\xc3\xce\xc8\x94\xd1\xcf\x33\x1f\xe0\x94\x33\xaa\x7a\x4c\xd2\x7b\x18\x67\xa6\xac\x7f\x93\xf4\x52\x5a\x55\x2d\x05\xcb\xed\x2a\x37\x5a\xb1\x21\xa5\x5f\xef\x4e\x53\x47\xfd\xf9\xde\x32\xe6\xa1\x4b\x0d\x47\x26\xdf\x65\xe8\x26\xea\x0f\x2e\xab\x7c\x7f\x8e\x67\x71\x55\x3f\x3c\xa5\xda\x55\xa9\x78\xbb\x22\xb4\x45\xc6\x16\xa2\xdf\xd0\xff\x8c\x1a\x49\xc9\xb0\xee\x2b\x4c\xf4\xc6\xf4\xcf\xea\x1c\xee\xcf\x1c\xb7\x87\xfd\xd5\x5c\x97\xab\xb0\xa5\xe2\xcb\x4d\x38\x13\xd7\xbb\xf0\x2e\xc4\x23\x81\xe3\x7a\xfc\xb1\x47\x92\x5b\xc5\x28\xb9\x20\xf4\x2b\x9e\x0a\x95\xa5\x0b\x95\x37\x84\xc7\x6f\xa9\xf6\x47\x9e\x8d\x65\x07\xa9\x3f\xfa\x05\x49\x63\xb6\x44\xb2\xca\x34\x14\x98\x88\x35\x66\xd6\x1f\x9b\x36\x16\x7e\x99\xfb\x9f\x7f\xcb\x04\x6f\x0c\xe5\xed\xd1\x63\x43\xff\x6b\xea\xd3\x54\x44\x5f\x9f\xa6\xb6\xfa\xc3\x59\xa9\x34\x6a\xbf\xc6\x26\x33\xa5\x53\xb5\x5d\xa5\x7e\x4f\xd4\x7b\x12\x56\x49\x87\x13\xe5\xd5\x07\xd7\xf3\xcb\xff\x52\xdd\x1b\x15\x96\x4d\x1e\x7a\x80\x6c\x48\xcd\x4d\x8a\x51\x6f\x34\x49\xe2\xe9\xa2\x37\xb8\x96\x1c\x2e\x93\xd6\x1e\x06\x6f\x20\xb0\x79\xd2\x2e\xfa\x5a\xd9\x6b\x22\xaf\x8d\xbb\x4e\xea\x9a\x99\x6b\x22\xce\xcf\x5b\x1b\x6d\x1d\xac\x75\x90\x76\x1b\x67\xdd\x94\x7d\x88\xb1\x7f\x84\xdd\x8a\x56\x0b\x58\x87\x73\xf7\x30\x08\xfe\x04\x00\x00\xff\xff\x47\x0a\xd1\x23\xca\x06\x00\x00") +// FileArtifactsDefinitionsServerInternalNotificationYaml is "artifacts/definitions/Server/Internal/Notification.yaml" +var FileArtifactsDefinitionsServerInternalNotificationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4c\xd1\x31\x6e\xc3\x30\x0c\x05\xd0\x5d\xa7\xf8\x07\x28\x7c\x80\x6c\x1d\x3c\x74\xc9\x90\x06\x01\x3a\x15\x8c\x4c\x43\x44\x65\x2a\xa0\x18\x1b\x01\x7a\xf8\xc2\x96\x13\x74\x14\x24\xbe\x4f\x7c\x29\x4d\x7c\xc0\x27\xdb\xcc\xd6\x7d\xa8\xb3\x29\xe5\xee\x58\x5c\x46\x89\xe4\x52\xb4\x86\x81\x6b\x34\xb9\xad\x87\x03\x7e\x03\x70\x4e\x52\xc1\x33\xab\x83\xcc\x65\xa4\xe8\x90\x0a\x52\xc8\x2e\xec\xb7\xd5\x8d\x69\x42\x99\xd9\xb0\x24\x89\x09\x31\x0b\xab\x07\x40\xff\x47\x80\x8c\x51\x59\xbd\xc3\x3b\x46\x2b\xea\xac\x03\x16\xc9\x19\x0b\x79\x4c\x18\x8b\x35\xb2\x36\xcc\x93\xd4\x80\xa7\x4f\x3a\x40\x46\xd0\xae\x6f\xbb\x44\x97\x99\xf3\x03\xb1\xa8\x72\x74\x1e\xe0\x65\x1b\x7b\xf9\x6f\xf0\xc4\x01\xcf\xa1\x2d\xed\xca\xfb\x62\xeb\xfb\x44\x0e\xe5\x05\x4b\xb1\x9f\xcd\x9c\x49\x32\x5d\x33\xaf\x94\x78\x17\x02\x70\x2c\xce\x87\x56\x48\x6b\x80\xee\x5e\x26\x5a\xf3\xea\xa3\x3a\x4f\xaf\x86\x3a\x7c\x95\x3b\x86\xb2\x06\x40\xb9\x2d\x54\x9d\xcc\x9b\xe5\x8f\xdb\xfa\x13\xfd\xe9\xd2\x9f\xbe\xfb\x4b\x7f\x3c\x87\xbf\x00\x00\x00\xff\xff\x2a\x81\xfe\x5f\x9d\x01\x00\x00") + // FileArtifactsDefinitionsServerMonitorHealthYaml is "artifacts/definitions/Server/Monitor/Health.yaml" var FileArtifactsDefinitionsServerMonitorHealthYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x56\x5d\x6f\xdb\x36\x14\x7d\xd7\xaf\xb8\x50\x57\xd4\x2e\x12\x25\x1d\xb6\x17\x23\x2a\x10\x18\xee\x3a\xc0\xc9\x32\xc7\xce\x1e\x8a\x42\x60\xa8\x9b\x88\x98\x48\xaa\xbc\x57\x4d\x3c\x27\xff\x7d\xa0\x28\x2b\x96\xed\x74\xcb\x30\x0c\xe3\x8b\x20\xe9\x7e\x1c\x9e\x7b\x0e\x25\x23\x34\x8e\xe0\x12\xdd\x57\x74\xc9\x99\x35\x8a\xad\x4b\x3e\xa2\x28\xb9\x88\x72\x24\xe9\x54\xc5\xca\x9a\x11\x3c\x44\x00\xf3\x42\x11\x28\x02\x2e\x10\xb4\x50\x06\xa8\xc9\x83\xa2\x89\x87\x5c\x50\x71\x6d\x85\xcb\x13\xf8\x99\x7d\x1c\x15\xf6\xce\x80\x35\x3e\x21\x02\x28\xac\xf6\x15\x11\x0d\x08\x93\x03\x1a\x71\x5d\x62\x0e\xd7\x4b\xc8\xf1\x46\xd4\x25\xfb\x50\x51\x96\x60\xf0\x0e\x94\x21\x16\x65\x49\x49\x14\xf1\xb2\xf2\x18\x27\xb3\xab\xc9\x2c\x9b\x5c\x4d\xce\xe7\x51\x54\x09\x27\x34\x32\x3a\x1a\x45\x00\x87\x10\xf6\xf1\xc1\xe1\x97\x1a\x8d\x5c\x46\x00\x00\x3d\xfc\x33\xe4\xda\x19\x20\x16\x4c\x80\x5f\xd1\x2d\x81\xfd\x76\xb4\x30\x4b\x20\x94\xd6\xe4\x94\xb4\x69\x0d\x98\x11\xc4\xef\x7e\x8c\xa3\x88\x6c\xed\x24\xf6\xda\x5c\x38\xab\x91\x0b\xac\xa9\x49\xf8\x52\xa3\x53\x21\xc2\xaf\xc3\x86\xab\xb0\xa6\x93\x39\x68\x64\xa7\x24\x65\xb5\x2b\xe1\x24\x85\xcb\xc9\x74\x32\x9e\xc3\x8d\x75\x5a\xf0\x20\x5c\xd2\x37\x05\x73\x35\x3a\x3a\x7a\x4d\xa3\xd7\xf9\x51\x9b\xf1\xe6\x00\x84\xbb\xa5\xf4\x53\x57\x2e\xac\xc0\x7a\x26\xad\xb9\x51\xb7\xeb\xa1\x29\x73\x9b\x5c\x2b\x93\x67\x22\xcf\x1d\x12\x1d\xbc\x28\xa9\xb2\x8e\x3f\x0f\x41\x10\x2c\x66\xd3\x2e\xf3\xc3\xec\x97\x33\x20\x69\x2b\x1c\x0c\xa3\x3d\xbb\x6b\xb7\xa2\x0c\x0f\x94\xe1\xd4\x09\xc6\xc1\x7d\x5a\x39\x2b\x91\x28\x93\x55\x9d\xb5\xc4\x66\x6c\x59\x94\x07\xb0\x4c\xe7\x4a\x23\xb1\xd0\xd5\x10\xde\xc2\xbb\xe3\xe3\x21\x9c\x12\x8c\x2f\x16\x17\xe8\x24\x1a\xde\x46\x0d\xeb\x62\x0e\x49\xe5\x68\x38\xd3\xa8\xad\x5b\x66\xd7\x4b\x46\x82\x23\x5f\xc2\x2f\x38\xbd\x84\xb3\xe6\xcd\x82\xf0\xd9\x22\xbb\x88\xb6\x23\x65\xa9\x7c\x13\x69\xb5\xa6\x4c\xd6\xce\x85\x3b\x63\x50\x7a\x19\xed\xb0\xba\x95\x60\x4d\xc8\x69\x05\xd8\x71\x78\x63\x1d\x0a\x59\x0c\x36\xb2\x9d\xbd\x4b\x57\xfd\x6a\x2d\x9d\x0b\xa3\xee\xcf\x85\xb1\x21\x55\x96\x56\xfe\x3e\xa8\xd0\x29\x9b\xa7\x82\xad\x1a\x10\xfb\xb9\xa5\x9d\xd6\x87\xc3\x8d\x32\x8f\x9b\x08\xbd\x30\x97\xcf\x74\x79\x1b\xca\x9f\x3a\x56\x37\x42\x72\xb2\x75\x04\x5c\x61\x69\xcf\x82\x0c\x07\xed\x75\x31\x9b\xa6\x1b\x5a\x4e\x16\xb3\xe9\xa7\xe3\xcf\xbd\xe6\x4f\x37\xbf\x7d\x9c\xcc\x26\x1b\x83\x85\xf7\x29\x1c\x47\x51\xe4\xd0\x4b\xad\x75\xd3\x1e\x63\x37\xe3\xea\x79\x3b\x88\xae\x3d\xa7\x84\xae\x4a\xec\x9a\x3c\x39\xf5\x87\x38\xe8\x93\x51\x57\xa5\x60\x1c\x75\x2a\x5d\xad\x7c\x98\x32\x08\xf1\xf8\x62\x11\xc3\xe3\x63\xf4\x2c\x19\xd4\x94\x1f\xf4\xf9\x32\x3d\xd6\x03\x82\xe1\x96\x0e\xf6\x12\xdd\x55\xcf\x98\xbc\xb3\x3a\xe9\xef\x88\x08\xe0\x5b\x16\x00\x78\x92\xf6\xf6\xbb\x00\xbb\x39\xa1\x06\xe1\x92\xc6\x4f\xa7\x53\xbc\xaf\x56\xb7\x44\x3b\xfa\x34\xde\x7b\xfc\xc7\xc3\x7e\x72\x37\xdc\xd5\x0a\xd0\xe4\x9e\xc8\x3d\x14\x07\xcb\x8c\x9f\x1c\xf3\xbf\x65\xfc\xc5\x56\xff\x3b\x6e\xff\xcf\xc7\xf2\xcd\xa9\x7c\xd7\x8e\xa3\x35\x30\x8c\x52\xf8\xd5\xf3\x06\xf1\xcb\x0f\x81\x61\xbc\x51\xfb\xd5\xab\xf6\x97\xa1\xf9\x9e\xd6\xb4\x7e\xde\xf6\x2b\xfd\xb7\x15\x1d\x82\x70\xe8\x71\xfc\x84\xbc\x83\x25\x3e\x4e\xfe\x6a\x02\xbe\x63\x4b\x39\x41\xfb\x1c\xf3\x64\xdd\xec\x84\x2a\x61\x40\x96\x82\x28\x8d\xa5\x35\x2c\x94\x41\x17\xbf\xef\xf8\xe9\x05\x38\x7b\xb7\xf1\x6a\x27\xbb\x3c\x24\x0d\x95\x30\x58\xf6\xa2\xbc\x2f\x9b\x9f\x95\xe0\x40\x58\xb0\x2a\xd5\x1f\xc2\xc3\xdb\x8c\x5a\xad\xd6\xc4\x36\x67\xcc\x03\x4c\x95\xc1\x71\x21\x1c\x43\x7c\x2f\xee\x15\x65\xda\xe6\x18\x43\xcc\x4a\xfb\xcb\xec\xf2\x32\x59\xfa\xe7\x31\x7c\xdf\xf7\xc7\xc9\x91\xc7\xf5\x4f\x80\x76\xd4\x8f\xd7\x4c\xc1\x38\x70\xf7\x0c\xd2\x3d\x56\xfd\x17\x81\xf7\xef\xd7\x77\x1b\x02\x5a\x10\x3a\xda\x10\x6b\x5f\x99\xe7\x42\xe3\x01\x5c\xa0\xd3\x8a\xc8\x83\x0b\x4a\xbd\xad\x55\x56\xfb\x44\xaf\xc7\x07\x98\xfb\x3f\x48\x8f\xe3\xcf\x00\x00\x00\xff\xff\xed\x43\x56\x16\xc7\x0a\x00\x00") @@ -204,11 +207,14 @@ var FileArtifactsDefinitionsServerMonitoringClientCountYaml = []byte("\x1f\x8b\x // FileArtifactsDefinitionsServerPowershellEncodedCommandYaml is "artifacts/definitions/Server/Powershell/EncodedCommand.yaml" var FileArtifactsDefinitionsServerPowershellEncodedCommandYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x7c\x53\x51\x4f\xdb\x3c\x14\x7d\xf7\xaf\x38\x82\x87\x24\xfa\x4a\xa4\x4f\x9a\x78\xa8\x96\xa1\xa9\x04\x0d\x89\x41\x69\x23\x78\x98\xb6\xc8\x24\xb7\x8d\xa5\xc4\x0e\xb6\x43\x41\xb4\xfb\xed\x93\xeb\x84\x76\x74\xa2\x2f\x89\x7b\xcf\x3d\x3e\xb9\xe7\x5c\xc9\x1b\x1a\x63\x4e\xfa\x89\x74\x3c\x55\x2b\xd2\xa6\xa2\xba\x8e\x53\x59\xa8\x92\xca\x89\x6a\x1a\x2e\x4b\x56\x92\x29\xb4\x68\xad\x50\x72\x8c\x35\x03\x2e\x2d\x84\x41\xab\x8c\x11\x0f\x35\xc1\x2a\xb4\xdc\xb8\x3f\x06\x06\x70\x09\xf2\x24\xf0\xbd\x31\xb2\x4a\x18\x70\x6d\xc5\x82\x17\x96\x01\x25\xb9\xba\x81\xad\xa8\xc7\x98\x98\x31\xe0\xfa\x26\x4b\xc7\xc8\x2a\x42\x51\x0b\x92\x16\x4d\x67\x2c\x1e\x08\xba\x93\x52\xc8\xe5\xb6\xe1\x5e\xc8\x52\xad\x4c\x9c\x3e\x91\xb4\x26\x9e\x6a\x55\x90\x31\x13\x4d\xdc\xa9\x64\x00\xb9\xc2\xdb\x75\x4e\xa2\x26\xab\x05\x3d\x11\x5a\x0f\x06\x3d\x53\xd1\x39\x38\x6a\xb5\x74\x77\xdb\x97\xd6\x8d\x23\x9d\xdd\xa5\xb3\x3c\xbd\x4b\xaf\x33\xc6\x8c\xea\x74\x41\x66\xcc\x80\x13\x3c\x76\xa4\x85\x3f\xc0\x9d\xd7\xfe\x05\x98\xa7\x57\xe9\x24\xc3\x64\x2b\xf8\xb2\x1c\x61\xca\xb5\x7b\x93\x0b\x35\x42\x3f\xc6\x2b\x21\x69\x84\x4c\x34\x64\x2c\x6f\xda\x11\x3a\xbb\xf8\xff\x34\x1c\x28\x00\x18\xab\x85\x5c\x26\x0f\xdc\xd0\xe9\x27\x3f\x9e\xfd\xf2\x0e\xd1\x72\x6d\x28\xf7\x87\x7c\x25\x6c\x95\x6b\x5a\xd2\xf3\x3b\xf0\x0e\xbf\xaf\xe0\x00\xb3\x6d\x4d\x82\x93\x30\x3c\x13\x51\x48\x72\x4d\xb2\x58\x7b\xf3\xfa\xc7\x10\x84\x28\x42\x78\x36\xfd\xdc\xa7\xe3\xcb\x8f\x5f\xf8\xf9\x5f\x14\xfc\x4d\x18\x0d\xe1\x89\x22\x7c\x9d\x63\xbe\x35\xf6\x0d\x72\x31\xbb\xf9\x8e\x15\xb7\x45\x95\x37\x4a\x0a\xab\x9c\xbc\x70\x70\x29\x09\x3e\xb6\x35\x88\xde\x88\xee\xbf\xa5\xb3\x74\x7f\xb2\x48\x7e\x23\x38\xf9\x58\x7e\xc0\x98\xa6\x56\x69\xdb\xdb\xf9\x0f\xbf\x1d\xb7\xa5\xa6\xad\xb9\x25\x97\xf5\xfe\xc2\xfe\x9b\xb0\xdb\x91\xbe\x90\x1c\xfc\x86\x96\xd7\x57\xc4\xe7\xbb\xc5\xc1\x66\x33\x54\x8e\x8f\x71\x4e\xef\xf9\x50\x78\x91\x7e\x07\x7a\x82\xdb\x8e\xf4\x0b\x8e\x0e\xd2\xf5\x3a\x04\x4e\x99\x5c\xc8\x85\x8a\x2f\x1e\x4b\x89\x85\x56\x4d\xbf\x33\x26\xf4\xcf\x5c\x94\xc9\xd0\x16\x61\xe3\x1c\xb9\xb8\x3d\xbf\x1e\xf5\xbe\x78\x3f\x7c\xc4\xc3\xe8\x08\x6b\x64\xdc\x2d\xf4\x66\xc3\xfe\x04\x00\x00\xff\xff\xbe\x8f\x95\xe4\x1b\x04\x00\x00") +// FileArtifactsDefinitionsSystemFlowArchiveYaml is "artifacts/definitions/System/Flow/Archive.yaml" +var FileArtifactsDefinitionsSystemFlowArchiveYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x24\xcd\x41\xaa\x83\x30\x10\x87\xf1\x7d\x4e\xf1\x3f\x81\x07\x70\x27\x0f\x1f\x14\x8a\x9b\x4a\xb7\x25\xc4\x11\x07\xe2\x4c\x98\x4c\x95\x40\x0f\x5f\xa4\xbb\x6f\xf5\xfd\x24\xee\xd4\xe3\xd1\xaa\xd3\xde\xfd\x67\x3d\xbb\xc1\xd2\xc6\x07\x85\x85\x6a\x32\x2e\xce\x2a\x3d\x3e\x01\x18\x04\x2c\x4e\x26\x31\x23\x9a\xf3\x1a\x93\xc3\xb7\xe8\x28\xa6\xcb\x3b\x51\x05\x1d\x24\x5e\xb1\xaa\x5d\x69\x0d\x6b\xd6\x13\x49\xf7\x92\xe9\x1a\x05\x80\x05\xbe\x11\xea\x4f\x0c\xc1\x5b\xa1\x1e\x7f\xf7\xdb\x38\xcd\xaf\xf1\x39\x4e\x73\xf8\x06\x00\x00\xff\xff\x45\x5c\x45\x29\x94\x00\x00\x00") + // FileArtifactsDefinitionsSystemFlowCompletionYaml is "artifacts/definitions/System/Flow/Completion.yaml" var FileArtifactsDefinitionsSystemFlowCompletionYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x44\xcd\x31\x8a\xc3\x30\x10\x46\xe1\x5e\xa7\xf8\x4f\xe0\x03\xb8\x5b\x8c\x17\x02\xc1\x4d\x4c\xda\x20\xe4\x11\x16\x48\x33\x62\x34\x89\x11\xe4\xf0\xc1\xa4\x48\xf7\x55\xef\xb1\x2f\x34\xe2\xd6\x9b\x51\x19\xfe\xb3\x1c\xc3\x24\xa5\x66\xb2\x24\xec\x36\x6a\x41\x53\x3d\x3d\xe2\xed\x80\x3f\x46\x62\x23\x65\x9f\xe1\xd5\x52\xf4\xc1\x60\xbb\x37\x54\x95\xed\x19\xa8\x81\x5e\xc4\xd6\x10\x45\x4f\x6a\x47\xcc\x72\x20\xfc\xa2\x40\x62\xd8\x4e\x68\xdf\xa9\x73\xd6\x2b\x8d\x98\xae\x97\x79\x59\x1f\xf3\x7d\x5e\x56\xf7\x09\x00\x00\xff\xff\x13\x41\x12\x2d\x97\x00\x00\x00") // FileArtifactsDefinitionsSystemHuntParticipationYaml is "artifacts/definitions/System/Hunt/Participation.yaml" -var FileArtifactsDefinitionsSystemHuntParticipationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x54\x5d\x6f\xda\x30\x14\x7d\xcf\xaf\x38\xa2\x93\x0a\xd3\x16\x6d\xaf\x68\x79\x40\x6d\xd6\x46\xa2\xb4\x85\xa8\x53\x9f\xc0\x4d\x2e\x8d\xa5\xc4\x0e\xf1\xcd\x10\xa2\xfc\xf7\x29\x4e\x02\xa1\x94\xd5\x4f\xb6\xef\xb9\xe7\x7e\xf8\x5c\x2b\x91\xd1\x10\xb3\x8d\x61\xca\xdc\xdb\x52\xb1\xfb\x20\x0a\x96\x91\xcc\x05\x4b\xad\x9c\x98\x4c\x54\xc8\xbc\xda\x0f\xf1\xe6\xa0\x5a\xbe\x8a\x73\x2d\x15\x1b\x64\x62\x83\x7c\x8f\x27\x48\x85\xa4\x54\x6c\x5c\x84\x89\x34\xa8\x2c\x4b\x11\x31\x22\x9d\xa6\x14\xb1\xc1\x3a\x91\x51\x62\x31\x35\x15\x89\x28\x81\xb1\xd1\xbb\x44\x31\xa4\x72\x9d\x1a\x32\xd1\x4c\xc3\x9a\xaf\xa2\x54\x10\x25\xeb\xcc\x82\x1a\xc7\x8a\xce\xc5\xb3\x2e\x11\x6b\x28\xcd\x50\x44\x31\x58\xc3\xb0\x28\x18\x92\x5d\xc7\xe1\x4d\x4e\x43\x5c\x8d\x03\x7f\x12\xce\xfd\x27\x7f\x12\x3a\x4e\x41\xb9\x2e\xd8\x0c\x1d\xe0\x3b\x6a\xc0\xdd\xfd\x24\x08\xef\xa7\xc1\xe4\x66\x7e\x3d\x0a\xc6\xcf\x36\x05\xa6\x2c\x4f\x45\x95\x45\xd3\x00\x6c\xb7\x88\x69\x29\x15\xa1\x27\xd2\x74\x6e\x6b\xee\x61\xb7\x1b\xfb\x21\x44\x9a\xda\x33\x7e\x79\x98\xf9\x63\xff\x2a\xc4\x57\xfc\x9e\xde\xdf\xd5\xad\xe9\x0f\xb6\x5b\x90\x8a\xb1\xdb\x9d\x92\xed\x89\x1a\x93\x5d\xc7\x24\x4b\x5d\x54\x4d\xeb\x77\x11\x40\xa1\xd7\xde\xb6\x45\xb2\xcc\xc8\xb0\xc8\xf2\x3e\xe5\x3a\x4a\xbc\xb0\x3d\x0f\x30\x9a\x61\x16\x25\x14\x97\x29\xc5\xdf\x8e\x29\x8e\x56\xa5\x84\x20\x86\x30\x78\xe8\xbc\x4a\x7d\xfb\x91\x9b\x4d\xcd\xe8\xb2\x88\xa8\x1f\xa5\x92\x14\xcf\x65\xec\x5d\xd9\x5d\x70\x3e\x52\x2b\x10\xef\xf2\xac\x02\x2f\x07\xd8\xbd\xf3\x5f\x95\x54\x6c\xbc\xed\x09\x69\x53\xfe\xe7\x05\xd6\x75\xfc\xd7\x7c\x7d\xd0\xfd\x59\xdc\xac\x92\xd7\x94\x56\x25\x19\x76\x47\xc5\xab\x71\x47\x4d\x3d\xc6\x9d\x88\x8c\xcc\x89\xa3\x6d\x53\x2b\x90\x13\xeb\x9f\x5b\x7f\xea\xb7\xad\xf7\x3e\xed\xfc\x6e\x70\x10\x50\x23\xa8\xc3\xc5\x97\xf6\x15\xd4\x52\x63\xe8\xe1\xb1\xea\x19\x7a\xc7\x5a\xaa\x31\xe6\x83\x17\x1b\x60\x1c\xdc\x05\x21\x7e\xf6\x3a\xb4\x17\x36\xb7\xce\x9c\x4a\xad\x2a\x3d\x56\xf1\x6e\x88\x8f\x63\xf6\x7e\xb8\xda\xd8\xad\xbb\x5c\xc5\xaa\xcb\x13\x26\xd4\x84\xc6\x5a\x72\x02\xd1\x9e\x82\x6b\xe8\xe5\x39\xb6\x7d\x92\x15\xd5\xfb\xcf\x02\x46\x67\x54\x0f\x18\x58\xc7\x62\xe3\x76\x7a\xd1\xd4\xde\x99\xd4\x76\xd0\xde\x10\x8a\x97\x94\xba\x35\x5e\xe0\xe9\x71\x5c\xbb\x74\xd2\x5d\xea\x34\xd5\x6b\xa9\x5e\xad\xd5\x0a\x10\x6b\x61\x50\x9a\xfa\xa7\xc9\x53\xcd\xe0\x84\xf0\x5a\x88\x3c\x81\x78\xd1\x7f\x69\x9f\xc2\x62\xb1\x30\xab\xf4\x90\x4f\xfb\x9f\x9c\xce\xfb\x62\xb1\x70\xfe\x05\x00\x00\xff\xff\x2e\x36\x7a\x5a\x92\x05\x00\x00") +var FileArtifactsDefinitionsSystemHuntParticipationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x54\x4f\x6f\xa3\x3e\x10\xbd\xf3\x29\x9e\xd2\x9f\xd4\xe4\xa7\x5d\xb4\x7b\x8d\x96\x43\xd5\xb2\x2d\x52\x9a\xfe\x09\xea\xaa\xa7\xc4\x85\x49\xb1\x64\x6c\x82\x87\x8d\xa2\x34\xdf\x7d\x85\x81\x94\x34\x8d\xca\xc9\x66\x9e\xdf\xcc\x3c\xbf\xb1\x16\x39\x8d\x31\xdb\x58\xa6\xdc\xbf\xa9\x34\xfb\xf7\xa2\x64\x99\xc8\x42\xb0\x34\xda\x4b\xc9\x26\xa5\x2c\xea\xf5\x18\x6f\x1e\xea\x2f\xd4\x69\x61\xa4\x66\x8b\x5c\x6c\x50\xec\xf1\x04\xa9\x91\x55\x9a\xad\x8f\x38\x93\x16\x75\x64\x29\x12\x46\x62\x94\xa2\x84\x2d\xd6\x99\x4c\x32\x87\x69\xa8\x48\x24\x19\xac\xcb\xde\x27\x4a\x21\xb5\xef\x35\x90\xa9\x61\x1a\x37\x7c\x35\xa5\x86\xa8\xd8\xe4\x0e\xd4\x1e\xec\xd2\xf8\x78\x36\x15\x52\x03\x6d\x18\x9a\x28\x05\x1b\x58\x16\x25\x43\xb2\xef\x79\xbc\x29\x68\x8c\xcb\x49\x14\x4e\xe3\x79\xf8\x14\x4e\x63\xcf\x2b\xa9\x30\x25\xdb\xb1\x07\x7c\x47\x03\xb8\xbd\x9b\x46\xf1\xdd\x63\x34\xbd\x9e\x5f\x5d\x44\x93\x67\x57\x06\x53\x5e\x28\x51\x57\xd2\x8a\x80\xed\x16\x29\x2d\xa5\x26\x0c\x84\x52\x73\xd7\xf7\x00\xbb\xdd\x24\x8c\x21\x94\x72\x7b\xfc\x0a\x30\x0b\x27\xe1\x65\x8c\xff\xf1\xfb\xf1\xee\xb6\x91\x67\x38\xda\x6e\x41\x3a\xc5\x6e\x77\x4c\xb6\x27\x6a\x43\xee\x3b\x24\x59\x9a\xb2\x16\x6e\xd8\x47\x00\xa5\x59\x07\xdb\x0e\xc9\x32\x27\xcb\x22\x2f\x86\x54\x98\x24\x0b\xe2\x6e\x3f\xc2\xc5\x0c\xb3\x24\xa3\xb4\x52\x94\x7e\x3b\xa4\x38\xf8\x6a\x37\x44\x29\x84\xc5\x7d\xef\x66\x9a\xbf\x9f\x1d\x73\xa5\x59\x53\x95\x09\x0d\x13\x25\x49\xf3\x5c\xa6\xc1\xa5\x5b\x45\xa7\x33\x75\xb7\x17\x9c\x9f\x74\xe1\xf9\x08\xbb\x0f\xe7\x57\x15\x95\x9b\x60\x7b\x44\xda\xb6\xff\x75\x83\xb5\xcc\x73\xf9\x45\xbc\xe7\xfe\x93\x40\x67\xb0\x79\x49\xab\x8a\x2c\xfb\x5d\x37\xf6\x08\xee\xe4\xe9\x8c\x71\x14\xfd\x73\x13\x3e\x86\x5d\x51\x08\xbe\x94\x7c\x37\x7a\x77\x4e\xeb\xa4\xf7\x1f\xff\x75\xf2\xeb\xa5\xc1\x38\xc0\x43\x2d\x16\x06\x87\x26\x6a\x30\xf6\x93\xab\x1a\x61\x12\xdd\x46\x31\x7e\x0e\x7a\xb4\x67\xce\x0f\xbd\x21\x95\x46\xd7\x46\xac\xf3\x5d\x13\x1f\xe6\x1c\xfc\xf0\x8d\x75\x4b\x7f\xb9\x4a\x75\x9f\x27\xce\xa8\x4d\x8d\xb5\xe4\x0c\xa2\xdb\x45\x57\x30\xcb\x53\x6c\xfb\x22\x6b\xaa\x8f\x2f\x05\xac\xc9\xa9\x99\x2c\xb0\x49\xc5\xc6\xef\x69\xd1\xf6\xde\x1b\xd1\x6e\xc2\xde\x10\x8b\x17\x45\xfd\x1e\xcf\xf0\xf4\x30\x69\x8e\xf4\xca\x5d\x1a\xa5\xcc\x5a\xea\x57\x17\x75\xce\xc3\x5a\x58\x54\xb6\x79\x62\x0a\x65\x18\x9c\x11\x5e\x4b\x51\x64\x10\x2f\xe6\x2f\xed\x4b\x58\x2c\x16\x76\xa5\xde\xeb\xe9\x1e\x92\xe3\x41\x5f\x2c\x16\xde\xbf\x00\x00\x00\xff\xff\x50\xbc\x64\x61\x8f\x05\x00\x00") // FileArtifactsDefinitionsSystemVFSDownloadFileYaml is "artifacts/definitions/System/VFS/DownloadFile.yaml" var FileArtifactsDefinitionsSystemVFSDownloadFileYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x54\x90\x5f\xab\xd4\x30\x10\xc5\xdf\xf3\x29\x0e\xfb\xa4\xd0\x5b\x41\xb8\x3e\x14\xee\x83\x5c\x77\x45\xf0\xa2\xd8\x55\xf0\x71\x36\x9d\x92\xe1\xa6\x49\x4d\x26\x2c\x15\x3f\xbc\xa4\xad\x7f\x36\x6f\x27\x33\xf3\x3b\x87\x13\x68\xe2\x0e\xfd\x92\x95\xa7\xf6\xdb\xa9\x6f\xdf\xc5\x6b\xf0\x91\x86\x93\x78\x36\x03\x67\x9b\x64\x56\x89\xa1\xc3\x2f\x03\x9c\x9d\x64\x48\x06\x05\x48\x50\x4e\x81\x3c\x28\xa9\x8c\x64\x15\x25\xf3\x80\xcb\x02\x75\x8c\xf7\x5f\x3f\x40\x23\xe6\x38\x17\x4f\xca\xf5\xcf\x00\xd5\x00\xdf\x63\xc1\x44\x0b\x52\x09\x10\xc5\x44\xa1\x90\xf7\x0b\x64\xc4\x12\x0b\xbc\x3c\x73\x83\x4b\x51\xe8\x32\x8b\xdd\x46\x0a\xc9\x06\xf0\x54\x82\x75\xb7\x2e\x57\xc7\x61\x15\x25\x73\x82\xf5\x62\x9f\xf3\xaa\x0f\x8f\xd1\x7b\xb6\x8a\x31\xc5\xa9\x0e\x38\xe8\xc1\xa0\xa2\x35\x06\x90\xae\x5b\xa3\x78\xc6\xa1\x57\xd2\x7c\x80\xd2\xa5\x35\x66\xa6\x44\x13\x2b\xa7\xdc\x19\xe0\x0e\x5b\x47\x9f\x49\x9d\x01\x80\x9b\x52\xce\x8e\x31\x93\x3a\xc4\xf1\x1f\x4e\x23\x86\xbd\xc6\x76\x3f\x19\xa9\x78\xed\xf0\xea\x3f\xe0\x5b\x6b\x39\xe7\x98\x6e\x37\x2a\xc0\x98\x1c\x4b\xb2\xbc\xfb\xff\x28\x9c\x64\x13\xf5\xdd\xa1\x3f\x7e\x3c\x3e\x9e\xd7\x44\xcd\x5f\x4c\x83\x5e\x7e\x72\x83\x63\x4a\xab\x70\xf4\xfa\xfe\x4d\x83\xa7\xe1\x7e\xbf\x03\x4e\x5f\x3e\x3d\xa1\xcc\x35\xd7\x8b\xea\x93\x1f\x36\x04\xed\x88\x87\x3f\xac\x97\xe6\x77\x00\x00\x00\xff\xff\xd1\xa0\x21\x6b\x18\x02\x00\x00") @@ -2756,6 +2762,40 @@ func Init() { + rb = bytes.NewReader(FileArtifactsDefinitionsServerInternalNotificationYaml) + r, err = gzip.NewReader(rb) + if err != nil { + panic(err) + } + + err = r.Close() + if err != nil { + panic(err) + } + + + + f, err = FS.OpenFile(CTX, "artifacts/definitions/Server/Internal/Notification.yaml", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777) + if err != nil { + panic(err) + } + + + + _, err = io.Copy(f, r) + if err != nil { + panic(err) + } + + + + err = f.Close() + if err != nil { + panic(err) + } + + + rb = bytes.NewReader(FileArtifactsDefinitionsServerMonitorHealthYaml) r, err = gzip.NewReader(rb) if err != nil { @@ -2926,6 +2966,40 @@ func Init() { + rb = bytes.NewReader(FileArtifactsDefinitionsSystemFlowArchiveYaml) + r, err = gzip.NewReader(rb) + if err != nil { + panic(err) + } + + err = r.Close() + if err != nil { + panic(err) + } + + + + f, err = FS.OpenFile(CTX, "artifacts/definitions/System/Flow/Archive.yaml", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777) + if err != nil { + panic(err) + } + + + + _, err = io.Copy(f, r) + if err != nil { + panic(err) + } + + + + err = f.Close() + if err != nil { + panic(err) + } + + + rb = bytes.NewReader(FileArtifactsDefinitionsSystemFlowCompletionYaml) r, err = gzip.NewReader(rb) if err != nil { diff --git a/artifacts/builder.go b/artifacts/builder.go index 7cbddbad3cb..cbd6a376102 100644 --- a/artifacts/builder.go +++ b/artifacts/builder.go @@ -6,7 +6,7 @@ import ( "github.com/Velocidex/ordereddict" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" - "www.velocidex.com/golang/velociraptor/uploads" + "www.velocidex.com/golang/velociraptor/file_store/api" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -16,7 +16,7 @@ import ( type ScopeBuilder struct { Config *config_proto.Config ACLManager vql_subsystem.ACLManager - Uploader uploads.Uploader + Uploader api.Uploader Logger *log.Logger Env *ordereddict.Dict } @@ -92,13 +92,13 @@ func GetServerConfig(scope *vfilter.Scope) (*config_proto.Config, bool) { return config, ok } -func GetUploader(scope *vfilter.Scope) (uploads.Uploader, bool) { +func GetUploader(scope *vfilter.Scope) (api.Uploader, bool) { scope_uploader, pres := scope.Resolve(constants.SCOPE_UPLOADER) if !pres { return nil, false } - config, ok := scope_uploader.(uploads.Uploader) + config, ok := scope_uploader.(api.Uploader) return config, ok } diff --git a/artifacts/definitions/Server/Internal/Notification.yaml b/artifacts/definitions/Server/Internal/Notification.yaml new file mode 100644 index 00000000000..7b68c564e9e --- /dev/null +++ b/artifacts/definitions/Server/Internal/Notification.yaml @@ -0,0 +1,10 @@ +name: Server.Internal.Notifications +description: | + This event artifact is an internal event stream over which client + notifications are sent. A frontend will watch for events over this + stream and if a client is actively connected to this frontend, the + client will be notified that new work is available to it. + + Note: This is an automated system artifact. You do not need to start it. + +type: SERVER_EVENT diff --git a/artifacts/definitions/System/Flow/Archive.yaml b/artifacts/definitions/System/Flow/Archive.yaml new file mode 100644 index 00000000000..8409141921d --- /dev/null +++ b/artifacts/definitions/System/Flow/Archive.yaml @@ -0,0 +1,6 @@ +name: System.Flow.Archive +description: | + An internal artifact that produces events for every flow completion + in the system. + +type: CLIENT_EVENT diff --git a/artifacts/definitions/System/Hunt/Participation.yaml b/artifacts/definitions/System/Hunt/Participation.yaml index 62c4a767375..1a648ad6a97 100644 --- a/artifacts/definitions/System/Hunt/Participation.yaml +++ b/artifacts/definitions/System/Hunt/Participation.yaml @@ -3,7 +3,7 @@ description: | Endpoints may participate in hunts. This artifact collects which hunt each system participated in. - Note: This is an automated system hunt. You do not need to start it. + Note: This is an automated system artifact. You do not need to start it. type: CLIENT_EVENT @@ -19,11 +19,11 @@ reports: artifact='System.Hunt.Participation') }, query={ SELECT Scheduled, - HuntId, - HuntDescription, - StartRequest.Args.Artifacts.Names + hunt_id, + hunt_description, + start_request.artifacts FROM allhunts - WHERE HuntId = ParticipatedHuntId + WHERE hunt_id = ParticipatedHuntId }) {{ end }} diff --git a/bin/inspect.go b/bin/inspect.go index 45c17527fd9..83a0abf8da6 100644 --- a/bin/inspect.go +++ b/bin/inspect.go @@ -33,7 +33,6 @@ import ( actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" - "www.velocidex.com/golang/velociraptor/constants" crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" datastore "www.velocidex.com/golang/velociraptor/datastore" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" @@ -48,7 +47,7 @@ var classifiers = map[string]proto.Message{ "/clients/C.[^/]+/vfs/.+": &actions_proto.VQLResponse{}, "/clients/C.[^/]+/collections/F\\.[^/]+$": &flows_proto.ArtifactCollectorContext{}, "/clients/C.[^/]+/tasks/[^/]+$": &crypto_proto.GrrMessage{}, - constants.HUNTS_URN + "H.[^/]+$": &api_proto.Hunt{}, + "/hunts/H.[^/]+$": &api_proto.Hunt{}, "/users/[^/]+$": &api_proto.VelociraptorUser{}, "/users/[^/]+/notifications/.+$": &api_proto.UserNotification{}, } diff --git a/constants/paths.go b/constants/paths.go deleted file mode 100644 index adf5c99a041..00000000000 --- a/constants/paths.go +++ /dev/null @@ -1,13 +0,0 @@ -package constants - -const ( - HUNTS_URN = "/hunts/" -) - -func GetHuntURN(hunt_id string) string { - return HUNTS_URN + hunt_id -} - -func GetHuntStatsPath(hunt_id string) string { - return HUNTS_URN + hunt_id + "/stats" -} diff --git a/crypto/resolver.go b/crypto/resolver.go index f36b776fd08..88fbb6df040 100644 --- a/crypto/resolver.go +++ b/crypto/resolver.go @@ -92,14 +92,16 @@ type serverPublicKeyResolver struct { func (self *serverPublicKeyResolver) GetPublicKey( client_id string) (*rsa.PublicKey, bool) { - subject := paths.GetClientKeyPath(client_id) + + client_path_manager := paths.NewClientPathManager(client_id) db, err := datastore.GetDB(self.config_obj) if err != nil { return nil, false } pem := &crypto_proto.PublicKey{} - err = db.GetSubject(self.config_obj, subject, pem) + err = db.GetSubject(self.config_obj, + client_path_manager.Key().Path(), pem) if err != nil { return nil, false } @@ -114,7 +116,8 @@ func (self *serverPublicKeyResolver) GetPublicKey( func (self *serverPublicKeyResolver) SetPublicKey( client_id string, key *rsa.PublicKey) error { - subject := paths.GetClientKeyPath(client_id) + + client_path_manager := paths.NewClientPathManager(client_id) db, err := datastore.GetDB(self.config_obj) if err != nil { return err @@ -124,7 +127,8 @@ func (self *serverPublicKeyResolver) SetPublicKey( Pem: PublicKeyToPem(key), EnrollTime: uint64(time.Now().Unix()), } - return db.SetSubject(self.config_obj, subject, pem) + return db.SetSubject(self.config_obj, + client_path_manager.Key().Path(), pem) } func (self *serverPublicKeyResolver) Clear() {} diff --git a/datastore/filebased.go b/datastore/filebased.go index 3979a8b75d3..2b9f87ba299 100644 --- a/datastore/filebased.go +++ b/datastore/filebased.go @@ -38,7 +38,6 @@ package datastore import ( "compress/gzip" - "fmt" "io" "io/ioutil" "os" @@ -76,11 +75,13 @@ func (self *FileBaseDataStore) GetClientTasks( client_id string, do_not_lease bool) ([]*crypto_proto.GrrMessage, error) { result := []*crypto_proto.GrrMessage{} - now := self.clock.Now().UTC().UnixNano() / 1000 - tasks_urn := paths.GetClientTasksPath(client_id) - now_urn := tasks_urn + fmt.Sprintf("/%d", now) + now := uint64(self.clock.Now().UTC().UnixNano() / 1000) - tasks, err := self.ListChildren(config_obj, tasks_urn, 0, 100) + client_path_manager := paths.NewClientPathManager(client_id) + now_urn := client_path_manager.Task(now).Path() + + tasks, err := self.ListChildren( + config_obj, client_path_manager.TasksDirectory().Path(), 0, 100) if err != nil { return nil, err } @@ -115,8 +116,9 @@ func (self *FileBaseDataStore) UnQueueMessageForClient( client_id string, message *crypto_proto.GrrMessage) error { + client_path_manager := paths.NewClientPathManager(client_id) return self.DeleteSubject(config_obj, - paths.GetClientTaskPath(client_id, message.TaskId)) + client_path_manager.Task(message.TaskId).Path()) } func (self *FileBaseDataStore) QueueMessageForClient( @@ -125,8 +127,9 @@ func (self *FileBaseDataStore) QueueMessageForClient( req *crypto_proto.GrrMessage) error { req.TaskId = uint64(self.clock.Now().UTC().UnixNano() / 1000) - subject := paths.GetClientTaskPath(client_id, req.TaskId) - return self.SetSubject(config_obj, subject, req) + client_path_manager := paths.NewClientPathManager(client_id) + return self.SetSubject(config_obj, + client_path_manager.Task(req.TaskId).Path(), req) } func (self *FileBaseDataStore) GetSubject( diff --git a/file_store/api/accessor.go b/file_store/api/accessor.go new file mode 100644 index 00000000000..80bb54b5339 --- /dev/null +++ b/file_store/api/accessor.go @@ -0,0 +1,153 @@ +package api + +// This implements a filesystem accessor which can be used to access +// the generic filestore. This allows us to run globs on the file +// store regardless of the specific filestore implementation. This +// accessor is for DirectoryFileStore + +import ( + "encoding/json" + "os" + "path" + "path/filepath" + "regexp" + "time" + + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/glob" + "www.velocidex.com/golang/vfilter" +) + +type FileStoreFileSystemAccessor struct { + file_store FileStore +} + +func NewFileStoreFileSystemAccessor( + config_obj *config_proto.Config, fs FileStore) *FileStoreFileSystemAccessor { + return &FileStoreFileSystemAccessor{fs} +} + +func (self FileStoreFileSystemAccessor) New( + scope *vfilter.Scope) glob.FileSystemAccessor { + return &FileStoreFileSystemAccessor{self.file_store} +} + +func (self FileStoreFileSystemAccessor) Lstat( + filename string) (glob.FileInfo, error) { + lstat, err := self.file_store.StatFile(filename) + if err != nil { + return nil, err + } + + return &FileStoreFileInfo{lstat, filename, nil}, nil +} + +func (self FileStoreFileSystemAccessor) ReadDir(path string) ([]glob.FileInfo, error) { + files, err := self.file_store.ListDirectory(path) + if err != nil { + return nil, err + } + + var result []glob.FileInfo + for _, f := range files { + result = append(result, + &FileStoreFileInfo{f, filepath.Join(path, f.Name()), nil}) + } + + return result, nil +} + +func (self FileStoreFileSystemAccessor) Open(path string) (glob.ReadSeekCloser, error) { + file, err := self.file_store.ReadFile(path) + if err != nil { + return nil, err + } + + return &FileReaderAdapter{file}, nil +} + +var FileStoreFileSystemAccessor_re = regexp.MustCompile("/") + +func (self FileStoreFileSystemAccessor) PathSplit(path string) []string { + return FileStoreFileSystemAccessor_re.Split(path, -1) +} + +func (self FileStoreFileSystemAccessor) PathJoin(root, stem string) string { + return path.Join(root, stem) +} + +func (self *FileStoreFileSystemAccessor) GetRoot(path string) (string, string, error) { + return "/", path, nil +} + +type FileStoreFileInfo struct { + os.FileInfo + FullPath_ string + Data_ interface{} +} + +func (self FileStoreFileInfo) Name() string { + return self.FileInfo.Name() +} + +func (self *FileStoreFileInfo) Data() interface{} { + return self.Data_ +} + +func (self *FileStoreFileInfo) FullPath() string { + return self.FullPath_ +} + +func (self *FileStoreFileInfo) Mtime() glob.TimeVal { + return glob.TimeVal{} +} + +func (self *FileStoreFileInfo) Ctime() glob.TimeVal { + return glob.TimeVal{} +} + +func (self *FileStoreFileInfo) Atime() glob.TimeVal { + return glob.TimeVal{} +} + +func (self *FileStoreFileInfo) IsLink() bool { + return self.Mode()&os.ModeSymlink != 0 +} + +func (self *FileStoreFileInfo) GetLink() (string, error) { + target, err := os.Readlink(self.FullPath_) + if err != nil { + return "", err + } + return target, nil +} + +func (self *FileStoreFileInfo) MarshalJSON() ([]byte, error) { + result, err := json.Marshal(&struct { + FullPath string + Size int64 + Mode os.FileMode + ModeStr string + ModTime time.Time + Sys interface{} + Mtime glob.TimeVal + Ctime glob.TimeVal + Atime glob.TimeVal + }{ + FullPath: self.FullPath(), + Size: self.Size(), + Mode: self.Mode(), + ModeStr: self.Mode().String(), + ModTime: self.ModTime(), + Sys: self.Sys(), + Mtime: self.Mtime(), + Ctime: self.Ctime(), + Atime: self.Atime(), + }) + + return result, err +} + +func (self *FileStoreFileInfo) UnmarshalJSON(data []byte) error { + return nil +} diff --git a/file_store/api/queues.go b/file_store/api/queues.go index 8c540b8f9c4..989ef91c273 100644 --- a/file_store/api/queues.go +++ b/file_store/api/queues.go @@ -1,17 +1,32 @@ package api import ( - "time" + "context" "github.com/Velocidex/ordereddict" - "www.velocidex.com/golang/vfilter" ) // A QueueManager writes query results into queues. The manager is // responsible for rotating the queue files as required. type QueueManager interface { - Push(queue_name, source string, mode int, rows []byte) error - PushRow(queue_name, source string, mode int, row *ordereddict.Dict) error - Read(queue_name, source string, start_time, endtime time.Time) <-chan vfilter.Row + PushEventRows(path_manager PathManager, rows []*ordereddict.Dict) error Watch(queue_name string) (output <-chan *ordereddict.Dict, cancel func()) } + +type ResultSetFileProperties struct { + Path string + StartTime, EndTime int64 +} + +// Path manager tells the filestore where to store things. +type PathManager interface { + // Gets a log path for writing new rows on. + GetPathForWriting() (string, error) + + // The name of the queue we will use to watch for any rows + // inserted into this result set. + GetQueueName() string + + // Generate paths for reading linked result sets. + GeneratePaths(ctx context.Context) <-chan *ResultSetFileProperties +} diff --git a/file_store/api/testsuite.go b/file_store/api/testsuite.go index f54952747de..014e3befc69 100644 --- a/file_store/api/testsuite.go +++ b/file_store/api/testsuite.go @@ -1,18 +1,25 @@ package api import ( + "context" "fmt" "io" + "io/ioutil" "os" "path" "sort" + "github.com/Velocidex/ordereddict" "github.com/alecthomas/assert" "github.com/stretchr/testify/suite" config_proto "www.velocidex.com/golang/velociraptor/config/proto" - "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/utils" ) +type Debugger interface { + Debug() +} + // An abstract test suite to ensure file store implementations all // comply with the API. type FileStoreTestSuite struct { @@ -180,34 +187,80 @@ type QueueManagerTestSuite struct { config_obj *config_proto.Config manager QueueManager + file_store FileStore +} + +func (self *QueueManagerTestSuite) Debug() { + switch t := self.manager.(type) { + case Debugger: + t.Debug() + } +} + +func (self *QueueManagerTestSuite) FilestoreGet(path string) string { + fd, _ := self.file_store.ReadFile(path) + value, _ := ioutil.ReadAll(fd) + return string(value) } func (self *QueueManagerTestSuite) TestPush() { artifact_name := "System.Hunt.Participation" - payload := []byte("[{\"foo\":1},{\"foo\":2}]") + payload := []*ordereddict.Dict{ + ordereddict.NewDict().Set("foo", 1), + ordereddict.NewDict().Set("foo", 2)} output, cancel := self.manager.Watch(artifact_name) defer cancel() - err := self.manager.Push(artifact_name, "C.123", - paths.MODE_MONITORING_DAILY, payload) + err := self.manager.PushEventRows( + MockPathManager{"log_path", artifact_name}, payload) assert.NoError(self.T(), err) for row := range output { - fmt.Printf("%v\n", row) value, _ := row.Get("foo") - if value.(int64) == 2 { + v, _ := utils.ToInt64(value) + if v == int64(2) { break } } + + // Make sure the manager wrote the event to the filestore as well. + assert.Contains(self.T(), self.FilestoreGet("log_path"), "foo") } -func NewQueueManagerTestSuite(config_obj *config_proto.Config, - manager QueueManager) *QueueManagerTestSuite { +func NewQueueManagerTestSuite( + config_obj *config_proto.Config, + manager QueueManager, + file_store FileStore) *QueueManagerTestSuite { return &QueueManagerTestSuite{ config_obj: config_obj, manager: manager, + file_store: file_store, } } + +type MockPathManager struct { + Path string + ArtifactName string +} + +func (self MockPathManager) GetPathForWriting() (string, error) { + return self.Path, nil +} + +func (self MockPathManager) GetQueueName() string { + return self.ArtifactName +} + +func (self MockPathManager) GeneratePaths(ctx context.Context) <-chan *ResultSetFileProperties { + output := make(chan *ResultSetFileProperties) + + go func() { + defer close(output) + + }() + + return output +} diff --git a/file_store/api/uploader.go b/file_store/api/uploader.go index f9abbdacdd1..ff1d86486a4 100644 --- a/file_store/api/uploader.go +++ b/file_store/api/uploader.go @@ -9,10 +9,29 @@ import ( "path" config_proto "www.velocidex.com/golang/velociraptor/config/proto" - "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/vfilter" ) +// Returned as the result of the query. +type UploadResponse struct { + Path string `json:"Path"` + Size uint64 `json:"Size"` + Error string `json:"Error,omitempty"` + Sha256 string `json:"sha256,omitempty"` + Md5 string `json:"md5,omitempty"` +} + +// Provide an uploader capable of uploading any reader object. +type Uploader interface { + Upload(ctx context.Context, + scope *vfilter.Scope, + filename string, + accessor string, + store_as_name string, + expected_size int64, + reader io.Reader) (*UploadResponse, error) +} + // An uploader into the filestore. type FileStoreUploader struct { file_store FileStore @@ -27,7 +46,7 @@ func (self *FileStoreUploader) Upload( store_as_name string, expected_size int64, reader io.Reader) ( - *uploads.UploadResponse, error) { + *UploadResponse, error) { if store_as_name == "" { store_as_name = filename @@ -79,7 +98,7 @@ loop: } scope.Log("Uploaded %v (%v bytes)", output_path, offset) - return &uploads.UploadResponse{ + return &UploadResponse{ Path: output_path, Size: uint64(offset), Sha256: hex.EncodeToString(sha_sum.Sum(nil)), diff --git a/file_store/directory/csv.go b/file_store/directory/csv.go new file mode 100644 index 00000000000..c2a7d356731 --- /dev/null +++ b/file_store/directory/csv.go @@ -0,0 +1,84 @@ +package directory + +import ( + "context" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/csv" + "www.velocidex.com/golang/velociraptor/utils" +) + +func row_to_dict(row_data []interface{}, headers []string) (*ordereddict.Dict, int64) { + row := ordereddict.NewDict() + var timestamp int64 + + for idx, row_item := range row_data { + if idx > len(headers) { + break + } + + // Event logs have a _ts column representing the time + // of each event. + column_name := headers[idx] + if column_name == "_ts" { + timestamp, _ = utils.ToInt64(row_item) + } + + row.Set(column_name, row_item) + } + + return row, timestamp +} + +func ReadRowsCSV( + ctx context.Context, + file_store api.FileStore, + log_path string, start_time, end_time int64) ( + <-chan *ordereddict.Dict, error) { + + fd, err := file_store.ReadFile(log_path) + if err != nil { + return nil, err + } + + // Read the headers which are the first row. + csv_reader := csv.NewReader(fd) + headers, err := csv_reader.Read() + if err != nil { + return nil, err + } + + output := make(chan *ordereddict.Dict) + + go func() { + defer close(output) + defer fd.Close() + + for { + select { + case <-ctx.Done(): + return + + default: + row_data, err := csv_reader.ReadAny() + if err != nil { + return + } + + dict, timestamp := row_to_dict(row_data, headers) + if timestamp < start_time { + continue + } + + if end_time > 0 && timestamp > end_time { + return + } + + output <- dict + } + } + }() + + return output, nil +} diff --git a/file_store/directory/directory.go b/file_store/directory/directory.go index 65e5f68e83b..87c2cfa7c9a 100644 --- a/file_store/directory/directory.go +++ b/file_store/directory/directory.go @@ -110,7 +110,7 @@ func (self *DirectoryFileStore) ListDirectory(dirname string) ( var result []os.FileInfo for _, fileinfo := range files { - result = append(result, &DirectoryFileStoreFileInfo{ + result = append(result, &api.FileStoreFileInfo{ fileinfo, utils.PathJoin(dirname, fileinfo.Name(), "/"), nil}) @@ -158,7 +158,7 @@ func (self *DirectoryFileStore) StatFile(filename string) (os.FileInfo, error) { return nil, err } - return &DirectoryFileStoreFileInfo{file, filename, nil}, nil + return &api.FileStoreFileInfo{file, filename, nil}, nil } func (self *DirectoryFileStore) WriteFile(filename string) (api.FileWriter, error) { @@ -203,6 +203,7 @@ func (self *DirectoryFileStore) FilenameToFileStorePath(filename string) string // prefix the LFN prefix to be able to access long paths but // then we must use \ as a separator. result := filepath.Join(components...) + if runtime.GOOS == "windows" { return WINDOWS_LFN_PREFIX + result } @@ -241,6 +242,6 @@ func (self *DirectoryFileStore) Walk(root string, walkFn filepath.WalkFunc) erro return err_1 } return walkFn(filename, - &DirectoryFileStoreFileInfo{info, path, nil}, err) + &api.FileStoreFileInfo{info, path, nil}, err) }) } diff --git a/file_store/directory/directory_accessor.go b/file_store/directory/directory_accessor.go deleted file mode 100644 index e41bd88e0d8..00000000000 --- a/file_store/directory/directory_accessor.go +++ /dev/null @@ -1,155 +0,0 @@ -package directory - -// This implements a filesystem accessor which can be used to access -// the filestore. This allows us to run globs on the file store -// regardless of the specific filestore implementation. -// This accessor is for DirectoryFileStore - -import ( - "encoding/json" - "os" - "path" - "path/filepath" - "regexp" - "time" - - config_proto "www.velocidex.com/golang/velociraptor/config/proto" - "www.velocidex.com/golang/velociraptor/datastore" - "www.velocidex.com/golang/velociraptor/file_store/api" - "www.velocidex.com/golang/velociraptor/glob" - "www.velocidex.com/golang/vfilter" -) - -type DirectoryFileStoreFileInfo struct { - os.FileInfo - _full_path string - _data interface{} -} - -func (self DirectoryFileStoreFileInfo) Name() string { - return datastore.UnsanitizeComponent(self.FileInfo.Name()) -} - -func (self *DirectoryFileStoreFileInfo) Data() interface{} { - return self._data -} - -func (self *DirectoryFileStoreFileInfo) FullPath() string { - return self._full_path -} - -func (self *DirectoryFileStoreFileInfo) Mtime() glob.TimeVal { - return glob.TimeVal{} -} - -func (self *DirectoryFileStoreFileInfo) Ctime() glob.TimeVal { - return glob.TimeVal{} -} - -func (self *DirectoryFileStoreFileInfo) Atime() glob.TimeVal { - return glob.TimeVal{} -} - -func (self *DirectoryFileStoreFileInfo) IsLink() bool { - return self.Mode()&os.ModeSymlink != 0 -} - -func (self *DirectoryFileStoreFileInfo) GetLink() (string, error) { - target, err := os.Readlink(self._full_path) - if err != nil { - return "", err - } - return target, nil -} - -func (self *DirectoryFileStoreFileInfo) MarshalJSON() ([]byte, error) { - result, err := json.Marshal(&struct { - FullPath string - Size int64 - Mode os.FileMode - ModeStr string - ModTime time.Time - Sys interface{} - Mtime glob.TimeVal - Ctime glob.TimeVal - Atime glob.TimeVal - }{ - FullPath: self.FullPath(), - Size: self.Size(), - Mode: self.Mode(), - ModeStr: self.Mode().String(), - ModTime: self.ModTime(), - Sys: self.Sys(), - Mtime: self.Mtime(), - Ctime: self.Ctime(), - Atime: self.Atime(), - }) - - return result, err -} - -func (self *DirectoryFileStoreFileInfo) UnmarshalJSON(data []byte) error { - return nil -} - -type DirectoryFileStoreFileSystemAccessor struct { - file_store *DirectoryFileStore -} - -func NewDirectoryFileStoreFileSystemAccessor(config_obj *config_proto.Config) *DirectoryFileStoreFileSystemAccessor { - return &DirectoryFileStoreFileSystemAccessor{ - &DirectoryFileStore{config_obj}} -} - -func (self DirectoryFileStoreFileSystemAccessor) New( - scope *vfilter.Scope) glob.FileSystemAccessor { - return &DirectoryFileStoreFileSystemAccessor{self.file_store} -} - -func (self DirectoryFileStoreFileSystemAccessor) Lstat( - filename string) (glob.FileInfo, error) { - lstat, err := self.file_store.StatFile(filename) - if err != nil { - return nil, err - } - - return &DirectoryFileStoreFileInfo{lstat, filename, nil}, nil -} - -func (self DirectoryFileStoreFileSystemAccessor) ReadDir(path string) ([]glob.FileInfo, error) { - files, err := self.file_store.ListDirectory(path) - if err != nil { - return nil, err - } - - var result []glob.FileInfo - for _, f := range files { - result = append(result, - &DirectoryFileStoreFileInfo{f, filepath.Join(path, f.Name()), nil}) - } - - return result, nil -} - -func (self DirectoryFileStoreFileSystemAccessor) Open(path string) (glob.ReadSeekCloser, error) { - file, err := self.file_store.ReadFile(path) - if err != nil { - return nil, err - } - - return &api.FileReaderAdapter{file}, nil -} - -var DirectoryFileStoreFileSystemAccessor_re = regexp.MustCompile("/") - -func (self DirectoryFileStoreFileSystemAccessor) PathSplit(path string) []string { - return DirectoryFileStoreFileSystemAccessor_re.Split(path, -1) -} - -func (self DirectoryFileStoreFileSystemAccessor) PathJoin(root, stem string) string { - return path.Join(root, stem) -} - -func (self *DirectoryFileStoreFileSystemAccessor) GetRoot(path string) (string, string, error) { - return "/", path, nil -} diff --git a/file_store/directory/json.go b/file_store/directory/json.go new file mode 100644 index 00000000000..cac4038289d --- /dev/null +++ b/file_store/directory/json.go @@ -0,0 +1,65 @@ +package directory + +import ( + "bufio" + "context" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/utils" +) + +func ReadRowsJSON( + ctx context.Context, + file_store api.FileStore, + log_path string, start_time, end_time int64) ( + <-chan *ordereddict.Dict, error) { + + fd, err := file_store.ReadFile(log_path) + if err != nil { + return nil, err + } + + output := make(chan *ordereddict.Dict) + + go func() { + defer close(output) + defer fd.Close() + + reader := bufio.NewReader(fd) + + for { + select { + case <-ctx.Done(): + return + + default: + row_data, err := reader.ReadBytes('\n') + if err != nil { + return + } + item := ordereddict.NewDict() + err = item.UnmarshalJSON(row_data) + if err != nil { + return + } + + ts, pres := item.Get("_ts") + if pres { + timestamp, _ := utils.ToInt64(ts) + if start_time > 0 && timestamp < start_time { + continue + } + + if end_time > 0 && timestamp > end_time { + return + } + } + + output <- item + } + } + }() + + return output, nil +} diff --git a/file_store/directory/queue.go b/file_store/directory/queue.go new file mode 100644 index 00000000000..3d3f7da12dc --- /dev/null +++ b/file_store/directory/queue.go @@ -0,0 +1,70 @@ +package directory + +import ( + "fmt" + + "github.com/Velocidex/ordereddict" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/memory" + "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/vfilter" +) + +var ( + pool = memory.NewQueuePool() +) + +type DirectoryQueueManager struct { + FileStore api.FileStore + scope *vfilter.Scope + Clock utils.Clock +} + +func (self *DirectoryQueueManager) PushEventRows( + path_manager api.PathManager, dict_rows []*ordereddict.Dict) error { + + for _, row := range dict_rows { + pool.Broadcast(path_manager.GetQueueName(), + row.Set("_ts", int(self.Clock.Now().Unix()))) + } + + log_path, err := path_manager.GetPathForWriting() + if err != nil { + return err + } + + fd, err := self.FileStore.WriteFile(log_path) + if err != nil { + fmt.Printf("Error: %v\n", err) + return err + } + defer fd.Close() + + for _, row := range dict_rows { + // Set a timestamp per event for easier querying. + row.Set("_ts", int(self.Clock.Now().Unix())) + } + + serialized, err := utils.DictsToJson(dict_rows) + if err != nil { + return err + } + + _, err = fd.Write(serialized) + return err +} + +func (self *DirectoryQueueManager) Watch( + queue_name string) (output <-chan *ordereddict.Dict, cancel func()) { + return pool.Register(queue_name) +} + +func NewDirectoryQueueManager(config_obj *config_proto.Config, + file_store api.FileStore) api.QueueManager { + return &DirectoryQueueManager{ + FileStore: file_store, + scope: vfilter.NewScope(), + Clock: utils.RealClock{}, + } +} diff --git a/file_store/directory/queue_test.go b/file_store/directory/queue_test.go new file mode 100644 index 00000000000..21cb937df1e --- /dev/null +++ b/file_store/directory/queue_test.go @@ -0,0 +1,28 @@ +package directory + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/alecthomas/assert" + "github.com/stretchr/testify/suite" + "www.velocidex.com/golang/velociraptor/config" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/memory" +) + +func TestDirectoryQueueManager(t *testing.T) { + dir, err := ioutil.TempDir("", "file_store_test") + assert.NoError(t, err) + + defer os.RemoveAll(dir) // clean up + + config_obj := config.GetDefaultConfig() + config_obj.Datastore.Implementation = "FileBaseDataStore" + config_obj.Datastore.FilestoreDirectory = dir + config_obj.Datastore.Location = dir + + manager := NewDirectoryQueueManager(config_obj, memory.Test_memory_file_store) + suite.Run(t, api.NewQueueManagerTestSuite(config_obj, manager, memory.Test_memory_file_store)) +} diff --git a/file_store/directory/result_sets.go b/file_store/directory/result_sets.go new file mode 100644 index 00000000000..3f6e8c7f693 --- /dev/null +++ b/file_store/directory/result_sets.go @@ -0,0 +1,71 @@ +package directory + +import ( + "context" + "strings" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/file_store/api" +) + +func GetTimeRange( + ctx context.Context, + file_store api.FileStore, + path_manager api.PathManager, + start_time, end_time int64) (<-chan *ordereddict.Dict, error) { + + output := make(chan *ordereddict.Dict) + + go func() { + defer close(output) + + sub_ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for prop := range path_manager.GeneratePaths(sub_ctx) { + if start_time > 0 && prop.EndTime < start_time { + continue + } + + if end_time > 0 && prop.StartTime > end_time { + return + } + + row_chan, err := ReadRows( + sub_ctx, file_store, prop.Path, + start_time, end_time) + if err != nil { + continue + } + + for item := range row_chan { + output <- item + } + } + + }() + return output, nil +} + +func ReadRows( + ctx context.Context, + file_store api.FileStore, + file_path string, + start_time, end_time int64) (<-chan *ordereddict.Dict, error) { + + // Backwards compatibility: We dont write .csv files any more + // but we can read them if they are there. + if strings.HasSuffix(file_path, ".csv") { + return ReadRowsCSV(ctx, file_store, file_path, start_time, end_time) + } + + // If we are supposed to read a json file but we cant find it + // - maybe it is a csv file instead - look for it. + row_chan, err := ReadRowsJSON(ctx, file_store, file_path, start_time, end_time) + if err != nil && strings.HasSuffix(file_path, ".json") { + file_path = strings.TrimSuffix(file_path, ".json") + ".csv" + return ReadRowsCSV(ctx, file_store, file_path, start_time, end_time) + } + + return row_chan, err +} diff --git a/file_store/file_store.go b/file_store/file_store.go index b5291609de2..813ffdfb568 100644 --- a/file_store/file_store.go +++ b/file_store/file_store.go @@ -18,6 +18,7 @@ package file_store import ( + "errors" "fmt" config_proto "www.velocidex.com/golang/velociraptor/config/proto" @@ -54,12 +55,23 @@ func GetFileStore(config_obj *config_proto.Config) api.FileStore { // Gets an accessor that can access the file store. func GetFileStoreFileSystemAccessor( config_obj *config_proto.Config) (glob.FileSystemAccessor, error) { - if config_obj.Datastore.Implementation == "MySQL" { + switch config_obj.Datastore.Implementation { + case "MySQL": datastore, err := mysql.NewSqlFileStore(config_obj) if err != nil { return nil, err } return mysql.NewSqlFileStoreAccessor(datastore.(*mysql.SqlFileStore)), nil + + case "FileBaseDataStore": + return api.NewFileStoreFileSystemAccessor( + config_obj, directory.NewDirectoryFileStore(config_obj)), nil + + case "Test": + return api.NewFileStoreFileSystemAccessor( + config_obj, memory.Test_memory_file_store), nil + } - return directory.NewDirectoryFileStoreFileSystemAccessor(config_obj), nil + + return nil, errors.New("Unknown file store implementation") } diff --git a/file_store/memory/memory.go b/file_store/memory/memory.go index 66f3781c610..642314e69a2 100644 --- a/file_store/memory/memory.go +++ b/file_store/memory/memory.go @@ -2,13 +2,13 @@ package memory import ( "bytes" + "fmt" "os" "path/filepath" "sync" "github.com/pkg/errors" "www.velocidex.com/golang/velociraptor/file_store/api" - "www.velocidex.com/golang/velociraptor/file_store/directory" "www.velocidex.com/golang/velociraptor/glob" ) @@ -63,6 +63,16 @@ type MemoryFileStore struct { Data map[string][]byte } +func (self *MemoryFileStore) Debug() { + self.mu.Lock() + defer self.mu.Unlock() + + fmt.Printf("MemoryFileStore: \n") + for k, v := range self.Data { + fmt.Printf("%v: %v\n", k, string(v)) + } +} + func (self *MemoryFileStore) ReadFile(filename string) (api.FileReader, error) { self.mu.Lock() defer self.mu.Unlock() @@ -92,7 +102,7 @@ func (self *MemoryFileStore) WriteFile(filename string) (api.FileWriter, error) } func (self *MemoryFileStore) StatFile(filename string) (os.FileInfo, error) { - return &directory.DirectoryFileStoreFileInfo{}, nil + return &api.FileStoreFileInfo{}, nil } func (self *MemoryFileStore) ListDirectory(dirname string) ([]os.FileInfo, error) { diff --git a/file_store/memory/queue.go b/file_store/memory/queue.go index 07be9781bea..ba56c5c2744 100644 --- a/file_store/memory/queue.go +++ b/file_store/memory/queue.go @@ -15,7 +15,6 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/utils" - "www.velocidex.com/golang/vfilter" ) var ( @@ -91,43 +90,44 @@ func NewQueuePool() *QueuePool { } type MemoryQueueManager struct { - file_store api.FileStore -} + FileStore api.FileStore -func (self *MemoryQueueManager) PushRow( - queue_name, source string, mode int, row *ordereddict.Dict) error { - pool.Broadcast(queue_name, - row.Set("ClientId", source). - Set("_ts", int(time.Now().Unix()))) + Clock utils.Clock +} - return nil +func (self *MemoryQueueManager) Debug() { + switch t := self.FileStore.(type) { + case api.Debugger: + t.Debug() + } } -func (self *MemoryQueueManager) Push( - queue_name, source string, mode int, serialized_rows []byte) error { - rows, err := utils.ParseJsonToDicts(serialized_rows) +func (self *MemoryQueueManager) PushEventRows( + path_manager api.PathManager, dict_rows []*ordereddict.Dict) error { + + for _, row := range dict_rows { + pool.Broadcast(path_manager.GetQueueName(), + row.Set("_ts", int(self.Clock.Now().Unix()))) + } + + log_path, err := path_manager.GetPathForWriting() if err != nil { return err } - for _, row := range rows { - pool.Broadcast(queue_name, - row.Set("ClientId", source). - Set("_ts", int(time.Now().Unix()))) + fd, err := self.FileStore.WriteFile(log_path) + if err != nil { + return err } + defer fd.Close() - return nil -} - -func (self *MemoryQueueManager) Read( - queue_name, source string, start_time, endtime time.Time) <-chan vfilter.Row { - output := make(chan vfilter.Row) - - go func() { - defer close(output) - }() + serialized, err := utils.DictsToJson(dict_rows) + if err != nil { + return err + } - return output + _, err = fd.Write(serialized) + return err } func (self *MemoryQueueManager) Watch( @@ -138,6 +138,7 @@ func (self *MemoryQueueManager) Watch( func NewMemoryQueueManager(config_obj *config_proto.Config, file_store api.FileStore) api.QueueManager { return &MemoryQueueManager{ - file_store: file_store, + FileStore: file_store, + Clock: utils.RealClock{}, } } diff --git a/file_store/memory/queue_test.go b/file_store/memory/queue_test.go index 0669b01cb1c..13c3ab49447 100644 --- a/file_store/memory/queue_test.go +++ b/file_store/memory/queue_test.go @@ -1,28 +1,15 @@ package memory import ( - "io/ioutil" - "os" "testing" - "github.com/alecthomas/assert" "github.com/stretchr/testify/suite" "www.velocidex.com/golang/velociraptor/config" "www.velocidex.com/golang/velociraptor/file_store/api" ) func TestMemoryQueueManager(t *testing.T) { - dir, err := ioutil.TempDir("", "file_store_test") - assert.NoError(t, err) - - defer os.RemoveAll(dir) // clean up - - //dir = "/tmp/file_store_test" - config_obj := config.GetDefaultConfig() - config_obj.Datastore.FilestoreDirectory = dir - config_obj.Datastore.Location = dir - manager := NewMemoryQueueManager(config_obj, Test_memory_file_store) - suite.Run(t, api.NewQueueManagerTestSuite(config_obj, manager)) + suite.Run(t, api.NewQueueManagerTestSuite(config_obj, manager, Test_memory_file_store)) } diff --git a/file_store/mysql/mysql.go b/file_store/mysql/mysql.go index 4d21879e1c9..13f9359eaf2 100644 --- a/file_store/mysql/mysql.go +++ b/file_store/mysql/mysql.go @@ -345,6 +345,10 @@ func (self SqlWriter) Close() error { } func (self *SqlWriter) Write(buff []byte) (int, error) { + return self.write_row("", buff) +} + +func (self *SqlWriter) write_row(channel string, buff []byte) (int, error) { if len(buff) == 0 { return 0, nil } @@ -359,12 +363,13 @@ func (self *SqlWriter) Write(buff []byte) (int, error) { defer tx.Rollback() insert, err := tx.Prepare(` -INSERT INTO filestore (id, part, start_offset, end_offset, data) +INSERT INTO filestore (id, part, start_offset, end_offset, data, channel) SELECT A.id AS id, A.part + 1 AS part, A.end_offset AS start_offset, A.end_offset + ? AS end_offset, - ? AS data + ? AS data, + ? AS channel FROM filestore AS A join ( SELECT max(part) AS max_part FROM filestore WHERE id=? ) AS B @@ -394,7 +399,8 @@ UPDATE filestore_metadata SET timestamp=now(), size=size + ? WHERE id = ?`) // Write this chunk only. chunk := snappy.Encode(nil, buff[:length]) - _, err = insert.Exec(length, chunk, self.file_id, self.file_id) + _, err = insert.Exec(length, chunk, channel, + self.file_id, self.file_id) if err != nil { return 0, err } @@ -724,8 +730,12 @@ func initializeDatabase( part int NOT NULL DEFAULT 0, start_offset int, end_offset int, + channel varchar(256), + part_id INT NOT NULL AUTO_INCREMENT, data blob, + PRIMARY KEY (part_id), unique INDEX(id, part, start_offset, end_offset), + INDEX(channel, part_id), INDEX(id, start_offset))`) if err != nil { return nil, err diff --git a/file_store/mysql/queue.go b/file_store/mysql/queue.go new file mode 100644 index 00000000000..1675f827d5e --- /dev/null +++ b/file_store/mysql/queue.go @@ -0,0 +1,166 @@ +// A Mysql queue implementation. + +// The goal of this implementation is to allow for efficient following +// of various queues (queue name is the channel name we follow). Since +// Mysql does not have an event driven query we need to poll the +// queues periodically by quering for new rows. This query needs to be +// as efficient as possible. + +// Since we also need to store the events in the client's monitoring +// space anyway we re-use the filestore table too. The filestore table +// looks like: + +// filestore(id int NOT NULL, +// part int NOT NULL DEFAULT 0, +// start_offset int, +// end_offset int, +// channel varchar, +// timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +// data blob, + +// Normally we access the table using the id (corresponding to the +// filename) but the client's monitoring log may be written in +// different places. For queuing purposes, we use the channel column +// to denote the queue_name and the timestamp corresponding to the +// time the part was written. + +// We then query for those data blobs that were written after the last +// poll time and have a channel we are interested in. + +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/Velocidex/ordereddict" + "github.com/golang/snappy" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/vfilter" +) + +type MysqlQueueManager struct { + file_store *SqlFileStore + scope *vfilter.Scope + config_obj *config_proto.Config +} + +func (self *MysqlQueueManager) PushEventRows( + path_manager api.PathManager, dict_rows []*ordereddict.Dict) error { + + if len(dict_rows) == 0 { + return nil + } + + log_path, err := path_manager.GetPathForWriting() + if err != nil { + return err + } + + serialized, err := utils.DictsToJson(dict_rows) + if err != nil { + return nil + } + + fd, err := self.file_store.WriteFile(log_path) + if err != nil { + return err + } + + // An internal write method to set the queue name in the table. + _, err = fd.(*SqlWriter).write_row(path_manager.GetQueueName(), serialized) + return err +} + +func (self *MysqlQueueManager) Watch(queue_name string) (<-chan *ordereddict.Dict, func()) { + output := make(chan *ordereddict.Dict, 1000) + ctx, cancel := context.WithCancel(context.Background()) + + // Keep track of the last part id + last_id := sql.NullInt64{} + + go func() { + defer close(output) + defer cancel() + + for { + err := self.emitEvents(queue_name, &last_id, output) + if err != nil { + return + } + + select { + case <-ctx.Done(): + return + + case <-time.After(time.Second): + continue + } + } + }() + + return output, cancel +} + +func (self *MysqlQueueManager) emitEvents(queue_name string, + last_id *sql.NullInt64, output chan *ordereddict.Dict) error { + + if !last_id.Valid { + err := db.QueryRow("SELECT max(part_id) FROM filestore WHERE channel = ?", + queue_name).Scan(last_id) + if err != sql.ErrNoRows { + // No entries exist yet. Start watching from 0. + last_id.Valid = true + return nil + } + return err + } + + fmt.Printf("Checking %v > %v\n", queue_name, last_id.Int64) + + rows, err := db.Query(` +SELECT data, part_id +FROM filestore +WHERE channel = ? AND part_id > ?`, queue_name, last_id.Int64) + if err != nil { + fmt.Printf("Error: %v\n", err) + return err + } + defer rows.Close() + + for rows.Next() { + var data []byte + err = rows.Scan(&data, last_id) + if err != nil { + continue + } + + decoded, err := snappy.Decode(nil, data) + if err != nil { + continue + } + + dict_rows, err := utils.ParseJsonToDicts(decoded) + if err != nil { + continue + } + + for _, row := range dict_rows { + output <- row + } + } + + return nil +} + +func NewMysqlQueueManager( + config_obj *config_proto.Config, file_store *SqlFileStore) (api.QueueManager, error) { + return &MysqlQueueManager{ + file_store, + vfilter.NewScope(), + config_obj}, nil +} diff --git a/file_store/queue.go b/file_store/queue.go index 58467f685f8..a79239964b2 100644 --- a/file_store/queue.go +++ b/file_store/queue.go @@ -1,26 +1,36 @@ package file_store import ( + "errors" "fmt" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/directory" "www.velocidex.com/golang/velociraptor/file_store/memory" + "www.velocidex.com/golang/velociraptor/file_store/mysql" ) // GetQueueManager selects an appropriate QueueManager object based on // config. -func GetQueueManager(config_obj *config_proto.Config) api.QueueManager { +func GetQueueManager(config_obj *config_proto.Config) (api.QueueManager, error) { file_store := GetFileStore(config_obj) switch config_obj.Datastore.Implementation { // For now everyone uses an in-memory queue manager. - case "FileBaseDataStore", "MySQL", "Test": - return memory.NewMemoryQueueManager(config_obj, file_store) + case "Test": + return memory.NewMemoryQueueManager(config_obj, file_store), nil + + case "FileBaseDataStore": + return directory.NewDirectoryQueueManager(config_obj, file_store), nil + + case "MySQL": + return mysql.NewMysqlQueueManager( + config_obj, file_store.(*mysql.SqlFileStore)) default: - panic(fmt.Sprintf("Unsupported QueueManager %v", + return nil, errors.New(fmt.Sprintf("Unsupported QueueManager %v", config_obj.Datastore.Implementation)) } } diff --git a/file_store/result_set.go b/file_store/result_set.go new file mode 100644 index 00000000000..2d8b97eeef9 --- /dev/null +++ b/file_store/result_set.go @@ -0,0 +1,33 @@ +// A Data store specific way of storing and managing result sets. + +// Velociraptor is essentially a query engine, therefore all the +// results are stored in result sets. This interface abstracts away +// where and how result sets are stored. + +package file_store + +import ( + "context" + + "github.com/Velocidex/ordereddict" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/directory" +) + +// GetTimeRange returns a channel thats feeds results from the +// filestore to the caller. Where and how the results are actually +// stored is abstracted and depends on the filestore implementation. +func GetTimeRange( + ctx context.Context, + config_obj *config_proto.Config, + path_manager api.PathManager, + start_time, end_time int64) (<-chan *ordereddict.Dict, error) { + file_store_factory := GetFileStore(config_obj) + + switch config_obj.Datastore.Implementation { + default: + return directory.GetTimeRange( + ctx, file_store_factory, path_manager, start_time, end_time) + } +} diff --git a/flows/api.go b/flows/api.go index ab31383c8b9..61e13fb6821 100644 --- a/flows/api.go +++ b/flows/api.go @@ -36,6 +36,7 @@ import ( "www.velocidex.com/golang/velociraptor/grpc_client" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" ) @@ -114,7 +115,8 @@ func GetFlowDetails( func availableDownloadFiles(config_obj *config_proto.Config, client_id string, flow_id string) (*api_proto.AvailableDownloads, error) { - download_file := paths.GetDownloadsFile(client_id, flow_id) + flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) + download_file := flow_path_manager.GetDownloadsFile().Path() download_path := path.Dir(download_file) return getAvailableDownloadFiles(config_obj, download_path) @@ -215,7 +217,7 @@ func CancelFlow( return nil, err } - err = services.NotifyClient(client_id) + err = services.NotifyClient(config_obj, client_id) if err != nil { return nil, err } @@ -255,10 +257,13 @@ func ArchiveFlow( Set("Timestamp", time.Now().UTC().Unix()). Set("Flow", collection_context) + path_manager := result_sets.NewArtifactPathManager(config_obj, + client_id, flow_id, "System.Flow.Archive") + return &api_proto.StartFlowResponse{ FlowId: flow_id, - }, services.GetJournal().PushRow( - "System.Flow.Archive", client_id, paths.MODE_MONITORING_DAILY, row) + }, services.GetJournal().PushRows(path_manager, + []*ordereddict.Dict{row}) } func GetFlowRequests( diff --git a/flows/artifacts.go b/flows/artifacts.go index 51a341aa617..441fca2083a 100644 --- a/flows/artifacts.go +++ b/flows/artifacts.go @@ -22,7 +22,6 @@ import ( "crypto/rand" "encoding/base32" "encoding/binary" - "encoding/json" "fmt" "path" "strings" @@ -42,13 +41,12 @@ import ( crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/csv" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" utils "www.velocidex.com/golang/velociraptor/utils" - vql_subsystem "www.velocidex.com/golang/velociraptor/vql" ) var ( @@ -174,16 +172,19 @@ func ScheduleArtifactCollectionFromCollectorArgs( CreateTime: uint64(time.Now().UnixNano() / 1000), State: flows_proto.ArtifactCollectorContext_RUNNING, Request: collector_request, + ClientId: client_id, } - collection_context.Urn = GetCollectionPath(client_id, collection_context.SessionId) - db, err := datastore.GetDB(config_obj) if err != nil { return "", err } // Save the collection context. - err = db.SetSubject(config_obj, collection_context.Urn, collection_context) + flow_path_manager := paths.NewFlowPathManager(client_id, + collection_context.SessionId) + err = db.SetSubject(config_obj, + flow_path_manager.Path(), + collection_context) if err != nil { return "", err } @@ -201,7 +202,7 @@ func ScheduleArtifactCollectionFromCollectorArgs( // Record the tasks for provenance of what we actually did. err = db.SetSubject(config_obj, - path.Join(collection_context.Urn, "task"), + flow_path_manager.Task().Path(), &api_proto.ApiFlowRequestDetails{ Items: []*crypto_proto.GrrMessage{task}}) if err != nil { @@ -246,8 +247,9 @@ func closeContext( collection_context.Status = err.Error() } - err = db.SetSubject(config_obj, collection_context.Urn, - collection_context) + flow_path_manager := paths.NewFlowPathManager( + collection_context.ClientId, collection_context.SessionId) + err = db.SetSubject(config_obj, flow_path_manager.Path(), collection_context) if err != nil { return err } @@ -259,11 +261,15 @@ func closeContext( row := ordereddict.NewDict(). Set("Timestamp", time.Now().UTC().Unix()). Set("Flow", collection_context). - Set("FlowId", collection_context.SessionId) + Set("FlowId", collection_context.SessionId). + Set("ClientId", collection_context.ClientId) - return services.GetJournal().PushRow("System.Flow.Completion", - collection_context.Request.ClientId, /* source */ - 0, row) + path_manager := result_sets.NewArtifactPathManager(config_obj, + collection_context.ClientId, collection_context.SessionId, + "System.Flow.Completion") + + return services.GetJournal().PushRows(path_manager, + []*ordereddict.Dict{row}) } return nil @@ -272,24 +278,19 @@ func closeContext( func flushContextLogs( config_obj *config_proto.Config, collection_context *flows_proto.ArtifactCollectorContext) error { - log_path := path.Join(collection_context.Urn, "logs") - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.WriteFile(log_path) - if err != nil { - return err - } - defer fd.Close() + flow_path_manager := paths.NewFlowPathManager( + collection_context.ClientId, + collection_context.SessionId).Log() - scope := vql_subsystem.MakeScope() - w, err := csv.GetCSVWriter(scope, fd) + rs_writer, err := result_sets.NewResultSetWriter(config_obj, flow_path_manager) if err != nil { return err } - defer w.Close() + defer rs_writer.Close() for _, row := range collection_context.Logs { - w.Write(ordereddict.NewDict(). + rs_writer.Write(ordereddict.NewDict(). Set("Timestamp", fmt.Sprintf("%v", row.Timestamp)). Set("time", time.Unix(int64(row.Timestamp)/1000000, 0).String()). Set("message", row.Message)) @@ -306,33 +307,26 @@ func flushContextUploadedFiles( config_obj *config_proto.Config, collection_context *flows_proto.ArtifactCollectorContext) error { - log_path := path.Join(collection_context.Urn, "uploads.csv") - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.WriteFile(log_path) - if err != nil { - return err - } - defer fd.Close() + flow_path_manager := paths.NewFlowPathManager( + collection_context.ClientId, + collection_context.SessionId).UploadMetadata() - scope := vql_subsystem.MakeScope() - w, err := csv.GetCSVWriter(scope, fd) + rs_writer, err := result_sets.NewResultSetWriter(config_obj, flow_path_manager) if err != nil { return err } - defer w.Close() + defer rs_writer.Close() for _, row := range collection_context.UploadedFiles { - w.Write(ordereddict.NewDict(). + rs_writer.Write(ordereddict.NewDict(). Set("Timestamp", fmt.Sprintf("%v", time.Now().UTC().Unix())). Set("started", time.Now().UTC().String()). Set("vfs_path", row.Name). Set("expected_size", fmt.Sprintf("%v", row.Size))) - } // Clear the logs from the flow object. collection_context.UploadedFiles = nil - return nil } @@ -343,20 +337,20 @@ func LoadCollectionContext( if flow_id == constants.MONITORING_WELL_KNOWN_FLOW { return &flows_proto.ArtifactCollectorContext{ - Urn: GetCollectionPath(client_id, flow_id), SessionId: flow_id, + ClientId: client_id, }, nil } - urn := GetCollectionPath(client_id, flow_id) - + flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) collection_context := &flows_proto.ArtifactCollectorContext{} db, err := datastore.GetDB(config_obj) if err != nil { return nil, err } - err = db.GetSubject(config_obj, urn, collection_context) + err = db.GetSubject(config_obj, flow_path_manager.Path(), + collection_context) if err != nil { return nil, err } @@ -366,7 +360,6 @@ func LoadCollectionContext( } collection_context.Dirty = false - collection_context.Urn = urn return collection_context, nil } @@ -410,52 +403,28 @@ func ArtifactCollectorProcessOneMessage( return err } - artifact_name, source_name := paths. - QueryNameToArtifactAndSource(response.Query.Name) - // Store the event log in the client's VFS. if response.Query.Name != "" { - log_path := paths.GetCSVPath( - collection_context.Request.ClientId, "", + path_manager := result_sets.NewArtifactPathManager(config_obj, + collection_context.Request.ClientId, collection_context.SessionId, - artifact_name, source_name, paths.MODE_CLIENT) + response.Query.Name) - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.WriteFile(log_path) + rs_writer, err := result_sets.NewResultSetWriter(config_obj, path_manager) if err != nil { fmt.Printf("Error: %v\n", err) return err } - defer fd.Close() + defer rs_writer.Close() - scope := vql_subsystem.MakeScope() - writer, err := csv.GetCSVWriter(scope, fd) + rows, err := utils.ParseJsonToDicts([]byte(response.Response)) if err != nil { fmt.Printf("Error: %v\n", err) return err } - defer writer.Close() - - // Decode the JSON data. - var rows []map[string]interface{} - err = json.Unmarshal([]byte(response.Response), &rows) - if err != nil { - return errors.WithStack(err) - } for _, row := range rows { - csv_row := ordereddict.NewDict() - - for _, column := range response.Columns { - item, pres := row[column] - if !pres { - csv_row.Set(column, "-") - } else { - csv_row.Set(column, item) - } - } - - writer.Write(csv_row) + rs_writer.Write(row) } // Update the artifacts with results in the @@ -558,12 +527,12 @@ func appendUploadDataToFile( file_store_factory := file_store.GetFileStore(config_obj) + flow_path_manager := paths.NewFlowPathManager( + message.Source, collection_context.SessionId) // Figure out where to store the file. - file_path := paths.GetUploadsFile( - message.Source, - collection_context.SessionId, + file_path := flow_path_manager.GetUploadsFile( file_buffer.Pathspec.Accessor, - file_buffer.Pathspec.Path) + file_buffer.Pathspec.Path).Path() fd, err := file_store_factory.WriteFile(file_path) if err != nil { @@ -620,8 +589,12 @@ func appendUploadDataToFile( Set("Accessor", file_buffer.Pathspec.Accessor). Set("Size", size) - return services.GetJournal().PushRow( - "System.Upload.Completion", message.Source, 0, row) + path_manager := result_sets.NewArtifactPathManager(config_obj, + message.Source, collection_context.SessionId, + "System.Upload.Completion") + + return services.GetJournal().PushRows(path_manager, + []*ordereddict.Dict{row}) } return nil @@ -700,8 +673,6 @@ func (self *FlowRunner) ProcessSingleMessage( collection_context, err = LoadCollectionContext( self.config_obj, job.Source, job.SessionId) if err != nil { - logger.Error(fmt.Sprintf("Unable to load flow %s: %v", job.SessionId, err)) - // Ignore logs and status messages from the // client. These are generated by cancel // requests anyway. @@ -709,6 +680,8 @@ func (self *FlowRunner) ProcessSingleMessage( return } + logger.Error(fmt.Sprintf("Unable to load flow %s: %v", job.SessionId, err)) + db, err := datastore.GetDB(self.config_obj) if err == nil { db.QueueMessageForClient(self.config_obj, job.Source, diff --git a/flows/foreman.go b/flows/foreman.go index 8a545befe89..1c5599d868f 100644 --- a/flows/foreman.go +++ b/flows/foreman.go @@ -121,7 +121,7 @@ func ForemanProcessMessage( return err } - return services.NotifyClient(client_id) + return services.NotifyClient(config_obj, client_id) }) } diff --git a/flows/hunts.go b/flows/hunts.go index 30cbf145a2f..80b42b36e99 100644 --- a/flows/hunts.go +++ b/flows/hunts.go @@ -38,6 +38,7 @@ import ( "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" ) @@ -120,10 +121,11 @@ func CreateHunt( // set it started. } else if hunt.State == api_proto.Hunt_RUNNING { hunt.StartTime = hunt.CreateTime - services.NotifyAll() + services.NotifyAll(config_obj) } - err = db.SetSubject(config_obj, constants.GetHuntURN(hunt.HuntId), hunt) + hunt_path_manager := paths.NewHuntPathManager(hunt.HuntId) + err = db.SetSubject(config_obj, hunt_path_manager.Path(), hunt) if err != nil { return nil, err } @@ -239,9 +241,11 @@ func ModifyHunt( Set("Hunt", hunt). Set("User", user) - services.GetJournal().PushRow( - "System.Hunt.Archive", "server", - paths.MODE_MONITORING_DAILY, row) + path_manager := result_sets.NewArtifactPathManager(config_obj, + "server", hunt_modification.HuntId, "System.Hunt.Archive") + + services.GetJournal().PushRows( + path_manager, []*ordereddict.Dict{row}) // We are trying to start the hunt. } else if hunt_modification.State == api_proto.Hunt_RUNNING { @@ -265,9 +269,9 @@ func ModifyHunt( return err } + hunt_path_manager := paths.NewHuntPathManager(hunt.HuntId) err = db.SetSubject( - config_obj, - constants.GetHuntURN(hunt.HuntId), hunt) + config_obj, hunt_path_manager.Path(), hunt) if err != nil { return err } @@ -282,5 +286,5 @@ func ModifyHunt( // Notify all the clients about the new hunt. New hunts are // not that common so notifying all the clients at once is // probably ok. - return services.NotifyAll() + return services.NotifyAll(config_obj) } diff --git a/flows/monitoring.go b/flows/monitoring.go index 3000945f931..796917441ae 100644 --- a/flows/monitoring.go +++ b/flows/monitoring.go @@ -6,10 +6,12 @@ import ( "www.velocidex.com/golang/velociraptor/constants" crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" - "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" + utils "www.velocidex.com/golang/velociraptor/utils" ) +// Receive monitoring messages from the client. func MonitoringProcessMessage( config_obj *config_proto.Config, collection_context *flows_proto.ArtifactCollectorContext, @@ -40,9 +42,22 @@ func MonitoringProcessMessage( // Store the event log in the client's VFS. if response.Query.Name != "" { - return services.GetJournal().Push( - response.Query.Name, message.Source, - paths.MODE_MONITORING_DAILY, []byte(response.Response)) + rows, err := utils.ParseJsonToDicts([]byte(response.Response)) + if err != nil { + return err + } + + // Mark the client this came from. Since message.Souce + // is cryptographically trusted, this column may also + // be trusted. + for _, row := range rows { + row.Set("ClientId", message.Source) + } + + path_manager := result_sets.NewArtifactPathManager(config_obj, + message.Source, message.SessionId, response.Query.Name) + + return services.GetJournal().PushRows(path_manager, rows) } diff --git a/flows/proto/artifact_collector.pb.go b/flows/proto/artifact_collector.pb.go index ef7175b23fd..04718686c85 100644 --- a/flows/proto/artifact_collector.pb.go +++ b/flows/proto/artifact_collector.pb.go @@ -340,8 +340,7 @@ func (m *ArtifactUploadedFileInfo) GetSize() uint64 { // This context is serialized into the data store. type ArtifactCollectorContext struct { - // Where we are stored in the data store. - Urn string `protobuf:"bytes,5,opt,name=urn,proto3" json:"urn,omitempty"` + ClientId string `protobuf:"bytes,27,opt,name=client_id,json=clientId,proto3" json:"client_id,omitempty"` SessionId string `protobuf:"bytes,13,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` Request *ArtifactCollectorArgs `protobuf:"bytes,11,opt,name=request,proto3" json:"request,omitempty"` // If an error occurs this is the backtrace. @@ -395,9 +394,9 @@ func (m *ArtifactCollectorContext) XXX_DiscardUnknown() { var xxx_messageInfo_ArtifactCollectorContext proto.InternalMessageInfo -func (m *ArtifactCollectorContext) GetUrn() string { +func (m *ArtifactCollectorContext) GetClientId() string { if m != nil { - return m.Urn + return m.ClientId } return "" } @@ -800,101 +799,100 @@ func init() { } var fileDescriptor_ecbffc830d4bd8f6 = []byte{ - // 1523 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x57, 0xed, 0x6e, 0x1b, 0x45, - 0x17, 0x7e, 0x1d, 0x27, 0x76, 0x3c, 0x69, 0xf2, 0xba, 0xf3, 0x26, 0xcd, 0x26, 0x6f, 0xab, 0x0e, - 0x6e, 0x0b, 0xa6, 0x2a, 0xeb, 0x34, 0xfd, 0x40, 0x2a, 0x02, 0x14, 0x27, 0x0e, 0xb5, 0x48, 0x93, - 0x74, 0xe3, 0xb4, 0x02, 0x81, 0xac, 0xf1, 0xee, 0x59, 0x7b, 0x9a, 0xf5, 0xce, 0x76, 0x66, 0xd6, - 0x6e, 0x7a, 0x0f, 0x5c, 0x03, 0x37, 0xc3, 0x7f, 0x7e, 0x71, 0x03, 0x70, 0x1b, 0x20, 0xa1, 0x99, - 0xdd, 0x8d, 0xed, 0x7c, 0x15, 0x24, 0x90, 0x9a, 0x3f, 0xde, 0x3d, 0x5f, 0xcf, 0x9c, 0x33, 0xe7, - 0x3c, 0x67, 0x83, 0x2c, 0x2a, 0x14, 0xf3, 0xa9, 0xab, 0xda, 0x2e, 0x0f, 0x02, 0x70, 0x15, 0x17, - 0x76, 0x24, 0xb8, 0xe2, 0x78, 0xc6, 0xfc, 0xac, 0x2e, 0x9a, 0x9f, 0x9a, 0x84, 0x3e, 0x0d, 0x15, - 0x73, 0x13, 0xe5, 0xea, 0x32, 0x75, 0x15, 0xe3, 0xa1, 0xac, 0x25, 0xda, 0xc1, 0xeb, 0x20, 0x53, - 0xb8, 0xe2, 0x38, 0x52, 0x3c, 0x95, 0xbf, 0xe2, 0x1d, 0x99, 0x28, 0x2a, 0x6f, 0x11, 0xde, 0x48, - 0xa1, 0xf6, 0xa9, 0xa0, 0x7d, 0x50, 0x20, 0x24, 0xf6, 0x50, 0x1e, 0xc2, 0x81, 0x95, 0x27, 0xf9, - 0xea, 0xdc, 0xfa, 0x7c, 0x62, 0x6a, 0xbf, 0x78, 0xbe, 0xd3, 0x08, 0x07, 0xf5, 0xcd, 0x5f, 0x7f, - 0xff, 0xed, 0xa7, 0xdc, 0xe7, 0xf8, 0x41, 0x23, 0x1c, 0x30, 0xc1, 0xc3, 0x3e, 0x84, 0x8a, 0x0c, - 0xa8, 0x60, 0xb4, 0x13, 0x80, 0x24, 0x8a, 0x93, 0x0e, 0x90, 0x48, 0xf0, 0x01, 0xf3, 0xc0, 0x23, - 0x3e, 0x17, 0x44, 0xf5, 0x80, 0xbc, 0x8e, 0x41, 0x1c, 0xdb, 0x95, 0x82, 0x01, 0x91, 0x8e, 0x0e, - 0x5f, 0xf9, 0x79, 0x06, 0x2d, 0x65, 0xe0, 0x9b, 0x59, 0x9a, 0x1b, 0xa2, 0x2b, 0xb1, 0x85, 0x8a, - 0xae, 0x00, 0xaa, 0xb8, 0xb0, 0x72, 0x24, 0x57, 0x2d, 0x39, 0xd9, 0x2b, 0xfe, 0x3f, 0x2a, 0xb9, - 0x01, 0x83, 0x50, 0xb5, 0x99, 0x67, 0xe5, 0x8d, 0x6e, 0x36, 0x11, 0x34, 0x3d, 0x7c, 0x0d, 0x15, - 0x62, 0xd1, 0x85, 0x50, 0x59, 0x4b, 0x24, 0x57, 0x9d, 0x75, 0xd2, 0x37, 0xbc, 0x8d, 0x4a, 0x59, - 0x3d, 0xa5, 0x35, 0x45, 0xf2, 0xd5, 0x52, 0xbd, 0x6a, 0xb2, 0xa8, 0x60, 0xab, 0xd5, 0x03, 0x72, - 0xa2, 0xd4, 0xa7, 0x0f, 0x68, 0x1c, 0xba, 0x3d, 0xbb, 0x52, 0xd8, 0x31, 0x0f, 0xce, 0xc8, 0x15, - 0x1f, 0x21, 0x14, 0x9d, 0x14, 0xc9, 0x9a, 0x21, 0xb9, 0xea, 0xdc, 0xfa, 0x4a, 0x5a, 0x9d, 0xb3, - 0x55, 0xac, 0xaf, 0x19, 0x8c, 0xbb, 0xf8, 0xfa, 0x48, 0xa6, 0x01, 0xd4, 0x38, 0xa2, 0x5d, 0x41, - 0x23, 0xad, 0x33, 0x16, 0x1e, 0xbf, 0x42, 0x0b, 0x3c, 0x92, 0xed, 0x08, 0x44, 0x5b, 0x82, 0xcb, - 0x43, 0xcf, 0x2a, 0x90, 0x5c, 0x75, 0xaa, 0xbe, 0x65, 0xa2, 0x7e, 0x81, 0x6f, 0xed, 0x45, 0x20, - 0xa8, 0xb9, 0x6e, 0x12, 0x81, 0x20, 0x89, 0x11, 0xa9, 0xb6, 0x7a, 0x82, 0x2b, 0x15, 0xb0, 0xb0, - 0xfb, 0xb1, 0x5d, 0x59, 0xd8, 0x8b, 0x24, 0xd9, 0x07, 0x41, 0x0e, 0x8c, 0x76, 0xbd, 0x78, 0x7f, - 0xcd, 0xfc, 0x39, 0x57, 0x78, 0x24, 0xf7, 0x41, 0x24, 0x62, 0x0c, 0xa8, 0xa8, 0x58, 0x1f, 0x78, - 0xac, 0xac, 0x22, 0xc9, 0x55, 0xa7, 0xeb, 0x5f, 0x1b, 0x90, 0x06, 0x7e, 0xb4, 0x1b, 0xf7, 0x3b, - 0x20, 0x08, 0xf7, 0xd3, 0xf8, 0x26, 0x03, 0x11, 0x87, 0xa4, 0x03, 0x3e, 0x17, 0x40, 0x5c, 0x1a, - 0xba, 0x10, 0x68, 0xb4, 0xf1, 0x6b, 0x2e, 0xb6, 0x92, 0x68, 0xeb, 0xf9, 0xc7, 0x6b, 0x6b, 0x4e, - 0x16, 0x1b, 0xff, 0x90, 0x43, 0xd7, 0x68, 0x10, 0xf0, 0x61, 0xdb, 0x8d, 0xa5, 0xe2, 0xfd, 0x36, - 0x1f, 0x80, 0x10, 0xcc, 0x03, 0x69, 0xcd, 0xea, 0x0b, 0xab, 0xbf, 0x34, 0xb0, 0xcf, 0xf1, 0x5e, - 0xd3, 0x27, 0x4a, 0xc4, 0x40, 0x86, 0x40, 0x86, 0x2c, 0x08, 0x48, 0x2c, 0x81, 0x50, 0x92, 0x78, - 0x9d, 0x14, 0x8f, 0x30, 0x9f, 0x44, 0x02, 0xa4, 0x6e, 0x40, 0x16, 0x4a, 0x05, 0xd4, 0xd3, 0x07, - 0xd5, 0xe7, 0x08, 0x69, 0x1f, 0xbc, 0x13, 0x43, 0xdb, 0x59, 0x34, 0xb0, 0x9b, 0xc6, 0x7f, 0x2f, - 0x03, 0xc5, 0x7b, 0x68, 0xd9, 0xe5, 0xfd, 0x88, 0x05, 0xe0, 0x8d, 0xe6, 0xac, 0x4d, 0x45, 0x57, - 0x5a, 0x8b, 0xe6, 0x72, 0x97, 0x47, 0xad, 0x3f, 0xd1, 0xa0, 0xce, 0x52, 0xe6, 0x37, 0x21, 0xae, - 0x04, 0x68, 0xe5, 0x4c, 0x43, 0x3b, 0x20, 0x23, 0x1e, 0x4a, 0xc0, 0xcb, 0xa8, 0xe8, 0xeb, 0xdc, - 0x99, 0x97, 0x36, 0x75, 0x41, 0xbf, 0x36, 0x3d, 0xfc, 0x18, 0x15, 0x05, 0xbc, 0x8e, 0x41, 0x2a, - 0x6b, 0xca, 0xc0, 0x5e, 0x3f, 0xd5, 0x53, 0x93, 0xd8, 0x99, 0x71, 0xe5, 0x7b, 0x64, 0x65, 0x16, - 0x87, 0x51, 0xc0, 0xa9, 0x07, 0xde, 0x36, 0x0b, 0xa0, 0x19, 0xfa, 0x1c, 0x63, 0x34, 0xad, 0x6b, - 0x90, 0x22, 0x99, 0x67, 0xbc, 0x82, 0x66, 0x07, 0xbe, 0x6c, 0x47, 0x54, 0xf5, 0x0c, 0x50, 0xc9, - 0x29, 0x0e, 0x7c, 0xb9, 0x4f, 0x55, 0x4f, 0x9b, 0x4b, 0xf6, 0x16, 0xcc, 0x44, 0x4d, 0x3b, 0xe6, - 0xb9, 0xf2, 0x4b, 0x71, 0x14, 0xff, 0xe4, 0x04, 0x9b, 0x3c, 0x54, 0xf0, 0x46, 0xe1, 0x32, 0xca, - 0xc7, 0x22, 0x34, 0x33, 0x50, 0x72, 0xf4, 0x23, 0xbe, 0x81, 0x90, 0x04, 0x29, 0x19, 0x0f, 0x75, - 0x86, 0xf3, 0x46, 0x51, 0x4a, 0x25, 0x93, 0x49, 0xce, 0xfd, 0x8d, 0x24, 0xf1, 0x75, 0x54, 0xea, - 0x50, 0xf7, 0x48, 0x09, 0xea, 0x66, 0xd9, 0x8c, 0x04, 0xf8, 0x26, 0x9a, 0x33, 0xcc, 0x00, 0x6d, - 0xdd, 0x63, 0xe9, 0xf1, 0x51, 0x22, 0xd2, 0x3d, 0xa8, 0x0d, 0x34, 0x27, 0x0e, 0x52, 0x83, 0xab, - 0x89, 0x41, 0x22, 0x32, 0x06, 0x77, 0xd0, 0xc2, 0x11, 0x0b, 0x02, 0xa3, 0x96, 0x8a, 0xf6, 0x23, - 0x6b, 0xda, 0xd8, 0xcc, 0x6b, 0x69, 0x2b, 0x13, 0xe2, 0x6d, 0xb4, 0x10, 0xa7, 0x35, 0x6e, 0xfb, - 0x2c, 0x00, 0x69, 0x59, 0x86, 0x1c, 0x6f, 0x9e, 0xca, 0xe2, 0xf4, 0x45, 0x38, 0xf3, 0xf1, 0x98, - 0x44, 0xe2, 0x35, 0xb4, 0xa8, 0xb8, 0xa2, 0x41, 0xfb, 0x54, 0xb4, 0x65, 0x03, 0x8a, 0x8d, 0xee, - 0x70, 0xc2, 0x63, 0x03, 0xdd, 0x48, 0x3c, 0xe0, 0x4d, 0x04, 0xae, 0x02, 0x6f, 0xe4, 0xda, 0x39, - 0x56, 0x20, 0xad, 0x15, 0xe3, 0xba, 0x6a, 0x8c, 0x1a, 0xa9, 0x4d, 0x16, 0xa2, 0xae, 0x2d, 0xce, - 0x01, 0x4d, 0x3c, 0x57, 0xcf, 0x01, 0x4d, 0x3c, 0xee, 0xa0, 0xe9, 0x80, 0x9b, 0x31, 0xd0, 0x49, - 0x5e, 0x4d, 0x93, 0xdc, 0xe1, 0xdd, 0x67, 0x20, 0x25, 0xed, 0x82, 0x63, 0xd4, 0xf8, 0x33, 0x34, - 0x23, 0x15, 0x55, 0x60, 0x2d, 0x90, 0x5c, 0x75, 0x61, 0xfd, 0xce, 0x45, 0x57, 0x9a, 0x76, 0x8d, - 0x7d, 0xa0, 0x8d, 0x9d, 0xc4, 0x07, 0xef, 0xa1, 0x82, 0x7e, 0x88, 0xa5, 0xf5, 0x5f, 0x7d, 0xad, - 0xf5, 0x4f, 0xcd, 0xf0, 0xdf, 0xc7, 0x35, 0x63, 0x1d, 0x2a, 0xa9, 0x87, 0x99, 0x86, 0x04, 0x84, - 0xe0, 0x82, 0x24, 0xa6, 0xc4, 0x0c, 0x7b, 0xe7, 0xd8, 0x0c, 0x79, 0xc2, 0xfb, 0xb6, 0x93, 0x86, - 0xc1, 0xb7, 0xd0, 0x7c, 0x2c, 0x41, 0xb4, 0x43, 0xae, 0x98, 0xcf, 0xc0, 0xb3, 0xca, 0x66, 0x0b, - 0x5c, 0xd1, 0xc2, 0xdd, 0x54, 0x86, 0x7f, 0xd4, 0x1c, 0x94, 0xb1, 0x6f, 0x7b, 0xc8, 0x54, 0xaf, - 0x2d, 0x40, 0xc6, 0x81, 0x92, 0xd6, 0x35, 0xb3, 0x19, 0x98, 0x39, 0x86, 0x8b, 0xa9, 0xde, 0x0c, - 0x7e, 0x1c, 0x04, 0x44, 0x0f, 0xcc, 0x19, 0xe2, 0x26, 0xda, 0x55, 0x8b, 0x98, 0x20, 0x69, 0x00, - 0x9b, 0xb4, 0x7a, 0x4c, 0x12, 0x45, 0x8f, 0xf4, 0x1e, 0xd4, 0xde, 0x5c, 0x8c, 0xb1, 0x95, 0x9e, - 0xc1, 0x9a, 0xe4, 0xb1, 0x70, 0x13, 0x7e, 0xd2, 0xac, 0x94, 0x45, 0x7b, 0xc9, 0x54, 0xcf, 0x49, - 0xa2, 0xe0, 0x45, 0x34, 0xe3, 0x31, 0xa1, 0x8e, 0xcd, 0x8c, 0xce, 0x3a, 0xc9, 0x4b, 0xe5, 0x29, - 0x9a, 0x31, 0xd5, 0xc3, 0x25, 0x34, 0x73, 0xb8, 0x7b, 0xd0, 0x68, 0x95, 0xff, 0x83, 0xe7, 0x50, - 0xd1, 0x39, 0xdc, 0xdd, 0x6d, 0xee, 0x7e, 0x55, 0xce, 0xe1, 0x05, 0x84, 0x5a, 0x0d, 0xe7, 0x59, - 0x73, 0x77, 0xa3, 0xd5, 0xd8, 0x2a, 0x4f, 0x69, 0xbb, 0x86, 0xe3, 0xec, 0x39, 0xe5, 0x3c, 0xbe, - 0x82, 0x66, 0x37, 0x9c, 0xcd, 0xa7, 0xcd, 0x17, 0x8d, 0xad, 0xf2, 0x74, 0xa5, 0x87, 0xca, 0x9b, - 0xa6, 0x72, 0x8d, 0x01, 0x84, 0xaa, 0xa5, 0xd7, 0xb6, 0x5e, 0xb8, 0x03, 0x10, 0x7a, 0x54, 0xcd, - 0x8c, 0x4d, 0x3b, 0xd9, 0x2b, 0x7e, 0x32, 0xb9, 0x3b, 0xdf, 0x3d, 0xb9, 0x23, 0xf3, 0xca, 0x21, - 0x2a, 0xff, 0x1b, 0xc4, 0xf4, 0x47, 0x09, 0xcd, 0x6d, 0x6b, 0x3a, 0x4f, 0xb9, 0xe8, 0x72, 0x8a, - 0x78, 0x78, 0x0e, 0x45, 0xd4, 0xff, 0x67, 0x2e, 0x79, 0x1e, 0xcd, 0x39, 0x5b, 0xdb, 0x5b, 0x54, - 0x81, 0x56, 0x4d, 0xf0, 0xc6, 0x93, 0x33, 0xb4, 0x50, 0xb8, 0xd8, 0xf1, 0x14, 0x57, 0xdc, 0x43, - 0x38, 0x04, 0x35, 0xe4, 0xe2, 0x28, 0x99, 0xb3, 0xb6, 0x6e, 0xda, 0x64, 0xb1, 0x3a, 0xe5, 0x54, - 0x63, 0xc6, 0xec, 0x40, 0x7f, 0x9c, 0xc4, 0x17, 0x30, 0x4b, 0xb6, 0x7b, 0x4e, 0x57, 0xb0, 0xfe, - 0xc8, 0x1c, 0xa1, 0x86, 0x3f, 0xd9, 0x20, 0x01, 0x93, 0x4a, 0x4f, 0x89, 0xf1, 0x23, 0x59, 0x18, - 0x42, 0x25, 0x89, 0xa8, 0x50, 0xd9, 0x32, 0xd4, 0x6b, 0xc6, 0x3e, 0x4d, 0x44, 0xdf, 0x5c, 0x46, - 0x44, 0xf5, 0x8f, 0x0c, 0xc6, 0x07, 0xf8, 0x66, 0x4b, 0xdb, 0x90, 0xf0, 0xe4, 0x2b, 0xe0, 0x04, - 0xc3, 0x58, 0xdb, 0xe7, 0x32, 0x16, 0xff, 0x4b, 0x8c, 0x55, 0xbf, 0x67, 0x30, 0x3e, 0xc4, 0xb7, - 0x9f, 0xf2, 0x21, 0xe9, 0xd3, 0xf0, 0x98, 0x18, 0xad, 0xde, 0xf9, 0x89, 0xa3, 0xf9, 0xd4, 0x00, - 0x17, 0xd8, 0x00, 0xec, 0x4b, 0xf9, 0xed, 0xbb, 0xcb, 0xf8, 0xad, 0x7e, 0xd7, 0xe0, 0xdc, 0xc6, - 0x95, 0xb3, 0x38, 0x69, 0x74, 0x8f, 0x48, 0x4e, 0x7c, 0x2a, 0xec, 0x73, 0xb9, 0xf0, 0xcb, 0x77, - 0x70, 0x61, 0xdd, 0x32, 0x00, 0x18, 0xcf, 0xef, 0xa4, 0xd7, 0xa1, 0xcd, 0xed, 0xd5, 0xdc, 0x54, - 0xca, 0x92, 0xef, 0xd8, 0x8c, 0xf6, 0x24, 0x89, 0x5a, 0x29, 0xc0, 0x58, 0x87, 0xbf, 0x0f, 0xbc, - 0xf9, 0xf0, 0x9c, 0x45, 0x7a, 0xc1, 0x18, 0x8d, 0x6d, 0xd7, 0xee, 0x38, 0x7b, 0x2c, 0x19, 0x7e, - 0x6d, 0x1a, 0x9f, 0x4d, 0xbc, 0xd1, 0x1c, 0x75, 0x27, 0x11, 0xa0, 0x62, 0x11, 0x4a, 0x42, 0x89, - 0x04, 0x53, 0xc2, 0xb1, 0x4f, 0x72, 0xcd, 0xa9, 0x3e, 0x83, 0xc0, 0x4b, 0x3e, 0x05, 0x4d, 0xcf, - 0xab, 0x1e, 0xf4, 0xed, 0xf1, 0x4f, 0xf3, 0xf7, 0x9e, 0xd6, 0xff, 0x39, 0x02, 0xaf, 0x3f, 0xf8, - 0xf6, 0xfe, 0x70, 0x38, 0xb4, 0x07, 0x10, 0x70, 0x97, 0x79, 0xf0, 0xc6, 0x76, 0x79, 0xbf, 0xd6, - 0xe5, 0x01, 0x0d, 0xbb, 0xb5, 0x44, 0x28, 0x68, 0xa4, 0xb8, 0xa8, 0xe9, 0xd2, 0xa6, 0xff, 0x0b, - 0x76, 0x0a, 0xe6, 0xe7, 0xc1, 0x9f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x6b, 0xb1, 0x3a, 0x88, 0x5a, - 0x0e, 0x00, 0x00, + // 1517 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x57, 0xeb, 0x6e, 0xdb, 0xc6, + 0x12, 0x3e, 0xf2, 0x45, 0xb2, 0xd6, 0xb1, 0x8f, 0xb2, 0xc7, 0x8e, 0x69, 0x27, 0x41, 0xf6, 0x28, + 0xc9, 0x39, 0x6a, 0x90, 0x52, 0x8e, 0x73, 0x29, 0x90, 0xa2, 0x2d, 0x2c, 0x5b, 0x6e, 0x84, 0x3a, + 0xb6, 0x43, 0xcb, 0x09, 0x5a, 0xb4, 0x10, 0x56, 0xe4, 0x50, 0xda, 0x98, 0xe2, 0x32, 0xbb, 0x4b, + 0x29, 0xce, 0x3b, 0xf4, 0x19, 0xfa, 0x32, 0xfd, 0xdf, 0x57, 0x28, 0xda, 0xd7, 0x68, 0x81, 0x62, + 0x97, 0xa4, 0x75, 0xf1, 0x25, 0x2d, 0xd0, 0x02, 0xf1, 0x1f, 0x91, 0x73, 0xfb, 0x38, 0xb3, 0x33, + 0xdf, 0xac, 0x91, 0x45, 0x85, 0x62, 0x3e, 0x75, 0x55, 0xcb, 0xe5, 0x41, 0x00, 0xae, 0xe2, 0xc2, + 0x8e, 0x04, 0x57, 0x1c, 0xcf, 0x9a, 0x9f, 0xb5, 0x25, 0xf3, 0x53, 0x95, 0xd0, 0xa3, 0xa1, 0x62, + 0x6e, 0xa2, 0x5c, 0x5b, 0xa1, 0xae, 0x62, 0x3c, 0x94, 0xd5, 0x44, 0xdb, 0x7f, 0x13, 0x64, 0x0a, + 0x57, 0x9c, 0x44, 0x8a, 0xa7, 0xf2, 0xd7, 0xbc, 0x2d, 0x13, 0x45, 0xf9, 0x1d, 0xc2, 0x9b, 0x29, + 0xd4, 0x01, 0x15, 0xb4, 0x07, 0x0a, 0x84, 0xc4, 0x1e, 0x9a, 0x86, 0xb0, 0x6f, 0x4d, 0x93, 0xe9, + 0xca, 0xfc, 0xc6, 0x42, 0x62, 0x6a, 0xbf, 0x7c, 0xb1, 0x5b, 0x0f, 0xfb, 0xb5, 0xad, 0x5f, 0x7e, + 0xfb, 0xf5, 0xc7, 0xdc, 0x67, 0xf8, 0x61, 0x3d, 0xec, 0x33, 0xc1, 0xc3, 0x1e, 0x84, 0x8a, 0xf4, + 0xa9, 0x60, 0xb4, 0x1d, 0x80, 0x24, 0x8a, 0x93, 0x36, 0x90, 0x48, 0xf0, 0x3e, 0xf3, 0xc0, 0x23, + 0x3e, 0x17, 0x44, 0x75, 0x81, 0xbc, 0x89, 0x41, 0x9c, 0xd8, 0xe5, 0xbc, 0x01, 0x91, 0x8e, 0x0e, + 0x5f, 0xfe, 0x69, 0x16, 0x2d, 0x67, 0xe0, 0x5b, 0x59, 0x9a, 0x9b, 0xa2, 0x23, 0xb1, 0x85, 0x0a, + 0xae, 0x00, 0xaa, 0xb8, 0xb0, 0x72, 0x24, 0x57, 0x29, 0x3a, 0xd9, 0x2b, 0xbe, 0x8e, 0x8a, 0x6e, + 0xc0, 0x20, 0x54, 0x2d, 0xe6, 0x59, 0xd3, 0x46, 0x37, 0x97, 0x08, 0x1a, 0x1e, 0xbe, 0x86, 0xf2, + 0xb1, 0xe8, 0x40, 0xa8, 0xac, 0x65, 0x92, 0xab, 0xcc, 0x39, 0xe9, 0x1b, 0xde, 0x41, 0xc5, 0xac, + 0x9e, 0xd2, 0x9a, 0x22, 0xd3, 0x95, 0x62, 0xad, 0x62, 0xb2, 0x28, 0x63, 0xab, 0xd9, 0x05, 0x72, + 0xaa, 0xd4, 0x5f, 0x1f, 0xd0, 0x38, 0x74, 0xbb, 0x76, 0x39, 0xbf, 0x6b, 0x1e, 0x9c, 0xa1, 0x2b, + 0x3e, 0x46, 0x28, 0x3a, 0x2d, 0x92, 0x35, 0x4b, 0x72, 0x95, 0xf9, 0x8d, 0xd5, 0xb4, 0x3a, 0x67, + 0xab, 0x58, 0x5b, 0x37, 0x18, 0xf7, 0xf0, 0x8d, 0xa1, 0x4c, 0x03, 0xa8, 0x51, 0x44, 0xbb, 0x8c, + 0x86, 0x5a, 0x67, 0x24, 0x3c, 0x7e, 0x8d, 0x16, 0x79, 0x24, 0x5b, 0x11, 0x88, 0x96, 0x04, 0x97, + 0x87, 0x9e, 0x95, 0x27, 0xb9, 0xca, 0x54, 0x6d, 0xdb, 0x44, 0xfd, 0x1c, 0xdf, 0xde, 0x8f, 0x40, + 0x50, 0x73, 0xdc, 0x24, 0x02, 0x41, 0x12, 0x23, 0x52, 0x69, 0x76, 0x05, 0x57, 0x2a, 0x60, 0x61, + 0xe7, 0x23, 0xbb, 0xbc, 0xb8, 0x1f, 0x49, 0x72, 0x00, 0x82, 0x1c, 0x1a, 0xed, 0x46, 0xe1, 0xc1, + 0xba, 0xf9, 0x73, 0xae, 0xf0, 0x48, 0x1e, 0x80, 0x48, 0xc4, 0x18, 0x50, 0x41, 0xb1, 0x1e, 0xf0, + 0x58, 0x59, 0x05, 0x92, 0xab, 0xcc, 0xd4, 0xbe, 0x32, 0x20, 0x75, 0xfc, 0x78, 0x2f, 0xee, 0xb5, + 0x41, 0x10, 0xee, 0xa7, 0xf1, 0x4d, 0x06, 0x22, 0x0e, 0x49, 0x1b, 0x7c, 0x2e, 0x80, 0xb8, 0x34, + 0x74, 0x21, 0xd0, 0x68, 0xa3, 0xc7, 0x5c, 0x68, 0x26, 0xd1, 0x36, 0xa6, 0x9f, 0xac, 0xaf, 0x3b, + 0x59, 0x6c, 0xfc, 0x7d, 0x0e, 0x5d, 0xa3, 0x41, 0xc0, 0x07, 0x2d, 0x37, 0x96, 0x8a, 0xf7, 0x5a, + 0xbc, 0x0f, 0x42, 0x30, 0x0f, 0xa4, 0x35, 0xa7, 0x0f, 0xac, 0xf6, 0xca, 0xc0, 0xbe, 0xc0, 0xfb, + 0x0d, 0x9f, 0x28, 0x11, 0x03, 0x19, 0x00, 0x19, 0xb0, 0x20, 0x20, 0xb1, 0x04, 0x42, 0x49, 0xe2, + 0x75, 0x5a, 0x3c, 0xc2, 0x7c, 0x12, 0x09, 0x90, 0xba, 0x01, 0x59, 0x28, 0x15, 0x50, 0x4f, 0x7f, + 0xa8, 0xfe, 0x8e, 0x90, 0xf6, 0xc0, 0x3b, 0x35, 0xb4, 0x9d, 0x25, 0x03, 0xbb, 0x65, 0xfc, 0xf7, + 0x33, 0x50, 0xbc, 0x8f, 0x56, 0x5c, 0xde, 0x8b, 0x58, 0x00, 0xde, 0x70, 0xce, 0x5a, 0x54, 0x74, + 0xa4, 0xb5, 0x64, 0x0e, 0x77, 0x65, 0xd8, 0xfa, 0x63, 0x0d, 0xea, 0x2c, 0x67, 0x7e, 0x63, 0xe2, + 0x72, 0x80, 0x56, 0xcf, 0x34, 0xb4, 0x03, 0x32, 0xe2, 0xa1, 0x04, 0xbc, 0x82, 0x0a, 0xbe, 0xce, + 0x9d, 0x79, 0x69, 0x53, 0xe7, 0xf5, 0x6b, 0xc3, 0xc3, 0x4f, 0x50, 0x41, 0xc0, 0x9b, 0x18, 0xa4, + 0xb2, 0xa6, 0x0c, 0xec, 0x8d, 0x89, 0x9e, 0x1a, 0xc7, 0xce, 0x8c, 0xcb, 0xdf, 0x21, 0x2b, 0xb3, + 0x38, 0x8a, 0x02, 0x4e, 0x3d, 0xf0, 0x76, 0x58, 0x00, 0x8d, 0xd0, 0xe7, 0x18, 0xa3, 0x19, 0x5d, + 0x83, 0x14, 0xc9, 0x3c, 0xe3, 0x55, 0x34, 0xd7, 0xf7, 0x65, 0x2b, 0xa2, 0xaa, 0x6b, 0x80, 0x8a, + 0x4e, 0xa1, 0xef, 0xcb, 0x03, 0xaa, 0xba, 0xda, 0x5c, 0xb2, 0x77, 0x60, 0x26, 0x6a, 0xc6, 0x31, + 0xcf, 0xe5, 0x9f, 0x0b, 0xc3, 0xf8, 0xa7, 0x5f, 0xb0, 0xc5, 0x43, 0x05, 0x6f, 0xd5, 0xf8, 0x1c, + 0x5e, 0x9f, 0x98, 0xc3, 0x9b, 0x08, 0x49, 0x90, 0x92, 0xf1, 0x50, 0x6b, 0x17, 0x8c, 0xb6, 0x98, + 0x4a, 0xc6, 0xf3, 0x9d, 0xff, 0x0b, 0xf9, 0xe2, 0x1b, 0xa8, 0xd8, 0xa6, 0xee, 0xb1, 0x12, 0xd4, + 0xcd, 0x12, 0x1b, 0x0a, 0xf0, 0x2d, 0x34, 0x6f, 0x48, 0x02, 0x5a, 0xba, 0xdd, 0xd2, 0x4c, 0x50, + 0x22, 0xd2, 0xed, 0xa8, 0x0d, 0x34, 0x3d, 0xf6, 0x53, 0x83, 0xab, 0x89, 0x41, 0x22, 0x32, 0x06, + 0x77, 0xd1, 0xe2, 0x31, 0x0b, 0x02, 0xa3, 0x96, 0x8a, 0xf6, 0x22, 0x6b, 0xc6, 0xd8, 0x2c, 0x68, + 0x69, 0x33, 0x13, 0xe2, 0x1d, 0xb4, 0x18, 0xa7, 0xe5, 0x6e, 0xf9, 0x2c, 0x00, 0x69, 0x59, 0x86, + 0x27, 0x6f, 0x4d, 0x64, 0x31, 0x79, 0x26, 0xce, 0x42, 0x3c, 0x22, 0x91, 0x78, 0x1d, 0x2d, 0x29, + 0xae, 0x68, 0xd0, 0x9a, 0x88, 0xb6, 0x62, 0x40, 0xb1, 0xd1, 0x1d, 0x8d, 0x79, 0x6c, 0xa2, 0x9b, + 0x89, 0x07, 0xbc, 0x8d, 0xc0, 0x55, 0xe0, 0x0d, 0x5d, 0xdb, 0x27, 0x0a, 0xa4, 0xb5, 0x6a, 0x5c, + 0xd7, 0x8c, 0x51, 0x3d, 0xb5, 0xc9, 0x42, 0xd4, 0xb4, 0xc5, 0x39, 0xa0, 0x89, 0xe7, 0xda, 0x39, + 0xa0, 0x89, 0xc7, 0x5d, 0x34, 0x13, 0x70, 0x33, 0x11, 0x3a, 0xc9, 0xab, 0x69, 0x92, 0xbb, 0xbc, + 0xf3, 0x1c, 0xa4, 0xa4, 0x1d, 0x70, 0x8c, 0x1a, 0x7f, 0x8a, 0x66, 0xa5, 0xa2, 0x0a, 0xac, 0x45, + 0x92, 0xab, 0x2c, 0x6e, 0xdc, 0xbd, 0xe8, 0x48, 0xd3, 0x06, 0xb2, 0x0f, 0xb5, 0xb1, 0x93, 0xf8, + 0xe0, 0x7d, 0x94, 0xd7, 0x0f, 0xb1, 0xb4, 0xfe, 0xad, 0x8f, 0xb5, 0xf6, 0x89, 0xe1, 0x81, 0x07, + 0xb8, 0x6a, 0xac, 0x43, 0x25, 0xf5, 0x5c, 0xd3, 0x90, 0x80, 0x10, 0x5c, 0x90, 0xc4, 0x94, 0x98, + 0xb9, 0x6f, 0x9f, 0x98, 0x79, 0x4f, 0x5a, 0xcf, 0x76, 0xd2, 0x30, 0xf8, 0x36, 0x5a, 0x88, 0x25, + 0x88, 0x56, 0xc8, 0x15, 0xf3, 0x19, 0x78, 0x56, 0xc9, 0x2c, 0x84, 0x2b, 0x5a, 0xb8, 0x97, 0xca, + 0xf0, 0x0f, 0x9a, 0x8e, 0x32, 0x22, 0x6e, 0x0d, 0x98, 0xea, 0xb6, 0x04, 0xc8, 0x38, 0x50, 0xd2, + 0xba, 0x66, 0x96, 0x04, 0x33, 0x9f, 0xe1, 0x62, 0xaa, 0x97, 0x84, 0x1f, 0x07, 0x01, 0xd1, 0xb3, + 0x73, 0x86, 0xc3, 0x89, 0x76, 0xd5, 0x22, 0x26, 0x48, 0x1a, 0xc0, 0x26, 0xcd, 0x2e, 0x93, 0x44, + 0xd1, 0x63, 0xbd, 0x12, 0xb5, 0x37, 0x17, 0x23, 0xc4, 0xa5, 0xc7, 0xb1, 0x2a, 0x79, 0x2c, 0xdc, + 0x84, 0xaa, 0x34, 0x41, 0x65, 0xd1, 0x5e, 0x31, 0xd5, 0x75, 0x92, 0x28, 0x78, 0x09, 0xcd, 0x7a, + 0x4c, 0xa8, 0x13, 0x33, 0xae, 0x73, 0x4e, 0xf2, 0x52, 0x7e, 0x86, 0x66, 0x4d, 0xf5, 0x70, 0x11, + 0xcd, 0x1e, 0xed, 0x1d, 0xd6, 0x9b, 0xa5, 0x7f, 0xe1, 0x79, 0x54, 0x70, 0x8e, 0xf6, 0xf6, 0x1a, + 0x7b, 0x5f, 0x96, 0x72, 0x78, 0x11, 0xa1, 0x66, 0xdd, 0x79, 0xde, 0xd8, 0xdb, 0x6c, 0xd6, 0xb7, + 0x4b, 0x53, 0xda, 0xae, 0xee, 0x38, 0xfb, 0x4e, 0x69, 0x1a, 0x5f, 0x41, 0x73, 0x9b, 0xce, 0xd6, + 0xb3, 0xc6, 0xcb, 0xfa, 0x76, 0x69, 0xa6, 0xdc, 0x45, 0xa5, 0x2d, 0x53, 0xb9, 0x7a, 0x1f, 0x42, + 0xd5, 0xd4, 0x1b, 0x5c, 0xef, 0xde, 0x3e, 0x08, 0x3d, 0xaa, 0x66, 0xc6, 0x66, 0x9c, 0xec, 0x15, + 0x3f, 0x1d, 0x5f, 0xa3, 0xef, 0x9f, 0xdc, 0xa1, 0x79, 0xf9, 0x08, 0x95, 0xfe, 0x09, 0x8e, 0xfa, + 0xbd, 0x88, 0xe6, 0x77, 0x34, 0xb3, 0xa7, 0xb4, 0x74, 0x39, 0x45, 0x3c, 0x3a, 0x87, 0x22, 0x6a, + 0xff, 0x31, 0x87, 0xbc, 0x80, 0xe6, 0x9d, 0xed, 0x9d, 0x6d, 0xaa, 0x40, 0xab, 0xc6, 0x78, 0xe3, + 0xe9, 0x19, 0x5a, 0xc8, 0x5f, 0xec, 0x38, 0xc1, 0x15, 0xf7, 0x11, 0x0e, 0x41, 0x0d, 0xb8, 0x38, + 0x4e, 0xe6, 0xac, 0xa5, 0x9b, 0x36, 0xd9, 0xb1, 0x4e, 0x29, 0xd5, 0x98, 0x31, 0x3b, 0xd4, 0xf7, + 0x94, 0xf8, 0x02, 0x66, 0xc9, 0xd6, 0xd0, 0x64, 0x05, 0x6b, 0x8f, 0xcd, 0x27, 0x54, 0xf1, 0xc7, + 0x9b, 0x24, 0x60, 0x52, 0xe9, 0x29, 0x31, 0x7e, 0x24, 0x0b, 0x43, 0xa8, 0x24, 0x11, 0x15, 0x2a, + 0xdb, 0x8b, 0x7a, 0xe3, 0xd8, 0x93, 0x44, 0xf4, 0xf5, 0x65, 0x44, 0x54, 0xfb, 0xbf, 0xc1, 0xf8, + 0x2f, 0xbe, 0xd5, 0xd4, 0x36, 0x24, 0x3c, 0xbd, 0x10, 0x9c, 0x62, 0x18, 0x6b, 0xfb, 0x5c, 0xc6, + 0xe2, 0x7f, 0x8a, 0xb1, 0x6a, 0xf7, 0x0d, 0xc6, 0xff, 0xf0, 0x9d, 0x67, 0x7c, 0x40, 0x7a, 0x34, + 0x3c, 0x21, 0x46, 0xab, 0xd7, 0x7f, 0xe2, 0x68, 0x6e, 0x1d, 0xe0, 0x02, 0xeb, 0x83, 0x7d, 0x29, + 0xbf, 0x7d, 0x7b, 0x19, 0xbf, 0xd5, 0xee, 0x19, 0x9c, 0x3b, 0xb8, 0x7c, 0x16, 0x27, 0x8d, 0xee, + 0x11, 0xc9, 0x89, 0x4f, 0x85, 0x7d, 0x2e, 0x17, 0x7e, 0xf1, 0x1e, 0x2e, 0xac, 0x59, 0x06, 0x00, + 0xe3, 0x85, 0xdd, 0xf4, 0x38, 0xb4, 0xb9, 0xbd, 0x96, 0x9b, 0x4a, 0x59, 0xf2, 0x3d, 0x9b, 0xd1, + 0x1e, 0x27, 0x51, 0x2b, 0x05, 0x18, 0xe9, 0xf0, 0x0f, 0x81, 0x37, 0x1f, 0x9d, 0xb3, 0x48, 0x2f, + 0x18, 0xa3, 0x91, 0xed, 0xda, 0x19, 0x65, 0x8f, 0x65, 0xc3, 0xaf, 0x0d, 0xe3, 0xb3, 0x85, 0x37, + 0x1b, 0xc3, 0xee, 0x24, 0x02, 0x54, 0x2c, 0x42, 0x49, 0x28, 0x91, 0x60, 0x4a, 0x38, 0x72, 0x3b, + 0xd7, 0x9c, 0xea, 0x33, 0x08, 0xbc, 0xe4, 0x56, 0x68, 0x7a, 0x5e, 0x75, 0xa1, 0x67, 0x8f, 0xde, + 0xd2, 0x3f, 0x78, 0x5a, 0xff, 0xfb, 0x08, 0xbc, 0xf6, 0xf0, 0x9b, 0x07, 0x83, 0xc1, 0xc0, 0xee, + 0x43, 0xc0, 0x5d, 0xe6, 0xc1, 0x5b, 0xdb, 0xe5, 0xbd, 0x6a, 0x87, 0x07, 0x34, 0xec, 0x54, 0x13, + 0xa1, 0xa0, 0x91, 0xe2, 0xa2, 0xaa, 0x4b, 0x9b, 0xfe, 0x5b, 0xd8, 0xce, 0x9b, 0x9f, 0x87, 0x7f, + 0x04, 0x00, 0x00, 0xff, 0xff, 0x30, 0x69, 0xfe, 0xc7, 0x65, 0x0e, 0x00, 0x00, } diff --git a/flows/proto/artifact_collector.proto b/flows/proto/artifact_collector.proto index 4afc2a141d0..7568f34f6b9 100644 --- a/flows/proto/artifact_collector.proto +++ b/flows/proto/artifact_collector.proto @@ -69,7 +69,9 @@ message ArtifactUploadedFileInfo { // This context is serialized into the data store. message ArtifactCollectorContext { // Where we are stored in the data store. - string urn = 5; + // string urn = 5; + + string client_id = 27; string session_id = 13; ArtifactCollectorArgs request = 11; diff --git a/go.mod b/go.mod index 45f8658e9da..e6d3b295acc 100644 --- a/go.mod +++ b/go.mod @@ -1,125 +1,125 @@ module www.velocidex.com/golang/velociraptor require ( - cloud.google.com/go v0.47.0 // indirect - cloud.google.com/go/bigquery v1.1.0 // indirect - cloud.google.com/go/storage v1.1.1 - github.com/Depado/bfchroma v1.2.0 - github.com/Masterminds/goutils v1.1.0 // indirect - github.com/Masterminds/semver v1.4.2 // indirect - github.com/Masterminds/sprig v2.20.0+incompatible - github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667 // indirect - github.com/Showmax/go-fqdn v0.0.0-20180501083314-6f60894d629f - github.com/Velocidex/ahocorasick v0.0.0-20180712114356-e1c353eeaaee - github.com/Velocidex/cgofuse v1.1.2 - github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b - github.com/Velocidex/go-yara v1.1.10-0.20200414034554-457848df11f9 - github.com/Velocidex/ordereddict v0.0.0-20200419183704-bc44e4ef79c7 - github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49 - github.com/Velocidex/yaml/v2 v2.2.5 - github.com/ZachtimusPrime/Go-Splunk-HTTP v0.0.0-20200420213219-094ff9e8d788 - github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 - github.com/alecthomas/chroma v0.7.2 - github.com/alecthomas/participle v0.4.1 - github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c // indirect - github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 - github.com/aws/aws-sdk-go v1.26.7 - github.com/c-bata/go-prompt v0.2.3 - github.com/cevaris/ordered_map v0.0.0-20180310183325-0efaee1733e3 - github.com/clbanning/mxj v1.8.4 - github.com/creack/pty v1.1.9 // indirect - github.com/crewjam/saml v0.3.1 - github.com/davecgh/go-spew v1.1.1 - github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 - github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/dustin/go-humanize v1.0.0 - github.com/elastic/go-elasticsearch/v7 v7.3.0 // indirect - github.com/elastic/go-libaudit v0.4.0 - github.com/evanphx/json-patch v4.5.0+incompatible - github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect - github.com/go-ole/go-ole v1.2.4 - github.com/go-sql-driver/mysql v1.5.0 - github.com/gogo/protobuf v1.1.1 - github.com/golang/mock v1.4.3 - github.com/golang/protobuf v1.3.5 - github.com/golang/snappy v0.0.1 - github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 // indirect - github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 // indirect - github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e // indirect - github.com/google/go-cmp v0.3.1 // indirect - github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf - github.com/google/uuid v1.1.1 // indirect - github.com/gorilla/csrf v1.6.2 - github.com/gorilla/schema v1.1.0 - github.com/grpc-ecosystem/grpc-gateway v1.11.3 - github.com/hanwen/go-fuse v1.0.1-0.20190726130028-2f298055551b - github.com/hillu/go-ntdll v0.0.0-20190226223014-dd4204aa705e - github.com/hillu/go-yara v1.2.2 // indirect - github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c // indirect - github.com/huandu/xstrings v1.2.0 // indirect - github.com/imdario/mergo v0.3.7 // indirect - github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect - github.com/jmoiron/sqlx v1.2.0 - github.com/jstemmer/go-junit-report v0.9.1 // indirect - github.com/kierdavis/dateparser v0.0.0-20171227112021-81e70b820720 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/kr/pretty v0.2.0 // indirect - github.com/kr/pty v1.1.8 // indirect - github.com/lestrrat-go/file-rotatelogs v2.2.0+incompatible - github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d // indirect - github.com/lib/pq v1.2.0 // indirect - github.com/magefile/mage v1.9.0 - github.com/mattn/go-pointer v0.0.0-20180825124634-49522c3f3791 - github.com/mattn/go-runewidth v0.0.8 // indirect - github.com/mattn/go-sqlite3 v1.11.0 - github.com/mattn/go-tty v0.0.0-20190424173100-523744f04859 // indirect - github.com/microcosm-cc/bluemonday v1.0.2 - github.com/olekukonko/tablewriter v0.0.2 - github.com/pkg/errors v0.9.1 - github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 // indirect - github.com/processout/grpc-go-pool v1.2.1 - github.com/prometheus/client_golang v1.2.1 - github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 - github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d - github.com/russross/blackfriday/v2 v2.0.1 - github.com/sebdah/goldie v1.0.0 - github.com/sergi/go-diff v1.0.0 - github.com/shirou/gopsutil v0.0.0-20190627142359-4c8b404ee5c5 - github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.5.1 - github.com/tebeka/strftime v0.1.3 // indirect - github.com/tink-ab/tempfile v0.0.0-20180226111222-33beb0518f1a - github.com/vjeantet/grok v1.0.0 - github.com/xor-gate/ar v0.0.0-20170530204233-5c72ae81e2b7 // indirect - github.com/xor-gate/debpkg v0.0.0-20181217150151-a0c70a3d4213 - golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 - golang.org/x/exp v0.0.0-20191014171548-69215a2ee97e // indirect - golang.org/x/mod v0.2.0 // indirect - golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e - golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect - golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 - golang.org/x/tools v0.0.0-20200207131002-533eb2654509 // indirect - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect - google.golang.org/api v0.11.0 - google.golang.org/appengine v1.6.5 // indirect - google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d - google.golang.org/grpc v1.28.1 - gopkg.in/alecthomas/kingpin.v2 v2.2.6 - gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df - gopkg.in/sourcemap.v1 v1.0.5 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb - www.velocidex.com/golang/evtx v0.0.2-0.20191118125331-191dc946afdf - www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321 - www.velocidex.com/golang/go-ntfs v0.0.0-20200110083657-950cbe916617 - www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6 - www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf - www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 - www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 - www.velocidex.com/golang/vfilter v0.0.0-20200422024715-0e09f92d1676 - www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b + cloud.google.com/go v0.47.0 // indirect + cloud.google.com/go/bigquery v1.1.0 // indirect + cloud.google.com/go/storage v1.1.1 + github.com/Depado/bfchroma v1.2.0 + github.com/Masterminds/goutils v1.1.0 // indirect + github.com/Masterminds/semver v1.4.2 // indirect + github.com/Masterminds/sprig v2.20.0+incompatible + github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667 // indirect + github.com/Showmax/go-fqdn v0.0.0-20180501083314-6f60894d629f + github.com/Velocidex/ahocorasick v0.0.0-20180712114356-e1c353eeaaee + github.com/Velocidex/cgofuse v1.1.2 + github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b + github.com/Velocidex/go-yara v1.1.10-0.20200414034554-457848df11f9 + github.com/Velocidex/ordereddict v0.0.0-20200428070154-4cd604876fe5 + github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49 + github.com/Velocidex/yaml/v2 v2.2.5 + github.com/ZachtimusPrime/Go-Splunk-HTTP v0.0.0-20200420213219-094ff9e8d788 + github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 + github.com/alecthomas/chroma v0.7.2 + github.com/alecthomas/participle v0.4.1 + github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c // indirect + github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 + github.com/aws/aws-sdk-go v1.26.7 + github.com/c-bata/go-prompt v0.2.3 + github.com/cevaris/ordered_map v0.0.0-20180310183325-0efaee1733e3 + github.com/clbanning/mxj v1.8.4 + github.com/creack/pty v1.1.9 // indirect + github.com/crewjam/saml v0.3.1 + github.com/davecgh/go-spew v1.1.1 + github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 + github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/dustin/go-humanize v1.0.0 + github.com/elastic/go-elasticsearch/v7 v7.3.0 // indirect + github.com/elastic/go-libaudit v0.4.0 + github.com/evanphx/json-patch v4.5.0+incompatible + github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect + github.com/go-ole/go-ole v1.2.4 + github.com/go-sql-driver/mysql v1.5.0 + github.com/gogo/protobuf v1.1.1 + github.com/golang/mock v1.4.3 + github.com/golang/protobuf v1.3.5 + github.com/golang/snappy v0.0.1 + github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 // indirect + github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 // indirect + github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e // indirect + github.com/google/go-cmp v0.3.1 // indirect + github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf + github.com/google/uuid v1.1.1 // indirect + github.com/gorilla/csrf v1.6.2 + github.com/gorilla/schema v1.1.0 + github.com/grpc-ecosystem/grpc-gateway v1.11.3 + github.com/hanwen/go-fuse v1.0.1-0.20190726130028-2f298055551b + github.com/hillu/go-ntdll v0.0.0-20190226223014-dd4204aa705e + github.com/hillu/go-yara v1.2.2 // indirect + github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c // indirect + github.com/huandu/xstrings v1.2.0 // indirect + github.com/imdario/mergo v0.3.7 // indirect + github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect + github.com/jmoiron/sqlx v1.2.0 + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/kierdavis/dateparser v0.0.0-20171227112021-81e70b820720 + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/kr/pretty v0.2.0 // indirect + github.com/kr/pty v1.1.8 // indirect + github.com/lestrrat-go/file-rotatelogs v2.2.0+incompatible + github.com/lestrrat-go/strftime v0.0.0-20190725011945-5c849dd2c51d // indirect + github.com/lib/pq v1.2.0 // indirect + github.com/magefile/mage v1.9.0 + github.com/mattn/go-pointer v0.0.0-20180825124634-49522c3f3791 + github.com/mattn/go-runewidth v0.0.8 // indirect + github.com/mattn/go-sqlite3 v1.11.0 + github.com/mattn/go-tty v0.0.0-20190424173100-523744f04859 // indirect + github.com/microcosm-cc/bluemonday v1.0.2 + github.com/olekukonko/tablewriter v0.0.2 + github.com/pkg/errors v0.9.1 + github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 // indirect + github.com/processout/grpc-go-pool v1.2.1 + github.com/prometheus/client_golang v1.2.1 + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 + github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d + github.com/russross/blackfriday/v2 v2.0.1 + github.com/sebdah/goldie v1.0.0 + github.com/sergi/go-diff v1.0.0 + github.com/shirou/gopsutil v0.0.0-20190627142359-4c8b404ee5c5 + github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/testify v1.5.1 + github.com/tebeka/strftime v0.1.3 // indirect + github.com/tink-ab/tempfile v0.0.0-20180226111222-33beb0518f1a + github.com/vjeantet/grok v1.0.0 + github.com/xor-gate/ar v0.0.0-20170530204233-5c72ae81e2b7 // indirect + github.com/xor-gate/debpkg v0.0.0-20181217150151-a0c70a3d4213 + golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 + golang.org/x/exp v0.0.0-20191014171548-69215a2ee97e // indirect + golang.org/x/mod v0.2.0 // indirect + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect + golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 + golang.org/x/tools v0.0.0-20200207131002-533eb2654509 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect + google.golang.org/api v0.11.0 + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d + google.golang.org/grpc v1.28.1 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df + gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb + www.velocidex.com/golang/evtx v0.0.2-0.20191118125331-191dc946afdf + www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321 + www.velocidex.com/golang/go-ntfs v0.0.0-20200110083657-950cbe916617 + www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6 + www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf + www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 + www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 + www.velocidex.com/golang/vfilter v0.0.0-20200422024715-0e09f92d1676 + www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b ) // replace www.velocidex.com/golang/vfilter => /home/mic/projects/vfilter diff --git a/go.sum b/go.sum index 00edf51684e..affd5047a99 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/Velocidex/ordereddict v0.0.0-20191103011020-3b5a5f6957d4/go.mod h1:px github.com/Velocidex/ordereddict v0.0.0-20191106020901-97c468e5e403/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo= github.com/Velocidex/ordereddict v0.0.0-20200419183704-bc44e4ef79c7 h1:kOuTDzcjsrEE+JM+/PkKLIfoyy+7nGxiG5idfZYKA5M= github.com/Velocidex/ordereddict v0.0.0-20200419183704-bc44e4ef79c7/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo= +github.com/Velocidex/ordereddict v0.0.0-20200428070154-4cd604876fe5 h1:mqiDJyva9oCVq+rl6gnofUDZW+1Q+V/xzrH2hYE/MAM= +github.com/Velocidex/ordereddict v0.0.0-20200428070154-4cd604876fe5/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo= github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49 h1:TJVN1zYl5sKJUq571gYqU/5VqFIap9MUo2KNujc0fcg= github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49/go.mod h1:kfPUQ2gP0xtIydiR52dirNYt4OvCr+iZuepL4XaIk58= github.com/Velocidex/yaml/v2 v2.2.5 h1:8XZwR8tmm5UWAXotI3fL17s9fjKEMqZ293fgqUiMhBw= diff --git a/gui/static/angular-components/artifact/server-events-directive.js b/gui/static/angular-components/artifact/server-events-directive.js index 266e57e8b0b..c49be9bdf92 100644 --- a/gui/static/angular-components/artifact/server-events-directive.js +++ b/gui/static/angular-components/artifact/server-events-directive.js @@ -48,7 +48,7 @@ const ServerEventsController = function($scope, $uibModal, grrApiService) { }; var timestamps = this.selectedArtifact.timestamps; - if (angular.isDefined(timestamps.length)) { + if (angular.isArray(timestamps) && timestamps.length) { for (var i=0; i= timestamp_start && ts <= timestamp_end) { @@ -78,7 +78,7 @@ const ServerEventsController = function($scope, $uibModal, grrApiService) { }; ServerEventsController.prototype.onDateChange = function() { - if (!angular.isDefined(this.selected_date)) { + if (!angular.isObject(this.selected_date)) { return; } @@ -106,15 +106,15 @@ ServerEventsController.prototype.openDatePicker = function() { }; ServerEventsController.prototype.selectArtifact = function(artifact) { - this.selectedArtifact = artifact; - this.selected_date = null; + this.selectedArtifact = artifact; + this.selected_date = null; - if (artifact.timestamps.length > 0) { - var last_timestamp = artifact.timestamps[artifact.timestamps.length-1]; - this.selected_date = new Date(last_timestamp * 1000); - } + if (angular.isArray(artifact.timestamps) && artifact.timestamps.length > 0) { + var last_timestamp = artifact.timestamps[artifact.timestamps.length-1]; + this.selected_date = new Date(last_timestamp * 1000); + } - return false; + return false; }; ServerEventsController.prototype.showHelp = function() { diff --git a/gui/static/angular-components/core/csv-viewer-directive.js b/gui/static/angular-components/core/csv-viewer-directive.js index 4a7882d446d..9a392f287a2 100644 --- a/gui/static/angular-components/core/csv-viewer-directive.js +++ b/gui/static/angular-components/core/csv-viewer-directive.js @@ -94,7 +94,8 @@ CsvViewerDirective.prototype.fetchText_ = function() { if (self.scope_.baseUrl && angular.isDefined(self.scope_.params)) { var url = self.scope_.baseUrl; - var params = Object.assign({}, self.scope_.params); + var params = Object.assign({}, self.scope_.params); + if (angular.isObject(params) && angular.isDefined(params.path)) { params['start_row'] = 0; params['rows'] = MAX_ROWS_PER_TABLE; diff --git a/gui/static/angular-components/flow/flow-inspector-directive.js b/gui/static/angular-components/flow/flow-inspector-directive.js index 5658fa0f721..e8124485c44 100644 --- a/gui/static/angular-components/flow/flow-inspector-directive.js +++ b/gui/static/angular-components/flow/flow-inspector-directive.js @@ -66,8 +66,10 @@ FlowInspectorController.prototype.startPolling = function(newValues, oldValues) this.pollPromise_ = undefined; this.uploadedFilesParams = { - path: '/clients/' + this.scope_['clientId'] + '/collections/' + - this.scope_['flowId'] + '/uploads.csv' + client_id: this.scope_['clientId'], + flow_id: this.scope_['flowId'] , + type: 'uploads', + path: this.scope_['flowId'], }; if (angular.isDefined(this.scope_['apiBasePath']) && diff --git a/gui/static/angular-components/flow/flow-log-directive.js b/gui/static/angular-components/flow/flow-log-directive.js index dd994e306dd..f09b044ee02 100644 --- a/gui/static/angular-components/flow/flow-log-directive.js +++ b/gui/static/angular-components/flow/flow-log-directive.js @@ -27,8 +27,12 @@ const FlowLogController = function($scope) { * @private */ FlowLogController.prototype.onChange_ = function() { - this.queryParams.path = '/clients/' + this.scope_["clientId"] + - '/collections/' + this.scope_['flowId'] + '/logs';; + this.queryParams = { + client_id: this.scope_["clientId"], + flow_id: this.scope_['flowId'], + type: 'log', + path: this.scope_['flowId'], + }; }; /** diff --git a/gui/static/angular-components/flow/flow-results-directive.js b/gui/static/angular-components/flow/flow-results-directive.js index 933d01a6914..591a19b8aab 100644 --- a/gui/static/angular-components/flow/flow-results-directive.js +++ b/gui/static/angular-components/flow/flow-results-directive.js @@ -63,25 +63,15 @@ FlowResultsController.prototype.onFlowIdOrBasePathChange_ = function( if (newValues != oldValues) { if (newValues.every(angular.isDefined)) { - var components = this.selectedArtifact.split("/"); - // Artifact has no source name. - if (components.length == 1) { - this.queryParams = { - path: '/clients/'+ this.scope_['clientId'] + - '/artifacts/' + this.selectedArtifact + - '/' + this.scope_.flowId + '.csv', - client_id: this.scope_['clientId'], - }; - } else { - // Artifact has a source name. - this.queryParams = { - path: '/clients/' + this.scope_['clientId'] + - '/artifacts/' + components[0] + - '/' + this.scope_.flowId + "/" + - components[1] + '.csv', - client_id: this.scope_['clientId'], - }; - } + this.queryParams = { + client_id: this.scope_['clientId'], + flow_id: this.scope_.flowId , + artifact: this.selectedArtifact, + // Path is ignored by is used by angular to trigger + // changes. + path: this.scope_.flowId+this.selectedArtifact + }; + } } }; diff --git a/gui/static/angular-components/hunt/hunt-clients-directive.js b/gui/static/angular-components/hunt/hunt-clients-directive.js index f3ef94ac508..bd53d25e082 100644 --- a/gui/static/angular-components/hunt/hunt-clients-directive.js +++ b/gui/static/angular-components/hunt/hunt-clients-directive.js @@ -27,7 +27,11 @@ const HuntClientsController = function($scope) { HuntClientsController.prototype.onContextChange_ = function(newValues, oldValues) { if (newValues != oldValues || this.pageData == null) { - this.params = {path: "hunts/" + this.scope_.huntId + ".csv"}; + this.params = { + hunt_id: this.scope_.huntId, + type: "clients", + path: this.scope_.huntId, + }; } }; diff --git a/http_comms/comms.go b/http_comms/comms.go index 83e0e2b7400..cfd1ebeeddb 100644 --- a/http_comms/comms.go +++ b/http_comms/comms.go @@ -394,6 +394,7 @@ func (self *NotificationReader) sendMessageList( // synchronization of endpoints. wait := self.maxPoll + time.Duration( rand.Intn(int(self.maxPollDev)))*time.Second + self.logger.Info("Sleeping for %v", wait) select { case <-ctx.Done(): diff --git a/paths/client.go b/paths/client.go new file mode 100644 index 00000000000..afc13c6941a --- /dev/null +++ b/paths/client.go @@ -0,0 +1,67 @@ +package paths + +import ( + "context" + "fmt" + "path" + + "www.velocidex.com/golang/velociraptor/file_store/api" +) + +type ClientPathManager struct { + path string + client_id string +} + +func (self ClientPathManager) Path() string { + return self.path +} + +func (self ClientPathManager) GetPathForWriting() (string, error) { + return self.path, nil +} + +func (self ClientPathManager) GetQueueName() string { + return self.client_id +} + +func (self ClientPathManager) GeneratePaths(ctx context.Context) <-chan *api.ResultSetFileProperties { + output := make(chan *api.ResultSetFileProperties) + go func() { + defer close(output) + + output <- &api.ResultSetFileProperties{ + Path: self.path, + EndTime: int64(1) << 62, + } + }() + return output +} + +func NewClientPathManager(client_id string) *ClientPathManager { + return &ClientPathManager{ + path: path.Join("/clients", client_id), + client_id: client_id, + } +} + +// Gets the flow's log file. +func (self ClientPathManager) Ping() *ClientPathManager { + self.path = path.Join(self.path, "ping") + return &self +} + +func (self ClientPathManager) Key() *ClientPathManager { + self.path = path.Join(self.path, "key") + return &self +} + +func (self ClientPathManager) TasksDirectory() *ClientPathManager { + self.path = path.Join(self.path, "tasks") + return &self +} + +func (self ClientPathManager) Task(task_id uint64) *ClientPathManager { + self.path = path.Join(self.path, "tasks", fmt.Sprintf("%d", task_id)) + return &self +} diff --git a/paths/clients.go b/paths/clients.go deleted file mode 100644 index 31d9b59c5ce..00000000000 --- a/paths/clients.go +++ /dev/null @@ -1,27 +0,0 @@ -package paths - -import ( - "fmt" - "path" -) - -func GetClientMetadataPath(client_id string) string { - return path.Join("/clients", client_id) -} - -func GetClientPingPath(client_id string) string { - return path.Join("/clients", client_id, "ping") -} - -func GetClientKeyPath(client_id string) string { - return path.Join("/clients", client_id, "key") -} - -func GetClientTasksPath(client_id string) string { - return path.Join("/clients", client_id, "tasks") -} - -func GetClientTaskPath(client_id string, task_id uint64) string { - return path.Join("/clients", client_id, "tasks", - fmt.Sprintf("%d", task_id)) -} diff --git a/paths/flow_metadata.go b/paths/flow_metadata.go new file mode 100644 index 00000000000..6b1ac27b726 --- /dev/null +++ b/paths/flow_metadata.go @@ -0,0 +1,134 @@ +package paths + +import ( + "context" + "path" + + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/utils" +) + +type FlowPathManager struct { + path string + client_id string + flow_id string +} + +func (self FlowPathManager) Path() string { + return self.path +} + +func (self FlowPathManager) GetPathForWriting() (string, error) { + return self.path, nil +} + +func (self FlowPathManager) GetQueueName() string { + return self.client_id + self.flow_id +} + +func (self FlowPathManager) GeneratePaths(ctx context.Context) <-chan *api.ResultSetFileProperties { + output := make(chan *api.ResultSetFileProperties) + go func() { + defer close(output) + + output <- &api.ResultSetFileProperties{ + Path: self.path, + EndTime: int64(1) << 62, + } + }() + return output +} + +func NewFlowPathManager(client_id, flow_id string) *FlowPathManager { + return &FlowPathManager{ + path: path.Join("/clients", client_id, "collections", flow_id), + client_id: client_id, + flow_id: flow_id, + } +} + +// Gets the flow's log file. +func (self FlowPathManager) Log() *FlowPathManager { + self.path = path.Join(self.path, "logs") + return &self +} + +func (self FlowPathManager) Task() *FlowPathManager { + self.path = path.Join(self.path, "task") + return &self +} + +func (self FlowPathManager) UploadMetadata() *FlowPathManager { + self.path = path.Join(self.path, "uploads.json") + return &self +} + +func (self FlowPathManager) GetDownloadsFile() *FlowPathManager { + self.path = path.Join("/downloads", self.client_id, self.flow_id, + self.flow_id+".zip") + return &self +} + +// Figure out where to store the VFSDownloadInfo file. We maintain a +// metadata file in the client's VFS area linking back to the +// collection which most recently uploaded this file. +func (self FlowPathManager) GetVFSDownloadInfoPath( + accessor, client_path string) *FlowPathManager { + components := []string{"clients", self.client_id, "vfs_files", accessor} + + if accessor == "ntfs" { + device, subpath, err := GetDeviceAndSubpath(client_path) + if err == nil { + components = append(components, device) + components = append(components, utils.SplitComponents(subpath)...) + self.path = utils.JoinComponents(components, "/") + return &self + } + } + + components = append(components, utils.SplitComponents(client_path)...) + self.path = utils.JoinComponents(components, "/") + return &self +} + +// GetVFSDownloadInfoPath returns the vfs path to the directory info +// file. +func (self FlowPathManager) GetVFSDirectoryInfoPath(accessor, client_path string) *FlowPathManager { + components := []string{"clients", self.client_id, "vfs", accessor} + + if accessor == "ntfs" { + device, subpath, err := GetDeviceAndSubpath(client_path) + if err == nil { + components = append(components, device) + components = append(components, utils.SplitComponents(subpath)...) + self.path = utils.JoinComponents(components, "/") + return &self + } + } + + components = append(components, utils.SplitComponents(client_path)...) + self.path = utils.JoinComponents(components, "/") + return &self +} + +// Currently only CLIENT artifacts upload files. We store the uploaded +// file inside the collection that uploaded it. +func (self FlowPathManager) GetUploadsFile(accessor, client_path string) *FlowPathManager { + components := []string{ + "clients", self.client_id, "collections", + self.flow_id, "uploads", accessor} + + if accessor == "ntfs" { + device, subpath, err := GetDeviceAndSubpath(client_path) + if err == nil { + components = append(components, device) + components = append(components, utils.SplitComponents(subpath)...) + self.path = utils.JoinComponents(components, "/") + return &self + } + } + + components = append(components, utils.SplitComponents(client_path)...) + self.path = utils.JoinComponents(components, "/") + return &self +} diff --git a/paths/hunt_metadata.go b/paths/hunt_metadata.go new file mode 100644 index 00000000000..b89dbcae9d5 --- /dev/null +++ b/paths/hunt_metadata.go @@ -0,0 +1,61 @@ +package paths + +import ( + "context" + "path" + + "www.velocidex.com/golang/velociraptor/file_store/api" +) + +type HuntPathManager struct { + path string + hunt_id string +} + +func (self HuntPathManager) Path() string { + return self.path +} + +func (self HuntPathManager) GetPathForWriting() (string, error) { + return self.path, nil +} + +func (self HuntPathManager) GetQueueName() string { + return self.hunt_id +} + +func (self HuntPathManager) GeneratePaths(ctx context.Context) <-chan *api.ResultSetFileProperties { + output := make(chan *api.ResultSetFileProperties) + go func() { + defer close(output) + + output <- &api.ResultSetFileProperties{ + Path: self.path, + EndTime: int64(1) << 62, + } + }() + return output +} + +func NewHuntPathManager(hunt_id string) *HuntPathManager { + return &HuntPathManager{ + path: path.Join("/hunts", hunt_id), + hunt_id: hunt_id, + } +} + +func (self HuntPathManager) Stats() *HuntPathManager { + self.path = path.Join(self.path, "stats") + return &self +} + +func (self HuntPathManager) HuntDirectory() *HuntPathManager { + self.path = "/hunts" + return &self +} + +// Get result set for storing participating clients. +func (self HuntPathManager) Clients() *HuntPathManager { + self.path = path.Join("/hunts", self.hunt_id+".json") + return &self +} diff --git a/paths/paths.go b/paths/paths.go index 5f1d4f7f78e..2af5b225cbc 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -4,23 +4,19 @@ package paths import ( - "fmt" "path" "regexp" "strings" "time" - - "www.velocidex.com/golang/velociraptor/utils" ) const ( // The different types of artifacts. - MODE_INVALID = 0 - MODE_CLIENT = 1 - MODE_SERVER = 2 - MODE_SERVER_EVENT = 3 - MODE_MONITORING_DAILY = 4 - MODE_JOURNAL_DAILY = 5 + MODE_INVALID = 0 + MODE_CLIENT = 1 + MODE_CLIENT_EVENT = 2 + MODE_SERVER = 3 + MODE_SERVER_EVENT = 4 ) func ModeNameToMode(name string) int { @@ -28,175 +24,16 @@ func ModeNameToMode(name string) int { switch name { case "CLIENT": return MODE_CLIENT + case "CLIENT_EVENT": + return MODE_CLIENT_EVENT case "SERVER": return MODE_SERVER case "SERVER_EVENT": return MODE_SERVER_EVENT - case "MONITORING_DAILY", "CLIENT_EVENT": - return MODE_MONITORING_DAILY - case "JOURNAL_DAILY": - return MODE_JOURNAL_DAILY } return 0 } -// Resolve the path relative to the filestore where the CVS files are -// stored. This depends on what kind of log it is (mode), and various -// other details depending on the mode. -// -// This function represents a map between the type of artifact and its -// location on disk. It is used by all code that needs to read or -// write artifact results. -func GetCSVPath( - client_id, day_name, flow_id, artifact_name, source_name string, - mode int) string { - - switch mode { - case MODE_CLIENT: - if source_name != "" { - return fmt.Sprintf( - "/clients/%s/artifacts/%s/%s/%s.csv", - client_id, artifact_name, - flow_id, source_name) - } else { - return fmt.Sprintf( - "/clients/%s/artifacts/%s/%s.csv", - client_id, artifact_name, - flow_id) - } - - case MODE_SERVER: - if source_name != "" { - return fmt.Sprintf( - "/clients/server/artifacts/%s/%s/%s.csv", - artifact_name, flow_id, source_name) - } else { - return fmt.Sprintf( - "/clients/server/artifacts/%s/%s.csv", - artifact_name, flow_id) - } - - case MODE_SERVER_EVENT: - if source_name != "" { - return fmt.Sprintf( - "/server_artifacts/%s/%s/%s.csv", - artifact_name, day_name, source_name) - } else { - return fmt.Sprintf( - "/server_artifacts/%s/%s.csv", - artifact_name, day_name) - } - - case MODE_JOURNAL_DAILY: - if source_name != "" { - return fmt.Sprintf( - "/journals/%s/%s/%s.csv", - artifact_name, day_name, source_name) - } else { - return fmt.Sprintf( - "/journals/%s/%s.csv", - artifact_name, day_name) - } - - case MODE_MONITORING_DAILY: - if client_id == "" { - return GetCSVPath( - client_id, day_name, - flow_id, artifact_name, - source_name, MODE_JOURNAL_DAILY) - - } else { - if source_name != "" { - return fmt.Sprintf( - "/clients/%s/monitoring/%s/%s/%s.csv", - client_id, artifact_name, - day_name, source_name) - } else { - return fmt.Sprintf( - "/clients/%s/monitoring/%s/%s.csv", - client_id, artifact_name, day_name) - } - } - } - - return "" -} - -// Currently only CLIENT artifacts upload files. We store the uploaded -// file inside the collection that uploaded it. -func GetUploadsFile(client_id, flow_id, accessor, client_path string) string { - components := []string{ - "clients", client_id, "collections", - flow_id, "uploads", accessor} - - if accessor == "ntfs" { - device, subpath, err := GetDeviceAndSubpath(client_path) - if err == nil { - components = append(components, device) - components = append(components, utils.SplitComponents(subpath)...) - return utils.JoinComponents(components, "/") - } - } - - components = append(components, utils.SplitComponents(client_path)...) - return utils.JoinComponents(components, "/") -} - -// Figure out where to store the VFSDownloadInfo file. We maintain a -// metadata file in the client's VFS area linking back to the -// collection which most recently uploaded this file. -func GetVFSDownloadInfoPath(client_id, accessor, client_path string) string { - components := []string{ - "clients", client_id, "vfs_files", - accessor} - - if accessor == "ntfs" { - device, subpath, err := GetDeviceAndSubpath(client_path) - if err == nil { - components = append(components, device) - components = append(components, utils.SplitComponents(subpath)...) - return utils.JoinComponents(components, "/") - } - } - - components = append(components, utils.SplitComponents(client_path)...) - return utils.JoinComponents(components, "/") -} - -// GetVFSDownloadInfoPath returns the vfs path to the directory info -// file. -func GetVFSDirectoryInfoPath(client_id, accessor, client_path string) string { - components := []string{ - "clients", client_id, "vfs", - accessor} - - if accessor == "ntfs" { - device, subpath, err := GetDeviceAndSubpath(client_path) - if err == nil { - components = append(components, device) - components = append(components, utils.SplitComponents(subpath)...) - return utils.JoinComponents(components, "/") - } - } - - components = append(components, utils.SplitComponents(client_path)...) - return utils.JoinComponents(components, "/") -} - -// GetUploadsMetadata returns the path to the metadata file that contains all the uploads. -func GetUploadsMetadata(client_id, flow_id string) string { - return path.Join( - "/clients", client_id, "collections", - flow_id, "uploads.csv") -} - -// Get the file store path for placing the download zip for the flow. -func GetDownloadsFile(client_id, flow_id string) string { - return path.Join( - "/downloads", client_id, flow_id, - flow_id+".zip") -} - // Get the file store path for placing the download zip for the flow. func GetHuntDownloadsFile(hunt_id string) string { return path.Join( @@ -231,12 +68,6 @@ func QueryNameToArtifactAndSource(query_name string) ( } } -func GetDayName() string { - now := time.Now() - return fmt.Sprintf("%d-%02d-%02d", now.Year(), - now.Month(), now.Day()) -} - var day_name_regex = regexp.MustCompile( `^\d\d\d\d-\d\d-\d\d`) diff --git a/reporting/container.go b/reporting/container.go index d3fb7af0570..8aa9b5cdbe2 100644 --- a/reporting/container.go +++ b/reporting/container.go @@ -18,8 +18,8 @@ import ( "github.com/pkg/errors" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/csv" - "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/vfilter" ) @@ -195,7 +195,7 @@ func (self *Container) Upload( accessor string, store_as_name string, expected_size int64, - reader io.Reader) (*uploads.UploadResponse, error) { + reader io.Reader) (*api.UploadResponse, error) { self.Lock() defer self.Unlock() @@ -229,12 +229,12 @@ func (self *Container) Upload( n, err := utils.Copy(ctx, utils.NewTee(writer, sha_sum, md5_sum), reader) if err != nil { - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), }, err } - return &uploads.UploadResponse{ + return &api.UploadResponse{ Path: sanitized_name, Size: uint64(n), Sha256: hex.EncodeToString(sha_sum.Sum(nil)), diff --git a/result_sets/paths.go b/result_sets/paths.go new file mode 100644 index 00000000000..9c0db398aef --- /dev/null +++ b/result_sets/paths.go @@ -0,0 +1,270 @@ +package result_sets + +import ( + "context" + "errors" + "fmt" + "path" + "regexp" + "sort" + "time" + + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/utils" +) + +// The path manager is responsible for telling the file store where to +// store the rows. +type ArtifactPathManager struct { + config_obj *config_proto.Config + client_id, flow_id, full_artifact_name string + + clock utils.Clock + file_store api.FileStore +} + +func NewArtifactPathManager( + config_obj *config_proto.Config, + client_id, flow_id, full_artifact_name string) *ArtifactPathManager { + file_store_factory := file_store.GetFileStore(config_obj) + return &ArtifactPathManager{ + config_obj: config_obj, + client_id: client_id, + flow_id: flow_id, + full_artifact_name: full_artifact_name, + clock: utils.RealClock{}, + file_store: file_store_factory, + } +} + +func (self *ArtifactPathManager) GetQueueName() string { + return self.full_artifact_name +} + +func (self *ArtifactPathManager) Path() string { + result, _ := self.GetPathForWriting() + return result +} + +func (self *ArtifactPathManager) GetPathForWriting() (string, error) { + artifact_name, artifact_source := paths.SplitFullSourceName(self.full_artifact_name) + mode, err := GetArtifactMode(self.config_obj, artifact_name) + if err != nil { + return "", err + } + + now := self.clock.Now().UTC() + day_name := fmt.Sprintf("%d-%02d-%02d", now.Year(), + now.Month(), now.Day()) + + result := get_back_path(self.client_id, day_name, self.flow_id, + artifact_name, artifact_source, mode) + + return result + ".json", nil +} + +// Get the result set files for event artifacts by listing the +// directory that contains all the daily files. NOTE: This is mostly +// used by the DirectoryFileStore to avoid having to scan large event +// files for small time ranges. The SqlFileStore does not need to do +// this because there is a timestamp index on the events themselves. +func (self *ArtifactPathManager) get_event_files() ([]*api.ResultSetFileProperties, error) { + artifact_name, source_name := paths.SplitFullSourceName(self.full_artifact_name) + mode, err := GetArtifactMode(self.config_obj, self.full_artifact_name) + if err != nil { + return nil, err + } + + dir_name := "" + + switch mode { + case paths.MODE_SERVER_EVENT: + if source_name != "" { + dir_name = fmt.Sprintf( + "/server_artifacts/%s/%s/", + artifact_name, source_name) + } else { + dir_name = fmt.Sprintf( + "/server_artifacts/%s/", + artifact_name) + } + + case paths.MODE_CLIENT_EVENT: + if self.client_id == "" { + return nil, errors.New("Client event without client id") + + } else { + if source_name != "" { + dir_name = fmt.Sprintf( + "/clients/%s/monitoring/%s/%s", + self.client_id, artifact_name, source_name) + } else { + dir_name = fmt.Sprintf( + "/clients/%s/monitoring/%s", + self.client_id, artifact_name) + } + } + + default: + path_for_writing, err := self.GetPathForWriting() + if err != nil { + return nil, err + } + return []*api.ResultSetFileProperties{ + &api.ResultSetFileProperties{ + Path: path_for_writing, + }}, nil + } + + children, err := self.file_store.ListDirectory(dir_name) + if err != nil { + return nil, err + } + + result := make([]*api.ResultSetFileProperties, 0, len(children)) + for _, child := range children { + full_path := path.Join(dir_name, child.Name()) + timestamp := DayNameToTimestamp(full_path) + result = append(result, &api.ResultSetFileProperties{ + Path: full_path, + StartTime: timestamp, + EndTime: timestamp + 60*60*24, + }) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].StartTime < result[j].StartTime + }) + + return result, nil +} + +func (self *ArtifactPathManager) GeneratePaths(ctx context.Context) <-chan *api.ResultSetFileProperties { + output := make(chan *api.ResultSetFileProperties) + + go func() { + defer close(output) + + // Find the directory over which we need to list. + children, _ := self.get_event_files() + for _, child := range children { + select { + case <-ctx.Done(): + return + + case output <- child: + } + } + }() + + return output +} + +var day_name_regex = regexp.MustCompile( + `\d\d\d\d-\d\d-\d\d`) + +func DayNameToTimestamp(name string) int64 { + matches := day_name_regex.FindAllString(name, -1) + if len(matches) == 1 { + time, err := time.Parse("2006-01-02 MST", + matches[0]+" UTC") + if err == nil { + return time.Unix() + } + } + return 0 +} + +// Resolve the path relative to the filestore where the CVS files are +// stored. This depends on what kind of log it is (mode), and various +// other details depending on the mode. +// +// This function represents a map between the type of artifact and its +// location on disk. It is used by all code that needs to read or +// write artifact results. +func get_back_path(client_id, day_name, flow_id, artifact_name, source_name string, + mode int) string { + + switch mode { + case paths.MODE_CLIENT: + if source_name != "" { + return fmt.Sprintf( + "/clients/%s/artifacts/%s/%s/%s", + client_id, artifact_name, + flow_id, source_name) + } else { + return fmt.Sprintf( + "/clients/%s/artifacts/%s/%s", + client_id, artifact_name, + flow_id) + } + + case paths.MODE_SERVER: + if source_name != "" { + return fmt.Sprintf( + "/clients/server/artifacts/%s/%s/%s", + artifact_name, flow_id, source_name) + } else { + return fmt.Sprintf( + "/clients/server/artifacts/%s/%s", + artifact_name, flow_id) + } + + case paths.MODE_SERVER_EVENT: + if source_name != "" { + return fmt.Sprintf( + "/server_artifacts/%s/%s/%s", + artifact_name, source_name, day_name) + } else { + return fmt.Sprintf( + "/server_artifacts/%s/%s", + artifact_name, day_name) + } + + case paths.MODE_CLIENT_EVENT: + if client_id == "" { + // Should never happen. + panic("Client event without client id") + + } else { + if source_name != "" { + return fmt.Sprintf( + "/clients/%s/monitoring/%s/%s/%s", + client_id, artifact_name, source_name, + day_name) + } else { + return fmt.Sprintf( + "/clients/%s/monitoring/%s/%s", + client_id, artifact_name, day_name) + } + } + } + + return "" +} + +type MonitoringArtifactPathManager struct { + path string +} + +func (self MonitoringArtifactPathManager) Path() string { + return self.path +} + +// Represents the directory where all the available monitoring logs +// are present - i.e. listing this directory reveals all logs +// currently available. +func NewMonitoringArtifactPathManager(client_id string) *MonitoringArtifactPathManager { + result := &MonitoringArtifactPathManager{} + + if client_id != "" { + result.path = path.Join("/clients", client_id, "monitoring") + } else { + result.path = "/server_artifacts" + } + + return result +} diff --git a/result_sets/paths_test.go b/result_sets/paths_test.go new file mode 100644 index 00000000000..4ab4d067de9 --- /dev/null +++ b/result_sets/paths_test.go @@ -0,0 +1,165 @@ +package result_sets + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/Velocidex/ordereddict" + "github.com/alecthomas/assert" + "www.velocidex.com/golang/velociraptor/artifacts" + "www.velocidex.com/golang/velociraptor/config" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/directory" + "www.velocidex.com/golang/velociraptor/file_store/memory" + "www.velocidex.com/golang/velociraptor/utils" +) + +type path_tests_t struct { + client_id, flow_id, full_artifact_name string + expected string +} + +var path_tests = []path_tests_t{ + // Regular client artifact + {"C.123", "F.123", "Windows.Sys.Users", + "/clients/C.123/artifacts/Windows.Sys.Users/F.123.json"}, + + // Artifact with source + {"C.123", "F.123", "Generic.Client.Info/Users", + "/clients/C.123/artifacts/Generic.Client.Info/F.123/Users.json"}, + + // Server artifacts + {"C.123", "F.123", "Windows.Utils.DownloadBinaries", + "/clients/server/artifacts/Windows.Utils.DownloadBinaries/F.123.json"}, + + // Server events + {"C.123", "F.123", "Elastic.Flows.Upload", + "/server_artifacts/Elastic.Flows.Upload/2020-04-25.json"}, + + // Client events + {"C.123", "F.123", "Windows.Events.ProcessCreation", + "/clients/C.123/monitoring/Windows.Events.ProcessCreation/2020-04-25.json"}, +} + +// The path manager maps artifacts, clients, flows etc into a file +// store path. For event artifacts, the path manager splits the files +// by day to ensure they are not too large and can be easily archived. +func TestPathManager(t *testing.T) { + config_obj := config.GetDefaultConfig() + ts := int64(1587800823) + artifacts.GetGlobalRepository(config_obj) + + for _, testcase := range path_tests { + path_manager := NewArtifactPathManager( + config_obj, + testcase.client_id, + testcase.flow_id, + testcase.full_artifact_name) + path_manager.clock = utils.MockClock{time.Unix(ts, 0)} + path, err := path_manager.GetPathForWriting() + assert.NoError(t, err) + assert.Equal(t, path, testcase.expected) + + file_store := memory.Test_memory_file_store + file_store.Clear() + + qm := memory.NewMemoryQueueManager(config_obj, file_store).(*memory.MemoryQueueManager) + qm.Clock = path_manager.clock + + qm.PushEventRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict()}) + + data, ok := file_store.Get(testcase.expected) + assert.Equal(t, ok, true) + assert.Equal(t, string(data), "{\"_ts\":1587800823}\n") + } +} + +// Test the path manager with DirectoryFileStore +func TestPathManagerDailyRotations(t *testing.T) { + dir, err := ioutil.TempDir("", "path_manager_test") + assert.NoError(t, err) + + defer os.RemoveAll(dir) // clean up + + config_obj := config.GetDefaultConfig() + config_obj.Datastore.Implementation = "FileBaseDataStore" + config_obj.Datastore.FilestoreDirectory = dir + config_obj.Datastore.Location = dir + + artifacts.GetGlobalRepository(config_obj) + + file_store_factory := file_store.GetFileStore(config_obj) + clock := &utils.MockClock{} + + path_manager := NewArtifactPathManager( + config_obj, + "C.123", + "F.123", + "Windows.Events.ProcessCreation") + path_manager.clock = clock + + qm := directory.NewDirectoryQueueManager( + config_obj, file_store_factory).(*directory.DirectoryQueueManager) + qm.Clock = clock + + // Write 3 different events in different days + timestamps := []int64{1587200823, 1587300823, 1587400823} + for _, ts := range timestamps { + clock.MockNow = time.Unix(ts, 0) + qm.PushEventRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict()}) + } + + ctx := context.Background() + results := []*api.ResultSetFileProperties{} + for child := range path_manager.GeneratePaths(ctx) { + results = append(results, child) + } + + assert.Equal(t, len(results), 3) + for idx, result := range results { + assert.True(t, timestamps[idx] > result.StartTime, + "Timestamp %v %v", timestamps[idx], result) + assert.True(t, timestamps[idx] < result.EndTime, + "Timestamp %v %v", timestamps[idx], result) + } + + // Test GetTimeRange - no time range specified should return + // all items. + times := []int64{} + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + path_manager, 0, 0) + assert.NoError(t, err) + for row := range row_chan { + ts, _ := row.Get("_ts") + times = append(times, ts.(int64)) + } + assert.Equal(t, times, timestamps) + + // Cover a small time range - no end time + times = nil + row_chan, err = file_store.GetTimeRange(ctx, config_obj, + path_manager, 1587300822, 0) + assert.NoError(t, err) + for row := range row_chan { + ts, _ := row.Get("_ts") + times = append(times, ts.(int64)) + } + assert.Equal(t, times, timestamps[1:]) + + // Cover a small time range - no start time + times = nil + row_chan, err = file_store.GetTimeRange(ctx, config_obj, + path_manager, 0, 1587300824) + assert.NoError(t, err) + for row := range row_chan { + ts, _ := row.Get("_ts") + times = append(times, ts.(int64)) + } + assert.Equal(t, times, timestamps[:2]) +} diff --git a/result_sets/result_sets.go b/result_sets/result_sets.go new file mode 100644 index 00000000000..66a4be21338 --- /dev/null +++ b/result_sets/result_sets.go @@ -0,0 +1,83 @@ +// Manage reading and writing result sets. + +// Velociraptor is essentially a VQL engine - all operations are +// simply queries and all queries return a result set. Result sets are +// essentially tables - containing columns specified by the query +// itself and rows. + +// Usually queries are encapsulated within artifacts so they contain a +// name and a type. Velociraptor writes these result sets to various +// places in the file store based on the artifact type and its name. + +// This module abstracts the specific location by simply providing an +// interface for code to read and write various artifact's result +// sets. + +package result_sets + +import ( + "errors" + "fmt" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/artifacts" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/utils" +) + +func GetArtifactMode(config_obj *config_proto.Config, artifact_name string) (int, error) { + repository, _ := artifacts.GetGlobalRepository(config_obj) + + artifact, pres := repository.Get(artifact_name) + if !pres { + return 0, errors.New(fmt.Sprintf("Artifact %s not known", artifact_name)) + } + + return paths.ModeNameToMode(artifact.Type), nil +} + +type ResultSetWriter struct { + rows []*ordereddict.Dict + fd api.FileWriter +} + +func (self *ResultSetWriter) Write(row *ordereddict.Dict) { + self.rows = append(self.rows, row) + if len(self.rows) > 10000 { + self.Flush() + } +} + +func (self *ResultSetWriter) Flush() { + serialized, err := utils.DictsToJson(self.rows) + + if err == nil { + self.fd.Write(serialized) + } + self.rows = nil +} + +func (self *ResultSetWriter) Close() { + self.Flush() + self.fd.Close() +} + +func NewResultSetWriter( + config_obj *config_proto.Config, + path_manager api.PathManager) (*ResultSetWriter, error) { + file_store_factory := file_store.GetFileStore(config_obj) + log_path, err := path_manager.GetPathForWriting() + if err != nil { + return nil, err + } + + fd, err := file_store_factory.WriteFile(log_path) + if err != nil { + return nil, err + } + + return &ResultSetWriter{fd: fd}, nil +} diff --git a/server/comms.go b/server/comms.go index edd0b603aec..62e1084392e 100644 --- a/server/comms.go +++ b/server/comms.go @@ -336,6 +336,7 @@ func control(server_obj *Server) http.Handler { // do this by streaming pad packets to the client, // while the flow is processed. w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Accel-Buffering", "no") w.WriteHeader(http.StatusOK) flusher.Flush() diff --git a/server/server.go b/server/server.go index b3a5c27891a..f2b76b71aff 100644 --- a/server/server.go +++ b/server/server.go @@ -155,9 +155,9 @@ func (self *Server) Process( IpAddress: message_info.RemoteAddr, } + client_path_manager := paths.NewClientPathManager(message_info.Source) err = self.db.SetSubject( - self.config, - paths.GetClientPingPath(message_info.Source), client_info) + self.config, client_path_manager.Ping().Path(), client_info) if err != nil { return nil, 0, err } diff --git a/server/server_test.go b/server/server_test.go index 149c44ebe6f..d68cde65c25 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -15,6 +15,7 @@ import ( "www.velocidex.com/golang/velociraptor/api" api_mock "www.velocidex.com/golang/velociraptor/api/mock" api_proto "www.velocidex.com/golang/velociraptor/api/proto" + "www.velocidex.com/golang/velociraptor/artifacts" "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" @@ -26,6 +27,7 @@ import ( "www.velocidex.com/golang/velociraptor/flows" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/server" "www.velocidex.com/golang/velociraptor/services" ) @@ -69,6 +71,7 @@ func (self *ServerTestSuite) SetupTest() { // Start the journaling service manually for tests. services.StartJournalService(self.config_obj) + artifacts.GetGlobalRepository(config_obj) self.server, err = server.NewServer(config_obj) require.NoError(self.T(), err) @@ -298,7 +301,8 @@ func (self *ServerTestSuite) TestMonitoring() { Source: self.client_id, SessionId: constants.MONITORING_WELL_KNOWN_FLOW, VQLResponse: &actions_proto.VQLResponse{ - Columns: []string{"ClientId", "Timestamp", "Fqdn", "HuntId", "Participate"}, + Columns: []string{"ClientId", "Timestamp", "Fqdn", + "HuntId", "Participate"}, Response: fmt.Sprintf( `[{"ClientId": "%s", "Participate": true, "HuntId": "H.123"}]`, self.client_id), @@ -309,9 +313,10 @@ func (self *ServerTestSuite) TestMonitoring() { }) runner.Close() - self.RequiredFilestoreContains( - "/clients/"+self.client_id+"/monitoring/System.Hunt.Participation/"+ - paths.GetDayName()+".csv", self.client_id) + path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, constants.MONITORING_WELL_KNOWN_FLOW, "System.Hunt.Participation") + + self.RequiredFilestoreContains(path_manager.Path(), self.client_id) } // An invalid monitoring response will log an error in the client's @@ -324,8 +329,9 @@ func (self *ServerTestSuite) TestInvalidMonitoringPacket() { Source: self.client_id, SessionId: constants.MONITORING_WELL_KNOWN_FLOW, VQLResponse: &actions_proto.VQLResponse{ - Columns: []string{"ClientId", "Timestamp", "Fqdn", "HuntId", "Participate"}, - Response: fmt.Sprintf(`}}}`), // Invalid json + Columns: []string{"ClientId", "Timestamp", + "Fqdn", "HuntId", "Participate"}, + Response: `}}}`, // Invalid json Query: &actions_proto.VQLRequest{ Name: "System.Hunt.Participation", }, @@ -520,13 +526,13 @@ func (self *ServerTestSuite) TestUploadBuffer() { }) runner.Close() + flow_path_manager := paths.NewFlowPathManager(self.client_id, flow_id) self.RequiredFilestoreContains( - "/clients/"+self.client_id+"/collections/"+flow_id+"/uploads/file/tmp/foobar", + flow_path_manager.GetUploadsFile("file", "/tmp/foobar").Path(), "hello world") self.RequiredFilestoreContains( - "/clients/"+self.client_id+"/collections/"+flow_id+"/uploads.csv", - flow_id) + flow_path_manager.UploadMetadata().Path(), flow_id) } // Test VQLResponse are written correctly. @@ -557,9 +563,9 @@ func (self *ServerTestSuite) TestVQLResponse() { }) runner.Close() - self.RequiredFilestoreContains( - "/clients/"+self.client_id+"/artifacts/Generic.Client.Info/"+flow_id+".csv", - self.client_id) + flow_path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, flow_id, "Generic.Client.Info") + self.RequiredFilestoreContains(flow_path_manager.Path(), self.client_id) } // Errors from the client kill the flow. diff --git a/services/hunt_dispatcher.go b/services/hunt_dispatcher.go index 2d7be7995f9..134e7b60d99 100644 --- a/services/hunt_dispatcher.go +++ b/services/hunt_dispatcher.go @@ -31,6 +31,7 @@ import ( "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/grpc_client" "www.velocidex.com/golang/velociraptor/logging" + "www.velocidex.com/golang/velociraptor/paths" ) var ( @@ -139,9 +140,9 @@ func (self *HuntDispatcher) _flush_stats() error { } for _, hunt_obj := range self.hunts { - err = db.SetSubject( - self.config_obj, - constants.GetHuntStatsPath(hunt_obj.HuntId), hunt_obj.Stats) + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) + err = db.SetSubject(self.config_obj, + hunt_path_manager.Stats().Path(), hunt_obj.Stats) if err != nil { logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent) logger.Error("Flushing %s to disk: %v", hunt_obj.HuntId, err) @@ -191,7 +192,9 @@ func (self *HuntDispatcher) Refresh() error { return err } - hunts, err := db.ListChildren(self.config_obj, constants.HUNTS_URN, 0, 1000) + hunt_path_manager := paths.NewHuntPathManager("") + hunts, err := db.ListChildren(self.config_obj, + hunt_path_manager.HuntDirectory().Path(), 0, 1000) if err != nil { return err } @@ -203,9 +206,9 @@ func (self *HuntDispatcher) Refresh() error { } hunt_obj := &api_proto.Hunt{} + hunt_path_manager := paths.NewHuntPathManager(hunt_id) err = db.GetSubject( - self.config_obj, - constants.GetHuntURN(hunt_id), hunt_obj) + self.config_obj, hunt_path_manager.Path(), hunt_obj) if err != nil { continue } @@ -213,16 +216,11 @@ func (self *HuntDispatcher) Refresh() error { // Re-read the stats into the hunt object. hunt_stats := &api_proto.HuntStats{} err := db.GetSubject(self.config_obj, - constants.GetHuntStatsPath(hunt_id), hunt_stats) + hunt_path_manager.Stats().Path(), hunt_stats) if err == nil { hunt_obj.Stats = hunt_stats } - // FIXME: Backwards compatibility - if path.Base(hunt_obj.HuntId) != hunt_id { - continue - } - // This hunt is newer than the last_timestamp, we need // to update it. if hunt_obj.StartTime > last_timestamp { diff --git a/services/hunt_manager.go b/services/hunt_manager.go index 3c9d3409607..cb0073a639e 100644 --- a/services/hunt_manager.go +++ b/services/hunt_manager.go @@ -20,7 +20,6 @@ package services import ( "context" "errors" - "fmt" "sync" "time" @@ -31,11 +30,10 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/datastore" - "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/csv" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" "www.velocidex.com/golang/velociraptor/grpc_client" "www.velocidex.com/golang/velociraptor/logging" + "www.velocidex.com/golang/velociraptor/paths" "www.velocidex.com/golang/vfilter" ) @@ -51,10 +49,6 @@ type ParticipationRecord struct { type HuntManager struct { mu sync.Mutex - // We keep a cache of hunt writers to write the output of each - // hunt. Note that each writer is responsible for its own - // flushing etc. - writers map[string]*csv.CSVWriter config_obj *config_proto.Config APIClientFactory grpc_client.APIClientFactory @@ -77,8 +71,10 @@ func (self *HuntManager) Start( for { select { - case row := <-qm_chan: - fmt.Printf("row %v\n", row) + case row, ok := <-qm_chan: + if !ok { + return + } self.ProcessRow(ctx, scope, row) case <-ctx.Done(): @@ -86,7 +82,7 @@ func (self *HuntManager) Start( } } - fmt.Printf("Exit\n") + logger.Info("Exiting hunt manager\n") }() return nil @@ -96,10 +92,6 @@ func (self *HuntManager) Start( func (self *HuntManager) Close() { self.mu.Lock() defer self.mu.Unlock() - for _, v := range self.writers { - v.Close() - } - logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent) logger.Info("Shutting down hunt manager service.") } @@ -149,28 +141,6 @@ func (self *HuntManager) ProcessRow( return } - // Fetch the CSV writer for this hunt or create a new one and - // cache it. - writer, pres := self.writers[participation_row.HuntId] - if !pres { - file_store_factory := file_store.GetFileStore(self.config_obj) - - // Hold references to all the writers for the life of - // the manager. - fd, err := file_store_factory.WriteFile( - constants.GetHuntURN(participation_row.HuntId) + ".csv") - if err != nil { - return - } - - writer, err = csv.GetCSVWriter(scope, fd) - if err != nil { - return - } - - self.writers[participation_row.HuntId] = writer - } - request := &flows_proto.ArtifactCollectorArgs{ ClientId: participation_row.ClientId, Creator: participation_row.HuntId, @@ -241,7 +211,9 @@ func (self *HuntManager) ProcessRow( dict_row.Set("FlowId", response.FlowId) dict_row.Set("Timestamp", time.Now().Unix()) - writer.Write(dict_row) + + path_manager := paths.NewHuntPathManager(participation_row.HuntId) + GetJournal().PushRows(path_manager.Clients(), []*ordereddict.Dict{dict_row}) } func startHuntManager( @@ -250,7 +222,6 @@ func startHuntManager( config_obj *config_proto.Config) (*HuntManager, error) { result := &HuntManager{ config_obj: config_obj, - writers: make(map[string]*csv.CSVWriter), APIClientFactory: grpc_client.GRPCAPIClient{}, } return result, result.Start(ctx, wg) diff --git a/services/hunt_manager_test.go b/services/hunt_manager_test.go index 1b6ab556053..6eaa81da7d4 100644 --- a/services/hunt_manager_test.go +++ b/services/hunt_manager_test.go @@ -21,6 +21,8 @@ import ( "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/memory" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" + "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" ) type HuntTestSuite struct { @@ -98,19 +100,21 @@ func (self *HuntTestSuite) TestHuntManager() { db, err := datastore.GetDB(self.config_obj) assert.NoError(t, err) - err = db.SetSubject(self.config_obj, - constants.GetHuntURN(hunt_obj.HuntId), hunt_obj) + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) + err = db.SetSubject(self.config_obj, hunt_path_manager.Path(), hunt_obj) assert.NoError(t, err) GetHuntDispatcher().Refresh() // Simulate a System.Hunt.Participation event - GetJournal().PushRow("System.Hunt.Participation", self.client_id, 0, - ordereddict.NewDict(). + path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, "", "System.Hunt.Participation") + GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict(). Set("HuntId", self.hunt_id). Set("ClientId", self.client_id). Set("Fqdn", "MyHost"). - Set("Participate", true)) + Set("Participate", true)}) // Make sure manager reacted. time.Sleep(1 * time.Second) @@ -153,19 +157,21 @@ func (self *HuntTestSuite) TestHuntWithLabelClientNoLabel() { db, err := datastore.GetDB(self.config_obj) assert.NoError(t, err) - err = db.SetSubject(self.config_obj, - constants.GetHuntURN(hunt_obj.HuntId), hunt_obj) + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) + err = db.SetSubject(self.config_obj, hunt_path_manager.Path(), hunt_obj) assert.NoError(t, err) GetHuntDispatcher().Refresh() // Simulate a System.Hunt.Participation event - GetJournal().PushRow("System.Hunt.Participation", self.client_id, 0, - ordereddict.NewDict(). + path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, "", "System.Hunt.Participation") + GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict(). Set("HuntId", self.hunt_id). Set("ClientId", self.client_id). Set("Fqdn", "MyHost"). - Set("Participate", true)) + Set("Participate", true)}) // Make sure manager reacted. time.Sleep(1 * time.Second) @@ -216,8 +222,8 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabelDifferentCase() { db, err := datastore.GetDB(self.config_obj) assert.NoError(t, err) - err = db.SetSubject(self.config_obj, - constants.GetHuntURN(hunt_obj.HuntId), hunt_obj) + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) + err = db.SetSubject(self.config_obj, hunt_path_manager.Path(), hunt_obj) assert.NoError(t, err) _, err = clients.LabelClients(self.config_obj, @@ -231,12 +237,14 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabelDifferentCase() { GetHuntDispatcher().Refresh() // Simulate a System.Hunt.Participation event - GetJournal().PushRow("System.Hunt.Participation", self.client_id, 0, - ordereddict.NewDict(). + path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, "", "System.Hunt.Participation") + GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict(). Set("HuntId", self.hunt_id). Set("ClientId", self.client_id). Set("Fqdn", "MyHost"). - Set("Participate", true)) + Set("Participate", true)}) // Make sure manager reacted. time.Sleep(1 * time.Second) @@ -287,8 +295,8 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabel() { db, err := datastore.GetDB(self.config_obj) assert.NoError(t, err) - err = db.SetSubject(self.config_obj, - constants.GetHuntURN(hunt_obj.HuntId), hunt_obj) + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) + err = db.SetSubject(self.config_obj, hunt_path_manager.Path(), hunt_obj) assert.NoError(t, err) _, err = clients.LabelClients(self.config_obj, @@ -302,12 +310,14 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabel() { GetHuntDispatcher().Refresh() // Simulate a System.Hunt.Participation event - GetJournal().PushRow("System.Hunt.Participation", self.client_id, 0, - ordereddict.NewDict(). + path_manager := result_sets.NewArtifactPathManager(self.config_obj, + self.client_id, "", "System.Hunt.Participation") + GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict(). Set("HuntId", self.hunt_id). Set("ClientId", self.client_id). Set("Fqdn", "MyHost"). - Set("Participate", true)) + Set("Participate", true)}) // Make sure manager reacted. time.Sleep(1 * time.Second) diff --git a/services/interrogation.go b/services/interrogation.go index c89651a2f3f..dfd05840a9a 100644 --- a/services/interrogation.go +++ b/services/interrogation.go @@ -52,12 +52,10 @@ func (self *InterrogationService) Start( vql, _ := vfilter.Parse("SELECT * FROM Artifact.Server.Internal.Interrogate()") go func() { for row := range vql.Eval(ctx, scope) { - row_dict, ok := row.(*ordereddict.Dict) - if ok { - err := self.ProcessRow(scope, row_dict) - if err != nil { - logger.Error("Interrogation Service: %v", err) - } + row_dict := vfilter.RowToDict(ctx, scope, row) + err := self.ProcessRow(row_dict) + if err != nil { + logger.Error("Interrogation Service: %v", err) } } }() @@ -65,15 +63,15 @@ func (self *InterrogationService) Start( return nil } -func (self *InterrogationService) ProcessRow(scope *vfilter.Scope, - row *ordereddict.Dict) error { - getter := func(field string) string { - return vql_subsystem.GetStringFromRow(scope, row, field) +func (self *InterrogationService) ProcessRow(row *ordereddict.Dict) error { + client_id, ok := row.GetString("ClientId") + if !ok { + return errors.New("Unknown ClientId") } - client_id := getter("ClientId") - if client_id == "" { - return errors.New("Unknown ClientId") + getter := func(field string) string { + result, _ := row.GetString(field) + return result } client_info := &actions_proto.ClientInfo{ @@ -87,28 +85,19 @@ func (self *InterrogationService) ProcessRow(scope *vfilter.Scope, LastInterrogateFlowId: getter("FlowId"), } - label_array_obj, ok := row.Get("Labels") + label_array, ok := row.GetStrings("Labels") if ok { - label_array, ok := label_array_obj.([]interface{}) - if ok { - for _, item := range label_array { - label, ok := item.(string) - if !ok { - continue - } - - client_info.Labels = append(client_info.Labels, label) - } - } + client_info.Labels = append(client_info.Labels, label_array...) } - client_urn := paths.GetClientMetadataPath(client_id) + client_path_manager := paths.NewClientPathManager(client_id) db, err := datastore.GetDB(self.config_obj) if err != nil { return err } - err = db.SetSubject(self.config_obj, client_urn, client_info) + err = db.SetSubject(self.config_obj, + client_path_manager.Path(), client_info) if err != nil { return err } diff --git a/services/journal.go b/services/journal.go index 0353226d334..53ea00e5790 100644 --- a/services/journal.go +++ b/services/journal.go @@ -10,19 +10,13 @@ package services import ( - "fmt" "sync" - "time" "github.com/Velocidex/ordereddict" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/api" - "www.velocidex.com/golang/velociraptor/file_store/csv" "www.velocidex.com/golang/velociraptor/logging" - "www.velocidex.com/golang/velociraptor/paths" - "www.velocidex.com/golang/velociraptor/utils" - vql_subsystem "www.velocidex.com/golang/velociraptor/vql" ) var ( @@ -44,61 +38,9 @@ func (self *JournalService) Watch(queue_name string) ( return self.qm.Watch(queue_name) } -func (self *JournalService) PushRow(queue_name, source string, mode int, row *ordereddict.Dict) error { - err := self.qm.PushRow(queue_name, source, mode, row) - if err != nil { - return err - } - - return self.write_rows(queue_name, source, mode, []*ordereddict.Dict{row}) -} - -func (self *JournalService) Push(queue_name, source string, mode int, rows []byte) error { - err := self.qm.Push(queue_name, source, mode, rows) - if err != nil { - return err - } - - dict_rows, err := utils.ParseJsonToDicts(rows) - if err != nil { - fmt.Printf("Error: %v\n", err) - return err - } - - return self.write_rows(queue_name, source, mode, dict_rows) -} - -func (self *JournalService) write_rows(queue_name, source string, mode int, - dict_rows []*ordereddict.Dict) error { - // Write the event into the client's monitoring log - file_store_factory := file_store.GetFileStore(self.config_obj) - artifact_name, source_name := paths.QueryNameToArtifactAndSource(queue_name) - - log_path := paths.GetCSVPath( - source, /* client_id */ - paths.GetDayName(), - "", artifact_name, source_name, mode) - - fd, err := file_store_factory.WriteFile(log_path) - if err != nil { - fmt.Printf("Error: %v\n", err) - return err - } - defer fd.Close() - - writer, err := csv.GetCSVWriter(vql_subsystem.MakeScope(), fd) - if err != nil { - fmt.Printf("Error: %v\n", err) - return err - } - defer writer.Close() - - for _, row := range dict_rows { - row.Set("_ts", int(time.Now().Unix())) - writer.Write(row) - } - - return nil +func (self *JournalService) PushRows( + path_manager api.PathManager, rows []*ordereddict.Dict) error { + return self.qm.PushEventRows(path_manager, rows) } func (self *JournalService) Start() error { @@ -119,10 +61,15 @@ func GetJournal() *JournalService { } func StartJournalService(config_obj *config_proto.Config) error { + qm, err := file_store.GetQueueManager(config_obj) + if err != nil { + return err + } + service := &JournalService{ config_obj: config_obj, logger: logging.GetLogger(config_obj, &logging.FrontendComponent), - qm: file_store.GetQueueManager(config_obj), + qm: qm, } return service.Start() diff --git a/services/notfications.go b/services/notfications.go index af2fb803f19..ddace0dafed 100644 --- a/services/notfications.go +++ b/services/notfications.go @@ -3,7 +3,12 @@ package services import ( "sync" + "github.com/Velocidex/ordereddict" + "github.com/pkg/errors" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/notifications" + "www.velocidex.com/golang/velociraptor/result_sets" ) var ( @@ -11,34 +16,58 @@ var ( notification_pool *notifications.NotificationPool ) -// For now very simple - the notifier service simply calls on the -// notfication pool because there is only a single frontend. In future -// manage notifications for multiple frontends. -func startNotificationService(notifier *notifications.NotificationPool) error { +// The notifier service watches for events from +// Server.Internal.Notifications and notifies the notification pool in +// the current process. This allows multiprocess communication as the +// notifications may arrive from other frontend processes through the +// journal service. +func startNotificationService( + config_obj *config_proto.Config, + notifier *notifications.NotificationPool) error { pool_mu.Lock() defer pool_mu.Unlock() + if notifier == nil { + return errors.New("Notifier must be specified.") + } + + logger := logging.GetLogger(config_obj, &logging.FrontendComponent) + logger.Info("Starting the notification service.") + notification_pool = notifier + go func() { + events, cancel := GetJournal().Watch("Server.Internal.Notifications") + defer cancel() + + for event := range events { + target, ok := event.GetString("Target") + if !ok { + continue + } + + if target == "All" { + notification_pool.NotifyAll() + } else { + notification_pool.Notify(target) + } + } + }() return nil } -func NotifyAll() error { - pool_mu.Lock() - defer pool_mu.Unlock() +func NotifyAll(config_obj *config_proto.Config) error { + path_manager := result_sets.NewArtifactPathManager( + config_obj, "server" /* client_id */, "", "Server.Internal.Notifications") - if notification_pool != nil { - notification_pool.NotifyAll() - } - return nil + return GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict().Set("Target", "All")}) } -func NotifyClient(client_id string) error { - pool_mu.Lock() - defer pool_mu.Unlock() +func NotifyClient(config_obj *config_proto.Config, client_id string) error { + path_manager := result_sets.NewArtifactPathManager( + config_obj, "server" /* client_id */, "", "Server.Internal.Notifications") - if notification_pool != nil { - notification_pool.Notify(client_id) - } - return nil + return GetJournal().PushRows(path_manager, + []*ordereddict.Dict{ordereddict.NewDict().Set("Target", client_id)}) } diff --git a/services/server_artifacts.go b/services/server_artifacts.go index 22077e39b50..a70e5e3a698 100644 --- a/services/server_artifacts.go +++ b/services/server_artifacts.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "path" "runtime/debug" "sync" "time" @@ -22,8 +21,8 @@ import ( "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/notifications" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/utils" - "www.velocidex.com/golang/velociraptor/vql" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -34,7 +33,7 @@ const ( type serverLogger struct { config_obj *config_proto.Config - w *csv.CSVWriter + w *result_sets.ResultSetWriter } func (self *serverLogger) Write(b []byte) (int, error) { @@ -134,14 +133,15 @@ func (self *ServerArtifactsRunner) process( func (self *ServerArtifactsRunner) processTask( ctx context.Context, task *crypto_proto.GrrMessage) error { - urn := path.Join("/clients", source, "collections", task.SessionId) + + flow_urn := paths.NewFlowPathManager(source, task.SessionId).Path() collection_context := &flows_proto.ArtifactCollectorContext{} db, err := datastore.GetDB(self.config_obj) if err != nil { return err } - err = db.GetSubject(self.config_obj, urn, collection_context) + err = db.GetSubject(self.config_obj, flow_urn, collection_context) if err != nil { return err } @@ -152,7 +152,7 @@ func (self *ServerArtifactsRunner) processTask( collection_context.State = flows_proto.ArtifactCollectorContext_TERMINATED collection_context.ActiveTime = uint64(time.Now().UnixNano() / 1000) - return db.SetSubject(self.config_obj, urn, collection_context) + return db.SetSubject(self.config_obj, flow_urn, collection_context) } func (self *ServerArtifactsRunner) runQuery( @@ -162,23 +162,17 @@ func (self *ServerArtifactsRunner) runQuery( self.mu.Lock() defer self.mu.Unlock() + flow_id := task.SessionId + // Set up the logger for writing query logs. Note this must be // destroyed last since we need to be able to receive logs // from scope destructors. - log_path := path.Join(collection_context.Urn, "logs") - - file_store_factory := file_store.GetFileStore(self.config_obj) - fd, err := file_store_factory.WriteFile(log_path) + path_manager := paths.NewFlowPathManager(source, flow_id).Log() + rs_writer, err := result_sets.NewResultSetWriter(self.config_obj, path_manager) if err != nil { return err } - defer fd.Close() - - log_sink, err := csv.GetCSVWriter(vql.MakeScope(), fd) - if err != nil { - return err - } - defer log_sink.Close() + defer rs_writer.Close() arg := task.VQLClientAction if arg == nil { @@ -189,8 +183,6 @@ func (self *ServerArtifactsRunner) runQuery( return errors.New("Query should be specified") } - flow_id := task.SessionId - // Cancel the query after this deadline deadline := time.After(self.timeout) started := time.Now().Unix() @@ -210,7 +202,9 @@ func (self *ServerArtifactsRunner) runQuery( self.config_obj, file_store.GetFileStore(self.config_obj), "/"), ACLManager: vql_subsystem.NewRoleACLManager("administrator"), - Logger: log.New(&serverLogger{self.config_obj, log_sink}, "server", 0), + Logger: log.New( + &serverLogger{self.config_obj, rs_writer}, + source, 0), }.Build() defer scope.Close() @@ -238,23 +232,16 @@ func (self *ServerArtifactsRunner) runQuery( } read_chan := vql.Eval(sub_ctx, scope) - - var write_chan chan vfilter.Row - + var rs_writer *result_sets.ResultSetWriter if query.Name != "" { name := artifacts.DeobfuscateString( self.config_obj, query.Name) - artifact_name, source_name := paths. - QueryNameToArtifactAndSource(name) - - log_path := paths.GetCSVPath( - /* client_id */ source, - "", - /* flow_id */ flow_id, - artifact_name, source_name, - paths.MODE_SERVER) - write_chan = self.GetWriter(scope, log_path) - defer close(write_chan) + + path_manager := result_sets.NewArtifactPathManager( + self.config_obj, source, flow_id, name) + rs_writer, err = result_sets.NewResultSetWriter( + self.config_obj, path_manager) + defer rs_writer.Close() // Update the artifacts with results in the // context. @@ -289,9 +276,10 @@ func (self *ServerArtifactsRunner) runQuery( if !ok { break process_query } - if write_chan != nil { + if rs_writer != nil { row_idx += 1 - write_chan <- row + rs_writer.Write(vfilter.RowToDict( + sub_ctx, scope, row)) } } } diff --git a/services/server_monitoring.go b/services/server_monitoring.go index 4fec2315621..e993775af98 100644 --- a/services/server_monitoring.go +++ b/services/server_monitoring.go @@ -7,7 +7,6 @@ package services import ( "context" "errors" - "fmt" "sync" "time" @@ -17,13 +16,9 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/datastore" - "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/api" - "www.velocidex.com/golang/velociraptor/file_store/csv" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" "www.velocidex.com/golang/velociraptor/logging" - "www.velocidex.com/golang/velociraptor/paths" - "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/velociraptor/result_sets" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -138,8 +133,7 @@ func (self *EventTable) Update( // Run each source concurrently. for _, source := range artifact.Sources { - err := self.RunQuery( - new_ctx, config_obj, + err := self.RunQuery(new_ctx, config_obj, scope, name, source) if err != nil { return err @@ -150,83 +144,6 @@ func (self *EventTable) Update( return nil } -func (self *EventTable) GetWriter( - ctx context.Context, - config_obj *config_proto.Config, - scope *vfilter.Scope, - artifact_name string, - source_name string) chan vfilter.Row { - - row_chan := make(chan vfilter.Row) - - go func() { - file_store_factory := file_store.GetFileStore(config_obj) - last_log := "" - var err error - var fd api.FileWriter - var writer *csv.CSVWriter - var columns []string - - closer := func() { - if writer != nil { - writer.Close() - } - if fd != nil { - fd.Close() - } - - writer = nil - fd = nil - } - - defer closer() - - for row := range row_chan { - log_path := paths.GetCSVPath( - /* client_id */ "", - paths.GetDayName(), - /* flow_id */ "", - artifact_name, source_name, - paths.MODE_SERVER_EVENT) - - // We need to rotate the log file. - if log_path != last_log { - closer() - - fd, err = file_store_factory.WriteFile(log_path) - if err != nil { - fmt.Printf("Error: %v\n", err) - continue - } - - writer, err = csv.GetCSVWriter(scope, fd) - if err != nil { - continue - } - } - - if columns == nil { - columns = scope.GetMembers(row) - } - - // First column is a row timestamp. This makes - // it easier to do a row scan for time ranges. - dict_row := ordereddict.NewDict(). - Set("_ts", int(time.Now().Unix())) - for _, column := range columns { - value, pres := scope.Associative(row, column) - if pres { - dict_row.Set(column, value) - } - } - - writer.Write(dict_row) - } - }() - - return row_chan -} - func (self *EventTable) RunQuery( ctx context.Context, config_obj *config_proto.Config, @@ -259,21 +176,28 @@ func (self *EventTable) RunQuery( go func() { defer self.wg.Done() - name := artifact_name if source.Name != "" { - name = utils.PathJoin(artifact_name, source.Name, "/") + artifact_name = artifact_name + "/" + source.Name } - row_chan := self.GetWriter(ctx, config_obj, scope, - artifact_name, source.Name) - defer close(row_chan) - logger := logging.GetLogger(config_obj, &logging.FrontendComponent) - logger.Info("Collecting Server Event Artifact: %s", name) + logger.Info("Collecting Server Event Artifact: %s", artifact_name) + + path_manager := result_sets.NewArtifactPathManager( + config_obj, "", "", artifact_name) + rs_writer, err := result_sets.NewResultSetWriter( + config_obj, path_manager) + if err != nil { + logger.Error("NewResultSetWriter", err) + return + } + defer rs_writer.Close() for _, vql := range vqls { for row := range vql.Eval(ctx, scope) { - row_chan <- row + rs_writer.Write(vfilter.RowToDict(ctx, scope, row). + Set("_ts", time.Now().Unix())) + rs_writer.Flush() } } }() diff --git a/services/services.go b/services/services.go index bf8c15dada2..4ef6c738f7b 100644 --- a/services/services.go +++ b/services/services.go @@ -43,7 +43,7 @@ func StartServices( return err } - err = startNotificationService(notifier) + err = startNotificationService(config_obj, notifier) if err != nil { return err } diff --git a/services/vfs_service.go b/services/vfs_service.go index 26802c8ca62..f0a9c8d0ae1 100644 --- a/services/vfs_service.go +++ b/services/vfs_service.go @@ -77,12 +77,14 @@ func (self *VFSService) ProcessDownloadFile( panic(err) } + flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) + for row := range vql.Eval(ctx, sub_scope) { Accessor := vql_subsystem.GetStringFromRow(scope, row, "Accessor") Path := vql_subsystem.GetStringFromRow(scope, row, "Path") // Figure out where the file was uploaded to. - vfs_path := paths.GetUploadsFile(client_id, flow_id, Accessor, Path) + vfs_path := flow_path_manager.GetUploadsFile(Accessor, Path).Path() // Check to make sure the file actually exists. file_store_factory := file_store.GetFileStore(self.config_obj) @@ -95,7 +97,7 @@ func (self *VFSService) ProcessDownloadFile( // We store a place holder in the VFS pointing at the // read vfs_path of the download. err = db.SetSubject(self.config_obj, - paths.GetVFSDownloadInfoPath(client_id, Accessor, Path), + flow_path_manager.GetVFSDownloadInfoPath(Accessor, Path).Path(), &flows_proto.VFSDownloadInfo{ VfsPath: vfs_path, Mtime: uint64(ts) * 1000000, diff --git a/uploads/api.go b/uploads/api.go index 05b00d0a204..522707c4e95 100644 --- a/uploads/api.go +++ b/uploads/api.go @@ -1,29 +1,2 @@ // Uploaders deliver files from accessors to the server (or another target). package uploads - -import ( - "context" - "io" - - "www.velocidex.com/golang/vfilter" -) - -// Returned as the result of the query. -type UploadResponse struct { - Path string `json:"Path"` - Size uint64 `json:"Size"` - Error string `json:"Error,omitempty"` - Sha256 string `json:"sha256,omitempty"` - Md5 string `json:"md5,omitempty"` -} - -// Provide an uploader capable of uploading any reader object. -type Uploader interface { - Upload(ctx context.Context, - scope *vfilter.Scope, - filename string, - accessor string, - store_as_name string, - expected_size int64, - reader io.Reader) (*UploadResponse, error) -} diff --git a/uploads/client_uploader.go b/uploads/client_uploader.go index 0327d2c7278..3f248f22aa4 100644 --- a/uploads/client_uploader.go +++ b/uploads/client_uploader.go @@ -11,6 +11,7 @@ import ( actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" "www.velocidex.com/golang/velociraptor/constants" crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/responder" "www.velocidex.com/golang/vfilter" ) @@ -29,8 +30,8 @@ func (self *VelociraptorUploader) Upload( store_as_name string, expected_size int64, reader io.Reader) ( - *UploadResponse, error) { - result := &UploadResponse{ + *api.UploadResponse, error) { + result := &api.UploadResponse{ Path: filename, } diff --git a/uploads/file_based.go b/uploads/file_based.go index a59a1b7ef0c..642caa27981 100644 --- a/uploads/file_based.go +++ b/uploads/file_based.go @@ -30,6 +30,7 @@ import ( "runtime" "www.velocidex.com/golang/velociraptor/datastore" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/vfilter" ) @@ -75,7 +76,7 @@ func (self *FileBasedUploader) Upload( store_as_name string, expected_size int64, reader io.Reader) ( - *UploadResponse, error) { + *api.UploadResponse, error) { if self.UploadDir == "" { scope.Log("UploadDir is not set") @@ -121,7 +122,7 @@ func (self *FileBasedUploader) Upload( } scope.Log("Uploaded %v (%v bytes)", file_path, offset) - return &UploadResponse{ + return &api.UploadResponse{ Path: file_path, Size: uint64(offset), Sha256: hex.EncodeToString(sha_sum.Sum(nil)), diff --git a/utils/clock.go b/utils/clock.go new file mode 100644 index 00000000000..3ab922844c2 --- /dev/null +++ b/utils/clock.go @@ -0,0 +1,21 @@ +package utils + +import "time" + +type Clock interface { + Now() time.Time +} + +type RealClock struct{} + +func (self RealClock) Now() time.Time { + return time.Now() +} + +type MockClock struct { + MockNow time.Time +} + +func (self MockClock) Now() time.Time { + return self.MockNow +} diff --git a/utils/json.go b/utils/json.go index c8af646756f..fddb41cc4ef 100644 --- a/utils/json.go +++ b/utils/json.go @@ -1,6 +1,7 @@ package utils import ( + "bytes" "encoding/json" "github.com/Velocidex/ordereddict" @@ -8,21 +9,79 @@ import ( ) func ParseJsonToDicts(serialized []byte) ([]*ordereddict.Dict, error) { - var raw_objects []json.RawMessage - err := json.Unmarshal(serialized, &raw_objects) - if err != nil { - return nil, errors.WithStack(err) + if len(serialized) == 0 { + return nil, nil } - result := make([]*ordereddict.Dict, 0, len(raw_objects)) - for _, raw_message := range raw_objects { - item := ordereddict.NewDict() - err = json.Unmarshal(raw_message, &item) + // Support decoding an array of objects. + if serialized[0] == '[' { + var raw_objects []json.RawMessage + err := json.Unmarshal(serialized, &raw_objects) if err != nil { + return nil, errors.WithStack(err) + } + + result := make([]*ordereddict.Dict, 0, len(raw_objects)) + for _, raw_message := range raw_objects { + item := ordereddict.NewDict() + err = json.Unmarshal(raw_message, &item) + if err != nil { + return nil, err + } + result = append(result, item) + } + + return result, nil + } + + // Otherwise, it must be JSONL + lines := bytes.Split(serialized, []byte{'\n'}) + result := make([]*ordereddict.Dict, 0, len(lines)) + for _, line := range lines { + if len(line) == 0 { continue } + + item := ordereddict.NewDict() + err := json.Unmarshal(line, &item) + if err != nil { + return nil, err + } result = append(result, item) } return result, nil } + +func DictsToJson(rows []*ordereddict.Dict) ([]byte, error) { + out := bytes.Buffer{} + for _, row := range rows { + serialized, err := json.Marshal(row) + if err != nil { + return nil, err + } + + out.Write(serialized) + out.Write([]byte{'\n'}) + } + + return out.Bytes(), nil +} + +// Convert old json format to jsonl. +func JsonToJsonl(rows []byte) ([]byte, error) { + if len(rows) == 0 { + return rows, nil + } + + // I am tempted to store the json directly in the database + // avoiding the roundtrip but this means that it might be + // possible to inject invalid json to the database. For now we + // take the performance hit and then think of something + // better. + dict_rows, err := ParseJsonToDicts(rows) + if err != nil { + return nil, err + } + return DictsToJson(dict_rows) +} diff --git a/vql/networking/upload.go b/vql/networking/upload.go index 890dc3b2a53..8d0600b9e00 100644 --- a/vql/networking/upload.go +++ b/vql/networking/upload.go @@ -22,8 +22,8 @@ import ( "github.com/Velocidex/ordereddict" "www.velocidex.com/golang/velociraptor/artifacts" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/glob" - "www.velocidex.com/golang/velociraptor/uploads" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -68,7 +68,7 @@ func (self *UploadFunction) Call(ctx context.Context, accessor, err := glob.GetAccessor(arg.Accessor, scope) if err != nil { scope.Log("upload: %v", err) - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), } } @@ -77,7 +77,7 @@ func (self *UploadFunction) Call(ctx context.Context, if err != nil { scope.Log("upload: Unable to open %s: %s", arg.File, err.Error()) - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), } } @@ -95,7 +95,7 @@ func (self *UploadFunction) Call(ctx context.Context, stat.Size(), // Expected size. file) if err != nil { - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), } } diff --git a/vql/server/flows.go b/vql/server/flows.go index afe2e181d1a..3009e600199 100644 --- a/vql/server/flows.go +++ b/vql/server/flows.go @@ -9,11 +9,10 @@ import ( "www.velocidex.com/golang/velociraptor/artifacts" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/csv" "www.velocidex.com/golang/velociraptor/flows" - "www.velocidex.com/golang/velociraptor/glob" "www.velocidex.com/golang/velociraptor/grpc_client" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -174,8 +173,6 @@ func (self EnumerateFlowPlugin) Call( return } - file_store_factory := file_store.GetFileStore(config_obj) - emit := func(item_type, target string) { output_chan <- ordereddict.NewDict(). Set("Type", item_type). @@ -189,58 +186,42 @@ func (self EnumerateFlowPlugin) Call( return } - // Delete all the uploads from the flow. - csv_file_path := paths.GetUploadsMetadata( + flow_path_manager := paths.NewFlowPathManager( arg.ClientId, arg.FlowId) - defer emit("UploadMetadata", csv_file_path) - fd, err := file_store_factory.ReadFile(csv_file_path) - if err == nil { - defer fd.Close() + upload_metadata_path, _ := flow_path_manager.UploadMetadata(). + GetPathForWriting() - for row := range csv.GetCSVReader(fd) { - upload, pres := row.Get("vfs_path") - if pres { - emit("Upload", upload.(string)) - } - } + defer emit("UploadMetadata", upload_metadata_path) + + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + flow_path_manager.UploadMetadata(), 0, 0) + if err != nil { + scope.Log("enumerate_flow: %v", err) + return } - repository, _ := artifacts.GetGlobalRepository(config_obj) - for _, full_artifact_name := range collection_context.ArtifactsWithResults { - artifact_name, source := paths.SplitFullSourceName(full_artifact_name) - artifact, pres := repository.Get(artifact_name) - if !pres { - scope.Log("Artifact %s not known", artifact_name) - continue + for row := range row_chan { + upload, pres := row.GetString("vfs_path") + if pres { + emit("Upload", upload) } + } - csv_path := paths.GetCSVPath( - arg.ClientId, "*", - arg.FlowId, artifact_name, source, - paths.ModeNameToMode(artifact.Type)) - - globber := make(glob.Globber) - accessor, err := file_store.GetFileStoreFileSystemAccessor(config_obj) + for _, artifact_name := range collection_context.ArtifactsWithResults { + result_path, err := result_sets.NewArtifactPathManager( + config_obj, arg.ClientId, arg.FlowId, artifact_name). + GetPathForWriting() if err != nil { scope.Log("enumerate_flow: %v", err) - return - } - - globber.Add(csv_path, accessor.PathSplit) - - // Expanding the glob is not sorted but we really need - // to go in order of dates. - for hit := range globber.ExpandWithContext( - ctx, config_obj, "", accessor) { - full_path := hit.FullPath() - emit("Result", full_path) + continue } + emit("Result", result_path) } // The flow's logs - log_path := path.Join(collection_context.Urn, "logs") + log_path, _ := flow_path_manager.Log().GetPathForWriting() emit("Log", log_path) // The flow's metadata diff --git a/vql/server/hunts.go b/vql/server/hunts.go index d5399e5992d..9dd7e3168ce 100644 --- a/vql/server/hunts.go +++ b/vql/server/hunts.go @@ -29,12 +29,12 @@ import ( "www.velocidex.com/golang/velociraptor/acls" api_proto "www.velocidex.com/golang/velociraptor/api/proto" "www.velocidex.com/golang/velociraptor/artifacts" - "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/csv" "www.velocidex.com/golang/velociraptor/flows" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" @@ -77,7 +77,9 @@ func (self HuntsPlugin) Call( return } - hunts, err := db.ListChildren(config_obj, constants.HUNTS_URN, 0, 100) + hunt_path_manager := paths.NewHuntPathManager("") + hunts, err := db.ListChildren(config_obj, + hunt_path_manager.HuntDirectory().Path(), 0, 100) if err != nil { scope.Log("Error: %v", err) return @@ -91,8 +93,10 @@ func (self HuntsPlugin) Call( } // Re-read the stats into the hunt object. + hunt_path_manager := paths.NewHuntPathManager(hunt_obj.HuntId) hunt_stats := &api_proto.HuntStats{} - err := db.GetSubject(config_obj, hunt_urn+"/stats", hunt_stats) + err := db.GetSubject(config_obj, + hunt_path_manager.Stats().Path(), hunt_stats) if err == nil { hunt_obj.Stats = hunt_stats } @@ -157,9 +161,10 @@ func (self HuntResultsPlugin) Call( return } + hunt_path_manager := paths.NewHuntPathManager(arg.HuntId) hunt_obj := &api_proto.Hunt{} err = db.GetSubject(config_obj, - path.Join(constants.HUNTS_URN, arg.HuntId), hunt_obj) + hunt_path_manager.Path(), hunt_obj) if err != nil { scope.Log("hunt_results: %v", err) return @@ -196,22 +201,20 @@ func (self HuntResultsPlugin) Call( } // Backwards compatibility. - file_path := path.Join("hunts", arg.HuntId+".csv") - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.ReadFile(file_path) + hunt_path_manager := paths.NewHuntPathManager(arg.HuntId).Clients() + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + hunt_path_manager, 0, 0) if err != nil { - scope.Log("Error %v: %v\n", err, file_path) return } - defer fd.Close() // Read each CSV file and emit it with // some extra columns for context. - for row := range csv.GetCSVReader(fd) { + for row := range row_chan { participation_row := &services.ParticipationRecord{} err := vfilter.ExtractArgs(scope, row, participation_row) if err != nil { - return + continue } if participation_row.Participate { @@ -223,36 +226,29 @@ func (self HuntResultsPlugin) Call( } // Read individual flow's - // results. Artifacts are by - // definition client artifacts - hunts - // only run on client artifacts. - result_path := paths.GetCSVPath( - participation_row.ClientId, "", + // results. + path_manager := result_sets.NewArtifactPathManager( + config_obj, + participation_row.ClientId, participation_row.FlowId, - arg.Artifact, arg.Source, - paths.MODE_CLIENT) - fd, err := file_store_factory.ReadFile(result_path) + arg.Artifact) + row_chan, err := file_store.GetTimeRange( + ctx, config_obj, path_manager, 0, 0) if err != nil { continue } - defer fd.Close() - - // Read each CSV file and emit it with - // some extra columns for context. - for row := range csv.GetCSVReader(fd) { - value := row. - Set("FlowId", participation_row.FlowId). - Set("ClientId", - participation_row.ClientId). - Set("Fqdn", - participation_row.Fqdn) + + // Read each result set and emit it + // with some extra columns for + // context. + for row := range row_chan { + value := row.Set("FlowId", participation_row.FlowId). + Set("ClientId", participation_row.ClientId). + Set("Fqdn", participation_row.Fqdn) if !arg.Brief { - value. - Set("HuntId", - participation_row.HuntId). - Set("Context", - collection_context) + value.Set("HuntId", participation_row.HuntId). + Set("Context", collection_context) } output_chan <- value } diff --git a/vql/server/monitoring.go b/vql/server/monitoring.go index c2e8bb590b9..e22dc7a519d 100644 --- a/vql/server/monitoring.go +++ b/vql/server/monitoring.go @@ -21,17 +21,14 @@ package server import ( "context" - "io" "time" "github.com/Velocidex/ordereddict" "www.velocidex.com/golang/velociraptor/acls" "www.velocidex.com/golang/velociraptor/artifacts" - config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/csv" - "www.velocidex.com/golang/velociraptor/glob" "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" @@ -67,104 +64,24 @@ func (self MonitoringPlugin) Call( return } - if arg.DayName == "" { - arg.DayName = "*" - } - - // Allow the source to be specified in - // artifact_name/Source notation. - artifact_name := arg.Artifact - source := arg.Source - if arg.Source == "" { - artifact_name, source = paths.SplitFullSourceName(arg.Artifact) - } - - // Figure out the mode by looking at the artifact type. - if arg.Mode == "" { - repository, _ := artifacts.GetGlobalRepository(config_obj) - artifact, pres := repository.Get(artifact_name) - if !pres { - scope.Log("Artifact %s not known", arg.Artifact) - return - } - arg.Mode = artifact.Type - } - - mode := paths.ModeNameToMode(arg.Mode) - if mode == 0 { - scope.Log("Unknown mode %v", arg.Mode) - return - } + path_manager := result_sets.NewArtifactPathManager( + config_obj, arg.ClientId, arg.FlowId, arg.Artifact) - log_path := paths.GetCSVPath( - arg.ClientId, arg.DayName, - arg.FlowId, artifact_name, - source, mode) - - globber := make(glob.Globber) - accessor, err := file_store.GetFileStoreFileSystemAccessor(config_obj) + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + path_manager, arg.StartTime, arg.EndTime) if err != nil { scope.Log("monitoring: %v", err) return } - globber.Add(log_path, accessor.PathSplit) - - for hit := range globber.ExpandWithContext( - ctx, config_obj, "", accessor) { - err := self.ScanLog(config_obj, - scope, output_chan, - hit.FullPath()) - if err != nil { - scope.Log( - "Error reading %v: %v", - hit.FullPath(), err) - } + for row := range row_chan { + output_chan <- row } }() return output_chan } -func (self MonitoringPlugin) ScanLog( - config_obj *config_proto.Config, - scope *vfilter.Scope, - output_chan chan<- vfilter.Row, - log_path string) error { - - fd, err := file_store.GetFileStore(config_obj).ReadFile(log_path) - if err != nil { - return err - } - defer fd.Close() - - csv_reader := csv.NewReader(fd) - headers, err := csv_reader.Read() - if err != nil { - return err - } - - for { - row := ordereddict.NewDict() - row_data, err := csv_reader.ReadAny() - if err != nil { - if err == io.EOF { - return nil - } - return err - } - - for idx, row_item := range row_data { - if idx > len(headers) { - break - } - row.Set(headers[idx], row_item) - } - - output_chan <- row - } -} - func (self MonitoringPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo { return &vfilter.PluginInfo{ Name: "monitoring", @@ -226,22 +143,14 @@ func (self WatchMonitoringPlugin) Call( return } - // Figure out the artifact's type by querying the - // artifact repository. - repository, _ := artifacts.GetGlobalRepository(config_obj) - artifact, pres := repository.Get(arg.Artifact) - if !pres { + mode, err := result_sets.GetArtifactMode(config_obj, arg.Artifact) + if err != nil { scope.Log("Artifact %s not known", arg.Artifact) return } - mode := paths.ModeNameToMode(artifact.Type) switch mode { - case paths.MODE_INVALID: - scope.Log("Unknown mode %v", artifact.Type) - return - - case paths.MODE_SERVER_EVENT, paths.MODE_MONITORING_DAILY, paths.MODE_JOURNAL_DAILY: + case paths.MODE_SERVER_EVENT, paths.MODE_CLIENT_EVENT: break default: @@ -258,7 +167,10 @@ func (self WatchMonitoringPlugin) Call( case <-ctx.Done(): return - case row := <-qm_chan: + case row, ok := <-qm_chan: + if !ok { + return + } output_chan <- row } } diff --git a/vql/server/results.go b/vql/server/results.go index d195883ac6d..6010682b898 100644 --- a/vql/server/results.go +++ b/vql/server/results.go @@ -21,21 +21,14 @@ package server import ( "context" - "io" - "sort" - "strings" - "time" "github.com/Velocidex/ordereddict" "www.velocidex.com/golang/velociraptor/acls" "www.velocidex.com/golang/velociraptor/artifacts" - config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store" - "www.velocidex.com/golang/velociraptor/file_store/csv" "www.velocidex.com/golang/velociraptor/flows" - "www.velocidex.com/golang/velociraptor/glob" "www.velocidex.com/golang/velociraptor/paths" - "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/velociraptor/result_sets" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -77,19 +70,15 @@ func (self UploadsPlugins) Call( return } - file_store_factory := file_store.GetFileStore(config_obj) - - // File uploads are stored in their own CSV file. - csv_file_path := paths.GetUploadsMetadata( - arg.ClientId, arg.FlowId) - fd, err := file_store_factory.ReadFile(csv_file_path) + flow_path_manager := paths.NewFlowPathManager(arg.ClientId, arg.FlowId) + row_chan, err := file_store.GetTimeRange(ctx, config_obj, + flow_path_manager.UploadMetadata(), 0, 0) if err != nil { scope.Log("uploads: %v", err) return } - defer fd.Close() - for row := range csv.GetCSVReader(fd) { + for row := range row_chan { output_chan <- row } }() @@ -155,17 +144,9 @@ func (self SourcePlugin) Call( return } - // If the artifact_name has a "/" it means an artifact and - // source - override the Source. - if strings.Contains(arg.Artifact, "/") { - components := strings.SplitN(arg.Artifact, "/", 2) - arg.Artifact = components[0] - arg.Source = components[1] - } - // Hunt mode is just a proxy for the hunt_results() // plugin. - if arg.Mode == "HUNT" || arg.HuntId != "" { + if arg.HuntId != "" { args := ordereddict.NewDict(). Set("hunt_id", arg.HuntId). Set("artifact", arg.Artifact). @@ -179,146 +160,29 @@ func (self SourcePlugin) Call( return } - // Figure out the mode by looking at the artifact type. - if arg.Mode == "" { - repository, _ := artifacts.GetGlobalRepository(config_obj) - artifact, pres := repository.Get(arg.Artifact) - if !pres { - scope.Log("Artifact %s not known", arg.Artifact) - return - } - arg.Mode = artifact.Type - } - - mode := paths.ModeNameToMode(arg.Mode) - if mode == 0 { - scope.Log("Invalid mode %v", arg.Mode) - return + if arg.Source != "" { + arg.Artifact = arg.Artifact + "/" + arg.Source + arg.Source = "" } - // Find the glob for the CSV files making up these results. - csv_path := paths.GetCSVPath( - arg.ClientId, "*", - arg.FlowId, arg.Artifact, arg.Source, mode) - if csv_path == "" { - scope.Log("Invalid mode %v", arg.Mode) - return - } + path_manager := result_sets.NewArtifactPathManager( + config_obj, arg.ClientId, arg.FlowId, arg.Artifact) - globber := make(glob.Globber) - accessor, err := file_store.GetFileStoreFileSystemAccessor(config_obj) + row_chan, err := file_store.GetTimeRange( + ctx, config_obj, path_manager, arg.StartTime, arg.EndTime) if err != nil { scope.Log("source: %v", err) return } - globber.Add(csv_path, accessor.PathSplit) - - // Expanding the glob is not sorted but we really need - // to go in order of dates. - hits := []string{} - for hit := range globber.ExpandWithContext( - ctx, config_obj, "", accessor) { - hits = append(hits, hit.FullPath()) - } - sort.Strings(hits) - - for _, hit := range hits { - ts_start, ts_end := parseFileTimestamp(hit) - - // Skip files modified before the required - // start time. - if ts_end < arg.StartTime { - continue - } - - if arg.EndTime > 0 && ts_start >= arg.EndTime { - return - } - - err := self.ScanLog(ctx, config_obj, - scope, arg, output_chan, hit) - if err != nil { - scope.Log("Error reading %v: %v", hit, err) - } + for row := range row_chan { + output_chan <- row } }() return output_chan } -func (self SourcePlugin) ScanLog( - ctx context.Context, - config_obj *config_proto.Config, - scope *vfilter.Scope, - arg *SourcePluginArgs, - output_chan chan<- vfilter.Row, - log_path string) error { - - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.ReadFile(log_path) - if err != nil { - return err - } - defer fd.Close() - - csv_reader := csv.NewReader(fd) - headers, err := csv_reader.Read() - if err != nil { - return err - } - - row_to_dict := func(row_data []interface{}) *ordereddict.Dict { - row := ordereddict.NewDict() - - for idx, row_item := range row_data { - if idx > len(headers) { - break - } - // Event logs have a _ts column representing - // the time of each event. - column_name := headers[idx] - if column_name == "_ts" { - timestamp, ok := row_item.(int) - if ok { - if timestamp < int(arg.StartTime) { - return nil - } - - if arg.EndTime > 0 && timestamp > int(arg.EndTime) { - return nil - } - } - } - - row.Set(column_name, row_item) - } - return row - } - - for { - select { - case <-ctx.Done(): - return nil - - default: - row_data, err := csv_reader.ReadAny() - if err != nil { - if err == io.EOF { - return nil - } - return err - } - - dict := row_to_dict(row_data) - if dict == nil { - break - } - output_chan <- dict - } - } -} - func (self SourcePlugin) Info( scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo { return &vfilter.PluginInfo{ @@ -328,19 +192,6 @@ func (self SourcePlugin) Info( } } -// Derive the unix timestamp from the filename. -func parseFileTimestamp(filename string) (int64, int64) { - for _, component := range utils.SplitComponents(filename) { - component = strings.Split(component, ".")[0] - ts, err := time.Parse("2006-01-02", component) - if err == nil { - start := ts.Unix() - return start, start + 60*60*24 - } - } - return 0, 0 -} - // Override SourcePluginArgs from the scope. func parseSourceArgsFromScope(arg *SourcePluginArgs, scope *vfilter.Scope) { client_id, pres := scope.Resolve("ClientId") @@ -416,7 +267,7 @@ func (self FlowResultsPlugin) Call( } // If no artifact is specified, get the first one from - // the hunt. + // the flow. if arg.Artifact == "" { flow, err := flows.GetFlowDetails(config_obj, arg.ClientId, arg.FlowId) if err != nil { @@ -430,46 +281,23 @@ func (self FlowResultsPlugin) Call( return } arg.Artifact = requested_artifacts[0] - - if arg.Source == "" { - arg.Artifact, arg.Source = paths.SplitFullSourceName( - arg.Artifact) - } - - // If the source is not specified find the - // first named source from the artifact - // definition. - if arg.Source == "" { - repo, err := artifacts.GetGlobalRepository(config_obj) - if err == nil { - artifact_def, ok := repo.Get(arg.Artifact) - if ok { - for _, source := range artifact_def.Sources { - if source.Name != "" { - arg.Source = source.Name - break - } - } - } - } - } } - result_path := paths.GetCSVPath( - arg.ClientId, "", - arg.FlowId, - arg.Artifact, arg.Source, - paths.MODE_CLIENT) + if arg.Source != "" { + arg.Artifact = arg.Artifact + "/" + arg.Source + arg.Source = "" + } - file_store_factory := file_store.GetFileStore(config_obj) - fd, err := file_store_factory.ReadFile(result_path) + path_manager := result_sets.NewArtifactPathManager( + config_obj, arg.ClientId, arg.FlowId, arg.Artifact) + row_chan, err := file_store.GetTimeRange( + ctx, config_obj, path_manager, 0, 0) if err != nil { - scope.Log("flow_results: %v", err) + scope.Log("source: %v", err) return } - defer fd.Close() - for row := range csv.GetCSVReader(fd) { + for row := range row_chan { output_chan <- row } }() diff --git a/vql/tools/gcs_upload.go b/vql/tools/gcs_upload.go index 5fdfc375cce..827dfd00df2 100644 --- a/vql/tools/gcs_upload.go +++ b/vql/tools/gcs_upload.go @@ -13,8 +13,8 @@ import ( "github.com/Velocidex/ordereddict" "golang.org/x/net/context" "google.golang.org/api/option" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/glob" - "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/velociraptor/utils" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" @@ -89,7 +89,7 @@ func upload_gcs(ctx context.Context, scope *vfilter.Scope, reader io.Reader, projectID, bucket, name string, credentials string) ( - *uploads.UploadResponse, error) { + *api.UploadResponse, error) { // Cache the bucket handle between invocations. var bucket_handle *storage.BucketHandle @@ -141,12 +141,12 @@ func upload_gcs(ctx context.Context, scope *vfilter.Scope, n, err := utils.Copy(ctx, utils.NewTee( writer, sha_sum, md5_sum, log_writer), reader) if err != nil { - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), }, err } - return &uploads.UploadResponse{ + return &api.UploadResponse{ Path: name, Size: uint64(n), Sha256: hex.EncodeToString(sha_sum.Sum(nil)), diff --git a/vql/tools/s3_upload.go b/vql/tools/s3_upload.go index 380752c2dd1..2384b615277 100644 --- a/vql/tools/s3_upload.go +++ b/vql/tools/s3_upload.go @@ -11,8 +11,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "golang.org/x/net/context" + "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/glob" - "www.velocidex.com/golang/velociraptor/uploads" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -94,7 +94,7 @@ func upload_S3(ctx context.Context, scope *vfilter.Scope, reader io.Reader, bucket, name string, credentialsKey string, credentialsSecret string, region string) ( - *uploads.UploadResponse, error) { + *api.UploadResponse, error) { scope.Log("upload_S3: Uploading %v to %v", name, bucket) @@ -102,7 +102,7 @@ func upload_S3(ctx context.Context, scope *vfilter.Scope, creds := credentials.NewStaticCredentials(credentialsKey, credentialsSecret, token) _, err := creds.Get() if err != nil { - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), }, err } @@ -118,12 +118,12 @@ func upload_S3(ctx context.Context, scope *vfilter.Scope, Body: reader, }) if err != nil { - return &uploads.UploadResponse{ + return &api.UploadResponse{ Error: err.Error(), }, err } - return &uploads.UploadResponse{ + return &api.UploadResponse{ Path: result.Location, }, nil }