From e9bc30aed32a1da0dcdfac8bcf0fc80898e0ccc9 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 5 Feb 2023 03:38:38 +1000 Subject: [PATCH] Brought back the pool client (#2418) --- actions/events.go | 20 +- .../Windows/Applications/Edge/Favicons.yaml | 67 +++++ artifacts/proto/artifact.pb.go | 278 +++++++++--------- artifacts/proto/artifact.proto | 4 + bin/pool.go | 8 +- config/proto/config.pb.go | 12 +- config/proto/config.proto | 2 +- executor/pool.go | 125 +++++--- flows/client_flow_runner.go | 21 +- go.mod | 2 +- go.sum | 4 +- .../components/widgets/preview_uploads.jsx | 2 - http_comms/comms.go | 4 + responder/pool.go | 30 +- services/hunt_dispatcher/hunt_dispatcher.go | 2 +- services/hunt_manager/hunt_manager.go | 51 +++- services/hunt_manager/hunt_manager_test.go | 84 ++---- services/launcher/launcher.go | 10 + startup/pool.go | 2 - vql/server/elastic.go | 13 +- 20 files changed, 444 insertions(+), 297 deletions(-) create mode 100644 artifacts/definitions/Windows/Applications/Edge/Favicons.yaml diff --git a/actions/events.go b/actions/events.go index f97ea5240ce..66832c5dbae 100644 --- a/actions/events.go +++ b/actions/events.go @@ -215,12 +215,16 @@ func (self UpdateEventTable) Run( // Start a new query for each event. action_obj := &VQLClientAction{} for _, event := range table.Events { - // Name of the query we are running. + + // Name of the query we are running. There must be at least + // one query with a name. artifact_name := GetQueryName(event.Query) - if artifact_name != "" { - logger.Info("Starting monitoring query %s", artifact_name) + if artifact_name == "" { + continue } + logger.Info("Starting monitoring query %s", artifact_name) + query_responder := responder.NewMonitoringResponder( ctx, config_obj, output_chan, artifact_name) @@ -311,9 +315,13 @@ func InitializeEventTable( config_obj, &actions_proto.VQLEventTable{}) // When the context is finished, tear down the event table. - go func(table *EventTable, ctx context.Context) { - <-ctx.Done() - table.Close() + go func(table *EventTable, service_ctx context.Context) { + select { + case <-service_ctx.Done(): + table.Close() + + case <-table.Ctx.Done(): + } }(GlobalEventTable, service_ctx) mu.Unlock() diff --git a/artifacts/definitions/Windows/Applications/Edge/Favicons.yaml b/artifacts/definitions/Windows/Applications/Edge/Favicons.yaml new file mode 100644 index 00000000000..0bbf775c847 --- /dev/null +++ b/artifacts/definitions/Windows/Applications/Edge/Favicons.yaml @@ -0,0 +1,67 @@ +name: Windows.Applications.Edge.Favicons +description: | + Enumerate the users edge favicons. + + Chrome Favicons are stored in the 'Favicons' SQLite database, within + the 'favicons', 'favicon_bitmaps' and 'icon_mapping' tables. Older + versions of Chrome stored Favicons in a 'Thumbnails' SQLite + database, within the 'favicons' table. + +references: + - https://www.foxtonforensics.com/browser-history-examiner/chrome-history-location + +author: Phill Moore, @phillmoore + +parameters: + - name: faviconsGlob + default: /AppData/Local/Microsoft/Edge/User Data/*/Favicons + + - name: faviconsQuery + default: | + SELECT favicons.id AS ID, + favicon_bitmaps.icon_id AS IconID, + favicon_bitmaps.image_data as _image, + datetime( favicon_bitmaps.last_updated / 1000000 + ( strftime( '%s', '1601-01-01' ) ), 'unixepoch', 'localtime' ) AS LastUpdated, + icon_mapping.page_url AS PageURL, + favicons.url AS FaviconURL + FROM favicons + INNER JOIN icon_mapping + INNER JOIN favicon_bitmaps + ON icon_mapping.icon_id = favicon_bitmaps.icon_id + AND favicons.id = favicon_bitmaps.icon_id + ORDER BY favicons.id ASC + - name: userRegex + default: . + type: regex + +precondition: | + SELECT OS From info() where OS = 'windows' + +sources: + - query: | + LET favicons_files = SELECT * from foreach( + row={ + SELECT Uid, Name AS User, + expand(path=Directory) AS HomeDirectory + FROM Artifact.Windows.Sys.Users() + WHERE Name =~ userRegex + }, + query={ + SELECT User, OSPath, Mtime + FROM glob(globs=faviconsGlob, root=HomeDirectory) + }) + + SELECT * FROM foreach(row=favicons_files, + query={ + SELECT ID, IconID, LastUpdated, PageURL, FaviconURL, + upload(accessor="data", + file=_image, + name=format(format="Image%v.png", args=ID)) AS Image + FROM sqlite( + file=OSPath, + query=faviconsQuery) + }) + +column_types: +- name: Image + type: preview_upload diff --git a/artifacts/proto/artifact.pb.go b/artifacts/proto/artifact.pb.go index 35229947969..dd1e7496b40 100644 --- a/artifacts/proto/artifact.pb.go +++ b/artifacts/proto/artifact.pb.go @@ -496,6 +496,7 @@ type Artifact struct { Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` Author string `protobuf:"bytes,12,opt,name=author,proto3" json:"author,omitempty"` Reference []string `protobuf:"bytes,5,rep,name=reference,proto3" json:"reference,omitempty"` + References []string `protobuf:"bytes,23,rep,name=references,proto3" json:"references,omitempty"` RequiredPermissions []string `protobuf:"bytes,13,rep,name=required_permissions,json=requiredPermissions,proto3" json:"required_permissions,omitempty"` // Default resource limits. Resources *Resources `protobuf:"bytes,19,opt,name=resources,proto3" json:"resources,omitempty"` @@ -594,6 +595,13 @@ func (x *Artifact) GetReference() []string { return nil } +func (x *Artifact) GetReferences() []string { + if x != nil { + return x.References + } + return nil +} + func (x *Artifact) GetRequiredPermissions() []string { if x != nil { return x.RequiredPermissions @@ -1171,7 +1179,7 @@ var file_artifact_proto_rawDesc = []byte{ 0x70, 0x6c, 0x65, 0x20, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x73, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x20, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x20, 0x6f, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, - 0x22, 0xb1, 0x0d, 0x0a, 0x08, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0xb1, 0x01, + 0x22, 0xf9, 0x0d, 0x0a, 0x08, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0xb1, 0x01, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x9c, 0x01, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x95, 0x01, 0x12, 0x92, 0x01, 0x54, 0x68, 0x65, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, @@ -1197,138 +1205,142 @@ var file_artifact_proto_rawDesc = []byte{ 0x65, 0x6e, 0x63, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x42, 0x26, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x20, 0x12, 0x1e, 0x41, 0x20, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x2e, 0x52, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x73, 0x0a, - 0x14, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x42, 0x40, 0xe2, 0xfc, 0xe3, - 0xc4, 0x01, 0x3a, 0x12, 0x38, 0x41, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x13, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, - 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x05, - 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x68, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x64, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x42, 0x44, 0xe2, 0xfc, 0xe3, - 0xc4, 0x01, 0x3e, 0x12, 0x3c, 0x41, 0x20, 0x56, 0x51, 0x4c, 0x20, 0x65, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, 0x20, 0x65, 0x76, 0x61, 0x6c, - 0x75, 0x61, 0x74, 0x65, 0x64, 0x20, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x75, - 0x73, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x52, 0x0c, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x6c, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x42, 0x32, 0xe2, - 0xfc, 0xe3, 0xc4, 0x01, 0x2c, 0x12, 0x2a, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, - 0x73, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, 0x20, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, - 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x2e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x6e, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x5a, 0xe2, 0xfc, 0xe3, - 0xc4, 0x01, 0x54, 0x12, 0x52, 0x54, 0x68, 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x20, 0x6f, 0x66, - 0x20, 0x74, 0x68, 0x65, 0x20, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x20, 0x43, 0x61, 0x6e, 0x20, - 0x62, 0x65, 0x20, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x2c, 0x20, 0x43, 0x4c, 0x49, 0x45, 0x4e, - 0x54, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x2c, 0x20, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x2c, - 0x20, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x2c, 0x20, 0x49, - 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x5a, 0x0a, - 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x29, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x23, 0x12, 0x21, 0x57, - 0x68, 0x65, 0x72, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x20, 0x67, 0x65, 0x74, 0x73, 0x20, 0x69, 0x74, 0x73, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x52, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x6a, 0x0a, 0x07, 0x69, 0x6d, 0x70, - 0x6f, 0x72, 0x74, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, 0x09, 0x42, 0x50, 0xe2, 0xfc, 0xe3, 0xc4, - 0x01, 0x4a, 0x12, 0x48, 0x41, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x6f, 0x74, - 0x68, 0x65, 0x72, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x27, 0x20, 0x65, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x56, 0x51, 0x4c, 0x20, 0x74, 0x6f, 0x20, 0x70, 0x72, 0x65, - 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x07, 0x69, 0x6d, - 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x7b, 0x0a, 0x06, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x12, 0x20, 0x01, 0x28, 0x09, 0x42, 0x63, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x5d, 0x12, 0x5b, 0x56, - 0x51, 0x4c, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x20, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x2e, 0x20, 0x49, - 0x74, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x62, 0x65, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x6d, 0x61, - 0x74, 0x69, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x20, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x06, 0x65, 0x78, 0x70, 0x6f, - 0x72, 0x74, 0x12, 0x6d, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0b, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x42, 0x44, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x3e, 0x12, 0x3c, 0x41, 0x20, 0x6c, 0x69, - 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x20, 0x74, 0x6f, - 0x20, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x70, 0x6f, 0x73, - 0x74, 0x20, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x73, 0x12, 0x34, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x03, 0x72, 0x61, 0x77, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x09, 0x42, 0x26, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x20, 0x12, 0x1e, 0x54, 0x68, - 0x65, 0x20, 0x72, 0x61, 0x77, 0x20, 0x59, 0x41, 0x4d, 0x4c, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x03, 0x72, 0x61, - 0x77, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x19, 0x0a, - 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x5f, 0x69, 0x6e, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x49, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x61, - 0x6c, 0x69, 0x61, 0x73, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x41, 0x6c, - 0x69, 0x61, 0x73, 0x3a, 0x7f, 0xda, 0xfc, 0xe3, 0xc4, 0x01, 0x79, 0x0a, 0x77, 0x41, 0x6e, 0x20, - 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x20, 0x77, 0x72, 0x61, 0x70, 0x73, 0x20, 0x61, - 0x20, 0x56, 0x51, 0x4c, 0x20, 0x71, 0x75, 0x65, 0x72, 0x79, 0x20, 0x69, 0x6e, 0x20, 0x72, 0x65, - 0x75, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x2c, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x65, 0x64, 0x20, 0x77, 0x61, 0x79, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, - 0x20, 0x61, 0x72, 0x65, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x73, - 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x69, 0x6e, 0x67, 0x20, 0x74, - 0x68, 0x65, 0x6d, 0x2e, 0x22, 0x3c, 0x0a, 0x13, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x05, 0x69, - 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x22, 0x82, 0x03, 0x0a, 0x04, 0x54, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, - 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x5f, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x5f, 0x61, 0x73, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x73, 0x73, 0x65, - 0x74, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x61, - 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0d, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x55, 0x72, 0x6c, 0x12, - 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x25, - 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x6d, 0x61, 0x74, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x4a, 0x0a, 0x0b, 0x74, 0x68, 0x69, 0x72, 0x64, - 0x5f, 0x70, 0x61, 0x72, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, - 0x6f, 0x6c, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x22, 0xcc, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6f, - 0x70, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x0c, 0x6f, 0x70, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, - 0x0a, 0x0a, 0x69, 0x6f, 0x70, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x02, 0x52, 0x09, 0x69, 0x6f, 0x70, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x19, 0x0a, - 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x07, 0x6d, 0x61, 0x78, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, - 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x79, 0x74, - 0x65, 0x73, 0x42, 0x37, 0x5a, 0x35, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, - 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, - 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x74, 0x2e, 0x52, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x46, 0x0a, + 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x17, 0x20, 0x03, 0x28, + 0x09, 0x42, 0x26, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x20, 0x12, 0x1e, 0x41, 0x20, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x73, 0x0a, 0x14, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x64, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, + 0x03, 0x28, 0x09, 0x42, 0x40, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x3a, 0x12, 0x38, 0x41, 0x20, 0x6c, + 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, + 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x63, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x13, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, + 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x6f, + 0x6f, 0x6c, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x68, 0x0a, + 0x0c, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x42, 0x44, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x3e, 0x12, 0x3c, 0x41, 0x20, 0x56, + 0x51, 0x4c, 0x20, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6f, + 0x20, 0x62, 0x65, 0x20, 0x65, 0x76, 0x61, 0x6c, 0x75, 0x61, 0x74, 0x65, 0x64, 0x20, 0x70, 0x72, + 0x69, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x0c, 0x70, 0x72, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x6c, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x65, 0x74, 0x65, 0x72, 0x42, 0x32, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x2c, 0x12, 0x2a, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, 0x20, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x6e, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x42, 0x5a, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x54, 0x12, 0x52, 0x54, 0x68, 0x65, + 0x20, 0x74, 0x79, 0x70, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x20, 0x43, 0x61, 0x6e, 0x20, 0x62, 0x65, 0x20, 0x43, 0x4c, 0x49, 0x45, 0x4e, + 0x54, 0x2c, 0x20, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x2c, + 0x20, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x2c, 0x20, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x5f, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x2c, 0x20, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x52, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x5a, 0x0a, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x29, 0xe2, + 0xfc, 0xe3, 0xc4, 0x01, 0x23, 0x12, 0x21, 0x57, 0x68, 0x65, 0x72, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x20, 0x67, 0x65, 0x74, 0x73, 0x20, 0x69, + 0x74, 0x73, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x12, 0x6a, 0x0a, 0x07, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x11, 0x20, 0x03, + 0x28, 0x09, 0x42, 0x50, 0xe2, 0xfc, 0xe3, 0xc4, 0x01, 0x4a, 0x12, 0x48, 0x41, 0x20, 0x6c, 0x69, + 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, 0x61, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x73, 0x27, 0x20, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x56, 0x51, + 0x4c, 0x20, 0x74, 0x6f, 0x20, 0x70, 0x72, 0x65, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x2e, 0x52, 0x07, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x7b, 0x0a, + 0x06, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x42, 0x63, 0xe2, + 0xfc, 0xe3, 0xc4, 0x01, 0x5d, 0x12, 0x5b, 0x56, 0x51, 0x4c, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x20, 0x65, 0x78, + 0x70, 0x6f, 0x72, 0x74, 0x73, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x62, + 0x65, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x20, + 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x74, 0x68, + 0x65, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x20, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x52, 0x06, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x6d, 0x0a, 0x07, 0x72, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x42, 0x44, 0xe2, 0xfc, 0xe3, 0xc4, + 0x01, 0x3e, 0x12, 0x3c, 0x41, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x72, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x70, 0x6f, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, + 0x52, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, + 0x38, 0x0a, 0x03, 0x72, 0x61, 0x77, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x42, 0x26, 0xe2, 0xfc, + 0xe3, 0xc4, 0x01, 0x20, 0x12, 0x1e, 0x54, 0x68, 0x65, 0x20, 0x72, 0x61, 0x77, 0x20, 0x59, 0x41, + 0x4d, 0x4c, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x2e, 0x52, 0x03, 0x72, 0x61, 0x77, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x5f, 0x69, + 0x6e, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x49, 0x6e, + 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x16, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x3a, 0x7f, 0xda, 0xfc, 0xe3, + 0xc4, 0x01, 0x79, 0x0a, 0x77, 0x41, 0x6e, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x20, 0x77, 0x72, 0x61, 0x70, 0x73, 0x20, 0x61, 0x20, 0x56, 0x51, 0x4c, 0x20, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x20, 0x69, 0x6e, 0x20, 0x72, 0x65, 0x75, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x2c, 0x20, + 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x77, 0x61, 0x79, 0x2e, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x61, 0x6c, 0x6c, + 0x20, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6e, + 0x67, 0x20, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x61, 0x6e, 0x61, + 0x6c, 0x79, 0x7a, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x65, 0x6d, 0x2e, 0x22, 0x3c, 0x0a, 0x13, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x82, 0x03, 0x0a, 0x04, 0x54, + 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x12, 0x2c, 0x0a, 0x12, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x5f, 0x61, 0x73, 0x73, 0x65, 0x74, + 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x41, 0x73, 0x73, 0x65, 0x74, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x23, + 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x66, 0x69, 0x6c, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, + 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x20, 0x0a, + 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, + 0x4a, 0x0a, 0x0b, 0x74, 0x68, 0x69, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x79, 0x12, 0x21, + 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xcc, 0x01, 0x0a, 0x09, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x70, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0c, 0x6f, 0x70, 0x73, + 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, 0x75, + 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x63, 0x70, + 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6f, 0x70, 0x73, 0x5f, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x69, 0x6f, 0x70, 0x73, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x6f, 0x77, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x52, 0x6f, 0x77, 0x73, + 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x55, + 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x42, 0x37, 0x5a, 0x35, 0x77, 0x77, + 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, + 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/artifacts/proto/artifact.proto b/artifacts/proto/artifact.proto index a808e7cb362..1ad9203e39e 100644 --- a/artifacts/proto/artifact.proto +++ b/artifacts/proto/artifact.proto @@ -139,6 +139,10 @@ message Artifact { description: "A reference for this artifact." }]; + repeated string references = 23 [(sem_type) = { + description: "A reference for this artifact." + }]; + repeated string required_permissions = 13 [(sem_type) = { description: "A list of required permissions to collect this artifact." }]; diff --git a/bin/pool.go b/bin/pool.go index 84151d9c0cf..dcbed165d2c 100644 --- a/bin/pool.go +++ b/bin/pool.go @@ -1,5 +1,3 @@ -// +build XXXX - /* Velociraptor - Dig Deeper Copyright (C) 2019-2022 Rapid7 Inc. @@ -114,9 +112,15 @@ func doPoolClient() error { client_config.Client.WritebackWindows = client_config.Client.WritebackLinux if client_config.Client.LocalBuffer != nil { client_config.Client.LocalBuffer.DiskSize = 0 + + // Limit the total size of the ring buffer. + client_config.Client.LocalBuffer.MemorySize = 100000 } client_config.Client.Concurrency = uint64(*pool_client_concurrency) + // Disable client info updates in pool clients + client_config.Client.ClientInfoUpdateTime = -1 + // Make sure the config is ok. err = crypto_utils.VerifyConfig(client_config) if err != nil { diff --git a/config/proto/config.pb.go b/config/proto/config.pb.go index 345346f22a8..988ebfc93d3 100644 --- a/config/proto/config.pb.go +++ b/config/proto/config.pb.go @@ -534,11 +534,11 @@ type ClientConfig struct { // How often clients inform the server about flow progress // (default 5 seconds). DefaultServerFlowStatsUpdate uint64 `protobuf:"varint,39,opt,name=default_server_flow_stats_update,json=defaultServerFlowStatsUpdate,proto3" json:"default_server_flow_stats_update,omitempty"` - // Clients will send a Client.Info.Update message to the server - // every this many seconds.This helps to keep the server info up - // to date about each client. This should not be sent too + // Clients will send a Server.Internal.ClientInfo message to the + // server every this many seconds.This helps to keep the server + // info up to date about each client. This should not be sent too // frequently. The default is 1 day (86400 seconds). - ClientInfoUpdateTime uint64 `protobuf:"varint,40,opt,name=client_info_update_time,json=clientInfoUpdateTime,proto3" json:"client_info_update_time,omitempty"` + ClientInfoUpdateTime int64 `protobuf:"varint,40,opt,name=client_info_update_time,json=clientInfoUpdateTime,proto3" json:"client_info_update_time,omitempty"` } func (x *ClientConfig) Reset() { @@ -783,7 +783,7 @@ func (x *ClientConfig) GetDefaultServerFlowStatsUpdate() uint64 { return 0 } -func (x *ClientConfig) GetClientInfoUpdateTime() uint64 { +func (x *ClientConfig) GetClientInfoUpdateTime() int64 { if x != nil { return x.ClientInfoUpdateTime } @@ -4122,7 +4122,7 @@ var file_config_proto_rawDesc = []byte{ 0x6c, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x46, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x74, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x28, 0x20, 0x01, 0x28, 0x04, 0x52, 0x14, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x6d, 0x65, 0x18, 0x28, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x22, 0xad, 0x04, 0x0a, 0x09, 0x41, 0x50, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, diff --git a/config/proto/config.proto b/config/proto/config.proto index 6e3983c26ba..23bb78b9f43 100644 --- a/config/proto/config.proto +++ b/config/proto/config.proto @@ -244,7 +244,7 @@ message ClientConfig { // server every this many seconds.This helps to keep the server // info up to date about each client. This should not be sent too // frequently. The default is 1 day (86400 seconds). - uint64 client_info_update_time = 40; + int64 client_info_update_time = 40; } message APIConfig { diff --git a/executor/pool.go b/executor/pool.go index 2652a9bc2ef..38b984070be 100644 --- a/executor/pool.go +++ b/executor/pool.go @@ -1,9 +1,7 @@ -// +build XXXX - /* The pool client pretends to be a large number of clients in order to exert a large load on the server. In reality each client is - running in an go routine in parallel. + running in a go routine in parallel. Therefore when we do a hunt, each pool client goroutine will receive the same VQL query and run the same code. This reduces the @@ -12,7 +10,7 @@ This pool executor memoizes the results from each query in memory so each query is run only once but the results are returned from - each goroutine fake client as it was unique. This increases the + each goroutine fake client as if it was unique. This increases the total number of pool clients we can support since most of the work is pushed out to the comms. */ @@ -23,7 +21,6 @@ import ( "context" "fmt" "sync" - "time" "google.golang.org/protobuf/proto" "www.velocidex.com/golang/velociraptor/actions" @@ -37,12 +34,12 @@ import ( var ( pool_mu sync.Mutex + // Get transactions by session id - session_id_cache = make(map[string]*transaction) + transaction_by_session_id = make(map[string]*transaction) - // Get transactions by query name - query_cache = make(map[string]*transaction) - ts = time.Now().UnixNano() + // Get transactions by a unique key for the FlowRequest message. + transaction_by_flow_key = make(map[string]*transaction) ) type transaction struct { @@ -55,14 +52,6 @@ type transaction struct { Done bool } -func getInc() int64 { - pool_mu.Lock() - defer pool_mu.Unlock() - - ts++ - return ts -} - // A wrapper around the standard client executor for use of pool // clients. When multiple requests come in for the same query and // parameters, we cache the results when the first request comes in @@ -79,38 +68,40 @@ func (self *PoolClientExecutor) ReadResponse() <-chan *crypto_proto.VeloMessage return self.Outbound } -// Inspect the request and decide if we will cache it under a query. -func getQueryName(message *crypto_proto.VeloMessage) string { - query_name := "" - if message.VQLClientAction != nil { - for _, query := range message.VQLClientAction.Query { - if query.Name != "" { - query_name = query.Name - } +// Inspect the request and derive a unique session key for it +func getRequestKey(req *crypto_proto.FlowRequest) string { + key := "" + + for _, action := range req.VQLClientActions { + for _, query := range action.Query { + key += query.Name } - // Cache it under the query name and the serialized parameters - serialized, _ := json.Marshal(message.VQLClientAction.Env) - return fmt.Sprintf("%v: %v", query_name, string(serialized)) + // Cache it under the query name and the serialized + // parameters. This way when any of the parameters change we + // recalculate the query. + key += json.MustMarshalString(action.Env) } - return "" -} - -func getSessionKey(message *crypto_proto.VeloMessage) string { - return fmt.Sprintf("%s/%d", message.SessionId, message.QueryId) + return key } +// Gets the transaction for this request or create a new transaction. func getCompletedTransaction(message *crypto_proto.VeloMessage) *transaction { pool_mu.Lock() defer pool_mu.Unlock() - query_name := getQueryName(message) + // We only cache FlowRequest messages. + if message.FlowRequest == nil { + return nil + } + + key := getRequestKey(message.FlowRequest) // Do not cache empty queries. - if query_name == "" { + if key == "" { return nil } - result, pres := query_cache[query_name] + result, pres := transaction_by_flow_key[key] // Transaction fully cached and completed. if pres { @@ -123,11 +114,14 @@ func getCompletedTransaction(message *crypto_proto.VeloMessage) *transaction { Request: message, IsDone: make(chan bool), } - session_id_cache_key := getSessionKey(message) - query_cache[query_name] = trans - session_id_cache[session_id_cache_key] = trans - fmt.Printf("Starting transaction for %v\n", session_id_cache_key) + // Cache it for the next + transaction_by_flow_key[key] = trans + transaction_by_session_id[message.SessionId] = trans + + fmt.Printf("Starting transaction for %v\n", message.SessionId) + + // Return nil for the next caller to start executing this transaction. return nil } @@ -151,10 +145,10 @@ func (self *PoolClientExecutor) maybeUpdateEventTable( fmt.Printf("Installing new event table for version %v\n", req.UpdateEventTable.Version) - g_responder := responder.GlobalPoolEventResponder - pool_responder := g_responder.NewResponder(ctx, self.config_obj, req) + g_responder := responder.GetPoolEventResponder(ctx) actions.UpdateEventTable{}.Run( - self.config_obj, ctx, pool_responder, req.UpdateEventTable) + self.config_obj, ctx, g_responder.EventTableInput, + req.UpdateEventTable) } @@ -174,15 +168,15 @@ func (self *PoolClientExecutor) ProcessRequest( <-tran.IsDone fmt.Printf("Getting %v responses from cache\n", len(tran.Responses)) + + // Replay the transaction into the output channel. for _, resp := range tran.Responses { response := &crypto_proto.VeloMessage{ SessionId: message.SessionId, RequestId: message.RequestId, - ResponseId: uint64(getInc()), - TaskId: message.TaskId, VQLResponse: self.maybeTransformResponse(resp.VQLResponse), LogMessage: resp.LogMessage, - Status: resp.Status, + FlowStats: resp.FlowStats, } select { case <-ctx.Done(): @@ -193,9 +187,14 @@ func (self *PoolClientExecutor) ProcessRequest( return } + // If we get here there is no cached transaction - just forward to + // the normal executor. self.ClientExecutor.ProcessRequest(ctx, message) } +// Inspect the response and transform it if needed. Currently we only +// need to replace the Hostname with the pool client's ID so it +// appears to be a different client. func (self *PoolClientExecutor) maybeTransformResponse( response *actions_proto.VQLResponse) *actions_proto.VQLResponse { @@ -232,6 +231,12 @@ func (self *PoolClientExecutor) maybeTransformResponse( return response } +// A Pool Client is a virtualized client running in a goroutine which +// emulates a full blown client. Flow Requests are cached globally in +// a transaction so they can be replayed back for all clients. This +// allows us to calculate any query once but return all the results at +// once from other clients immediately therefore increasing the load +// on the server. func NewPoolClientExecutor( ctx context.Context, client_id string, @@ -270,18 +275,24 @@ func NewPoolClientExecutor( }, nil } +// Compare messages from the real client executor against the cached +// transactions and add them to the transactions. If we detect the +// flow is complete we mark the transactions as done and other clients +// may replay it. func maybeCacheResult(response *crypto_proto.VeloMessage) { pool_mu.Lock() defer pool_mu.Unlock() - session_id := getSessionKey(response) + session_id := response.SessionId // Check if the transaction is tracked - tran, pres := session_id_cache[session_id] + tran, pres := transaction_by_session_id[session_id] if pres { fmt.Printf("%v\n", response) tran.Responses = append(tran.Responses, response) - if response.Status != nil && !tran.Done { + + // Determine if the flow is completed by looking at the FlowStat + if !tran.Done && isFlowComplete(response) { fmt.Printf("Completing transaction for session_id %v\n", session_id) // The transaction is now done. @@ -289,5 +300,21 @@ func maybeCacheResult(response *crypto_proto.VeloMessage) { tran.Done = true } } +} + +// Detect if this is a FlowStats message which represents the flow is +// compelte. +func isFlowComplete(message *crypto_proto.VeloMessage) bool { + if message.FlowStats == nil { + return false + } + for _, s := range message.FlowStats.QueryStatus { + // Flow is not completed as one of the queries is still + // running. + if s.Status == crypto_proto.VeloStatus_PROGRESS { + return false + } + } + return true } diff --git a/flows/client_flow_runner.go b/flows/client_flow_runner.go index b6d0313e65d..2eacd57cac6 100644 --- a/flows/client_flow_runner.go +++ b/flows/client_flow_runner.go @@ -3,6 +3,7 @@ package flows import ( "context" "fmt" + "strings" "time" "github.com/Velocidex/ordereddict" @@ -98,8 +99,13 @@ func (self *ClientFlowRunner) MonitoringLogMessage( artifact_name := artifacts.DeobfuscateString( self.config_obj, response.Artifact) - if artifact_name == "" { - artifact_name = "Unknown" + + // If we are not able to deobfuscate the artifact name properly + // (e.g. due to server keys changing) we really can not store the + // data anywhere so drop it. + if artifact_name == "" || + strings.HasPrefix(artifact_name, "$") { + return nil } log_path_manager, err := artifact_paths.NewArtifactLogPathManager( @@ -139,7 +145,13 @@ func (self *ClientFlowRunner) MonitoringVQLResponse( // Deobfuscate the response if needed. _ = artifacts.Deobfuscate(self.config_obj, response) + // If we are not able to deobfuscate the artifact name properly + // (e.g. due to server keys changing) we really can not store the + // data anywhere so drop it. query_name := response.Query.Name + if query_name == "" || strings.HasPrefix(query_name, "$") { + return nil + } journal, err := services.GetJournal(self.config_obj) if err != nil { @@ -402,6 +414,11 @@ func (self *ClientFlowRunner) VQLResponse( return err } + if response.Query.Name == "" || + strings.HasPrefix(response.Query.Name, "$") { + return nil + } + path_manager, err := artifact_paths.NewArtifactPathManager( self.config_obj, client_id, flow_id, response.Query.Name) if err != nil { diff --git a/go.mod b/go.mod index 02f53d96a40..22fa759c4e0 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,7 @@ require ( howett.net/plist v1.0.0 www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 - www.velocidex.com/golang/go-ntfs v0.1.2-0.20221117122413-b97c856cb140 + www.velocidex.com/golang/go-ntfs v0.1.2-0.20230203145252-d0e4bdc73bc3 www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a www.velocidex.com/golang/oleparse v0.0.0-20220617011920-94df2342d0b7 diff --git a/go.sum b/go.sum index 315064bd7f3..84105f970b1 100644 --- a/go.sum +++ b/go.sum @@ -1226,8 +1226,8 @@ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e h1:AhcXPgNKhJ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e/go.mod h1:ykEQ7AUF9AL+mfCefDmLwmZOnU2So6wM3qKM8xdsHhU= www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 h1:pfvo7NFo0eJj6Zr7d+4vMx/Zr2JriMMPEWRHUf1YjUw= www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671/go.mod h1:qnzHyB9yD2khtYO+wf3ck9FQxX3wFhXeJHFBnuUIZcc= -www.velocidex.com/golang/go-ntfs v0.1.2-0.20221117122413-b97c856cb140 h1:cRKPEhkLDHT98D3lS7SvRGgLkvhpiHJFGgQXRlCOJ9w= -www.velocidex.com/golang/go-ntfs v0.1.2-0.20221117122413-b97c856cb140/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= +www.velocidex.com/golang/go-ntfs v0.1.2-0.20230203145252-d0e4bdc73bc3 h1:pLixKq8hAf90YS/gH6VGFWKy5pTilGDI1R/MxGqLUho= +www.velocidex.com/golang/go-ntfs v0.1.2-0.20230203145252-d0e4bdc73bc3/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de/go.mod h1:j9Xy8Z9wxzY2SCB8CqDkkoSzy+eUwevnOrRm/XM2q/A= www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d h1:OQKwxK0O4a/8YTmfkQNzUspyrvlpRbLi318L08DC0oY= www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d/go.mod h1:TPJ3phbAuZIu7XuPyNqgoP2k3P+eNHfHHGcivhcsxaA= diff --git a/gui/velociraptor/src/components/widgets/preview_uploads.jsx b/gui/velociraptor/src/components/widgets/preview_uploads.jsx index 7a21f7b96d3..9c31db0f5af 100644 --- a/gui/velociraptor/src/components/widgets/preview_uploads.jsx +++ b/gui/velociraptor/src/components/widgets/preview_uploads.jsx @@ -8,7 +8,6 @@ import axios from 'axios'; import api from '../core/api-service.jsx'; import { HexViewDialog } from '../utils/hex.jsx'; import Button from 'react-bootstrap/Button'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import qs from 'qs'; import Modal from 'react-bootstrap/Modal'; import HexView from '../utils/hex.jsx'; @@ -306,7 +305,6 @@ export default class PreviewUpload extends Component { size="sm" onClick={()=>this.setState({showDialog: true})} variant="outline-info"> - {string_data} diff --git a/http_comms/comms.go b/http_comms/comms.go index 55d3a26d50d..cfc2bdf9ed5 100644 --- a/http_comms/comms.go +++ b/http_comms/comms.go @@ -618,6 +618,10 @@ func NewNotificationReader( if config_obj.Client.ClientInfoUpdateTime > 0 { last_update_period = time.Duration( config_obj.Client.ClientInfoUpdateTime) * time.Second + + // Set to a negative number to disable Server.Internal.ClientInfo + } else if config_obj.Client.ClientInfoUpdateTime == -1 { + last_update_period = 0 } return &NotificationReader{ diff --git a/responder/pool.go b/responder/pool.go index 2af3bf29c21..f3a30179b51 100644 --- a/responder/pool.go +++ b/responder/pool.go @@ -1,5 +1,3 @@ -// +build XXXX - package responder import ( @@ -7,10 +5,8 @@ import ( "fmt" "sync" - config_proto "www.velocidex.com/golang/velociraptor/config/proto" crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" "www.velocidex.com/golang/velociraptor/json" - "www.velocidex.com/golang/velociraptor/logging" ) // The pool event responder is a singleton which distributes any @@ -38,6 +34,10 @@ type PoolEventResponder struct { ctx context.Context + // Event table will push messages to this channel and we will + // distribute them to all the other clients. + EventTableInput chan *crypto_proto.VeloMessage + client_responders map[int]chan *crypto_proto.VeloMessage } @@ -51,9 +51,12 @@ func GetPoolEventResponder(ctx context.Context) *PoolEventResponder { result := &PoolEventResponder{ ctx: ctx, + EventTableInput: make(chan *crypto_proto.VeloMessage), client_responders: make(map[int]chan *crypto_proto.VeloMessage), } + result.Start() + GlobalPoolEventResponder = result return result } @@ -67,26 +70,15 @@ func (self *PoolEventResponder) RegisterPoolClientResponder( } // Gets a new responder which is feeding the GlobalPoolEventResponder -func (self *PoolEventResponder) NewResponder( - ctx context.Context, - config_obj *config_proto.Config, - req *crypto_proto.VeloMessage) *Responder { - // The PoolEventResponder input - in := make(chan *crypto_proto.VeloMessage) - - // Prepare a new responder that will feed us. - result := &Responder{ - ctx: ctx, - output: in, - logger: logging.GetLogger(config_obj, &logging.ClientComponent), - } +func (self *PoolEventResponder) Start() { go func() { for { select { case <-self.ctx.Done(): return - case message, ok := <-in: + + case message, ok := <-self.EventTableInput: if !ok { return } @@ -114,6 +106,4 @@ func (self *PoolEventResponder) NewResponder( } } }() - - return result } diff --git a/services/hunt_dispatcher/hunt_dispatcher.go b/services/hunt_dispatcher/hunt_dispatcher.go index db27ae2c65c..32996c811cc 100644 --- a/services/hunt_dispatcher/hunt_dispatcher.go +++ b/services/hunt_dispatcher/hunt_dispatcher.go @@ -461,9 +461,9 @@ func (self *HuntDispatcher) Refresh(config_obj *config_proto.Config) error { atomic.StoreUint64(&self.last_timestamp, hunt_obj.StartTime) dispatcherCurrentTimestamp.Set(float64(last_timestamp)) } - self.hunts[hunt_id] = &HuntRecord{Hunt: hunt_obj} } + return nil } diff --git a/services/hunt_manager/hunt_manager.go b/services/hunt_manager/hunt_manager.go index b3da89800fb..ef695a6264a 100644 --- a/services/hunt_manager/hunt_manager.go +++ b/services/hunt_manager/hunt_manager.go @@ -59,8 +59,8 @@ import ( "google.golang.org/protobuf/proto" api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" - "www.velocidex.com/golang/velociraptor/constants" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" "www.velocidex.com/golang/velociraptor/services" @@ -331,24 +331,48 @@ func (self *HuntManager) ProcessInterrogation( } // Watch for all flows created by a hunt and maintain the list of hunt -// completions. +// completions. TODO: This is inefficient because we are forced to +// open the flow object from disk to get at the request. We need to +// denote flows created by hunts by their own unique flow id. func (self *HuntManager) ProcessFlowCompletion( ctx context.Context, config_obj *config_proto.Config, row *ordereddict.Dict) error { - flow, err := journal.GetFlowFromQueue(config_obj, row) - if err != nil { - return err + flow_any, pres := row.Get("Flow") + if !pres { + return nil + } + + flow, ok := flow_any.(*flows_proto.ArtifactCollectorContext) + if !ok || flow == nil { + serialized, err := json.Marshal(flow_any) + if err != nil { + return err + } + flow = &flows_proto.ArtifactCollectorContext{} + err = json.Unmarshal(serialized, flow) + if err != nil { + return err + } } - // We only care about flows that were launched by hunts here. The - // flow creator is the hunt id. - hunt_id := flow.Request.Creator - if !strings.HasPrefix(hunt_id, constants.HUNT_PREFIX) { + // Sessions IDs that come from a hunt have a special format with + // the hunt id and flow id joined. This allows us to quickly + // identify the flow that belongs to a hunt without needing to + // read the original request from the datastore. + flow_id, pres := row.GetString("FlowId") + if !pres { + return errors.New("FlowId not found") + } + + parts := strings.Split(flow_id, ".H.") + if len(parts) != 2 { return nil } + hunt_id := "H." + parts[1] + // Flow is complete so add it to the hunt stats. We send a // mutation to the hunt dispatcher to mediate internal hunt state // manipulation. @@ -369,7 +393,7 @@ func (self *HuntManager) ProcessFlowCompletion( // status, so we dont bother broadcasting a mutation for them. We // only need to update the local hunt dispatcher on the master // node which will flush to disk eventually. - err = self.processMutation(config_obj, mutation) + err := self.processMutation(config_obj, mutation) if err != nil { return err } @@ -384,7 +408,7 @@ func (self *HuntManager) ProcessFlowCompletion( []*ordereddict.Dict{ordereddict.NewDict(). Set("ClientId", flow.ClientId). Set("FlowId", flow.SessionId). - Set("StartTime", time.Unix(0, int64(flow.CreateTime*1000))). + Set("StartTime", time.Unix(0, int64(flow.StartTime*1000))). Set("EndTime", time.Unix(0, int64(flow.ActiveTime*1000))). Set("Status", flow.State.String()). Set("Error", flow.Status)}) @@ -537,10 +561,11 @@ func (self *HuntManager) ProcessParticipation( Stats: &api_proto.HuntStats{Stopped: true}}) } - // Use hunt information to launch the flow against this - // client. + // Control rate of hunt recruitment to balance server load. self.limiter.Wait(ctx) + // Use hunt information to launch the flow against this + // client. return scheduleHuntOnClient(ctx, config_obj, hunt_obj, participation_row.ClientId) } diff --git a/services/hunt_manager/hunt_manager_test.go b/services/hunt_manager/hunt_manager_test.go index 75faf2a4dcf..4b72f203bae 100644 --- a/services/hunt_manager/hunt_manager_test.go +++ b/services/hunt_manager/hunt_manager_test.go @@ -31,6 +31,7 @@ type HuntTestSuite struct { client_id string hunt_id string + flow_id string expected *flows_proto.ArtifactCollectorArgs } @@ -45,13 +46,19 @@ func (self *HuntTestSuite) SetupTest() { self.hunt_id += "A" self.expected.Creator = self.hunt_id + launcher, err := services.GetLauncher(self.ConfigObj) + assert.NoError(self.T(), err) + + launcher.SetFlowIdForTests("F.1234") + self.flow_id = "F.1234." + self.hunt_id + // Write a client record. client_info_obj := &actions_proto.ClientInfo{ ClientId: self.client_id, } client_path_manager := paths.NewClientPathManager(self.client_id) db, _ := datastore.GetDB(self.ConfigObj) - err := db.SetSubject(self.ConfigObj, + err = db.SetSubject(self.ConfigObj, client_path_manager.Path(), client_info_obj) assert.NoError(self.T(), err) } @@ -59,11 +66,6 @@ func (self *HuntTestSuite) SetupTest() { func (self *HuntTestSuite) TestHuntManager() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -106,13 +108,13 @@ func (self *HuntTestSuite) TestHuntManager() { return false } _, err = LoadCollectionContext(self.ConfigObj, - self.client_id, "F.1234") + self.client_id, self.flow_id) return err == nil }) // Check that a flow was launched. collection_context, err := LoadCollectionContext(self.ConfigObj, - self.client_id, "F.1234") + self.client_id, self.flow_id) assert.NoError(t, err) assert.Equal(t, collection_context.Request.Artifacts, self.expected.Artifacts) } @@ -120,11 +122,6 @@ func (self *HuntTestSuite) TestHuntManager() { func (self *HuntTestSuite) TestHuntWithLabelClientNoLabel() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -166,7 +163,7 @@ func (self *HuntTestSuite) TestHuntWithLabelClientNoLabel() { time.Sleep(time.Second) // No flow should be launched. - _, err = LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err = LoadCollectionContext(self.ConfigObj, self.client_id, self.flow_id) assert.Error(t, err) // Now add the label to the client. The hunt will now be @@ -187,16 +184,12 @@ func (self *HuntTestSuite) TestHuntWithLabelClientNoLabel() { }) // The flow is now created. - _, err = LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err = LoadCollectionContext(self.ConfigObj, self.client_id, self.flow_id) assert.NoError(t, err) } func (self *HuntTestSuite) TestHuntWithLabelClientHasLabelDifferentCase() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ @@ -254,23 +247,19 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabelDifferentCase() { if err != nil { return false } - _, err := LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err := LoadCollectionContext( + self.ConfigObj, self.client_id, self.flow_id) return err == nil }) collection_context, err := LoadCollectionContext(self.ConfigObj, - self.client_id, "F.1234") + self.client_id, self.flow_id) assert.Equal(t, collection_context.Request.Artifacts, self.expected.Artifacts) } func (self *HuntTestSuite) TestHuntWithOverride() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // Hunt is paused so normally will not receive any clients. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -315,12 +304,13 @@ func (self *HuntTestSuite) TestHuntWithOverride() { return false } - _, err := LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err := LoadCollectionContext( + self.ConfigObj, self.client_id, self.flow_id) return err == nil }) collection_context, err := LoadCollectionContext(self.ConfigObj, - self.client_id, "F.1234") + self.client_id, self.flow_id) assert.NoError(t, err) assert.Equal(t, collection_context.Request.Artifacts, self.expected.Artifacts) } @@ -328,11 +318,6 @@ func (self *HuntTestSuite) TestHuntWithOverride() { func (self *HuntTestSuite) TestHuntWithLabelClientHasLabel() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -389,12 +374,13 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabel() { return false } - _, err := LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err := LoadCollectionContext( + self.ConfigObj, self.client_id, self.flow_id) return err == nil }) collection_context, err := LoadCollectionContext(self.ConfigObj, - self.client_id, "F.1234") + self.client_id, self.flow_id) assert.NoError(t, err) assert.Equal(t, collection_context.Request.Artifacts, self.expected.Artifacts) } @@ -402,11 +388,6 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasLabel() { func (self *HuntTestSuite) TestHuntWithLabelClientHasExcludedLabel() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -463,18 +444,14 @@ func (self *HuntTestSuite) TestHuntWithLabelClientHasExcludedLabel() { time.Sleep(time.Second) // No flow should be launched. - _, err = LoadCollectionContext(self.ConfigObj, self.client_id, "F.1234") + _, err = LoadCollectionContext( + self.ConfigObj, self.client_id, self.flow_id) assert.Error(t, err) } func (self *HuntTestSuite) TestHuntClientOSCondition() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -539,12 +516,12 @@ func (self *HuntTestSuite) TestHuntClientOSCondition() { vtesting.WaitUntil(5*time.Second, self.T(), func() bool { // Flow should be launched on client id because it is a Windows client. - _, err = LoadCollectionContext(self.ConfigObj, client_id_1, "F.1234") + _, err = LoadCollectionContext(self.ConfigObj, client_id_1, self.flow_id) return err == nil }) // No flow should be launched on client_id_2 because it is a Linux client. - _, err = LoadCollectionContext(self.ConfigObj, client_id_2, "F.1234") + _, err = LoadCollectionContext(self.ConfigObj, client_id_2, self.flow_id) assert.Error(t, err) } @@ -555,9 +532,6 @@ func (self *HuntTestSuite) TestHuntClientOSCondition() { func (self *HuntTestSuite) TestHuntClientOSConditionInterrogation() { t := self.T() - launcher, err := services.GetLauncher(self.ConfigObj) - assert.NoError(t, err) - db, err := datastore.GetDB(self.ConfigObj) assert.NoError(t, err) @@ -571,8 +545,6 @@ func (self *HuntTestSuite) TestHuntClientOSConditionInterrogation() { }) assert.NoError(t, err) - launcher.SetFlowIdForTests("F.1234") - // The hunt will launch the Generic.Client.Info on the client. hunt_obj := &api_proto.Hunt{ HuntId: self.hunt_id, @@ -632,7 +604,7 @@ func (self *HuntTestSuite) TestHuntClientOSConditionInterrogation() { mdb := test_utils.GetMemoryDataStore(self.T(), self.ConfigObj) vtesting.WaitUntil(time.Second, self.T(), func() bool { task := &crypto_proto.VeloMessage{} - path_manager := paths.NewFlowPathManager(self.client_id, "F.1234") + path_manager := paths.NewFlowPathManager(self.client_id, self.flow_id) err := mdb.GetSubject(self.ConfigObj, path_manager.Task(), task) return err != nil @@ -697,7 +669,7 @@ func (self *HuntTestSuite) TestHuntManagerMutations() { []*ordereddict.Dict{ordereddict.NewDict(). Set("Timestamp", time.Now().UTC().Unix()). Set("Flow", flow_obj). - Set("FlowId", flow_obj.SessionId). + Set("FlowId", self.flow_id). Set("ClientId", self.client_id), }, "System.Flow.Completion", self.client_id, "")) @@ -776,7 +748,7 @@ func (self *HuntTestSuite) TestHuntManagerErrors() { []*ordereddict.Dict{ordereddict.NewDict(). Set("Timestamp", time.Now().UTC().Unix()). Set("Flow", flow_obj). - Set("FlowId", flow_obj.SessionId). + Set("FlowId", self.flow_id). Set("ClientId", self.client_id), }, "System.Flow.Completion", self.client_id, "")) diff --git a/services/launcher/launcher.go b/services/launcher/launcher.go index 2bd762f41af..0d29ae5f1ea 100644 --- a/services/launcher/launcher.go +++ b/services/launcher/launcher.go @@ -121,6 +121,7 @@ import ( "encoding/base32" "encoding/binary" "fmt" + "strings" "sync" "time" @@ -556,6 +557,15 @@ func (self *Launcher) ScheduleArtifactCollectionFromCollectorArgs( session_id := NewFlowId(client_id) + // If the flow was created by a hunt, we encode the hunt id in the + // session id. The session id will be returned by the client, and + // the ingestor will be able to tie the session to the hunt + // without consulting the datastore. The ID will look something + // like F.1233.H.1234 + if strings.HasPrefix(collector_request.Creator, "H.") { + session_id += "." + collector_request.Creator + } + // How long to batch log messages for on the client. batch_delay := uint64(2000) if collector_request.LogBatchTime > 0 { diff --git a/startup/pool.go b/startup/pool.go index cf127f0342f..ee5802b18bc 100644 --- a/startup/pool.go +++ b/startup/pool.go @@ -1,5 +1,3 @@ -// +build XXXX - package startup import ( diff --git a/vql/server/elastic.go b/vql/server/elastic.go index ac3387ce99c..75af576aff8 100644 --- a/vql/server/elastic.go +++ b/vql/server/elastic.go @@ -78,6 +78,7 @@ type _ElasticPluginArgs struct { PipeLine string `vfilter:"optional,field=pipeline,doc=Pipeline for uploads"` DisableSSLSecurity bool `vfilter:"optional,field=disable_ssl_security,doc=Disable ssl certificate verifications."` RootCerts string `vfilter:"optional,field=root_ca,doc=As a better alternative to disable_ssl_security, allows root ca certs to be added here."` + MaxMemoryBuffer uint64 `vfilter:"optional,field=max_memory_buffer,doc=How large we allow the memory buffer to grow to while we are trying to contact the Elastic server (default 100mb)."` } type _ElasticPlugin struct{} @@ -189,6 +190,14 @@ func upload_rows( next_send_id := id + arg.ChunkSize next_send_time := time.After(wait_time) + // If the buffer is too large we need to drop the data on the + // floor. This might happen if the elastic server is not reachable + // for example. + max_buffer_size := uint64(100 * 1024 * 1024) + if arg.MaxMemoryBuffer > 0 { + max_buffer_size = arg.MaxMemoryBuffer + } + // Flush any remaining rows defer send_to_elastic(ctx, scope, output_chan, client, &buf) @@ -213,7 +222,8 @@ func upload_rows( continue } - if id > next_send_id { + if id > next_send_id || + buf.Len() > int(max_buffer_size) { send_to_elastic(ctx, scope, output_chan, client, &buf) next_send_id = id + arg.ChunkSize @@ -298,6 +308,7 @@ func send_to_elastic( select { case <-ctx.Done(): return + case output_chan <- ordereddict.NewDict(). Set("StatusCode", res.StatusCode). Set("Response", response):