diff --git a/api/artifacts.go b/api/artifacts.go index 2c233b3833a..042be57fa75 100644 --- a/api/artifacts.go +++ b/api/artifacts.go @@ -44,6 +44,7 @@ import ( "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" "www.velocidex.com/golang/velociraptor/result_sets" + "www.velocidex.com/golang/velociraptor/services" users "www.velocidex.com/golang/velociraptor/users" "www.velocidex.com/golang/velociraptor/utils" ) @@ -115,60 +116,30 @@ func setArtifactFile(config_obj *config_proto.Config, required_prefix string) ( *artifacts_proto.Artifact, error) { - // First ensure that the artifact is correct. - tmp_repository := artifacts.NewRepository() - artifact_definition, err := tmp_repository.LoadYaml( - in.Artifact, true /* validate */) - if err != nil { - return nil, err - } - - if !strings.HasPrefix(artifact_definition.Name, required_prefix) { - return nil, errors.New( - "Modified or custom artifacts must start with '" + - required_prefix + "'") - } - - file_store_factory := file_store.GetFileStore(config_obj) - vfs_path := paths.GetArtifactDefintionPath(artifact_definition.Name) - - // Load the new artifact into the global repo so it is - // immediately available. - global_repository, err := artifacts.GetGlobalRepository(config_obj) - if err != nil { - return nil, err - } + repository_manager := services.GetRepositoryManager() switch in.Op { - case api_proto.SetArtifactRequest_DELETE: - global_repository.Del(artifact_definition.Name) - err = file_store_factory.Delete(vfs_path) - return artifact_definition, err - case api_proto.SetArtifactRequest_SET: - // Now write it into the filestore. - fd, err := file_store_factory.WriteFile(vfs_path) + // First ensure that the artifact is correct. + tmp_repository := artifacts.NewRepository() + artifact_definition, err := tmp_repository.LoadYaml( + in.Artifact, true /* validate */) if err != nil { return nil, err } - defer fd.Close() - // We want to completely replace the content of the file. - err = fd.Truncate() - if err != nil { - return nil, err + if !strings.HasPrefix(artifact_definition.Name, required_prefix) { + return nil, errors.New( + "Modified or custom artifacts must start with '" + + required_prefix + "'") } - _, err = fd.Write([]byte(in.Artifact)) - if err != nil { - return nil, err - } + return artifact_definition, repository_manager.DeleteArtifactFile( + artifact_definition.Name) - // Load the artifact into the currently running repository. - // Artifact is already valid - no need to revalidate it again. - _, err = global_repository.LoadYaml(in.Artifact, false /* validate */) - return artifact_definition, err + case api_proto.SetArtifactRequest_SET: + return repository_manager.SetArtifactFile(in.Artifact, required_prefix) } return nil, errors.New("Unknown op") diff --git a/artifacts/definitions/Generic/Client/Stats.yaml b/artifacts/definitions/Generic/Client/Stats.yaml index ca186271bdf..d90b0a7def8 100644 --- a/artifacts/definitions/Generic/Client/Stats.yaml +++ b/artifacts/definitions/Generic/Client/Stats.yaml @@ -20,7 +20,7 @@ sources: FROM pslist(pid=getpid()) }) - - precondition: SELECT OS From info() where OS = 'linux' + - precondition: SELECT OS From info() where OS != 'windows' queries: - SELECT * from foreach( row={ diff --git a/artifacts/definitions/Server/Internal/ArtifactModification.yaml b/artifacts/definitions/Server/Internal/ArtifactModification.yaml new file mode 100644 index 00000000000..57fdcf90e9e --- /dev/null +++ b/artifacts/definitions/Server/Internal/ArtifactModification.yaml @@ -0,0 +1,10 @@ +name: Server.Internal.ArtifactModification +description: | + This event artifact is an internal event stream over which + notifications of artifact modifications are sent. Interested parties + can watch for new artifact modification events and rebuild caches + etc. + + Note: This is an automated system artifact. You do not need to start it. + +type: SERVER_EVENT diff --git a/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml b/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml index 686183e8461..be5db51950e 100644 --- a/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml +++ b/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml @@ -1,18 +1,18 @@ name: Windows.EventLogs.PowershellModule description: | - This Artifact will search and extract Module events (Event ID 4103) from + This Artifact will search and extract Module events (Event ID 4103) from Powershell-Operational Event Logs. - - Powershell is commonly used by attackers accross all stages of the attack + + Powershell is commonly used by attackers accross all stages of the attack lifecycle. Although quite noisy Module logging can provide valuable insight. - - There are several parameter's availible for search leveraging regex. - - DateAfter enables search for events after this date. - - DateBefore enables search for events before this date. - - ContextRegex enables regex search over ContextInfo text field. - - PayloadRegex enables a regex search over Payload text field. + + There are several parameter's availible for search leveraging regex. + - DateAfter enables search for events after this date. + - DateBefore enables search for events before this date. + - ContextRegex enables regex search over ContextInfo text field. + - PayloadRegex enables a regex search over Payload text field. - SearchVSS enables VSS search - + author: Matt Green - @mgreen27 @@ -35,14 +35,21 @@ parameters: description: "regex search over Payload text field." - name: SearchVSS description: "Add VSS into query." - type: bool - + type: bool + - name: LogLevelMap + type: hidden + default: | + Choice,Regex + All,"." + Warning,"3" + Verbose,"5" + sources: - query: | -- Build time bounds - LET DateAfterTime <= if(condition=DateAfter, + LET DateAfterTime <= if(condition=DateAfter, then=timestamp(epoch=DateAfter), else=timestamp(epoch="1600-01-01")) - LET DateBeforeTime <= if(condition=DateBefore, + LET DateBeforeTime <= if(condition=DateBefore, then=timestamp(epoch=DateBefore), else=timestamp(epoch="2200-01-01")) -- Parse Log level dropdown selection @@ -67,7 +74,7 @@ sources: FROM foreach( row=files, query={ - SELECT + SELECT timestamp(epoch=System.TimeCreated.SystemTime) As EventTime, System.EventID.Value as EventID, System.Computer as Computer, @@ -90,14 +97,14 @@ sources: then=ContextInfo=~PayloadRegex,else=TRUE) }) ORDER BY Source DESC - + -- Group results for deduplication LET grouped = SELECT * FROM hits GROUP BY EventRecordID -- Output results - SELECT + SELECT EventTime, EventID, Computer, @@ -110,4 +117,4 @@ sources: Opcode, Task, Source - FROM grouped \ No newline at end of file + FROM grouped diff --git a/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml b/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml index d54f04e9bc1..89daa8207fd 100644 --- a/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml +++ b/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml @@ -1,23 +1,23 @@ name: Windows.EventLogs.PowershellScriptblock description: | - This Artifact will search and extract ScriptBlock events (Event ID 4104) from + This Artifact will search and extract ScriptBlock events (Event ID 4104) from Powershell-Operational Event Logs. - - Powershell is commonly used by attackers accross all stages of the attack - lifecycle. A valuable hunt is to search Scriptblock logs for signs of + + Powershell is commonly used by attackers accross all stages of the attack + lifecycle. A valuable hunt is to search Scriptblock logs for signs of malicious content. - - There are several parameter's availible for search leveraging regex. - - DateAfter enables search for events after this date. - - DateBefore enables search for events before this date. - - SearchStrings enables regex search over scriptblock text field. - - StringWhiteList enables a regex whitelist for scriptblock text field. - - PathWhitelist enables a regex whitelist for path of scriptblock. - - LogLevel enables searching on type of log. Default is Warning level - which is logged even if ScriptBlock logging is turned off when - suspicious keywords detected in Powershell interpreter. See second + + There are several parameter's availible for search leveraging regex. + - DateAfter enables search for events after this date. + - DateBefore enables search for events before this date. + - SearchStrings enables regex search over scriptblock text field. + - StringWhiteList enables a regex whitelist for scriptblock text field. + - PathWhitelist enables a regex whitelist for path of scriptblock. + - LogLevel enables searching on type of log. Default is Warning level + which is logged even if ScriptBlock logging is turned off when + suspicious keywords detected in Powershell interpreter. See second reference for list of keywords. - - SearchVSS enables VSS search + - SearchVSS enables VSS search author: Matt Green - @mgreen27 @@ -40,7 +40,7 @@ parameters: description: "Regex of string to witelist" - name: PathWhitelist description: "Regex of path to whitelist." - + - name: LogLevel description: "Log level. Warning is Powershell default bad keyword list." type: choices @@ -58,14 +58,14 @@ parameters: Verbose,"5" - name: SearchVSS description: "Add VSS into query." - type: bool - + type: bool + sources: - query: | -- Build time bounds - LET DateAfterTime <= if(condition=DateAfter, + LET DateAfterTime <= if(condition=DateAfter, then=timestamp(epoch=DateAfter), else=timestamp(epoch="1600-01-01")) - LET DateBeforeTime <= if(condition=DateBefore, + LET DateBeforeTime <= if(condition=DateBefore, then=timestamp(epoch=DateBefore), else=timestamp(epoch="2200-01-01")) -- Parse Log level dropdown selection @@ -81,7 +81,8 @@ sources: FROM Artifact.Windows.Search.VSS(SearchFilesGlob=EventLog) }, else= { - SELECT * FROM glob(globs=EventLog) + SELECT *, FullPath AS Source + FROM glob(globs=EventLog) }) -- Main query @@ -101,19 +102,19 @@ sources: System.Level as Level, System.Opcode as Opcode, System.Task as Task, - if(condition=Source, then=Source, else=FullPath) as Source + Source FROM parse_evtx(filename=FullPath) WHERE System.EventID.Value = 4104 AND EventTime < DateBeforeTime - AND EventTime > DateAfterTime + AND EventTime > DateAfterTime AND format(format="%d", args=System.Level) =~ LogLevelRegex.value[0] - AND if(condition=SearchStrings, + AND if(condition=SearchStrings, then=ScriptBlockText =~ SearchStrings, - else=TRUE) - AND if(condition=StringWhitelist, + else=TRUE) + AND if(condition=StringWhitelist, then= NOT ScriptBlockText =~ StringWhitelist, - else=TRUE) - AND if(condition=PathWhitelist, + else=TRUE) + AND if(condition=PathWhitelist, then= NOT Path =~ PathWhitelist, else=TRUE) }) @@ -139,4 +140,4 @@ sources: Opcode, Task, Source - FROM grouped \ No newline at end of file + FROM grouped diff --git a/bin/query.go b/bin/query.go index dbb86b4376d..6156ae84fe8 100644 --- a/bin/query.go +++ b/bin/query.go @@ -39,6 +39,7 @@ import ( "www.velocidex.com/golang/velociraptor/services/journal" "www.velocidex.com/golang/velociraptor/services/labels" "www.velocidex.com/golang/velociraptor/services/launcher" + "www.velocidex.com/golang/velociraptor/services/repository" "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/velociraptor/utils" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -199,6 +200,11 @@ func startEssentialServices(config_obj *config_proto.Config, sm *services.Servic return err } + err = sm.Start(repository.StartRepositoryManager) + if err != nil { + return err + } + if config_obj.Datastore != nil { err = sm.Start(journal.StartJournalService) if err != nil { diff --git a/constants/constants.go b/constants/constants.go index e2f2a3688e9..e5d8b2d877e 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -20,7 +20,7 @@ package constants import "regexp" const ( - VERSION = "0.4.7" + VERSION = "0.4.8" ENROLLMENT_WELL_KNOWN_FLOW = "E:Enrol" MONITORING_WELL_KNOWN_FLOW = FLOW_PREFIX + "Monitoring" diff --git a/go.mod b/go.mod index d83c9769a0e..31ceceddfc0 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0 github.com/golang/snappy v0.0.1 github.com/google/rpmpack v0.0.0-20200615183209-0c831d19bd44 - github.com/google/uuid v1.1.1 // indirect + github.com/google/uuid v1.1.1 github.com/gookit/color v1.2.7 github.com/gorilla/csrf v1.6.2 github.com/gorilla/schema v1.1.0 diff --git a/gui/static/angular-components/artifact/client-event-directive.js b/gui/static/angular-components/artifact/client-event-directive.js index 86021032496..8cff293441d 100644 --- a/gui/static/angular-components/artifact/client-event-directive.js +++ b/gui/static/angular-components/artifact/client-event-directive.js @@ -142,6 +142,29 @@ ClientEventController.prototype.showHelp = function() { return false; }; +ClientEventController.prototype.showClientMonitoringTables = function() { + var self = this; + var url = 'v1/GetClientMonitoringState'; + + this.error = ""; + this.grrApiService_.get(url).then(function(response) { + self.state = response['data']; + + var modalScope = self.scope_.$new(); + var modalInstance = self.uibModal_.open({ + template: '', + scope: modalScope, + windowClass: 'wide-modal high-modal', + size: 'lg' + }); + + modalScope["json"] = JSON.stringify(self.state, null, 2); + modalScope["resolve"] = modalInstance.close; + }); + + return false; +}; // When the label changes, we need to switch the current table. ClientEventController.prototype.onLabelChange = function() { diff --git a/gui/static/angular-components/artifact/client-event.html b/gui/static/angular-components/artifact/client-event.html index 92dab8f5c64..a25583acfa6 100644 --- a/gui/static/angular-components/artifact/client-event.html +++ b/gui/static/angular-components/artifact/client-event.html @@ -21,6 +21,13 @@ + +
diff --git a/json/debug.go b/json/debug.go index 2c5ec96f4c8..7373253e16b 100644 --- a/json/debug.go +++ b/json/debug.go @@ -3,5 +3,9 @@ package json import "fmt" func Debug(v interface{}) { - fmt.Println(string(MustMarshalIndent(v))) + fmt.Println(StringIndent(v)) +} + +func Dump(v interface{}) { + fmt.Println(StringIndent(v)) } diff --git a/json/wrappers.go b/json/wrappers.go index 9dbb484a5e0..f735fc3cf70 100644 --- a/json/wrappers.go +++ b/json/wrappers.go @@ -27,6 +27,14 @@ func MustMarshalIndent(v interface{}) []byte { return result } +func StringIndent(v interface{}) string { + result, err := MarshalIndent(v) + if err != nil { + panic(err) + } + return string(result) +} + func MarshalIndent(v interface{}) ([]byte, error) { opts := NewEncOpts() return MarshalIndentWithOptions(v, opts) diff --git a/server/startup.go b/server/startup.go index 30c1ec2c07f..e548cd34414 100644 --- a/server/startup.go +++ b/server/startup.go @@ -16,6 +16,7 @@ import ( "www.velocidex.com/golang/velociraptor/services/journal" "www.velocidex.com/golang/velociraptor/services/labels" "www.velocidex.com/golang/velociraptor/services/launcher" + "www.velocidex.com/golang/velociraptor/services/repository" "www.velocidex.com/golang/velociraptor/services/sanity" "www.velocidex.com/golang/velociraptor/services/server_artifacts" "www.velocidex.com/golang/velociraptor/services/server_monitoring" @@ -75,6 +76,11 @@ func StartFrontendServices(config_obj *config_proto.Config, return err } + err = sm.Start(repository.StartRepositoryManager) + if err != nil { + return err + } + // Check everything is ok before we can start. err = sm.Start(sanity.StartSanityCheckService) if err != nil { diff --git a/services/client_monitoring/client_monitoring.go b/services/client_monitoring/client_monitoring.go index 8870b184fdc..b5faf2d98ef 100644 --- a/services/client_monitoring/client_monitoring.go +++ b/services/client_monitoring/client_monitoring.go @@ -10,6 +10,7 @@ import ( "math/rand" "sync" + "github.com/Velocidex/ordereddict" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" "www.velocidex.com/golang/velociraptor/artifacts" config_proto "www.velocidex.com/golang/velociraptor/config/proto" @@ -17,7 +18,9 @@ import ( crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" "www.velocidex.com/golang/velociraptor/datastore" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" + "www.velocidex.com/golang/velociraptor/result_sets" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/utils" ) @@ -167,7 +170,12 @@ func (self *ClientEventTable) setClientMonitoringState( // Notify all the client monitoring tables that we got // updated. This should cause all frontends to refresh. - return services.NotifyListener(self.config_obj, constants.ClientMonitoringFlowURN) + artifact_path_manager := result_sets.NewArtifactPathManager( + self.config_obj, "", "", "Server.Internal.ArtifactModification") + return services.GetJournal().PushRows(artifact_path_manager, []*ordereddict.Dict{ + ordereddict.NewDict().Set("artifact", "ClientEventTable"). + Set("op", "set"), + }) } func (self *ClientEventTable) GetClientUpdateEventTableMessage( @@ -206,6 +214,53 @@ func (self *ClientEventTable) GetClientUpdateEventTableMessage( } } +func (self *ClientEventTable) ProcessArtifactModificationEvent(event *ordereddict.Dict) { + self.mu.Lock() + defer self.mu.Unlock() + + json.Debug(event) + + modified_name, pres := event.GetString("artifact") + if !pres || modified_name == "" { + return + } + + // Determine if the modified artifact affects us. + is_relevant := func() bool { + if modified_name == "ClientEventTable" { + return true + } + + if utils.InString(self.state.Artifacts.Artifacts, modified_name) { + return true + } + + for _, label_event := range self.state.LabelEvents { + if utils.InString(label_event.Artifacts.Artifacts, modified_name) { + return true + } + } + return false + } + + if is_relevant() { + // Recompile artifacts and update the version. + self.state.Version = uint64(self.clock.Now().UnixNano()) + + clear_caches(self.state) + self.compileState(self.state) + } +} + +// Clear all the pre-compiled VQLCollectorArgs - next time we sent the +// artifact it will be rebuilt. +func clear_caches(state *flows_proto.ClientEventTable) { + state.Artifacts.CompiledCollectorArgs = nil + for _, event := range state.LabelEvents { + event.Artifacts.CompiledCollectorArgs = nil + } +} + func (self *ClientEventTable) LoadFromFile() error { self.mu.Lock() defer self.mu.Unlock() @@ -235,6 +290,7 @@ func (self *ClientEventTable) LoadFromFile() error { return self.setClientMonitoringState(self.state) } + clear_caches(self.state) return self.compileState(self.state) } @@ -259,34 +315,28 @@ func StartClientMonitoringService( services.RegisterClientEventManager(event_table) logger := logging.GetLogger(config_obj, &logging.FrontendComponent) - - notification, cancel := services.ListenForNotification( - constants.ClientMonitoringFlowURN) - defer cancel() + logger.Info("Starting Client Monitoring Service") wg.Add(1) go func() { defer wg.Done() + events, cancel := services.GetJournal().Watch("Server.Internal.ArtifactModification") + defer cancel() + for { select { case <-ctx.Done(): return - case <-notification: - err := event_table.LoadFromFile() - if err != nil { - logger.Error("StartClientMonitoringService: ", err) + case event, ok := <-events: + if !ok { return } + event_table.ProcessArtifactModificationEvent(event) } - - cancel() - notification, cancel = services.ListenForNotification( - constants.ClientMonitoringFlowURN) } }() - logger.Info("Starting Client Monitoring Service") return event_table.LoadFromFile() } diff --git a/services/client_monitoring/client_monitoring_test.go b/services/client_monitoring/client_monitoring_test.go index 67ebe999d76..4d6ccb2d9b6 100644 --- a/services/client_monitoring/client_monitoring_test.go +++ b/services/client_monitoring/client_monitoring_test.go @@ -14,11 +14,14 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/test_utils" flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/services/journal" "www.velocidex.com/golang/velociraptor/services/labels" "www.velocidex.com/golang/velociraptor/services/launcher" + "www.velocidex.com/golang/velociraptor/services/repository" "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/velociraptor/vtesting" ) type ClientMonitoringTestSuite struct { @@ -48,6 +51,7 @@ func (self *ClientMonitoringTestSuite) SetupTest() { require.NoError(self.T(), self.sm.Start(services.StartNotificationService)) require.NoError(self.T(), self.sm.Start(labels.StartLabelService)) require.NoError(self.T(), self.sm.Start(launcher.StartLauncherService)) + require.NoError(self.T(), self.sm.Start(repository.StartRepositoryManager)) require.NoError(self.T(), self.sm.Start(StartClientMonitoringService)) self.client_id = "C.12312" @@ -59,6 +63,134 @@ func (self *ClientMonitoringTestSuite) TearDownTest() { test_utils.GetMemoryDataStore(self.T(), self.config_obj).Clear() } +// Check that monitoring tables eventually follow when artifact +// definitions are updated. +func (self *ClientMonitoringTestSuite) TestUpdatingArtifacts() { + current_clock := &utils.IncClock{NowTime: 10} + + repository_manager := services.GetRepositoryManager() + repository_manager.SetArtifactFile(` +name: TestArtifact +sources: +- query: + SELECT * FROM info() +`, "") + + manager := services.ClientEventManager().(*ClientEventTable) + manager.clock = current_clock + + err := manager.SetClientMonitoringState(&flows_proto.ClientEventTable{ + Artifacts: &flows_proto.ArtifactCollectorArgs{ + Artifacts: []string{"TestArtifact"}, + }, + }) + assert.NoError(self.T(), err) + + old_table := manager.GetClientUpdateEventTableMessage(self.client_id) + assert.NotContains(self.T(), json.StringIndent(old_table), "Crib") + + // Now update the artifact. + repository_manager.SetArtifactFile(` +name: TestArtifact +sources: +- query: + SELECT *, Crib FROM info() +`, "") + + // The table should magically be updated! + vtesting.WaitUntil(5*time.Second, self.T(), func() bool { + table := manager.GetClientUpdateEventTableMessage(self.client_id) + return strings.Contains(json.StringIndent(table), "Crib") + }) +} + +func (self *ClientMonitoringTestSuite) TestUpdatingClientTable() { + current_clock := &utils.IncClock{NowTime: 10} + + repository_manager := services.GetRepositoryManager() + repository_manager.SetArtifactFile(` +name: TestArtifact +sources: +- query: + SELECT * FROM info() +`, "") + + manager := services.ClientEventManager().(*ClientEventTable) + manager.clock = current_clock + + // Set the initial table. + err := manager.SetClientMonitoringState(&flows_proto.ClientEventTable{ + Artifacts: &flows_proto.ArtifactCollectorArgs{ + Artifacts: []string{"TestArtifact"}, + }, + }) + assert.NoError(self.T(), err) + + // Get the client's event table + old_table := manager.GetClientUpdateEventTableMessage(self.client_id) + + // Now update the monitoring state + err = manager.SetClientMonitoringState(&flows_proto.ClientEventTable{ + Artifacts: &flows_proto.ArtifactCollectorArgs{ + Artifacts: []string{"TestArtifact"}, + }, + }) + assert.NoError(self.T(), err) + + // Get the client event table again + table := manager.GetClientUpdateEventTableMessage(self.client_id) + + // Make sure the client event table version is updated. + assert.True(self.T(), + table.UpdateEventTable.Version > old_table.UpdateEventTable.Version) +} + +func (self *ClientMonitoringTestSuite) TestUpdatingClientTableMultiFrontend() { + current_clock := &utils.IncClock{NowTime: 10} + + repository_manager := services.GetRepositoryManager() + repository_manager.SetArtifactFile(` +name: TestArtifact +sources: +- query: + SELECT * FROM info() +`, "") + + manager1 := services.ClientEventManager().(*ClientEventTable) + manager1.clock = current_clock + + // Set the initial table. + err := manager1.SetClientMonitoringState(&flows_proto.ClientEventTable{ + Artifacts: &flows_proto.ArtifactCollectorArgs{ + Artifacts: []string{"TestArtifact"}, + }, + }) + assert.NoError(self.T(), err) + + // Get the client's event table + old_table := manager1.GetClientUpdateEventTableMessage(self.client_id) + + // Now another frontend sets the client monitoring state + require.NoError(self.T(), self.sm.Start(StartClientMonitoringService)) + manager2 := services.ClientEventManager().(*ClientEventTable) + manager2.clock = current_clock + + // Now update the monitoring state + err = manager2.SetClientMonitoringState(&flows_proto.ClientEventTable{ + Artifacts: &flows_proto.ArtifactCollectorArgs{ + Artifacts: []string{"TestArtifact"}, + }, + }) + assert.NoError(self.T(), err) + + // Get the client event table again + table := manager2.GetClientUpdateEventTableMessage(self.client_id) + + // Make sure the client event table version is updated. + assert.True(self.T(), + table.UpdateEventTable.Version > old_table.UpdateEventTable.Version) +} + func (self *ClientMonitoringTestSuite) TestClientMonitoringCompiling() { // Every time the clock gives time.Now() it is forced to // increment. diff --git a/services/repository.go b/services/repository.go new file mode 100644 index 00000000000..17bad27eabe --- /dev/null +++ b/services/repository.go @@ -0,0 +1,32 @@ +package services + +import ( + "sync" + + artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" +) + +var ( + repository_mu sync.Mutex + grepository RepositoryManager +) + +func GetRepositoryManager() RepositoryManager { + repository_mu.Lock() + defer repository_mu.Unlock() + + return grepository +} + +func RegisterRepositoryManager(repository RepositoryManager) { + repository_mu.Lock() + defer repository_mu.Unlock() + + grepository = repository +} + +// Manages the global artifact repository +type RepositoryManager interface { + SetArtifactFile(data, required_prefix string) (*artifacts_proto.Artifact, error) + DeleteArtifactFile(name string) error +} diff --git a/services/repository/repository.go b/services/repository/repository.go new file mode 100644 index 00000000000..cbef82a414c --- /dev/null +++ b/services/repository/repository.go @@ -0,0 +1,114 @@ +package repository + +import ( + "context" + "errors" + "strings" + "sync" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/artifacts" + artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store" + "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/result_sets" + "www.velocidex.com/golang/velociraptor/services" +) + +type RepositoryManager struct { + mu sync.Mutex + config_obj *config_proto.Config +} + +func (self *RepositoryManager) SetArtifactFile(data, required_prefix string) ( + *artifacts_proto.Artifact, error) { + + // First ensure that the artifact is correct. + tmp_repository := artifacts.NewRepository() + artifact_definition, err := tmp_repository.LoadYaml( + data, true /* validate */) + if err != nil { + return nil, err + } + + if !strings.HasPrefix(artifact_definition.Name, required_prefix) { + return nil, errors.New( + "Modified or custom artifacts must start with '" + + required_prefix + "'") + } + + file_store_factory := file_store.GetFileStore(self.config_obj) + + // Load the new artifact into the global repo so it is + // immediately available. + global_repository, err := artifacts.GetGlobalRepository(self.config_obj) + if err != nil { + return nil, err + } + + vfs_path := paths.GetArtifactDefintionPath(artifact_definition.Name) + + // Now write it into the filestore. + fd, err := file_store_factory.WriteFile(vfs_path) + if err != nil { + return nil, err + } + defer fd.Close() + + // We want to completely replace the content of the file. + err = fd.Truncate() + if err != nil { + return nil, err + } + + _, err = fd.Write([]byte(data)) + if err != nil { + return nil, err + } + + // Load the artifact into the currently running repository. + // Artifact is already valid - no need to revalidate it again. + artifact, err := global_repository.LoadYaml(data, false /* validate */) + if err != nil { + return nil, err + } + + artifact_path_manager := result_sets.NewArtifactPathManager( + self.config_obj, "", "", "Server.Internal.ArtifactModification") + services.GetJournal().PushRows(artifact_path_manager, []*ordereddict.Dict{ + ordereddict.NewDict().Set("artifact", artifact.Name). + Set("op", "set"), + }) + + return artifact, nil +} + +func (self *RepositoryManager) DeleteArtifactFile(name string) error { + global_repository, err := artifacts.GetGlobalRepository(self.config_obj) + if err != nil { + return err + } + + _, pres := global_repository.Get(name) + if !pres { + return nil + } + + file_store_factory := file_store.GetFileStore(self.config_obj) + + global_repository.Del(name) + + vfs_path := paths.GetArtifactDefintionPath(name) + return file_store_factory.Delete(vfs_path) + +} + +func StartRepositoryManager(ctx context.Context, wg *sync.WaitGroup, + config_obj *config_proto.Config) error { + + services.RegisterRepositoryManager(&RepositoryManager{ + config_obj: config_obj, + }) + return nil +} diff --git a/vql/functions/uuid.go b/vql/functions/uuid.go new file mode 100644 index 00000000000..d68d2267dfd --- /dev/null +++ b/vql/functions/uuid.go @@ -0,0 +1,29 @@ +package functions + +import ( + "context" + + "github.com/Velocidex/ordereddict" + "github.com/google/uuid" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/vfilter" +) + +type _UUIDFunc struct{} + +func (self _UUIDFunc) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo { + return &vfilter.FunctionInfo{ + Name: "uuid", + Doc: "Generate a UUID.", + } +} + +func (self _UUIDFunc) Call(ctx context.Context, scope *vfilter.Scope, + args *ordereddict.Dict) vfilter.Any { + + return uuid.New() +} + +func init() { + vql_subsystem.RegisterFunction(&_UUIDFunc{}) +}