Update client monitoring tables when relevant artifacts are modified (V…

…elocidex#554) When an event artifact is modified through the GUI we need to recompile the client monitoring tables, or we risk caching stale data. This change broadcasts artifact modification events to the Server.Internal.ArtifactModification artifact. The client event manager then listens to changes in this artifact and rebuilds the tables (and increments versions) when the artifact changes.
simiaoxiaoseng · Aug 14, 2020 · 0fb9c95 · 0fb9c95
1 parent 982f69e
commit 0fb9c95
Show file tree

Hide file tree

Showing 20 changed files with 510 additions and 109 deletions.
diff --git a/api/artifacts.go b/api/artifacts.go
@@ -44,6 +44,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/result_sets"
+	"www.velocidex.com/golang/velociraptor/services"
 	users "www.velocidex.com/golang/velociraptor/users"
 	"www.velocidex.com/golang/velociraptor/utils"
 )
@@ -115,60 +116,30 @@ func setArtifactFile(config_obj *config_proto.Config,
 	required_prefix string) (
 	*artifacts_proto.Artifact, error) {
 
-	// First ensure that the artifact is correct.
-	tmp_repository := artifacts.NewRepository()
-	artifact_definition, err := tmp_repository.LoadYaml(
-		in.Artifact, true /* validate */)
-	if err != nil {
-		return nil, err
-	}
-
-	if !strings.HasPrefix(artifact_definition.Name, required_prefix) {
-		return nil, errors.New(
-			"Modified or custom artifacts must start with '" +
-				required_prefix + "'")
-	}
-
-	file_store_factory := file_store.GetFileStore(config_obj)
-	vfs_path := paths.GetArtifactDefintionPath(artifact_definition.Name)
-
-	// Load the new artifact into the global repo so it is
-	// immediately available.
-	global_repository, err := artifacts.GetGlobalRepository(config_obj)
-	if err != nil {
-		return nil, err
-	}
+	repository_manager := services.GetRepositoryManager()
 
 	switch in.Op {
-
 	case api_proto.SetArtifactRequest_DELETE:
-		global_repository.Del(artifact_definition.Name)
-		err = file_store_factory.Delete(vfs_path)
-		return artifact_definition, err
 
-	case api_proto.SetArtifactRequest_SET:
-		// Now write it into the filestore.
-		fd, err := file_store_factory.WriteFile(vfs_path)
+		// First ensure that the artifact is correct.
+		tmp_repository := artifacts.NewRepository()
+		artifact_definition, err := tmp_repository.LoadYaml(
+			in.Artifact, true /* validate */)
 		if err != nil {
 			return nil, err
 		}
-		defer fd.Close()
 
-		// We want to completely replace the content of the file.
-		err = fd.Truncate()
-		if err != nil {
-			return nil, err
+		if !strings.HasPrefix(artifact_definition.Name, required_prefix) {
+			return nil, errors.New(
+				"Modified or custom artifacts must start with '" +
+					required_prefix + "'")
 		}
 
-		_, err = fd.Write([]byte(in.Artifact))
-		if err != nil {
-			return nil, err
-		}
+		return artifact_definition, repository_manager.DeleteArtifactFile(
+			artifact_definition.Name)
 
-		// Load the artifact into the currently running repository.
-		// Artifact is already valid - no need to revalidate it again.
-		_, err = global_repository.LoadYaml(in.Artifact, false /* validate */)
-		return artifact_definition, err
+	case api_proto.SetArtifactRequest_SET:
+		return repository_manager.SetArtifactFile(in.Artifact, required_prefix)
 	}
 
 	return nil, errors.New("Unknown op")

diff --git a/artifacts/definitions/Generic/Client/Stats.yaml b/artifacts/definitions/Generic/Client/Stats.yaml
@@ -20,7 +20,7 @@ sources:
            FROM pslist(pid=getpid())
          })
 
-  - precondition: SELECT OS From info() where OS = 'linux'
+  - precondition: SELECT OS From info() where OS != 'windows'
     queries:
       - SELECT * from foreach(
          row={

diff --git a/artifacts/definitions/Server/Internal/ArtifactModification.yaml b/artifacts/definitions/Server/Internal/ArtifactModification.yaml
@@ -0,0 +1,10 @@
+name: Server.Internal.ArtifactModification
+description: |
+  This event artifact is an internal event stream over which
+  notifications of artifact modifications are sent. Interested parties
+  can watch for new artifact modification events and rebuild caches
+  etc.
+
+  Note: This is an automated system artifact. You do not need to start it.
+
+type: SERVER_EVENT
diff --git a/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml b/artifacts/definitions/Windows/EventLogs/PowershellModule.yaml
@@ -1,18 +1,18 @@
 name: Windows.EventLogs.PowershellModule
 description: |
-  This Artifact will search and extract Module events (Event ID 4103) from 
+  This Artifact will search and extract Module events (Event ID 4103) from
   Powershell-Operational Event Logs.
-  
-  Powershell is commonly used by attackers accross all stages of the attack 
+
+  Powershell is commonly used by attackers accross all stages of the attack
   lifecycle. Although quite noisy Module logging can provide valuable insight.
-  
-  There are several parameter's availible for search leveraging regex. 
-    - DateAfter enables search for events after this date.  
-    - DateBefore enables search for events before this date.   
-    - ContextRegex enables regex search over ContextInfo text field.  
-    - PayloadRegex enables a regex search over Payload text field.  
+
+  There are several parameter's availible for search leveraging regex.
+    - DateAfter enables search for events after this date.
+    - DateBefore enables search for events before this date.
+    - ContextRegex enables regex search over ContextInfo text field.
+    - PayloadRegex enables a regex search over Payload text field.
     - SearchVSS enables VSS search
-  
+
 
 author: Matt Green - @mgreen27
 
@@ -35,14 +35,21 @@ parameters:
     description: "regex search over Payload text field."
   - name: SearchVSS
     description: "Add VSS into query."
-    type: bool     
-
+    type: bool
+  - name: LogLevelMap
+    type: hidden
+    default: |
+      Choice,Regex
+      All,"."
+      Warning,"3"
+      Verbose,"5"
+
 sources:
   - query: |
         -- Build time bounds
-        LET DateAfterTime <= if(condition=DateAfter, 
+        LET DateAfterTime <= if(condition=DateAfter,
             then=timestamp(epoch=DateAfter), else=timestamp(epoch="1600-01-01"))
-        LET DateBeforeTime <= if(condition=DateBefore, 
+        LET DateBeforeTime <= if(condition=DateBefore,
             then=timestamp(epoch=DateBefore), else=timestamp(epoch="2200-01-01"))
 
         -- Parse Log level dropdown selection
@@ -67,7 +74,7 @@ sources:
           FROM foreach(
             row=files,
             query={
-              SELECT 
+              SELECT
                 timestamp(epoch=System.TimeCreated.SystemTime) As EventTime,
                 System.EventID.Value as EventID,
                 System.Computer as Computer,
@@ -90,14 +97,14 @@ sources:
                     then=ContextInfo=~PayloadRegex,else=TRUE)
             })
           ORDER BY Source DESC
-          
+
         -- Group results for deduplication
         LET grouped = SELECT *
           FROM hits
           GROUP BY EventRecordID
 
         -- Output results
-        SELECT 
+        SELECT
             EventTime,
             EventID,
             Computer,
@@ -110,4 +117,4 @@ sources:
             Opcode,
             Task,
             Source
-        FROM grouped
+        FROM grouped
diff --git a/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml b/artifacts/definitions/Windows/EventLogs/PowershellScriptblock.yaml
@@ -1,23 +1,23 @@
 name: Windows.EventLogs.PowershellScriptblock
 description: |
-  This Artifact will search and extract ScriptBlock events (Event ID 4104) from 
+  This Artifact will search and extract ScriptBlock events (Event ID 4104) from
   Powershell-Operational Event Logs.
-  
-  Powershell is commonly used by attackers accross all stages of the attack 
-  lifecycle. A valuable hunt is to search Scriptblock logs for signs of 
+
+  Powershell is commonly used by attackers accross all stages of the attack
+  lifecycle. A valuable hunt is to search Scriptblock logs for signs of
   malicious content.
-  
-  There are several parameter's availible for search leveraging regex. 
-    - DateAfter enables search for events after this date.  
-    - DateBefore enables search for events before this date.   
-    - SearchStrings enables regex search over scriptblock text field.  
-    - StringWhiteList enables a regex whitelist for scriptblock text field.  
-    - PathWhitelist enables a regex whitelist for path of scriptblock. 
-    - LogLevel enables searching on type of log. Default is Warning level 
-      which is logged even if ScriptBlock logging is turned off when 
-      suspicious keywords detected in Powershell interpreter. See second 
+
+  There are several parameter's availible for search leveraging regex.
+    - DateAfter enables search for events after this date.
+    - DateBefore enables search for events before this date.
+    - SearchStrings enables regex search over scriptblock text field.
+    - StringWhiteList enables a regex whitelist for scriptblock text field.
+    - PathWhitelist enables a regex whitelist for path of scriptblock.
+    - LogLevel enables searching on type of log. Default is Warning level
+      which is logged even if ScriptBlock logging is turned off when
+      suspicious keywords detected in Powershell interpreter. See second
       reference for list of keywords.
-    - SearchVSS enables VSS search  
+    - SearchVSS enables VSS search
 
 author: Matt Green - @mgreen27
 
@@ -40,7 +40,7 @@ parameters:
     description: "Regex of string to witelist"
   - name: PathWhitelist
     description: "Regex of path to whitelist."
-    
+
   - name: LogLevel
     description: "Log level. Warning is Powershell default bad keyword list."
     type: choices
@@ -58,14 +58,14 @@ parameters:
       Verbose,"5"
   - name: SearchVSS
     description: "Add VSS into query."
-    type: bool     
-      
+    type: bool
+
 sources:
   - query: |
         -- Build time bounds
-        LET DateAfterTime <= if(condition=DateAfter, 
+        LET DateAfterTime <= if(condition=DateAfter,
             then=timestamp(epoch=DateAfter), else=timestamp(epoch="1600-01-01"))
-        LET DateBeforeTime <= if(condition=DateBefore, 
+        LET DateBeforeTime <= if(condition=DateBefore,
             then=timestamp(epoch=DateBefore), else=timestamp(epoch="2200-01-01"))
 
         -- Parse Log level dropdown selection
@@ -81,7 +81,8 @@ sources:
               FROM Artifact.Windows.Search.VSS(SearchFilesGlob=EventLog)
             },
             else= {
-              SELECT * FROM glob(globs=EventLog)
+              SELECT *, FullPath AS Source
+              FROM glob(globs=EventLog)
             })
 
         -- Main query
@@ -101,19 +102,19 @@ sources:
                 System.Level as Level,
                 System.Opcode as Opcode,
                 System.Task as Task,
-                if(condition=Source, then=Source, else=FullPath) as Source
+                Source
               FROM parse_evtx(filename=FullPath)
               WHERE System.EventID.Value = 4104
                 AND EventTime < DateBeforeTime
-                AND EventTime > DateAfterTime 
+                AND EventTime > DateAfterTime
                 AND format(format="%d", args=System.Level) =~ LogLevelRegex.value[0]
-                AND if(condition=SearchStrings, 
+                AND if(condition=SearchStrings,
                     then=ScriptBlockText =~ SearchStrings,
-                    else=TRUE) 
-                AND if(condition=StringWhitelist, 
+                    else=TRUE)
+                AND if(condition=StringWhitelist,
                     then= NOT ScriptBlockText =~ StringWhitelist,
-                    else=TRUE) 
-                AND if(condition=PathWhitelist, 
+                    else=TRUE)
+                AND if(condition=PathWhitelist,
                     then= NOT Path =~ PathWhitelist,
                     else=TRUE)
           })
@@ -139,4 +140,4 @@ sources:
             Opcode,
             Task,
             Source
-        FROM grouped
+        FROM grouped
diff --git a/bin/query.go b/bin/query.go
@@ -39,6 +39,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/journal"
 	"www.velocidex.com/golang/velociraptor/services/labels"
 	"www.velocidex.com/golang/velociraptor/services/launcher"
+	"www.velocidex.com/golang/velociraptor/services/repository"
 	"www.velocidex.com/golang/velociraptor/uploads"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -199,6 +200,11 @@ func startEssentialServices(config_obj *config_proto.Config, sm *services.Servic
 		return err
 	}
 
+	err = sm.Start(repository.StartRepositoryManager)
+	if err != nil {
+		return err
+	}
+
 	if config_obj.Datastore != nil {
 		err = sm.Start(journal.StartJournalService)
 		if err != nil {

diff --git a/constants/constants.go b/constants/constants.go
@@ -20,7 +20,7 @@ package constants
 import "regexp"
 
 const (
-	VERSION                    = "0.4.7"
+	VERSION                    = "0.4.8"
 	ENROLLMENT_WELL_KNOWN_FLOW = "E:Enrol"
 	MONITORING_WELL_KNOWN_FLOW = FLOW_PREFIX + "Monitoring"
 

diff --git a/go.mod b/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0
 	github.com/golang/snappy v0.0.1
 	github.com/google/rpmpack v0.0.0-20200615183209-0c831d19bd44
-	github.com/google/uuid v1.1.1 // indirect
+	github.com/google/uuid v1.1.1
 	github.com/gookit/color v1.2.7
 	github.com/gorilla/csrf v1.6.2
 	github.com/gorilla/schema v1.1.0

diff --git a/gui/static/angular-components/artifact/client-event-directive.js b/gui/static/angular-components/artifact/client-event-directive.js
@@ -142,6 +142,29 @@ ClientEventController.prototype.showHelp = function() {
   return false;
 };
 
+ClientEventController.prototype.showClientMonitoringTables = function() {
+    var self = this;
+    var url = 'v1/GetClientMonitoringState';
+
+    this.error = "";
+    this.grrApiService_.get(url).then(function(response) {
+        self.state = response['data'];
+
+        var modalScope = self.scope_.$new();
+        var modalInstance = self.uibModal_.open({
+            template: '<grr-inspect-json json="json" '+
+                'on-resolve="resolve()" title="Raw Client Monitoring Tables"/>',
+            scope: modalScope,
+            windowClass: 'wide-modal high-modal',
+            size: 'lg'
+        });
+
+        modalScope["json"] = JSON.stringify(self.state, null, 2);
+        modalScope["resolve"] = modalInstance.close;
+    });
+
+    return false;
+};
 
 // When the label changes, we need to switch the current table.
 ClientEventController.prototype.onLabelChange = function() {

diff --git a/gui/static/angular-components/artifact/client-event.html b/gui/static/angular-components/artifact/client-event.html
@@ -21,6 +21,13 @@
           <i class="fa fa-edit"></i>
         </button>
 
+        <button class="btn btn-default "
+                tabindex="0" type="button" title="Show client monitoring tables"
+                ng-click="controller.showClientMonitoringTables()"
+                >
+          <span><i class="fa fa-binoculars"></i></span>
+        </button>
+
         <div class="btn-group" uib-dropdown is-open="status.isopen">
           <button id="single-button" type="button"
                   class="btn btn-default"

diff --git a/gui/static/angular-components/core/inspect-json-directive.js b/gui/static/angular-components/core/inspect-json-directive.js
@@ -31,6 +31,7 @@ InspectJsonController.prototype.showSettings = function() {
 exports.InspectJsonDirective = function() {
   return {
       scope: {
+          title: "@",
           json: "=",
           onResolve: '&',
       },

diff --git a/gui/static/angular-components/core/inspect-json.html b/gui/static/angular-components/core/inspect-json.html
@@ -1,5 +1,5 @@
 <div class="modal-header">
-  <h3>Raw Response JSON</h3>
+  <h3>{{ title || "Raw Response JSON" }}</h3>
 </div>
 
 <div class="modal-body d-flex flex-column">