Artifact collector can now produce a HTML report. (Velocidex#463)

* Artifact collector can now produce a HTML report.

Reports are produced by evaluating the templates in each collected
artifact's report section. We hook the source() plugin to make sure
the results are read from the produced zip file.

The reports allow one to add notes and interpretation hints to the
collected artifacts.

Currently this is only supported in the standalone collector, but we
should also provide a GUI feature to allow these reports to be added
to a notebook.

* Fix test

* Fix test

* fix test

* Added test

* Fix test.

* .

* .

acls/acls.go

@@ -258,7 +258,7 @@ func SetPolicy(
 func CheckAccess(
 	config_obj *config_proto.Config,
 	principal string,
-	permission ACL_PERMISSION, args ...string) (bool, error) {
+	permissions ...ACL_PERMISSION) (bool, error) {
 
 	// Internal calls from the server are allowed to do anything.
 	if principal == config_obj.Client.PinnedServerName {
@@ -274,7 +274,14 @@ func CheckAccess(
 		return false, err
 	}
 
-	return CheckAccessWithToken(acl_obj, permission, args...)
+	for _, permission := range permissions {
+		ok, err := CheckAccessWithToken(acl_obj, permission)
+		if !ok || err != nil {
+			return ok, err
+		}
+	}
+
+	return true, nil
 }
 
 func CheckAccessWithToken(

api/api.go

@@ -56,6 +56,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/services"
 	users "www.velocidex.com/golang/velociraptor/users"
 	"www.velocidex.com/golang/velociraptor/utils"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
 
 type ApiServer struct {
@@ -147,7 +148,9 @@ func (self *ApiServer) GetReport(
 			"User is not allowed to view reports.")
 	}
 
-	return getReport(ctx, self.config, user_name, in)
+	acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name)
+
+	return getReport(ctx, self.config, acl_manager, in)
 }
 
 func (self *ApiServer) CollectArtifact(
@@ -793,9 +796,13 @@ func (self *ApiServer) WriteEvent(
 
 		peer_name := peer_cert.Subject.CommonName
 
+		token, err := acls.GetEffectivePolicy(self.config, peer_name)
+		if err != nil {
+			return nil, err
+		}
+
 		// Check that the principal is allowed to push to the queue.
-		ok, err := acls.CheckAccess(self.config, peer_name,
-			acls.PUBLISH, in.Query.Name)
+		ok, err := acls.CheckAccessWithToken(token, acls.PUBLISH, in.Query.Name)
 		if err != nil {
 			return nil, err
 		}

api/notebooks.go

@@ -29,6 +29,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/services"
 	users "www.velocidex.com/golang/velociraptor/users"
 	"www.velocidex.com/golang/velociraptor/utils"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
 
 func (self *ApiServer) ExportNotebook(
@@ -445,8 +446,10 @@ func (self *ApiServer) UpdateNotebookCell(
 	// Run the actual query independently.
 	query_ctx, query_cancel := context.WithCancel(context.Background())
 
+	acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name)
+
 	tmpl, err := reporting.NewGuiTemplateEngine(
-		self.config, query_ctx, user_name, /* principal */
+		self.config, query_ctx, nil, acl_manager,
 		notebook_path_manager.Cell(in.CellId),
 		"Server.Internal.ArtifactDescription")
 	if err != nil {

api/reports.go

@@ -10,21 +10,24 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
 
 func getReport(ctx context.Context,
 	config_obj *config_proto.Config,
-	principal string,
+	acl_manager vql_subsystem.ACLManager,
 	in *api_proto.GetReportRequest) (
 	*api_proto.GetReportResponse, error) {
 
 	template_engine, err := reporting.NewGuiTemplateEngine(
-		config_obj, ctx, principal, nil, in.Artifact)
+		config_obj, ctx, nil, /* default scope */
+		acl_manager, nil, in.Artifact)
 	if err != nil {
 		if strings.HasPrefix(in.Artifact,
 			constants.ARTIFACT_CUSTOM_NAME_PREFIX) {
 			template_engine, err = reporting.NewGuiTemplateEngine(
-				config_obj, ctx, principal, nil,
+				config_obj, ctx, nil, /* default scope */
+				acl_manager, nil,
 				strings.TrimPrefix(in.Artifact,
 					constants.ARTIFACT_CUSTOM_NAME_PREFIX))
 		}

artifacts/artifacts.go

@@ -119,8 +119,7 @@ func sanitize_artifact_yaml(data string) string {
 
 	result := query_regexp.ReplaceAllStringFunc(data, func(m string) string {
 		parts := query_regexp.FindStringSubmatch(m)
-		return parts[1] + "|\n" + strings.Repeat(" ", len(parts[1])) +
-			parts[2]
+		return parts[1] + "|\n" + strings.Repeat(" ", len(parts[1])) + parts[2]
 	})
 	return result
 
@@ -231,21 +230,31 @@ func (self *Repository) Get(name string) (*artifacts_proto.Artifact, bool) {
 	self.Lock()
 	defer self.Unlock()
 
-	artifact_name, source_name := paths.SplitFullSourceName(name)
-
-	res, pres := self.Data[artifact_name]
+	result, pres := self.get(name)
 	if !pres {
 		return nil, false
 	}
 
 	// Delay processing until we need it. This means loading
 	// artifacts is faster.
-	compileArtifact(res)
+	compileArtifact(result)
+
+	// Return a copy to keep the repository pristine.
+	return proto.Clone(result).(*artifacts_proto.Artifact), true
+}
+
+func (self *Repository) get(name string) (*artifacts_proto.Artifact, bool) {
+	artifact_name, source_name := paths.SplitFullSourceName(name)
+
+	res, pres := self.Data[artifact_name]
+	if !pres {
+		return nil, false
+	}
 
 	// Caller did not specify a source - just give them a copy of
 	// the complete artifact.
 	if source_name == "" {
-		return proto.Clone(res).(*artifacts_proto.Artifact), pres
+		return res, pres
 	}
 
 	// Caller asked for only a specific source in the artifact -
@@ -292,7 +301,11 @@ func (self *Repository) List() []string {
 	self.Lock()
 	defer self.Unlock()
 
-	result := []string{}
+	return self.list()
+}
+
+func (self *Repository) list() []string {
+	result := make([]string, 0, len(self.Data))
 	for k := range self.Data {
 		result = append(result, k)
 	}

artifacts/definitions/Linux/Mounts.yaml

@@ -1,15 +1,25 @@
 name: Linux.Mounts
 description: List mounted filesystems by reading /proc/mounts
+
 parameters:
   - name: ProcMounts
     default: /proc/mounts
+
+precondition: |
+   SELECT OS From info() where OS = 'linux'
+
 sources:
-  - precondition: |
-      SELECT OS From info() where OS = 'linux'
-    queries:
-      - |
-        SELECT Device, Mount, FSType, split(string=Opts, sep=",") As Options
-               FROM parse_records_with_regex(
-                   file=ProcMounts,
-                   regex='(?m)^(?P<Device>[^ ]+) (?P<Mount>[^ ]+) (?P<FSType>[^ ]+) '+
-                         '(?P<Opts>[^ ]+)')
+  - query: |
+      SELECT Device, Mount, FSType, split(string=Opts, sep=",") As Options
+      FROM parse_records_with_regex(
+         file=ProcMounts,
+         regex='(?m)^(?P<Device>[^ ]+) (?P<Mount>[^ ]+) (?P<FSType>[^ ]+) '+
+             '(?P<Opts>[^ ]+)')
+
+
+reports:
+  - type: CLIENT
+    template: |
+      # Mounted filesystems
+
+      {{ Query "SELECT * FROM source()" | Table }}

artifacts/definitions/Windows/Search/FileFinder.yaml

@@ -70,7 +70,7 @@ sources:
                Mode.String AS Mode, Size,
                Mtime.Sec AS Modified,
                timestamp(epoch=Atime.Sec) AS ATime,
-               timestamp(epoch=Mtime.Sec) AS MTime,
+               timestamp(epoch=Mtime.Sec) AS MTime, "" AS Keywords,
                timestamp(epoch=Ctime.Sec) AS CTime, IsDir
         FROM glob(globs=SearchFilesGlob, accessor=Accessor)
 

artifacts/plugin.go

@@ -255,27 +255,33 @@ func (self _ArtifactRepositoryPluginAssociativeProtocol) Associative(
 }
 
 func NewArtifactRepositoryPlugin(repository *Repository) vfilter.PluginGeneratorInterface {
+	repository.Lock()
+	defer repository.Unlock()
 
 	if repository.artifact_plugin != nil {
 		return repository.artifact_plugin
 	}
 
-	// Cache it for next time.
-	repository.artifact_plugin = _NewArtifactRepositoryPlugin(repository, nil)
+	name_listing := repository.list()
+
+	// Cache it for next time and return it.
+	repository.artifact_plugin = _NewArtifactRepositoryPlugin(repository, name_listing, nil)
 
 	return repository.artifact_plugin
 }
 
 func _NewArtifactRepositoryPlugin(
-	repository *Repository, prefix []string) vfilter.PluginGeneratorInterface {
+	repository *Repository,
+	name_listing []string,
+	prefix []string) vfilter.PluginGeneratorInterface {
 
 	result := &ArtifactRepositoryPlugin{
 		repository: repository,
 		children:   make(map[string]vfilter.PluginGeneratorInterface),
 		prefix:     prefix,
 	}
 
-	for _, name := range repository.List() {
+	for _, name := range name_listing {
 		components := strings.Split(name, ".")
 		if len(components) < len(prefix) ||
 			!utils.SlicesEqual(components[:len(prefix)], prefix) {
@@ -286,15 +292,15 @@ func _NewArtifactRepositoryPlugin(
 
 		// We are at a leaf node.
 		if len(components) == 0 {
-			artifact, _ := repository.Get(name)
+			artifact, _ := repository.get(name)
 			result.leaf = artifact
 			return result
 		}
 
 		_, pres := result.children[components[0]]
 		if !pres {
 			result.children[components[0]] = _NewArtifactRepositoryPlugin(
-				repository, append(prefix, components[0]))
+				repository, name_listing, append(prefix, components[0]))
 		}
 	}
 

artifacts/testdata/server/testcases/file_finder.out.yaml

@@ -4,7 +4,7 @@ SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifa
   "Hash": null,
   "Size": 926,
   "Upload": null,
-  "Keywords": null
+  "Keywords": ""
  }
 ]SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifact.Windows.Search.FileFinder( Calculate_Hash="Y", SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[
  {
@@ -16,15 +16,15 @@ SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifa
   },
   "Size": 926,
   "Upload": null,
-  "Keywords": null
+  "Keywords": ""
  }
 ]SELECT basename(path=FullPath) AS File, Hash, Size, Upload.md5, Keywords FROM Artifact.Windows.Search.FileFinder( Upload_File="Y", SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[
  {
   "File": "test.zip",
   "Hash": null,
   "Size": 926,
   "Upload.md5": "fa2e382dedf895a251fcc369faa8818c",
-  "Keywords": null
+  "Keywords": ""
  }
 ]SELECT basename(path=FullPath) AS File, Keywords FROM Artifact.Windows.Search.FileFinder( YaraRule="wide nocase ascii:supercalifragilisticexpialidocious", SearchFilesGlob=srcDir + "/artifacts/testdata/**/*.in.yaml")[
  {

artifacts/testdata/windows/artifact_collector.in.yaml

@@ -1,8 +1,12 @@
 Queries:
   # Make sure we can collect artifacts with uploads
   - SELECT * FROM collect(artifacts='Windows.Search.FileFinder',
+          report="C:/1.html",
+          format='csv',
           args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'),
           output='c:/1.zip')
 
-  - SELECT FullPath from glob(globs="file:///c:/1.zip#/*",
+  - SELECT Size > 10, FullPath FROM glob(globs="C:/1.{zip,html}")
+
+  - SELECT FullPath from glob(globs="file:///c:/1.zip#/**",
          accessor='zip') WHERE not IsDir

artifacts/testdata/windows/artifact_collector.out.yaml

@@ -1,9 +1,25 @@
-SELECT * FROM collect(artifacts='Windows.Search.FileFinder', args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'), output='c:/1.zip')[
+SELECT * FROM collect(artifacts='Windows.Search.FileFinder', report="C:/1.html", format='csv', args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'), output='c:/1.zip')[
  {
-  "Container": "c:/1.zip"
+  "Container": "c:/1.zip",
+  "Report": "C:/1.html"
  }
-]SELECT FullPath from glob(globs="file:///c:/1.zip#/*", accessor='zip') WHERE not IsDir[
+]SELECT Size > 10, FullPath FROM glob(globs="C:/1.{zip,html}")[
+ {
+  "Size \u003e 10": false,
+  "FullPath": "C:\\1.html"
+ },
+ {
+  "Size \u003e 10": true,
+  "FullPath": "C:\\1.zip"
+ }
+]SELECT FullPath from glob(globs="file:///c:/1.zip#/**", accessor='zip') WHERE not IsDir[
+ {
+  "FullPath": "file:///c:/1.zip#Windows.Search.FileFinder.csv"
+ },
  {
   "FullPath": "file:///c:/1.zip#Windows.Search.FileFinder.json"
+ },
+ {
+  "FullPath": "file:///c:/1.zip#auto/C/Windows/notepad.exe"
  }
 ]