diff --git a/acls/acls.go b/acls/acls.go index 03f9953fd42..ed1c7b695f1 100644 --- a/acls/acls.go +++ b/acls/acls.go @@ -258,7 +258,7 @@ func SetPolicy( func CheckAccess( config_obj *config_proto.Config, principal string, - permission ACL_PERMISSION, args ...string) (bool, error) { + permissions ...ACL_PERMISSION) (bool, error) { // Internal calls from the server are allowed to do anything. if principal == config_obj.Client.PinnedServerName { @@ -274,7 +274,14 @@ func CheckAccess( return false, err } - return CheckAccessWithToken(acl_obj, permission, args...) + for _, permission := range permissions { + ok, err := CheckAccessWithToken(acl_obj, permission) + if !ok || err != nil { + return ok, err + } + } + + return true, nil } func CheckAccessWithToken( diff --git a/api/api.go b/api/api.go index f71e70584de..bd0003c7ec0 100644 --- a/api/api.go +++ b/api/api.go @@ -56,6 +56,7 @@ import ( "www.velocidex.com/golang/velociraptor/services" users "www.velocidex.com/golang/velociraptor/users" "www.velocidex.com/golang/velociraptor/utils" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" ) type ApiServer struct { @@ -147,7 +148,9 @@ func (self *ApiServer) GetReport( "User is not allowed to view reports.") } - return getReport(ctx, self.config, user_name, in) + acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name) + + return getReport(ctx, self.config, acl_manager, in) } func (self *ApiServer) CollectArtifact( @@ -793,9 +796,13 @@ func (self *ApiServer) WriteEvent( peer_name := peer_cert.Subject.CommonName + token, err := acls.GetEffectivePolicy(self.config, peer_name) + if err != nil { + return nil, err + } + // Check that the principal is allowed to push to the queue. - ok, err := acls.CheckAccess(self.config, peer_name, - acls.PUBLISH, in.Query.Name) + ok, err := acls.CheckAccessWithToken(token, acls.PUBLISH, in.Query.Name) if err != nil { return nil, err } diff --git a/api/notebooks.go b/api/notebooks.go index d8282406fc6..2ce2af0b31c 100644 --- a/api/notebooks.go +++ b/api/notebooks.go @@ -29,6 +29,7 @@ import ( "www.velocidex.com/golang/velociraptor/services" users "www.velocidex.com/golang/velociraptor/users" "www.velocidex.com/golang/velociraptor/utils" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" ) func (self *ApiServer) ExportNotebook( @@ -445,8 +446,10 @@ func (self *ApiServer) UpdateNotebookCell( // Run the actual query independently. query_ctx, query_cancel := context.WithCancel(context.Background()) + acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name) + tmpl, err := reporting.NewGuiTemplateEngine( - self.config, query_ctx, user_name, /* principal */ + self.config, query_ctx, nil, acl_manager, notebook_path_manager.Cell(in.CellId), "Server.Internal.ArtifactDescription") if err != nil { diff --git a/api/reports.go b/api/reports.go index 55831826970..3cc387f8a53 100644 --- a/api/reports.go +++ b/api/reports.go @@ -10,21 +10,24 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/reporting" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" ) func getReport(ctx context.Context, config_obj *config_proto.Config, - principal string, + acl_manager vql_subsystem.ACLManager, in *api_proto.GetReportRequest) ( *api_proto.GetReportResponse, error) { template_engine, err := reporting.NewGuiTemplateEngine( - config_obj, ctx, principal, nil, in.Artifact) + config_obj, ctx, nil, /* default scope */ + acl_manager, nil, in.Artifact) if err != nil { if strings.HasPrefix(in.Artifact, constants.ARTIFACT_CUSTOM_NAME_PREFIX) { template_engine, err = reporting.NewGuiTemplateEngine( - config_obj, ctx, principal, nil, + config_obj, ctx, nil, /* default scope */ + acl_manager, nil, strings.TrimPrefix(in.Artifact, constants.ARTIFACT_CUSTOM_NAME_PREFIX)) } diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go index 963a3f9dea6..46cab719e0d 100644 --- a/artifacts/artifacts.go +++ b/artifacts/artifacts.go @@ -119,8 +119,7 @@ func sanitize_artifact_yaml(data string) string { result := query_regexp.ReplaceAllStringFunc(data, func(m string) string { parts := query_regexp.FindStringSubmatch(m) - return parts[1] + "|\n" + strings.Repeat(" ", len(parts[1])) + - parts[2] + return parts[1] + "|\n" + strings.Repeat(" ", len(parts[1])) + parts[2] }) return result @@ -231,21 +230,31 @@ func (self *Repository) Get(name string) (*artifacts_proto.Artifact, bool) { self.Lock() defer self.Unlock() - artifact_name, source_name := paths.SplitFullSourceName(name) - - res, pres := self.Data[artifact_name] + result, pres := self.get(name) if !pres { return nil, false } // Delay processing until we need it. This means loading // artifacts is faster. - compileArtifact(res) + compileArtifact(result) + + // Return a copy to keep the repository pristine. + return proto.Clone(result).(*artifacts_proto.Artifact), true +} + +func (self *Repository) get(name string) (*artifacts_proto.Artifact, bool) { + artifact_name, source_name := paths.SplitFullSourceName(name) + + res, pres := self.Data[artifact_name] + if !pres { + return nil, false + } // Caller did not specify a source - just give them a copy of // the complete artifact. if source_name == "" { - return proto.Clone(res).(*artifacts_proto.Artifact), pres + return res, pres } // Caller asked for only a specific source in the artifact - @@ -292,7 +301,11 @@ func (self *Repository) List() []string { self.Lock() defer self.Unlock() - result := []string{} + return self.list() +} + +func (self *Repository) list() []string { + result := make([]string, 0, len(self.Data)) for k := range self.Data { result = append(result, k) } diff --git a/artifacts/definitions/Linux/Mounts.yaml b/artifacts/definitions/Linux/Mounts.yaml index 2fd5c9dae07..72b984c5a6a 100644 --- a/artifacts/definitions/Linux/Mounts.yaml +++ b/artifacts/definitions/Linux/Mounts.yaml @@ -1,15 +1,25 @@ name: Linux.Mounts description: List mounted filesystems by reading /proc/mounts + parameters: - name: ProcMounts default: /proc/mounts + +precondition: | + SELECT OS From info() where OS = 'linux' + sources: - - precondition: | - SELECT OS From info() where OS = 'linux' - queries: - - | - SELECT Device, Mount, FSType, split(string=Opts, sep=",") As Options - FROM parse_records_with_regex( - file=ProcMounts, - regex='(?m)^(?P[^ ]+) (?P[^ ]+) (?P[^ ]+) '+ - '(?P[^ ]+)') + - query: | + SELECT Device, Mount, FSType, split(string=Opts, sep=",") As Options + FROM parse_records_with_regex( + file=ProcMounts, + regex='(?m)^(?P[^ ]+) (?P[^ ]+) (?P[^ ]+) '+ + '(?P[^ ]+)') + + +reports: + - type: CLIENT + template: | + # Mounted filesystems + + {{ Query "SELECT * FROM source()" | Table }} diff --git a/artifacts/definitions/Windows/Search/FileFinder.yaml b/artifacts/definitions/Windows/Search/FileFinder.yaml index 46b88a615fd..18a0f747e68 100644 --- a/artifacts/definitions/Windows/Search/FileFinder.yaml +++ b/artifacts/definitions/Windows/Search/FileFinder.yaml @@ -70,7 +70,7 @@ sources: Mode.String AS Mode, Size, Mtime.Sec AS Modified, timestamp(epoch=Atime.Sec) AS ATime, - timestamp(epoch=Mtime.Sec) AS MTime, + timestamp(epoch=Mtime.Sec) AS MTime, "" AS Keywords, timestamp(epoch=Ctime.Sec) AS CTime, IsDir FROM glob(globs=SearchFilesGlob, accessor=Accessor) diff --git a/artifacts/plugin.go b/artifacts/plugin.go index 1d1ab462019..bcc872e4342 100644 --- a/artifacts/plugin.go +++ b/artifacts/plugin.go @@ -255,19 +255,25 @@ func (self _ArtifactRepositoryPluginAssociativeProtocol) Associative( } func NewArtifactRepositoryPlugin(repository *Repository) vfilter.PluginGeneratorInterface { + repository.Lock() + defer repository.Unlock() if repository.artifact_plugin != nil { return repository.artifact_plugin } - // Cache it for next time. - repository.artifact_plugin = _NewArtifactRepositoryPlugin(repository, nil) + name_listing := repository.list() + + // Cache it for next time and return it. + repository.artifact_plugin = _NewArtifactRepositoryPlugin(repository, name_listing, nil) return repository.artifact_plugin } func _NewArtifactRepositoryPlugin( - repository *Repository, prefix []string) vfilter.PluginGeneratorInterface { + repository *Repository, + name_listing []string, + prefix []string) vfilter.PluginGeneratorInterface { result := &ArtifactRepositoryPlugin{ repository: repository, @@ -275,7 +281,7 @@ func _NewArtifactRepositoryPlugin( prefix: prefix, } - for _, name := range repository.List() { + for _, name := range name_listing { components := strings.Split(name, ".") if len(components) < len(prefix) || !utils.SlicesEqual(components[:len(prefix)], prefix) { @@ -286,7 +292,7 @@ func _NewArtifactRepositoryPlugin( // We are at a leaf node. if len(components) == 0 { - artifact, _ := repository.Get(name) + artifact, _ := repository.get(name) result.leaf = artifact return result } @@ -294,7 +300,7 @@ func _NewArtifactRepositoryPlugin( _, pres := result.children[components[0]] if !pres { result.children[components[0]] = _NewArtifactRepositoryPlugin( - repository, append(prefix, components[0])) + repository, name_listing, append(prefix, components[0])) } } diff --git a/artifacts/testdata/server/testcases/file_finder.out.yaml b/artifacts/testdata/server/testcases/file_finder.out.yaml index 23cba19e5a3..d1d9ecc5cc3 100644 --- a/artifacts/testdata/server/testcases/file_finder.out.yaml +++ b/artifacts/testdata/server/testcases/file_finder.out.yaml @@ -4,7 +4,7 @@ SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifa "Hash": null, "Size": 926, "Upload": null, - "Keywords": null + "Keywords": "" } ]SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifact.Windows.Search.FileFinder( Calculate_Hash="Y", SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[ { @@ -16,7 +16,7 @@ SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifa }, "Size": 926, "Upload": null, - "Keywords": null + "Keywords": "" } ]SELECT basename(path=FullPath) AS File, Hash, Size, Upload.md5, Keywords FROM Artifact.Windows.Search.FileFinder( Upload_File="Y", SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[ { @@ -24,7 +24,7 @@ SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifa "Hash": null, "Size": 926, "Upload.md5": "fa2e382dedf895a251fcc369faa8818c", - "Keywords": null + "Keywords": "" } ]SELECT basename(path=FullPath) AS File, Keywords FROM Artifact.Windows.Search.FileFinder( YaraRule="wide nocase ascii:supercalifragilisticexpialidocious", SearchFilesGlob=srcDir + "/artifacts/testdata/**/*.in.yaml")[ { diff --git a/artifacts/testdata/windows/artifact_collector.in.yaml b/artifacts/testdata/windows/artifact_collector.in.yaml index ad7bcd21f13..f038a24be72 100644 --- a/artifacts/testdata/windows/artifact_collector.in.yaml +++ b/artifacts/testdata/windows/artifact_collector.in.yaml @@ -1,8 +1,12 @@ Queries: # Make sure we can collect artifacts with uploads - SELECT * FROM collect(artifacts='Windows.Search.FileFinder', + report="C:/1.html", + format='csv', args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'), output='c:/1.zip') - - SELECT FullPath from glob(globs="file:///c:/1.zip#/*", + - SELECT Size > 10, FullPath FROM glob(globs="C:/1.{zip,html}") + + - SELECT FullPath from glob(globs="file:///c:/1.zip#/**", accessor='zip') WHERE not IsDir diff --git a/artifacts/testdata/windows/artifact_collector.out.yaml b/artifacts/testdata/windows/artifact_collector.out.yaml index 6186c302af4..a8a8f4119bd 100644 --- a/artifacts/testdata/windows/artifact_collector.out.yaml +++ b/artifacts/testdata/windows/artifact_collector.out.yaml @@ -1,9 +1,25 @@ -SELECT * FROM collect(artifacts='Windows.Search.FileFinder', args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'), output='c:/1.zip')[ +SELECT * FROM collect(artifacts='Windows.Search.FileFinder', report="C:/1.html", format='csv', args=dict(SearchFilesGlob='c:/Windows/notepad.exe', Upload_File='Y'), output='c:/1.zip')[ { - "Container": "c:/1.zip" + "Container": "c:/1.zip", + "Report": "C:/1.html" } -]SELECT FullPath from glob(globs="file:///c:/1.zip#/*", accessor='zip') WHERE not IsDir[ +]SELECT Size > 10, FullPath FROM glob(globs="C:/1.{zip,html}")[ + { + "Size \u003e 10": false, + "FullPath": "C:\\1.html" + }, + { + "Size \u003e 10": true, + "FullPath": "C:\\1.zip" + } +]SELECT FullPath from glob(globs="file:///c:/1.zip#/**", accessor='zip') WHERE not IsDir[ + { + "FullPath": "file:///c:/1.zip#Windows.Search.FileFinder.csv" + }, { "FullPath": "file:///c:/1.zip#Windows.Search.FileFinder.json" + }, + { + "FullPath": "file:///c:/1.zip#auto/C/Windows/notepad.exe" } ] \ No newline at end of file diff --git a/bin/artifacts.go b/bin/artifacts.go index 793ee47cb77..81ec645710b 100644 --- a/bin/artifacts.go +++ b/bin/artifacts.go @@ -20,7 +20,6 @@ package main import ( "fmt" "log" - "os" "regexp" "strings" "time" @@ -33,10 +32,8 @@ import ( "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" logging "www.velocidex.com/golang/velociraptor/logging" - "www.velocidex.com/golang/velociraptor/reporting" "www.velocidex.com/golang/velociraptor/server" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" - "www.velocidex.com/golang/vfilter" ) var ( @@ -69,6 +66,10 @@ var ( "store all output in it."). Default("").String() + artifact_command_collect_report = artifact_command_collect.Flag( + "report", "When specified we create a report html file."). + Default("").String() + artifact_command_collect_output_password = artifact_command_collect.Flag( "password", "When specified we encrypt zip file with this password."). Default("").String() @@ -77,10 +78,6 @@ var ( "format", "Output format to use (text,json,csv,jsonl)."). Default("json").Enum("text", "json", "csv", "jsonl") - artifact_command_collect_details = artifact_command_collect.Flag( - "details", "Show more details (Use -d -dd for even more)"). - Short('d').Counter() - artifact_command_collect_name = artifact_command_collect.Arg( "artifact_name", "The artifact name to collect."). Required().HintAction(listArtifactsHint).Strings() @@ -101,128 +98,6 @@ func listArtifactsHint() []string { return result } -func collectArtifact( - config_obj *config_proto.Config, - repository *artifacts.Repository, - artifact_name string, - request *actions_proto.VQLCollectorArgs) { - - builder := artifacts.ScopeBuilder{ - Config: config_obj, - ACLManager: vql_subsystem.NullACLManager{}, - Logger: log.New(&LogWriter{config_obj}, "Velociraptor: ", log.Lshortfile), - Env: ordereddict.NewDict(), - } - - if *run_as != "" { - builder.ACLManager = vql_subsystem.NewServerACLManager(config_obj, *run_as) - } - - for _, request_env := range request.Env { - builder.Env.Set(request_env.Key, request_env.Value) - } - - scope := builder.Build() - defer scope.Close() - - if *trace_vql_flag { - scope.Tracer = logging.NewPlainLogger(config_obj, - &logging.ToolComponent) - } - - ctx := InstallSignalHandler(scope) - - for _, query := range request.Query { - vql, err := vfilter.Parse(query.VQL) - kingpin.FatalIfError(err, "Parse VQL") - - switch *artifact_command_collect_format { - case "text": - var rows []vfilter.Row - for row := range vql.Eval(ctx, scope) { - rows = append(rows, row) - } - - if *artifact_command_collect_details > 0 { - if query.Name != "" { - fmt.Printf("# %s\n\n", query.Name) - } - if query.Description != "" { - fmt.Printf("%s\n\n", reporting.FormatDescription( - config_obj, query.Description, rows)) - } - } - - // Queries without a name do not produce - // interesting results. - table := reporting.OutputRowsToTable(scope, rows, os.Stdout) - if query.Name == "" { - continue - } - table.SetCaption(true, query.Name) - if table.NumLines() > 0 { - table.Render() - } - fmt.Println("") - - case "json": - outputJSON(ctx, scope, vql, os.Stdout) - - case "jsonl": - outputJSONL(ctx, scope, vql, os.Stdout) - - case "csv": - outputCSV(ctx, scope, vql, os.Stdout) - } - } -} - -func collectArtifactToContainer( - config_obj *config_proto.Config, - repository *artifacts.Repository, - artifact_name string, - container *reporting.Container, - format string, - request *actions_proto.VQLCollectorArgs) { - - builder := artifacts.ScopeBuilder{ - Config: config_obj, - ACLManager: vql_subsystem.NullACLManager{}, - Logger: log.New(&LogWriter{config_obj}, "Velociraptor: ", log.Lshortfile), - Env: ordereddict.NewDict(), - Uploader: container, - } - - if *run_as != "" { - builder.ACLManager = vql_subsystem.NewServerACLManager(config_obj, *run_as) - } - - for _, request_env := range request.Env { - builder.Env.Set(request_env.Key, request_env.Value) - } - - scope := builder.Build() - defer scope.Close() - - ctx := InstallSignalHandler(scope) - - for _, query := range request.Query { - vql, err := vfilter.Parse(query.VQL) - kingpin.FatalIfError(err, "Parse VQL") - - // Store query output in the container. - err = container.StoreArtifact( - config_obj, ctx, scope, vql, query, - format) - kingpin.FatalIfError(err, "container.StoreArtifact") - - if query.Name != "" { - logging.GetLogger(config_obj, &logging.ToolComponent). - Info("Collected %s", query.Name) - } - } -} - func getRepository(config_obj *config_proto.Config) (*artifacts.Repository, error) { repository, err := server.GetGlobalRepository(config_obj) kingpin.FatalIfError(err, "Artifact GetGlobalRepository ") @@ -271,25 +146,11 @@ func printParameters(artifacts []string, repository *artifacts.Repository) { } } -// Check if the user specified an unknown parameter. -func valid_parameter(param_name string, repository *artifacts.Repository) bool { - for _, name := range *artifact_command_collect_name { - artifact, _ := repository.Get(name) - for _, param := range artifact.Parameters { - if param.Name == param_name { - return true - } - } - } - - return false -} - func doArtifactCollect() { config_obj, err := DefaultConfigLoader.WithNullLoader().LoadAndValidate() kingpin.FatalIfError(err, "Load Config ") - repository, err := getRepository(config_obj) + _, err = getRepository(config_obj) kingpin.FatalIfError(err, "Loading extra artifacts") now := time.Now() @@ -300,77 +161,41 @@ func doArtifactCollect() { }() - var container *reporting.Container - - if *artifact_command_collect_output != "" { - // Create an output container. - container, err = reporting.NewContainer(*artifact_command_collect_output) - kingpin.FatalIfError(err, "Can not create output container") - defer container.Close() + collect_args := ordereddict.NewDict() + for _, item := range *artifact_command_collect_args { + parts := strings.SplitN(item, "=", 2) + arg_name := parts[0] - // Set the password if there is one. - container.Password = *artifact_command_collect_output_password - } - - for _, name := range *artifact_command_collect_name { - artifact, pres := repository.Get(name) - if !pres { - kingpin.Fatalf("Artifact %v not known.", name) - } - - request := &actions_proto.VQLCollectorArgs{ - MaxWait: uint64(*max_wait), - } - - err := repository.Compile(artifact, request) - kingpin.FatalIfError( - err, fmt.Sprintf("Unable to compile artifact %s.", - name)) - - if env_map != nil { - for k, v := range *env_map { - if !valid_parameter(k, repository) { - kingpin.Fatalf( - "Param %v not known for %s.", - k, strings.Join(*artifact_command_collect_name, ",")) - } - request.Env = append( - request.Env, &actions_proto.VQLEnv{ - Key: k, Value: v, - }) - } + if len(parts) < 2 { + collect_args.Set(arg_name, "Y") + } else { + collect_args.Set(arg_name, parts[1]) } + } - for _, item := range *artifact_command_collect_args { - parts := strings.SplitN(item, "=", 2) - arg_name := parts[0] - var arg string - - if len(parts) < 2 { - arg = "Y" - } else { - arg = parts[1] - } - if !valid_parameter(arg_name, repository) { - printParameters(*artifact_command_collect_name, - repository) - kingpin.Fatalf("Param %v not known for any artifacts.", - arg_name) - } - request.Env = append( - request.Env, &actions_proto.VQLEnv{ - Key: arg_name, Value: arg, - }) - } + scope := artifacts.ScopeBuilder{ + Config: config_obj, + ACLManager: vql_subsystem.NullACLManager{}, + Logger: log.New(&LogWriter{config_obj}, " ", log.Lshortfile), + Env: ordereddict.NewDict(). + Set("Artifacts", *artifact_command_collect_name). + Set("Output", *artifact_command_collect_output). + Set("Password", *artifact_command_collect_output_password). + Set("Report", *artifact_command_collect_report). + Set("Args", collect_args). + Set("Format", *artifact_command_collect_format), + }.Build() + defer scope.Close() - if *artifact_command_collect_output == "" { - collectArtifact(config_obj, repository, name, request) - } else { - collectArtifactToContainer( - config_obj, repository, name, container, - *artifact_command_collect_format, request) - } + if *trace_vql_flag { + scope.Tracer = logging.NewPlainLogger(config_obj, + &logging.ToolComponent) } + + query := ` + SELECT * FROM collect(artifacts=Artifacts, output=Output, report=Report, + password=Password, args=Args, format=Format)` + eval_local_query(config_obj, *artifact_command_collect_format, query, scope) } func getFilterRegEx(pattern string) (*regexp.Regexp, error) { diff --git a/bin/fs.go b/bin/fs.go index fe84b9b56d3..da28561527b 100644 --- a/bin/fs.go +++ b/bin/fs.go @@ -73,32 +73,42 @@ var ( ) func eval_query( - config_obj *config_proto.Config, query string, scope *vfilter.Scope, + config_obj *config_proto.Config, format, query string, scope *vfilter.Scope, env *ordereddict.Dict) { if config_obj.ApiConfig != nil && config_obj.ApiConfig.Name != "" { logging.GetLogger(config_obj, &logging.ToolComponent). Info("API Client configuration loaded - will make gRPC connection.") - doRemoteQuery(config_obj, *fs_command_format, []string{query}, env) + doRemoteQuery(config_obj, format, []string{query}, env) return } - vql, err := vfilter.Parse(query) - if err != nil { - kingpin.FatalIfError(err, "Unable to parse VQL Query") - } + eval_local_query(config_obj, *fs_command_format, query, scope) +} + +func eval_local_query( + config_obj *config_proto.Config, format string, + query string, scope *vfilter.Scope) { + + vqls, err := vfilter.MultiParse(query) + kingpin.FatalIfError(err, "Unable to parse VQL Query") ctx := InstallSignalHandler(scope) - switch *fs_command_format { - case "text": - table := reporting.EvalQueryToTable(ctx, scope, vql, os.Stdout) - table.Render() + for _, vql := range vqls { + switch format { + case "text": + table := reporting.EvalQueryToTable(ctx, scope, vql, os.Stdout) + table.Render() + + case "csv": + outputCSV(ctx, scope, vql, os.Stdout) - case "jsonl": - outputJSONL(ctx, scope, vql, os.Stdout) + case "jsonl": + outputJSONL(ctx, scope, vql, os.Stdout) - case "json": - outputJSON(ctx, scope, vql, os.Stdout) + case "json": + outputJSON(ctx, scope, vql, os.Stdout) + } } } @@ -142,7 +152,7 @@ func doLS(path, accessor string) { query += " WHERE Sys.name_type != 'DOS' " } - eval_query(config_obj, query, scope, builder.Env) + eval_query(config_obj, *fs_command_format, query, scope, builder.Env) } func doRM(path, accessor string) { @@ -179,7 +189,7 @@ func doRM(path, accessor string) { "file_store_delete(path=FullPath) AS Deletion " + "FROM glob(globs=path, accessor=accessor) " - eval_query(config_obj, query, scope, builder.Env) + eval_query(config_obj, *fs_command_format, query, scope, builder.Env) } func doCp(path, accessor string, dump_dir string) { @@ -241,7 +251,7 @@ func doCp(path, accessor string, dump_dir string) { scope.Log("Copy from %v (%v) to %v (%v)", path, accessor, output_path, output_accessor) - eval_query(config_obj, ` + eval_query(config_obj, *fs_command_format, ` SELECT * from foreach( row={ SELECT Name, Size, Mode.String AS Mode, diff --git a/bin/report.go b/bin/report.go new file mode 100644 index 00000000000..7d8fe184e88 --- /dev/null +++ b/bin/report.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "fmt" + + kingpin "gopkg.in/alecthomas/kingpin.v2" + "www.velocidex.com/golang/velociraptor/flows" + "www.velocidex.com/golang/velociraptor/reporting" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" +) + +var ( + report_command = app.Command("report", "Generate a report.") + + report_command_flow = report_command.Command("flow", "Report on a collection") + + report_command_flow_client = report_command_flow.Arg( + "client_id", "The client id to generate the report for."). + Required().String() + + report_command_flow_flow_id = report_command_flow.Arg( + "flow_id", "The flow id to generate the report for."). + Required().String() +) + +func doHTMLReport() { + config_obj, err := DefaultConfigLoader.WithRequiredFrontend().LoadAndValidate() + kingpin.FatalIfError(err, "Unable to load config file") + + _, err = getRepository(config_obj) + kingpin.FatalIfError(err, "Unable to load artifacts") + + result, err := flows.GetFlowDetails(config_obj, *report_command_flow_client, + *report_command_flow_flow_id) + kingpin.FatalIfError(err, "Unable to load flow") + + if result.Context == nil { + kingpin.Fatalf("Unable to open flow %v", *report_command_flow_flow_id) + } + + fmt.Println(reporting.HtmlPreable) + defer fmt.Println(reporting.HtmlPostscript) + + for _, artifact_name := range result.Context.Request.Artifacts { + template_engine, err := reporting.NewHTMLTemplateEngine( + config_obj, context.Background(), nil, /* default scope */ + vql_subsystem.NullACLManager{}, artifact_name) + kingpin.FatalIfError(err, "Generating report") + + template_engine.SetEnv("ClientId", *report_command_flow_client) + template_engine.SetEnv("FlowId", *report_command_flow_flow_id) + + for k, v := range *env_map { + template_engine.SetEnv(k, v) + } + + res, err := reporting.GenerateClientReport( + template_engine, + *report_command_flow_client, *report_command_flow_flow_id, nil) + kingpin.FatalIfError(err, "Generating report") + fmt.Println(res) + } +} + +func init() { + command_handlers = append(command_handlers, func(command string) bool { + switch command { + case report_command_flow.FullCommand(): + doHTMLReport() + + default: + return false + } + return true + }) +} diff --git a/file_store/utils.go b/file_store/utils.go deleted file mode 100644 index a879d910336..00000000000 --- a/file_store/utils.go +++ /dev/null @@ -1,12 +0,0 @@ -package file_store - -/* -type XWriterAdapter struct { - FileWriter -} - -func (self *XWriterAdapter) Write(data []byte) (int, error) { - err := self.Append(data) - return len(data), err -} -*/ diff --git a/reporting/container.go b/reporting/container.go index da9e0c5b65c..fbd2cedd8c5 100644 --- a/reporting/container.go +++ b/reporting/container.go @@ -14,6 +14,7 @@ import ( "strings" "sync" + "github.com/Velocidex/ordereddict" "github.com/alexmullins/zip" "github.com/pkg/errors" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" @@ -29,10 +30,53 @@ type Container struct { fd io.WriteCloser zip *zip.Writer + tempfiles map[string]*os.File + Password string delegate_zip *zip.Writer } +func (self *Container) writeToContainer( + ctx context.Context, + tmpfile *os.File, + scope *vfilter.Scope, + artifact, format string) error { + + path_manager := NewContainerPathManager(artifact) + + // Copy the original json file into the container. + writer, closer, err := self.getZipFileWriter(path_manager.Path()) + if err != nil { + return err + } + + tmpfile.Seek(0, 0) + + _, err = utils.Copy(ctx, writer, tmpfile) + closer() + + switch format { + case "csv", "": + writer, closer, err := self.getZipFileWriter(path_manager.CSVPath()) + if err != nil { + return err + } + + csv_writer := csv.GetCSVAppender( + scope, writer, true /* write_headers */) + defer csv_writer.Close() + + // Convert from the json to csv. + tmpfile.Seek(0, 0) + + for item := range utils.ReadJsonFromFile(ctx, tmpfile) { + csv_writer.Write(item) + } + closer() + } + return nil +} + func (self *Container) StoreArtifact( config_obj *config_proto.Config, ctx context.Context, @@ -41,8 +85,10 @@ func (self *Container) StoreArtifact( query *actions_proto.VQLRequest, format string) error { + artifact_name := query.Name + // Dont store un-named queries but run them anyway. - if query.Name == "" { + if artifact_name == "" { for _ = range vql.Eval(ctx, scope) { } return nil @@ -56,58 +102,26 @@ func (self *Container) StoreArtifact( return errors.WithStack(err) } - defer os.Remove(tmpfile.Name()) // clean up - - var sanitized_name string + self.tempfiles[artifact_name] = tmpfile - switch format { - case "csv", "": - // In this instance we want to make / unescaped. - sanitized_name = query.Name + ".csv" - - csv_writer := csv.GetCSVAppender( - scope, tmpfile, true /* write_headers */) - defer csv_writer.Close() - - for row := range vql.Eval(ctx, scope) { - csv_writer.Write(row) + for row := range vql.Eval(ctx, scope) { + // Re-serialize it as compact json. + serialized, err := json.Marshal(row) + if err != nil { + continue } - case "jsonl", "json": - // In this instance we want to make / unescaped. - sanitized_name = query.Name + ".json" - - for row := range vql.Eval(ctx, scope) { - // Re-serialize it as compact json. - serialized, err := json.Marshal(row) - if err != nil { - continue - } - - _, err = tmpfile.Write(serialized) - if err != nil { - return errors.WithStack(err) - } - - // Separate lines with \n - tmpfile.Write([]byte("\n")) + _, err = tmpfile.Write(serialized) + if err != nil { + return errors.WithStack(err) } - default: - return errors.New("Format not supported") + // Separate lines with \n + tmpfile.Write([]byte("\n")) } - writer, closer, err := self.getZipFileWriter(string(sanitized_name)) - if err != nil { - return err - } - - tmpfile.Seek(0, 0) - - _, err = utils.Copy(ctx, writer, tmpfile) - closer() - - return err + return self.writeToContainer( + ctx, tmpfile, scope, artifact_name, format) } func (self *Container) getZipFileWriter(name string) (io.Writer, func(), error) { @@ -193,6 +207,24 @@ func sanitize(component string) string { return component } +func (self *Container) ReadArtifactResults( + ctx context.Context, + scope *vfilter.Scope, artifact string) chan *ordereddict.Dict { + output_chan := make(chan *ordereddict.Dict) + + // Get the tempfile we hold open with the results for this artifact + fd, pres := self.tempfiles[artifact] + if !pres { + scope.Log("Trying to access results for artifact %v, "+ + "but we did not collect it!", artifact) + close(output_chan) + return output_chan + } + + fd.Seek(0, 0) + return utils.ReadJsonFromFile(ctx, fd) +} + func (self *Container) Upload( ctx context.Context, scope *vfilter.Scope, @@ -251,6 +283,11 @@ func (self *Container) Close() error { self.delegate_zip.Close() } + // Remove all the tempfiles we still hold open + for _, file := range self.tempfiles { + os.Remove(file.Name()) + } + self.zip.Close() return self.fd.Close() } @@ -264,8 +301,9 @@ func NewContainer(path string) (*Container, error) { zip_writer := zip.NewWriter(fd) return &Container{ - fd: fd, - zip: zip_writer, + fd: fd, + tempfiles: make(map[string]*os.File), + zip: zip_writer, }, nil } diff --git a/reporting/gui.go b/reporting/gui.go index d13a6a22a9a..be155b19843 100644 --- a/reporting/gui.go +++ b/reporting/gui.go @@ -23,6 +23,7 @@ import ( actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/result_sets" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -392,13 +393,14 @@ func (self *logWriter) Messages() []string { func NewGuiTemplateEngine( config_obj *config_proto.Config, ctx context.Context, - principal string, + scope *vfilter.Scope, + acl_manager vql_subsystem.ACLManager, notebook_cell_path_manager *NotebookCellPathManager, artifact_name string) ( *GuiTemplateEngine, error) { base_engine, err := newBaseTemplateEngine( - config_obj, principal, artifact_name) + config_obj, scope, acl_manager, artifact_name) if err != nil { return nil, err } diff --git a/reporting/html.go b/reporting/html.go new file mode 100644 index 00000000000..be1094a5454 --- /dev/null +++ b/reporting/html.go @@ -0,0 +1,203 @@ +// This implements a template renderer for the GUI environment. + +package reporting + +import ( + "bytes" + "context" + "fmt" + "html" + "log" + "strings" + "text/template" + + "github.com/Depado/bfchroma" + "github.com/Masterminds/sprig" + "github.com/Velocidex/ordereddict" + + chroma_html "github.com/alecthomas/chroma/formatters/html" + blackfriday "github.com/russross/blackfriday/v2" + actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/vfilter" +) + +type HTMLTemplateEngine struct { + *BaseTemplateEngine + tmpl *template.Template + ctx context.Context + log_writer *logWriter + Data map[string]*actions_proto.VQLResponse +} + +func (self *HTMLTemplateEngine) Table(values ...interface{}) interface{} { + _, argv := parseOptions(values) + // Not enough args. + if len(argv) != 1 { + return "" + } + + switch t := argv[0].(type) { + default: + return t + + case chan *ordereddict.Dict: + columns := []string{} + + result := "\n" + + for item := range t { + if len(columns) == 0 { + columns = item.Keys() + result += " \n" + for _, name := range columns { + result += " \n" + } + result += " \n" + } + + result += " \n" + for _, name := range columns { + value, _ := item.Get(name) + result += fmt.Sprintf(" \n", value) + } + result += " \n" + } + result += "
" + name + "
%v
\n" + return result + } +} + +func (self *HTMLTemplateEngine) Noop(values ...interface{}) string { + return "" +} + +func (self *HTMLTemplateEngine) Execute(template_string string) (string, error) { + tmpl, err := self.tmpl.Parse(template_string) + if err != nil { + return "", err + } + + buffer := &bytes.Buffer{} + err = tmpl.Execute(buffer, self.Artifact) + if err != nil { + self.Error("Template Erorr: %v", err) + return "", err + } + + // We expect the template to be in markdown format, so now + // generate the HTML + output := blackfriday.Run( + buffer.Bytes(), + blackfriday.WithRenderer(bfchroma.NewRenderer( + bfchroma.ChromaOptions( + chroma_html.ClassPrefix("chroma"), + chroma_html.WithClasses(true), + chroma_html.WithLineNumbers(true)), + bfchroma.Style("github"), + ))) + + // Add classes to various tags + output_string := strings.ReplaceAll(string(output), + "", "
") + + // Sanitize the HTML. + return bm_policy.Sanitize(output_string), nil +} + +func (self *HTMLTemplateEngine) getMultiLineQuery(query string) (string, error) { + t := self.tmpl.Lookup(query) + if t == nil { + return query, nil + } + + buf := &bytes.Buffer{} + err := t.Execute(buf, self.Artifact) + if err != nil { + return "", err + } + + // html/template escapes its template but this + // is the wrong thing to do for us because we + // use the template as a work around for + // text/template actions not spanning multiple + // lines. + return html.UnescapeString(buf.String()), nil +} + +func (self *HTMLTemplateEngine) Query(queries ...string) interface{} { + output_chan := make(chan *ordereddict.Dict) + + go func() { + defer close(output_chan) + + for _, multiquery := range queries { + query, err := self.getMultiLineQuery(multiquery) + if err != nil { + self.Error("VQL Error: %v", err) + return + } + + multi_vql, err := vfilter.MultiParse(query) + if err != nil { + self.Error("VQL Error: %v", err) + return + } + + ctx, cancel := context.WithCancel(self.ctx) + defer cancel() + + for _, vql := range multi_vql { + for row := range vql.Eval(ctx, self.Scope) { + output_chan <- vfilter.RowToDict(ctx, self.Scope, row) + } + } + } + }() + + return output_chan +} + +func (self *HTMLTemplateEngine) Error(fmt_str string, argv ...interface{}) string { + self.Scope.Log(fmt_str, argv...) + return "" +} + +func (self *HTMLTemplateEngine) Messages() []string { + return self.log_writer.Messages() +} + +func NewHTMLTemplateEngine( + config_obj *config_proto.Config, + ctx context.Context, + scope *vfilter.Scope, + acl_manager vql_subsystem.ACLManager, + artifact_name string) ( + *HTMLTemplateEngine, error) { + + base_engine, err := newBaseTemplateEngine( + config_obj, scope, acl_manager, artifact_name) + if err != nil { + return nil, err + } + + log_writer := &logWriter{} + base_engine.Scope.Logger = log.New(log_writer, "", 0) + template_engine := &HTMLTemplateEngine{ + BaseTemplateEngine: base_engine, + ctx: ctx, + log_writer: log_writer, + } + template_engine.tmpl = template.New("").Funcs(sprig.TxtFuncMap()).Funcs( + template.FuncMap{ + "Query": template_engine.Query, + "Scope": template_engine.GetScope, + "Table": template_engine.Table, + "LineChart": template_engine.Noop, + "Timeline": template_engine.Noop, + "Get": template_engine.getFunction, + "str": strval, + }) + return template_engine, nil +} diff --git a/reporting/notebooks.go b/reporting/notebooks.go index 52a1ce2f689..27f030a8238 100644 --- a/reporting/notebooks.go +++ b/reporting/notebooks.go @@ -27,7 +27,7 @@ var ( ) const ( - htmlPreable = ` + HtmlPreable = ` @@ -147,7 +147,7 @@ pre { ` - htmlPostscript = ` + HtmlPostscript = ` @@ -267,8 +267,8 @@ func ExportNotebookToHTML( return err } - output.Write([]byte(fmt.Sprintf(htmlPreable, notebook.Name))) - defer output.Write([]byte(htmlPostscript)) + output.Write([]byte(fmt.Sprintf(HtmlPreable, notebook.Name))) + defer output.Write([]byte(HtmlPostscript)) cell := &api_proto.NotebookCell{} for _, cell_md := range notebook.CellMetadata { diff --git a/reporting/paths.go b/reporting/paths.go index f3befadc4c3..7917eb32db7 100644 --- a/reporting/paths.go +++ b/reporting/paths.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path" + "strings" "time" "github.com/Velocidex/ordereddict" @@ -130,3 +131,22 @@ func (self *NotebookExportPathManager) CellItem(cell_id, name string) string { func NewNotebookExportPathManager(notebook_id string) *NotebookExportPathManager { return &NotebookExportPathManager{notebook_id} } + +type ContainerPathManager struct { + artifact string +} + +func (self *ContainerPathManager) Path() string { + return self.artifact + ".json" +} + +func (self *ContainerPathManager) CSVPath() string { + return self.artifact + ".csv" +} + +func NewContainerPathManager(artifact string) *ContainerPathManager { + // Zip paths must not have leading / + artifact = strings.TrimPrefix(artifact, "/") + + return &ContainerPathManager{artifact: artifact} +} diff --git a/reporting/report.go b/reporting/report.go index 4571f32ab9c..2adb12db7e5 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -267,7 +267,8 @@ func GenerateHuntReport(template_engine TemplateEngine, func newBaseTemplateEngine( config_obj *config_proto.Config, - principal string, + scope *vfilter.Scope, + acl_manager vql_subsystem.ACLManager, artifact_name string) ( *BaseTemplateEngine, error) { repository, err := artifacts.GetGlobalRepository(config_obj) @@ -285,10 +286,13 @@ func newBaseTemplateEngine( // whole processing. Keep a reference to the environment so // SetEnv() can update it later. env := ordereddict.NewDict() - scope := artifacts.ScopeBuilder{ - Config: config_obj, - ACLManager: vql_subsystem.NewServerACLManager(config_obj, principal), - }.Build().AppendVars(env) + if scope == nil { + scope = artifacts.ScopeBuilder{ + Config: config_obj, + ACLManager: acl_manager, + }.Build() + } + scope.AppendVars(env) // Closing the scope is deferred to closing the template. diff --git a/reporting/text_expander.go b/reporting/text_expander.go index 51df9fb949b..cb1325c809f 100644 --- a/reporting/text_expander.go +++ b/reporting/text_expander.go @@ -8,6 +8,7 @@ import ( "github.com/olekukonko/tablewriter" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/utils" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -113,10 +114,11 @@ func (self *TextTemplateEngine) Table(values ...interface{}) string { func NewTextTemplateEngine( config_obj *config_proto.Config, - principal string, + scope *vfilter.Scope, + acl_manager vql_subsystem.ACLManager, artifact_name string) (*TextTemplateEngine, error) { base_engine, err := newBaseTemplateEngine( - config_obj, principal, artifact_name) + config_obj, scope, acl_manager, artifact_name) if err != nil { return nil, err } diff --git a/utils/json.go b/utils/json.go index fddb41cc4ef..e9862f7d695 100644 --- a/utils/json.go +++ b/utils/json.go @@ -1,8 +1,11 @@ package utils import ( + "bufio" "bytes" + "context" "encoding/json" + "io" "github.com/Velocidex/ordereddict" errors "github.com/pkg/errors" @@ -85,3 +88,35 @@ func JsonToJsonl(rows []byte) ([]byte, error) { } return DictsToJson(dict_rows) } + +func ReadJsonFromFile(ctx context.Context, fd io.Reader) chan *ordereddict.Dict { + output_chan := make(chan *ordereddict.Dict) + + go func() { + defer close(output_chan) + + reader := bufio.NewReader(fd) + + for { + select { + case <-ctx.Done(): + return + + default: + row_data, err := reader.ReadBytes('\n') + if len(row_data) == 0 || err != nil { + return + } + item := ordereddict.NewDict() + err = item.UnmarshalJSON(row_data) + if err != nil { + continue + } + + output_chan <- item + } + } + }() + + return output_chan +} diff --git a/vql/acls.go b/vql/acls.go index ff378c61a51..c88ca631274 100644 --- a/vql/acls.go +++ b/vql/acls.go @@ -17,7 +17,7 @@ const ( ) type ACLManager interface { - CheckAccess(permission acls.ACL_PERMISSION, args ...string) (bool, error) + CheckAccess(permission ...acls.ACL_PERMISSION) (bool, error) } // NullACLManager is an acl manager which allows everything. This is @@ -26,7 +26,7 @@ type ACLManager interface { type NullACLManager struct{} func (self NullACLManager) CheckAccess( - permission acls.ACL_PERMISSION, args ...string) (bool, error) { + permission ...acls.ACL_PERMISSION) (bool, error) { return true, nil } @@ -38,8 +38,15 @@ type ServerACLManager struct { } func (self *ServerACLManager) CheckAccess( - permission acls.ACL_PERMISSION, args ...string) (bool, error) { - return acls.CheckAccessWithToken(self.Token, permission, args...) + permissions ...acls.ACL_PERMISSION) (bool, error) { + for _, permission := range permissions { + ok, err := acls.CheckAccessWithToken(self.Token, permission) + if !ok || err != nil { + return ok, err + } + } + + return true, nil } // NewRoleACLManager creates an ACL manager with only the assigned @@ -73,20 +80,20 @@ func NewServerACLManager( // from within VQL so this is a safe assumption - if a user was to // override the ACL_MANAGER_VAR with something else this will lock // down the entire VQL ACL system and deny all permissions. -func CheckAccess(scope *vfilter.Scope, permission acls.ACL_PERMISSION) error { +func CheckAccess(scope *vfilter.Scope, permissions ...acls.ACL_PERMISSION) error { manager_any, pres := scope.Resolve(ACL_MANAGER_VAR) if !pres { - return errors.New(fmt.Sprintf("Permission denied: %s", permission)) + return errors.New(fmt.Sprintf("Permission denied: %v", permissions)) } manager, ok := manager_any.(ACLManager) if !ok { - return errors.New(fmt.Sprintf("Permission denied: %s", permission)) + return errors.New(fmt.Sprintf("Permission denied: %v", permissions)) } - perm, err := manager.CheckAccess(permission) + perm, err := manager.CheckAccess(permissions...) if !perm || err != nil { - return errors.New(fmt.Sprintf("Permission denied: %s", permission)) + return errors.New(fmt.Sprintf("Permission denied: %v", permissions)) } return nil diff --git a/vql/server/results.go b/vql/server/results.go index 6010682b898..8c1e7d82692 100644 --- a/vql/server/results.go +++ b/vql/server/results.go @@ -135,7 +135,7 @@ func (self SourcePlugin) Call( // parameters. This allows its use to be more concise in // reports etc where many parameters can be inferred from // context. - parseSourceArgsFromScope(arg, scope) + ParseSourceArgsFromScope(arg, scope) // Allow the plugin args to override the environment scope. err = vfilter.ExtractArgs(scope, args, arg) @@ -193,7 +193,7 @@ func (self SourcePlugin) Info( } // Override SourcePluginArgs from the scope. -func parseSourceArgsFromScope(arg *SourcePluginArgs, scope *vfilter.Scope) { +func ParseSourceArgsFromScope(arg *SourcePluginArgs, scope *vfilter.Scope) { client_id, pres := scope.Resolve("ClientId") if pres { arg.ClientId, _ = client_id.(string) diff --git a/vql/tools/collector.go b/vql/tools/collector.go index f708448b295..64dbc0b303e 100644 --- a/vql/tools/collector.go +++ b/vql/tools/collector.go @@ -1,13 +1,18 @@ +// +build server_vql + package tools import ( "context" "encoding/json" + "os" "github.com/Velocidex/ordereddict" "www.velocidex.com/golang/velociraptor/acls" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" "www.velocidex.com/golang/velociraptor/artifacts" + artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" + "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/reporting" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -17,6 +22,7 @@ import ( type CollectPluginArgs struct { Artifacts []string `vfilter:"required,field=artifacts,doc=A list of artifacts to collect."` Output string `vfilter:"required,field=output,doc=A path to write the output file on."` + Report string `vfilter:"optional,field=report,doc=A path to write the report on."` Args vfilter.Any `vfilter:"optional,field=args,doc=Optional parameters."` Password string `vfilter:"optional,field=password,doc=An optional password to encrypt the collection zip."` Format string `vfilter:"optional,field=format,doc=Output format (csv, jsonl)."` @@ -34,9 +40,15 @@ func (self CollectPlugin) Call( go func() { defer close(output_chan) - // ACLs will be carried through to the collected - // artifacts from this plugin. - err := vql_subsystem.CheckAccess(scope, acls.COLLECT_CLIENT) + var container *reporting.Container + + // This plugin allows one to create files, collect + // artifacts and also define new artifacts. It is very + // privileged. + err := vql_subsystem.CheckAccess(scope, + acls.COLLECT_SERVER, acls.ARTIFACT_WRITER, + acls.FILESYSTEM_WRITE, + acls.SERVER_ARTIFACT_WRITER) if err != nil { scope.Log("collect: %s", err) return @@ -58,25 +70,61 @@ func (self CollectPlugin) Call( return } - config_obj := &config_proto.Config{} - container, err := reporting.NewContainer(arg.Output) + config_obj, ok := artifacts.GetServerConfig(scope) + if !ok { + config_obj = config.GetDefaultConfig() + } + repository, err := getRepository(config_obj, arg.ArtifactDefinitions) if err != nil { scope.Log("collect: %v", err) return } - defer func() { - container.Close() - output_chan <- ordereddict.NewDict(). - Set("Container", arg.Output) - }() - // Should we encrypt it? - container.Password = arg.Password + artifact_definitions := []*artifacts_proto.Artifact{} + definitions := []*artifacts_proto.Artifact{} + for _, name := range arg.Artifacts { + artifact, pres := repository.Get(name) + if !pres { + scope.Log("Artifact %v not known.", name) + return + } + definitions = append(definitions, artifact) + } - repository, err := getRepository(config_obj, arg.ArtifactDefinitions) - if err != nil { - scope.Log("collect: %v", err) - return + if arg.Output != "" { + container, err = reporting.NewContainer(arg.Output) + if err != nil { + scope.Log("collect: %v", err) + return + } + + scope.Log("Will create container at %s", arg.Output) + + defer func() { + container.Close() + + if arg.Report != "" { + fd, err := os.OpenFile( + arg.Report, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700) + if err != nil { + scope.Log("Error creating report: %v", err) + return + } + + produceReport(config_obj, container, fd, + artifact_definitions, + scope, arg) + } + output_chan <- ordereddict.NewDict(). + Set("Container", arg.Output). + Set("Report", arg.Report) + }() + + // Should we encrypt it? + if arg.Password != "" { + container.Password = arg.Password + scope.Log("Will password protect container") + } } builder := artifacts.ScopeBuilderFromScope(scope) @@ -89,6 +137,7 @@ func (self CollectPlugin) Call( continue } + artifact_definitions = append(artifact_definitions, artifact) request := &actions_proto.VQLCollectorArgs{} err := repository.Compile(artifact, request) @@ -98,34 +147,19 @@ func (self CollectPlugin) Call( continue } - // First set defaulst + // First set defaults builder.Env = ordereddict.NewDict() for _, e := range request.Env { builder.Env.Set(e.Key, e.Value) } - in_params := func(key string) bool { - for _, param := range artifact.Parameters { - if param.Name == key { - return true - } - } - return false - } - // Now override provided parameters for _, key := range scope.GetMembers(arg.Args) { - if !in_params(key) { - // This is not an error - it - // just means that there are - // muliple artifacts to - // collect with different - // parameters. - continue + if !valid_parameter(key, definitions) { + scope.Log("Unknown parameter %s - ignoring", key) } - value, pres := scope.Associative(arg.Args, - key) + value, pres := scope.Associative(arg.Args, key) if pres { builder.Env.Set(key, value) } @@ -143,9 +177,26 @@ func (self CollectPlugin) Call( return } + // Dont store un-named queries but run them anyway. + if query.Name == "" { + for _ = range vql.Eval(ctx, subscope) { + } + continue + } + + // If no container is specified, just + // push the result to the output + // channel. + if container == nil { + for row := range vql.Eval(ctx, subscope) { + output_chan <- row + } + continue + } + + // Otherwise push the results into the container. err = container.StoreArtifact( - config_obj, - ctx, subscope, vql, query, arg.Format) + config_obj, ctx, subscope, vql, query, arg.Format) if err != nil { subscope.Log("collect: %v", err) return @@ -231,6 +282,18 @@ func (self CollectPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) } } +// Check if the user specified an unknown parameter. +func valid_parameter(param_name string, definitions []*artifacts_proto.Artifact) bool { + for _, definition := range definitions { + for _, param := range definition.Parameters { + if param.Name == param_name { + return true + } + } + } + return false +} + func init() { vql_subsystem.RegisterPlugin(&CollectPlugin{}) } diff --git a/vql/tools/reporting.go b/vql/tools/reporting.go new file mode 100644 index 00000000000..49d383a4fdc --- /dev/null +++ b/vql/tools/reporting.go @@ -0,0 +1,114 @@ +// +build server_vql + +package tools + +import ( + "context" + "io" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/artifacts" + artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/reporting" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/velociraptor/vql/server" + "www.velocidex.com/golang/vfilter" +) + +// Produce a collector report. +func produceReport( + config_obj *config_proto.Config, + container *reporting.Container, + writer io.Writer, + definitions []*artifacts_proto.Artifact, + scope *vfilter.Scope, + arg *CollectPluginArgs) error { + + builder := artifacts.ScopeBuilderFromScope(scope) + builder.Uploader = nil + + // Build scope from scratch and replace the source() + // plugin. We hook the source plugin to read results from the + // collection container. + subscope := builder.BuildFromScratch() + defer subscope.Close() + + // Reports can query the container directly. + subscope.AppendPlugins(&ContainerSourcePlugin{Container: container}) + + writer.Write([]byte(reporting.HtmlPreable)) + defer writer.Write([]byte(reporting.HtmlPostscript)) + + for _, definition := range definitions { + for _, report := range definition.Reports { + if report.Type != "client" { + continue + } + + template_engine, err := reporting.NewHTMLTemplateEngine( + config_obj, context.Background(), subscope, + vql_subsystem.NullACLManager{}, definition.Name) + if err != nil { + return err + } + + for _, param := range report.Parameters { + template_engine.SetEnv(param.Name, param.Default) + } + + res, err := reporting.GenerateClientReport( + template_engine, "", "", nil) + if err != nil { + return err + } + + writer.Write([]byte(res)) + } + } + return nil +} + +// A special implementation of the source() plugin which retrieves +// data stored in reporting containers. This only exists in generating +// reports from zip files. +type ContainerSourcePlugin struct { + server.SourcePlugin + Container *reporting.Container +} + +func (self *ContainerSourcePlugin) Call( + ctx context.Context, + scope *vfilter.Scope, + args *ordereddict.Dict) <-chan vfilter.Row { + output_chan := make(chan vfilter.Row) + + go func() { + defer close(output_chan) + + // This plugin will take parameters from environment + // parameters. This allows its use to be more concise in + // reports etc where many parameters can be inferred from + // context. + arg := &server.SourcePluginArgs{} + server.ParseSourceArgsFromScope(arg, scope) + + // Allow the plugin args to override the environment scope. + err := vfilter.ExtractArgs(scope, args, arg) + if err != nil { + scope.Log("source: %v", err) + return + } + + if arg.Source != "" { + arg.Artifact = arg.Artifact + "/" + arg.Source + arg.Source = "" + } + + for row := range self.Container.ReadArtifactResults(ctx, scope, arg.Artifact) { + output_chan <- row + } + }() + + return output_chan +}