Special Associative protocol for protobufs.

medtemo · Dec 5, 2018 · ad041c1 · ad041c1
1 parent ae91826
commit ad041c1
Show file tree

Hide file tree

Showing 33 changed files with 316 additions and 110 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/artifacts/plugin.go b/artifacts/plugin.go
@@ -14,7 +14,7 @@ import (
 
 type ArtifactRepositoryPlugin struct {
 	repository *Repository
-	children   map[string]*ArtifactRepositoryPlugin
+	children   map[string]vfilter.PluginGeneratorInterface
 	prefix     []string
 	leaf       *artifacts_proto.Artifact
 }
@@ -27,7 +27,7 @@ func (self *ArtifactRepositoryPlugin) Print() {
 	fmt.Printf("prefix '%v', Children %v, Leaf %v\n",
 		self.prefix, children, self.leaf != nil)
 	for _, v := range self.children {
-		v.Print()
+		v.(*ArtifactRepositoryPlugin).Print()
 	}
 }
 
@@ -97,9 +97,10 @@ func (self *ArtifactRepositoryPlugin) Name() string {
 }
 
 func (self *ArtifactRepositoryPlugin) Info(
-	type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+	scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name: self.Name(),
+		Doc:  "A pseudo plugin for accessing the artifacts repository from VQL.",
 	}
 }
 
@@ -161,10 +162,10 @@ func (self _ArtifactRepositoryPluginAssociativeProtocol) Associative(
 }
 
 func NewArtifactRepositoryPlugin(
-	repository *Repository, prefix []string) *ArtifactRepositoryPlugin {
+	repository *Repository, prefix []string) vfilter.PluginGeneratorInterface {
 	result := &ArtifactRepositoryPlugin{
 		repository: repository,
-		children:   make(map[string]*ArtifactRepositoryPlugin),
+		children:   make(map[string]vfilter.PluginGeneratorInterface),
 		prefix:     prefix,
 	}
 

diff --git a/artifacts/testdata/server/testcases/clients.in.yaml b/artifacts/testdata/server/testcases/clients.in.yaml
@@ -0,0 +1,15 @@
+Queries:
+  # Just get a list of all clients and their hostnames.
+  - SELECT OsInfo.fqdn as Hostname, ClientId, LastSeenAt / 1000000  as LastSeen FROM clients() order by Hostname
+
+  # Check that the clients plugin allows searching by indexes.
+  - SELECT OsInfo.fqdn as Hostname, OsInfo.system as System, ClientId FROM clients(search='host:testcomputer')
+
+  - SELECT * from clients()
+
+  - |
+    SELECT client_id, context.create_time as CreateTime,
+           runner_args.args.artifacts.names as Artifacts,
+           runner_args.flow_name as Flow
+    FROM flows(client_id='C.11a3013cca8f826e')
+    WHERE Flow = 'ArtifactCollector'
diff --git a/artifacts/testdata/server/testcases/clients.out.yaml b/artifacts/testdata/server/testcases/clients.out.yaml
@@ -0,0 +1,77 @@
+SELECT OsInfo.fqdn as Hostname, ClientId, LastSeenAt / 1000000  as LastSeen FROM clients() order by Hostname[
+ {
+  "ClientId": "C.c916a7e445eb0868",
+  "Hostname": "DESKTOP-IOME2K5",
+  "LastSeen": 1541049785.896051
+ },
+ {
+  "ClientId": "C.11a3013cca8f826e",
+  "Hostname": "TestComputer",
+  "LastSeen": 1542151833.433916
+ },
+ {
+  "ClientId": "C.952156a4b022ddee",
+  "Hostname": "trek",
+  "LastSeen": 1540368672.777812
+ }
+]SELECT OsInfo.fqdn as Hostname, OsInfo.system as System, ClientId FROM clients(search='host:testcomputer')[
+ {
+  "ClientId": "C.11a3013cca8f826e",
+  "Hostname": "TestComputer",
+  "System": "windows"
+ }
+]SELECT * from clients()[
+ {
+  "client_id": "C.11a3013cca8f826e",
+  "agent_information": {},
+  "os_info": {
+   "system": "windows",
+   "release": "Microsoft Windows 10 Pro N10.0.15063 Build 15063",
+   "machine": "amd64",
+   "fqdn": "TestComputer"
+  },
+  "last_seen_at": 1542151833433916,
+  "last_ip": "192.168.0.11:51087",
+  "last_ip_class": 1
+ },
+ {
+  "client_id": "C.952156a4b022ddee",
+  "agent_information": {},
+  "os_info": {
+   "system": "linux",
+   "release": "ubuntu18.10",
+   "machine": "amd64",
+   "fqdn": "trek"
+  },
+  "last_seen_at": 1540368672777812,
+  "last_ip": "192.168.0.5:33510",
+  "last_ip_class": 1
+ },
+ {
+  "client_id": "C.c916a7e445eb0868",
+  "agent_information": {},
+  "os_info": {
+   "system": "windows",
+   "release": "Microsoft Windows 10 Pro N10.0.17134 Build 17134",
+   "machine": "amd64",
+   "fqdn": "DESKTOP-IOME2K5"
+  },
+  "last_seen_at": 1541049785896051,
+  "last_ip": "192.168.0.18:49749",
+  "last_ip_class": 1
+ }
+]SELECT client_id, context.create_time as CreateTime,
+       runner_args.args.artifacts.names as Artifacts,
+       runner_args.flow_name as Flow
+FROM flows(client_id='C.11a3013cca8f826e')
+WHERE Flow = 'ArtifactCollector'[
+ {
+  "Artifacts": [
+   "Windows.Applications.ChocolateyPackages",
+   "Windows.Applications.Chrome.Extensions"
+  ],
+  "CreateTime": 1541550146808437,
+  "Flow": "ArtifactCollector",
+  "client_id": "C.11a3013cca8f826e"
+ }
+]
diff --git a/bin/vql.go b/bin/vql.go
@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"strings"
+
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	vfilter "www.velocidex.com/golang/vfilter"
 )
@@ -20,7 +21,7 @@ func doVQLList() {
 	fmt.Println("VQL Functions:")
 	for _, item := range info.Functions {
 		fmt.Printf("%s: %s\n", item.Name, item.Doc)
-		arg_desc, pres := type_map.Get(item.ArgType)
+		arg_desc, pres := type_map.Get(scope, item.ArgType)
 		if pres {
 			fmt.Printf("  Args:\n")
 			for k, v := range arg_desc.Fields {
@@ -42,7 +43,7 @@ func doVQLList() {
 	fmt.Println("VQL Plugins:")
 	for _, item := range info.Plugins {
 		fmt.Printf("%s: %s\n", item.Name, item.Doc)
-		arg_desc, pres := type_map.Get(item.ArgType)
+		arg_desc, pres := type_map.Get(scope, item.ArgType)
 		if pres {
 			fmt.Printf("  Args:\n")
 			for k, v := range arg_desc.Fields {

diff --git a/vql/binary.go b/vql/binary.go
@@ -188,11 +188,11 @@ func (self _BinaryParserPlugin) Name() string {
 	return "binary_parse"
 }
 
-func (self _BinaryParserPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self _BinaryParserPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "binary_parse",
 		Doc:     "Parse binary files using a profile.",
-		ArgType: type_map.AddType(&_BinaryParserPluginArg{}),
+		ArgType: type_map.AddType(scope, &_BinaryParserPluginArg{}),
 	}
 }
 
@@ -277,11 +277,11 @@ func (self _BinaryParserFunction) Name() string {
 	return "binary_parse"
 }
 
-func (self _BinaryParserFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+func (self _BinaryParserFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "binary_parse",
 		Doc:     "Parse a binary string with profile based parser.",
-		ArgType: type_map.AddType(&_BinaryParserFunctionArg{}),
+		ArgType: type_map.AddType(scope, &_BinaryParserFunctionArg{}),
 	}
 }
 

diff --git a/vql/common/clock.go b/vql/common/clock.go
@@ -54,7 +54,7 @@ func (self ClockPlugin) Call(
 	return output_chan
 }
 
-func (self ClockPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self ClockPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name: "clock",
 		Doc: "Generate a timestamp periodically. This is mostly " +

diff --git a/vql/common/env.go b/vql/common/env.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"strings"
+
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	vfilter "www.velocidex.com/golang/vfilter"
 )
@@ -31,11 +32,11 @@ func (self *EnvFunction) Call(ctx context.Context,
 	return os.Getenv(arg.Var)
 }
 
-func (self *EnvFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+func (self *EnvFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "environ",
 		Doc:     "Get an environment variable.",
-		ArgType: type_map.AddType(&EnvFunctionArgs{}),
+		ArgType: type_map.AddType(scope, &EnvFunctionArgs{}),
 	}
 }
 

diff --git a/vql/common/monitoring.go b/vql/common/monitoring.go
@@ -33,11 +33,11 @@ func (self WatchPlugin) Call(
 	return output_chan
 }
 
-func (self WatchPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self WatchPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "watch",
 		Doc:     "Run query periodically and watch for changes in output.",
-		ArgType: type_map.AddType(&WatchPluginArgs{}),
+		ArgType: type_map.AddType(scope, &WatchPluginArgs{}),
 	}
 }
 

diff --git a/vql/common/shell.go b/vql/common/shell.go
@@ -124,7 +124,7 @@ func (self ShellPlugin) Call(
 	return output_chan
 }
 
-func (self ShellPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self ShellPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "execve",
 		Doc:     "Execute the commands given by argv.",

diff --git a/vql/common/yara.go b/vql/common/yara.go
@@ -161,7 +161,7 @@ func (self YaraScanPlugin) Call(
 	return output_chan
 }
 
-func (self YaraScanPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self YaraScanPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "yara",
 		Doc:     "Scan files using yara rules.",
@@ -177,7 +177,7 @@ type YaraProcPluginArgs struct {
 
 type YaraProcPlugin struct{}
 
-func (self YaraProcPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self YaraProcPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "proc_yara",
 		Doc:     "Scan processes using yara rules.",

diff --git a/vql/filesystem/filesystem.go b/vql/filesystem/filesystem.go
@@ -64,12 +64,12 @@ func (self GlobPlugin) Call(
 	return output_chan
 }
 
-func (self GlobPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self GlobPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "glob",
 		Doc:     "Retrieve files based on a list of glob expressions",
-		RowType: type_map.AddType(glob.NewVirtualDirectoryPath("", nil)),
-		ArgType: type_map.AddType(&GlobPluginArgs{}),
+		RowType: type_map.AddType(scope, glob.NewVirtualDirectoryPath("", nil)),
+		ArgType: type_map.AddType(scope, &GlobPluginArgs{}),
 	}
 }
 
@@ -163,12 +163,12 @@ func (self ReadFilePlugin) Name() string {
 	return "read_file"
 }
 
-func (self ReadFilePlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self ReadFilePlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "read_file",
 		Doc:     "Read files in chunks.",
-		RowType: type_map.AddType(ReadFileResponse{}),
-		ArgType: type_map.AddType(&ReadFileArgs{}),
+		RowType: type_map.AddType(scope, ReadFileResponse{}),
+		ArgType: type_map.AddType(scope, &ReadFileArgs{}),
 	}
 }
 
@@ -211,7 +211,7 @@ func (self StatPlugin) Name() string {
 	return "stat"
 }
 
-func (self StatPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+func (self StatPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
 		Name:    "stat",
 		Doc:     "Get file information. Unlike glob() this does not support wildcards.",

diff --git a/vql/filesystem/grep.go b/vql/filesystem/grep.go
@@ -94,11 +94,11 @@ func (self *GrepFunction) Call(ctx context.Context,
 	}
 }
 
-func (self GrepFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+func (self GrepFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "grep",
 		Doc:     "Search a file for keywords.",
-		ArgType: type_map.AddType(&GrepFunctionArgs{}),
+		ArgType: type_map.AddType(scope, &GrepFunctionArgs{}),
 	}
 }
 

diff --git a/vql/filesystem/path.go b/vql/filesystem/path.go
@@ -3,6 +3,7 @@ package filesystem
 import (
 	"context"
 	"regexp"
+
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -33,11 +34,11 @@ func (self _Basename) Call(
 	return "/"
 }
 
-func (self _Basename) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+func (self _Basename) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "basename",
 		Doc:     "Splits the path on separator and return the basename.",
-		ArgType: type_map.AddType(&_BasenameArgs{}),
+		ArgType: type_map.AddType(scope, &_BasenameArgs{}),
 	}
 }
 

diff --git a/vql/functions/format.go b/vql/functions/format.go
@@ -42,11 +42,11 @@ func (self *FormatFunction) Call(ctx context.Context,
 	return fmt.Sprintf(arg.Format, format_args...)
 }
 
-func (self FormatFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+func (self FormatFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "format",
 		Doc:     "Format one or more items according to a format string.",
-		ArgType: type_map.AddType(&FormatArgs{}),
+		ArgType: type_map.AddType(scope, &FormatArgs{}),
 	}
 }