Updating to the latest vfilter (Velocidex#861)

api/reflect.go

@@ -28,6 +28,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/services"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
+	"www.velocidex.com/golang/vfilter/types"
 )
 
 var (
@@ -53,7 +54,7 @@ func (self *ApiServer) GetKeywordCompletions(
 	scope := vql_subsystem.MakeScope()
 	defer scope.Close()
 
-	type_map := vfilter.NewTypeMap()
+	type_map := types.NewTypeMap()
 	info := scope.Describe(type_map)
 
 	for _, item := range info.Functions {
@@ -94,14 +95,16 @@ func (self *ApiServer) GetKeywordCompletions(
 	return result, nil
 }
 
-func getArgDescriptors(arg_type string, type_map *vfilter.TypeMap,
-	scope *vfilter.Scope) []*api_proto.ArgDescriptor {
+func getArgDescriptors(
+	arg_type string,
+	type_map *vfilter.TypeMap,
+	scope vfilter.Scope) []*api_proto.ArgDescriptor {
 	args := []*api_proto.ArgDescriptor{}
 	arg_desc, pres := type_map.Get(scope, arg_type)
 	if pres && arg_desc != nil && arg_desc.Fields != nil {
 		for _, k := range arg_desc.Fields.Keys() {
 			v_any, _ := arg_desc.Fields.Get(k)
-			v, ok := v_any.(*vfilter.TypeReference)
+			v, ok := v_any.(*types.TypeReference)
 			if !ok {
 				continue
 			}

artifacts/definitions/Admin/Client/Upgrade.yaml

@@ -15,7 +15,8 @@ tools:
 
 parameters:
   - name: SleepDuration
-    default: 600
+    default: "600"
+    type: int
     description: |
       The MSI file is typically very large and we do not want to
       overwhelm the server so we stagger the download over this many
@@ -26,7 +27,11 @@ sources:
       SELECT OS From info() where OS = 'windows'
 
     query:  |
-      LET bin <= SELECT * FROM Artifact.Generic.Utils.FetchBinary(
+      // Force the file to be copied to the real temp directory since
+      // we are just about to remove the Tools directory.
+      LET bin <= SELECT copy(filename=FullPath,
+          dest=expand(path="%SYSTEMROOT%\\Temp\\") + basename(path=FullPath)) AS Dest
+      FROM Artifact.Generic.Utils.FetchBinary(
          ToolName="WindowsMSI", IsExecutable=FALSE,
          SleepDuration=SleepDuration)
 
@@ -35,6 +40,6 @@ sources:
       SELECT * FROM foreach(row=bin,
       query={
          SELECT * FROM execve(
-              argv=["msiexec.exe", "/i", FullPath, "/q"],
+              argv=["msiexec.exe", "/i", Dest, "/q"],
               length=10000000)
       })

artifacts/definitions/Windows/Applications/TeamViewer/Incoming.yaml

@@ -73,10 +73,11 @@ sources:
                     AND Record.TeamViewerID =~ TeamViewerIDRegex
                     AND Record.SourceHost =~ SourceHostRegex
                     AND Record.User =~ UserRegex
-                    AND ( timestamp(epoch=Record.StartTime) < DateBeforeTime
-                        AND timestamp(epoch=Record.StartTime) > DateAfterTime)
-                    AND ( timestamp(epoch=Record.EndTime) < DateBeforeTime
-                        AND timestamp(epoch=Record.EndTime) > DateAfterTime )
+                    AND timestamp(epoch=Record.StartTime) < DateBeforeTime
+                    AND timestamp(epoch=Record.StartTime) > DateAfterTime
+                    AND timestamp(epoch=Record.EndTime) < DateBeforeTime
+                    AND timestamp(epoch=Record.EndTime) > DateAfterTime
+
             })
 
       -- include VSS in calculation and deduplicate with GROUP BY by file

artifacts/definitions/Windows/Remediation/Sinkhole.yaml

@@ -59,12 +59,14 @@ sources:
 
       -- Check for backup to determine if sinkhole applied
       LET check_backup = SELECT FullPath FROM stat(filename=HostsFileBackup)
+      WHERE log(message="Found backup at " + FullPath)
 
       -- Backup old config
       LET backup = copy(filename=HostsFile,dest=HostsFileBackup)
 
       -- Restore old config
       LET restore = SELECT * FROM chain(
+            z=log(message="Will restore from backup"),
             a=copy(filename=HostsFileBackup,dest=HostsFile),
             b={
                 SELECT *
@@ -159,18 +161,23 @@ sources:
       LET HostsData = join(array=build_lines.Line,sep='\r\n')
 
       -- Force start of backup or restore if applicable
-      LET backup_restore <= if(condition= RestoreBackup,
-                then= if(condition= check_backup,
-                        then= restore,
-                        else= log(message='Can not restore hosts file as backup does not exist.')),
-                else= if(condition= check_backup,
-                        then={
-                            SELECT * FROM chain(
-                                a= log(message='Backup hosts file already exists.'),
-                                b= restore)
-                            },
-                        else= backup)
-                )
+      LET backup_restore <= if(
+         condition= RestoreBackup AND log(message="Will attempt to restore backup"),
+         then= if(
+            condition= check_backup,
+            then= restore,
+            -- then= { SELECT * FROM restore },
+            else= log(message='Can not restore hosts file as backup does not exist.')),
+
+         else= if(
+           condition= check_backup,
+           then={
+              SELECT * FROM chain(
+                 a= log(message='Backup hosts file already exists.'),
+                 b= restore)
+              },
+          else= backup)
+        )
 
       -- Do kick off logic
       LET do_it <= SELECT * FROM if(condition= NOT RestoreBackup,
@@ -179,7 +186,6 @@ sources:
                     a= log(message='Adding hosts entries.'),
                     b= write(DataBlob=HostsData),
                     c= flushdns
-
                 )})
 
       -- Finally show resultant HostsFile

artifacts/testdata/server/testcases/artifacts.in.yaml

@@ -9,6 +9,15 @@ Queries:
         env=dict(Device ='C:', VSSAnalysis='Y', KapeTriage='Y')).request AS Flow
     FROM scope()
 
+  # Schedule using a new style spec decleration.
+  - SELECT collect_client(
+        client_id='C.11a3013ccaXXXXX',
+        artifacts='Windows.KapeFiles.Targets',
+        spec=dict(`Windows.KapeFiles.Targets`=dict(
+            Device ='C:', VSSAnalysis='Y', KapeTriage='Y'))).request AS Flow
+    FROM scope()
+
+
   # Check that artifact_definitions work
   - SELECT name FROM artifact_definitions(names='Windows.KapeFiles.Targets') ORDER BY name
 

artifacts/testdata/server/testcases/artifacts.out.yaml

@@ -43,6 +43,44 @@ SELECT * FROM uploads(client_id='C.4f5e52adf0a337a9', flow_id='F.BN2HJCPOF5U7U')
    "compiled_collector_args": []
   }
  }
+]SELECT collect_client( client_id='C.11a3013ccaXXXXX', artifacts='Windows.KapeFiles.Targets', spec=dict(`Windows.KapeFiles.Targets`=dict( Device ='C:', VSSAnalysis='Y', KapeTriage='Y'))).request AS Flow FROM scope()[
+ {
+  "Flow": {
+   "creator": "",
+   "client_id": "C.11a3013ccaXXXXX",
+   "urgent": false,
+   "artifacts": [
+    "Windows.KapeFiles.Targets"
+   ],
+   "specs": [
+    {
+     "artifact": "Windows.KapeFiles.Targets",
+     "parameters": {
+      "env": [
+       {
+        "key": "Device",
+        "value": "C:"
+       },
+       {
+        "key": "VSSAnalysis",
+        "value": "Y"
+       },
+       {
+        "key": "KapeTriage",
+        "value": "Y"
+       }
+      ]
+     }
+    }
+   ],
+   "ops_per_second": 0,
+   "timeout": 0,
+   "max_rows": 0,
+   "max_upload_bytes": 0,
+   "allow_custom_overrides": false,
+   "compiled_collector_args": []
+  }
+ }
 ]SELECT name FROM artifact_definitions(names='Windows.KapeFiles.Targets') ORDER BY name[
  {
   "name": "Windows.Collectors.File"

artifacts/testdata/server/testcases/yara.out.yaml

@@ -158,7 +158,7 @@ FROM scope()
    "Name1",
    "Name2"
   ],
-  "IOCdata": "[]vfilter.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
+  "IOCdata": "[]types.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
   "_Source": "Windows.Detection.RemoteYara.Process"
  }
 ]SELECT mock_check(plugin='proc_dump', expected_calls=0), mock_check(function='upload', expected_calls=0), mock_check(plugin="proc_yara", expected_calls=1) FROM scope()[
@@ -189,7 +189,7 @@ FROM scope()
    "Name1",
    "Name2"
   ],
-  "IOCdata": "[]vfilter.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
+  "IOCdata": "[]types.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
   "_Source": "Windows.Detection.RemoteYara.Process"
  }
 ]SELECT mock_check(plugin='proc_dump', expected_calls=0), mock_check(function='upload', expected_calls=0), mock_check(plugin="proc_yara", expected_calls=1) FROM scope()[
@@ -220,7 +220,7 @@ FROM scope()
    "Name1",
    "Name2"
   ],
-  "IOCdata": "[]vfilter.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
+  "IOCdata": "[]types.Any{\"Foobar\", \"Other Hit\"} [Foobar Other Hit]",
   "_Source": "Windows.Detection.RemoteYara.Process"
  },
  {

artifacts/utils.go

@@ -10,7 +10,7 @@ import (
 )
 
 // Gets the client config from the scope.
-func GetConfig(scope *vfilter.Scope) (*config_proto.ClientConfig, bool) {
+func GetConfig(scope vfilter.Scope) (*config_proto.ClientConfig, bool) {
 	scope_config, pres := scope.Resolve(constants.SCOPE_CONFIG)
 	if !pres {
 		return nil, false
@@ -23,7 +23,7 @@ func GetConfig(scope *vfilter.Scope) (*config_proto.ClientConfig, bool) {
 	return config, ok
 }
 
-func GetUploader(scope *vfilter.Scope) (api.Uploader, bool) {
+func GetUploader(scope vfilter.Scope) (api.Uploader, bool) {
 	scope_uploader, pres := scope.Resolve(constants.SCOPE_UPLOADER)
 	if !pres {
 		return nil, false
@@ -37,7 +37,7 @@ func GetUploader(scope *vfilter.Scope) (api.Uploader, bool) {
 	return config, ok
 }
 
-func GetACLManager(scope *vfilter.Scope) (vql_subsystem.ACLManager, bool) {
+func GetACLManager(scope vfilter.Scope) (vql_subsystem.ACLManager, bool) {
 	scope_manager, pres := scope.Resolve(vql_subsystem.ACL_MANAGER_VAR)
 	if !pres {
 		return nil, false

bin/artifacts.go

@@ -174,8 +174,8 @@ func doArtifactCollect() {
 	}()
 
 	if *trace_vql_flag {
-		scope.Tracer = logging.NewPlainLogger(config_obj,
-			&logging.ToolComponent)
+		scope.SetTracer(logging.NewPlainLogger(config_obj,
+			&logging.ToolComponent))
 	}
 
 	query := `

bin/fs.go

@@ -73,7 +73,7 @@ var (
 )
 
 func eval_query(
-	config_obj *config_proto.Config, format, query string, scope *vfilter.Scope,
+	config_obj *config_proto.Config, format, query string, scope vfilter.Scope,
 	env *ordereddict.Dict) {
 	if config_obj.ApiConfig != nil && config_obj.ApiConfig.Name != "" {
 		logging.GetLogger(config_obj, &logging.ToolComponent).
@@ -87,7 +87,7 @@ func eval_query(
 
 func eval_local_query(
 	config_obj *config_proto.Config, format string,
-	query string, scope *vfilter.Scope) {
+	query string, scope vfilter.Scope) {
 
 	vqls, err := vfilter.MultiParse(query)
 	kingpin.FatalIfError(err, "Unable to parse VQL Query")
@@ -309,7 +309,7 @@ type FileStoreAccessorFactory struct {
 	config_obj *config_proto.Config
 }
 
-func (self FileStoreAccessorFactory) New(scope *vfilter.Scope) (glob.FileSystemAccessor, error) {
+func (self FileStoreAccessorFactory) New(scope vfilter.Scope) (glob.FileSystemAccessor, error) {
 	return file_store.GetFileStoreFileSystemAccessor(self.config_obj)
 }
 

bin/query.go

@@ -63,7 +63,7 @@ var (
 )
 
 func outputJSON(ctx context.Context,
-	scope *vfilter.Scope,
+	scope vfilter.Scope,
 	vql *vfilter.VQL,
 	out io.Writer) {
 	for result := range vfilter.GetResponseChannel(
@@ -76,7 +76,7 @@ func outputJSON(ctx context.Context,
 }
 
 func outputJSONL(ctx context.Context,
-	scope *vfilter.Scope,
+	scope vfilter.Scope,
 	vql *vfilter.VQL,
 	out io.Writer) {
 	for result := range vfilter.GetResponseChannel(
@@ -89,7 +89,7 @@ func outputJSONL(ctx context.Context,
 }
 
 func outputCSV(ctx context.Context,
-	scope *vfilter.Scope,
+	scope vfilter.Scope,
 	vql *vfilter.VQL,
 	out io.Writer) {
 	result_chan := vfilter.GetResponseChannel(vql, ctx, scope,
@@ -267,7 +267,7 @@ func doQuery() {
 	ctx := InstallSignalHandler(scope)
 
 	if *trace_vql_flag {
-		scope.Tracer = log.New(os.Stderr, "VQL Trace: ", 0)
+		scope.SetTracer(log.New(os.Stderr, "VQL Trace: ", 0))
 	}
 	for _, query := range *queries {
 		statements, err := vfilter.MultiParse(query)

bin/utils.go

@@ -32,7 +32,7 @@ import (
 	vfilter "www.velocidex.com/golang/vfilter"
 )
 
-func InstallSignalHandler(scope *vfilter.Scope) context.Context {
+func InstallSignalHandler(scope vfilter.Scope) context.Context {
 
 	// Wait for signal. When signal is received we shut down the
 	// server.