Refactor Raw Registry accessor to use paged reader. (Velocidex#925)

Fixes a file handle leak in the raw registry parser which would cause
errors in mounting hives for the windows profile service.

Fixes Velocidex#924

Using ntfs accessor in raw_reg artifacts allows us to avoid the file
handle leak issue. This fixes this behavior on old clients which do
not have the fix yet.

.wwhrd.yml

@@ -4,6 +4,8 @@ blacklist:
 
 whitelist:
   - Apache-2.0
+  - BSD-2-Clause
+  - BSD-3-Clause
   - MIT
   - NewBSD
   - FreeBSD

artifacts/definitions/Windows/Forensics/Prefetch.yaml

@@ -11,7 +11,7 @@ description: |
     - dateBefore enables search for prefetch evidence before this date.
     - binaryRegex enables to filter on binary name, e.g evil.exe.
     - hashRegex enables to filter on prefetch hash.
-    
+
 reference:
   - https://www.forensicswiki.org/wiki/Prefetch
 
@@ -51,7 +51,7 @@ sources:
                     // FilesAccessed,
                     FullPath,
                     Name AS PrefetchFileName,
-                    timestamp(epoch=Ctime.sec) as CreationTime,
+                    timestamp(epoch=Btime.sec) as CreationTime,
                     timestamp(epoch=Mtime.sec) as ModificationTime
                  FROM prefetch(filename=FullPath)
                  WHERE

artifacts/definitions/Windows/Forensics/RecentApps.yaml

@@ -38,7 +38,7 @@ sources:
                       string=Key.FullPath,
                       regex="/Users/(?P<User>[^/]+)/ntuser.dat").User AS User
             FROM read_reg_key(
-               globs=url(scheme="file",
+               globs=url(scheme="ntfs",
                   path=FullPath,
                   fragment=RecentAppsKey).String,
                accessor="raw_reg")

artifacts/definitions/Windows/Registry/NTUser.yaml

@@ -44,7 +44,8 @@ sources:
             Description,
             UUID,
             {
-                SELECT FullPath FROM glob(globs=expand(path=Directory) + "//NTUSER.DAT", accessor="file")
+                SELECT FullPath FROM glob(
+                   globs=expand(path=Directory) + "//NTUSER.DAT", accessor="file")
             } as FullPath,
             expand(path=Directory) as Directory
         FROM Artifact.Windows.Sys.Users()
@@ -55,9 +56,10 @@ sources:
                 SELECT * FROM UserProfiles
             },
             query={
-                SELECT FullPath, Data, Mtime.Sec AS Mtime, Username, Description, Uid, Gid, UUID, Directory
+                SELECT FullPath, Data, Mtime.Sec AS Mtime,
+                       Username, Description, Uid, Gid, UUID, Directory
                 FROM glob(
-                    globs=url(scheme="file",
+                    globs=url(scheme="ntfs",
                     path=FullPath,
                     fragment=KeyGlob).String,
                     accessor="raw_reg")

artifacts/definitions/Windows/Search/Yara.yaml

@@ -9,6 +9,9 @@ parameters:
     - name: nameRegex
       description: Only file names that match this regular expression will be scanned.
       default: "(exe|txt|dll|php)$"
+    - name: AlsoUpload
+      type: bool
+      description: Also upload matching files.
 
 precondition:
   SELECT * FROM info() WHERE OS =~ "windows"
@@ -44,6 +47,6 @@ sources:
         -- Only do something when yara rules are available.
         SELECT * FROM if(condition=yara_rules,
         then={
-          SELECT *, upload(file=FileName) AS Upload
+          SELECT *, if(condition=AlsoUpload, then=upload(file=FileName)) AS Upload
           FROM foreach(row=fileList, query=search)
         })

executor/pool.go

@@ -59,6 +59,12 @@ func getInc() int64 {
 	return ts
 }
 
+// A wrapper around the standard client executor for use of pool
+// clients. When multiple requests come in for the same query and
+// parameters, we cache the results when the first request comes in
+// and then feed the results to all other requests from memory. This
+// allows us to increase the load on the server simulating a large
+// fleet of independent clients.
 type PoolClientExecutor struct {
 	*ClientExecutor
 	Outbound chan *crypto_proto.GrrMessage
@@ -78,7 +84,7 @@ func getQueryName(message *crypto_proto.GrrMessage) string {
 				query_name = query.Name
 			}
 		}
-		// Cache it under the query name and the
+		// Cache it under the query name and the serialized parameters
 		serialized, _ := json.Marshal(message.VQLClientAction.Env)
 		return fmt.Sprintf("%v: %v", query_name, string(serialized))
 
@@ -121,6 +127,11 @@ func (self *PoolClientExecutor) ProcessRequest(
 	ctx context.Context,
 	message *crypto_proto.GrrMessage) {
 
+	if message.UpdateEventTable != nil {
+		fmt.Println("Will update event table")
+		return
+	}
+
 	tran := getCompletedTransaction(message)
 	if tran != nil {
 		// Wait until the transaction is done.

go.mod

@@ -118,7 +118,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20210128083254-ad86cfee337b
+	www.velocidex.com/golang/vfilter v0.0.0-20210220121641-879064f4499e
 	www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8
 )
 

go.sum

@@ -815,5 +815,7 @@ www.velocidex.com/golang/vfilter v0.0.0-20210122161546-3af91fe41a9b h1:bTVx4l9lF
 www.velocidex.com/golang/vfilter v0.0.0-20210122161546-3af91fe41a9b/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
 www.velocidex.com/golang/vfilter v0.0.0-20210128083254-ad86cfee337b h1:DxvxCkOD6ZHAXHL/nrMKdPZ0c2dltgwzvgkPD/8M+44=
 www.velocidex.com/golang/vfilter v0.0.0-20210128083254-ad86cfee337b/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU=
+www.velocidex.com/golang/vfilter v0.0.0-20210220121641-879064f4499e h1:h7uErYK4QOhsxTlle+6jxXgjCIwZ/wx44h9397fd3TM=
+www.velocidex.com/golang/vfilter v0.0.0-20210220121641-879064f4499e/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU=
 www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8 h1:QLV9zSccnI0IcfU0eIswx0G7g+umEGoCtaS+Hr9zRzk=
 www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8/go.mod h1:34AZRfhNvJ1QAwPpYrDxjCyOFys+NbSmH6LLVSjsAEg=

responder/pool.go

@@ -0,0 +1,7 @@
+package responder
+
+import "sync"
+
+type PoolResponder struct {
+	mu sync.Mutex
+}

vql/common/yara.go

@@ -114,7 +114,8 @@ func (self YaraScanPlugin) Call(
 			// into memory avoiding the need for
 			// buffering.
 			if arg.Accessor == "" {
-				err := scanFile(filename, arg.Context, arg.NumberOfHits,
+				err := scanFile(ctx, filename, arg.Context,
+					arg.NumberOfHits,
 					rules, output_chan, scope)
 
 				// Fall back to accessor scanning if
@@ -248,6 +249,7 @@ func scanFileByAccessor(
 }
 
 func scanFile(
+	ctx context.Context,
 	filename string,
 	context int,
 	total_number_of_hits int64,
@@ -275,6 +277,7 @@ func scanFile(
 		file_info:      stat,
 		filename:       filename,
 		reader:         fd,
+		ctx:            ctx,
 	}
 
 	err = rules.ScanFileWithCallback(

vql/filesystem/raw_registry.go

@@ -32,7 +32,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
 	"net/url"
 	"os"
 	"path"
@@ -48,6 +47,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/velociraptor/vql/readers"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -193,61 +193,51 @@ func NewRawValueBuffer(buf string, stat *RawRegValueInfo) *RawValueBuffer {
 	}
 }
 
-type RawRegistryFileCache struct {
-	registry *regparser.Registry
-	fd       glob.ReadSeekCloser
-}
-
 type RawRegFileSystemAccessor struct {
-	mu       sync.Mutex
-	fd_cache map[string]*RawRegistryFileCache
-	scope    vfilter.Scope
+	mu sync.Mutex
+
+	// Maintain a cache of already parsed hives
+	hive_cache map[string]*regparser.Registry
+	scope      vfilter.Scope
 }
 
 func (self *RawRegFileSystemAccessor) getRegHive(
-	file_path string) (*RawRegistryFileCache, *url.URL, error) {
-	url, err := url.Parse(file_path)
+	file_path string) (*regparser.Registry, *url.URL, error) {
+
+	// The file path is a url specifying the path to a key:
+	// Scheme is the underlying accessor
+	// Path is the path to be provided to the underlying accessor
+	// Fragment is the path within the reg hive that we need to open.
+	full_url, err := url.Parse(file_path)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	base_url := *url
+	// Cache the parsed hive under the underlying file.
+	base_url := *full_url
 	base_url.Fragment = ""
+	cache_key := base_url.String()
 
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
-	file_cache, pres := self.fd_cache[base_url.String()]
+	hive, pres := self.hive_cache[cache_key]
 	if !pres {
-		accessor, err := glob.GetAccessor(url.Scheme, self.scope)
+		paged_reader := readers.NewPagedReader(
+			self.scope,
+			base_url.Scheme, // Accessor
+			base_url.Path,   // Path to underlying file
+		)
+		hive, err = regparser.NewRegistry(paged_reader)
 		if err != nil {
+			paged_reader.Close()
 			return nil, nil, err
 		}
 
-		fd, err := accessor.Open(url.Path)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		reader, ok := fd.(io.ReaderAt)
-		if !ok {
-			return nil, nil, errors.New("file is not seekable")
-		}
-
-		registry, err := regparser.NewRegistry(reader)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		file_cache = &RawRegistryFileCache{
-			registry: registry,
-			fd:       fd,
-		}
-
-		self.fd_cache[url.String()] = file_cache
+		self.hive_cache[cache_key] = hive
 	}
 
-	return file_cache, url, nil
+	return hive, full_url, nil
 }
 
 const RawRegFileSystemTag = "_RawReg"
@@ -258,33 +248,24 @@ func (self *RawRegFileSystemAccessor) New(scope vfilter.Scope) (
 	result_any := vql_subsystem.CacheGet(scope, RawRegFileSystemTag)
 	if result_any == nil {
 		result := &RawRegFileSystemAccessor{
-			fd_cache: make(map[string]*RawRegistryFileCache),
-			scope:    scope,
+			hive_cache: make(map[string]*regparser.Registry),
+			scope:      scope,
 		}
 		vql_subsystem.CacheSet(scope, RawRegFileSystemTag, result)
-
-		// When scope is destroyed, we close all the filehandles.
-		err := scope.AddDestructor(func() {
-			result.mu.Lock()
-			defer result.mu.Unlock()
-
-			for _, v := range result.fd_cache {
-				v.fd.Close()
-			}
-		})
-		return result, err
+		return result, nil
 	}
 
 	return result_any.(glob.FileSystemAccessor), nil
 }
 
 func (self *RawRegFileSystemAccessor) ReadDir(key_path string) ([]glob.FileInfo, error) {
 	var result []glob.FileInfo
-	file_cache, url, err := self.getRegHive(key_path)
+	hive, url, err := self.getRegHive(key_path)
 	if err != nil {
 		return nil, err
 	}
-	key := file_cache.registry.OpenKey(url.Fragment)
+
+	key := hive.OpenKey(url.Fragment)
 	if key == nil {
 		return nil, errors.New("Key not found")
 	}