Allow NTFS cluster cache size to be specified in VQL. (Velocidex#1042)

Setting VQL scope variable NTFS_CACHE_SIZE will allow the NTFS cluster
cache to be this large. This reduces disk IO in favor of higher memory
usage. The default cache size is 100 clusters which results in a lot
of cache misses on the typical MFT (but only consumes 800kb).

Also automatically consume BOM for parse_csv() - the BOM is an
aBOMination and often set by Windows software (see
https://pkg.go.dev/github.com/spkg/bom)

artifacts/definitions/Windows/Detection/EnvironmentVariables.yaml

@@ -0,0 +1,33 @@
+name: Windows.Detection.EnvironmentVariables
+description: |
+   Find processes with the specified environment variables.
+
+parameters:
+   - name: ProcessNameRegex
+     default: .
+   - name: EnvironmentVariableRegex
+     default: COMSPEC
+   - name: FilterValueRegex
+     default: .
+   - name: WhitelistValueRegex
+     description: Ignore these values
+     default: ^C:\\Windows\\.+cmd.exe$
+
+sources:
+  - precondition:
+      SELECT OS From info() where OS = 'windows'
+
+    query: |
+      SELECT * FROM foreach(
+      row={
+          SELECT * FROM Artifact.Windows.Forensics.ProcessInfo(
+             ProcessNameRegex=ProcessNameRegex)
+      },
+      query={
+          SELECT Name, ImagePathName, CommandLine,
+             _key AS Var, _value AS Value
+          FROM items(item=Env)
+      })
+      WHERE Var =~ EnvironmentVariableRegex
+        AND Value =~ FilterValueRegex
+        AND NOT Value =~ WhitelistValueRegex

artifacts/definitions/Windows/System/Amcache.yaml

@@ -18,6 +18,9 @@ parameters:
     default: "%SYSTEMROOT%/appcompat/Programs/Amcache.hve"
   - name: amCacheRegPath
     default: /Root/InventoryApplicationFile/*
+  - name: NTFS_CACHE_SIZE
+    type: int
+    default: 1000
 
 precondition: |
   SELECT OS From info() where OS = 'windows'

constants/constants.go

@@ -83,10 +83,17 @@ const (
 	// USER record encoded in grpc context
 	GRPC_USER_CONTEXT key = iota
 
-	// Configuration for VQL plugins
+	// Configuration for VQL plugins. These can be set in an
+	// artifact to control the way VQL works.
 
-	// How often to expire the ntfs cache
+	// How often to expire the ntfs cache.
 	NTFS_CACHE_TIME = "NTFS_CACHE_TIME"
+
+	// Number of clusters to cache in memory (default 100).
+	NTFS_CACHE_SIZE = "NTFS_CACHE_SIZE"
+
+	RAW_REG_CACHE_SIZE = "RAW_REG_CACHE_SIZE"
+	BINARY_CACHE_SIZE  = "BINARY_CACHE_SIZE"
 )
 
 type key int

file_store/csv/reader.go

@@ -82,6 +82,12 @@ import (
 	"www.velocidex.com/golang/velociraptor/json"
 )
 
+const (
+	bom0 = 0xef
+	bom1 = 0xbb
+	bom2 = 0xbf
+)
+
 // A ParseError is returned for parsing errors.
 // Line numbers are 1-indexed and columns are 0-indexed.
 type ParseError struct {
@@ -197,11 +203,19 @@ type Reader struct {
 
 // NewReader returns a new Reader that reads from r.
 func NewReader(r io.ReadSeeker) *Reader {
-	return &Reader{
+	result := &Reader{
 		Comma:      ',',
 		r:          bufio.NewReader(r),
 		raw_reader: r,
 	}
+
+	// Swallow the BOM if possible.
+	b, err := result.r.Peek(3)
+	if err == nil && b[0] == bom0 && b[1] == bom1 && b[2] == bom2 {
+		result.Seek(3)
+	}
+
+	return result
 }
 
 // Read reads one record (a slice of fields) from r.

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/crewjam/saml v0.4.5
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elastic/go-elasticsearch/v7 v7.3.0 // indirect
 	github.com/elastic/go-libaudit v0.4.0

go.sum

@@ -170,6 +170,8 @@ github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk=
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=

gui/velociraptor/src/components/clients/client-summary.js

@@ -30,6 +30,9 @@ export default class VeloClientSummary extends Component {
     }
 
     getClientInfo = () => {
+        this.source.cancel();
+        this.source = axios.CancelToken.source();
+
         let client_id = this.props.client && this.props.client.client_id;
         if (client_id) {
             api.get("v1/GetClient/" + client_id).then(

vql/filesystem/raw_registry.go

@@ -43,6 +43,7 @@ import (
 	errors "github.com/pkg/errors"
 	"www.velocidex.com/golang/regparser"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/glob"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -217,12 +218,15 @@ func (self *RawRegFileSystemAccessor) getRegHive(
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	lru_size := vql_subsystem.GetIntFromRow(
+		self.scope, self.scope, constants.RAW_REG_CACHE_SIZE)
 	hive, pres := self.hive_cache[cache_key]
 	if !pres {
 		paged_reader := readers.NewPagedReader(
 			self.scope,
 			base_url.Scheme, // Accessor
 			base_url.Path,   // Path to underlying file
+			int(lru_size),
 		)
 		hive, err = regparser.NewRegistry(paged_reader)
 		if err != nil {

vql/parsers/binary.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/Velocidex/ordereddict"
+	"www.velocidex.com/golang/velociraptor/constants"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/readers"
 	vfilter "www.velocidex.com/golang/vfilter"
@@ -60,7 +61,8 @@ func (self ParseBinaryFunction) Call(
 		vql_subsystem.CacheSet(scope, arg.Profile, profile)
 	}
 
-	paged_reader := readers.NewPagedReader(scope, arg.Accessor, arg.Filename)
+	lru_size := vql_subsystem.GetIntFromRow(scope, scope, constants.BINARY_CACHE_SIZE)
+	paged_reader := readers.NewPagedReader(scope, arg.Accessor, arg.Filename, int(lru_size))
 	obj, err := profile.Parse(scope, arg.Struct, paged_reader, arg.Offset)
 	if err != nil {
 		scope.Log("parse_binary: %v", err)

vql/parsers/pe.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/Velocidex/ordereddict"
 	pe "www.velocidex.com/golang/go-pe"
+	"www.velocidex.com/golang/velociraptor/constants"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/readers"
 	vfilter "www.velocidex.com/golang/vfilter"
@@ -58,7 +59,8 @@ func (self _PEFunction) Call(
 		return &vfilter.Null{}
 	}
 
-	paged_reader := readers.NewPagedReader(scope, arg.Accessor, arg.Filename)
+	lru_size := vql_subsystem.GetIntFromRow(scope, scope, constants.BINARY_CACHE_SIZE)
+	paged_reader := readers.NewPagedReader(scope, arg.Accessor, arg.Filename, int(lru_size))
 	pe_file, err := pe.NewPEFile(paged_reader)
 	if err != nil {
 		scope.Log("parse_pe: %v for %v", err, arg.Filename)

vql/parsers/syslog/scanner.go

@@ -7,6 +7,7 @@ import (
 	"io"
 
 	"github.com/Velocidex/ordereddict"
+	"github.com/dimchansky/utfbom"
 	"www.velocidex.com/golang/velociraptor/glob"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
@@ -57,7 +58,8 @@ func (self ScannerPlugin) Call(
 				}
 				defer fd.Close()
 
-				scanner := bufio.NewScanner(fd)
+				// Support a BOM just incase
+				scanner := bufio.NewScanner(utfbom.SkipOnly(fd))
 				for scanner.Scan() {
 					select {
 					case <-ctx.Done():

vql/readers/ntfs.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	ntfs "www.velocidex.com/golang/go-ntfs/parser"
+	"www.velocidex.com/golang/velociraptor/constants"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -37,7 +38,8 @@ func GetNTFSContext(scope vfilter.Scope, device string) (*ntfs.NTFSContext, erro
 		return cache_ctx, nil
 	}
 
-	paged_reader := NewPagedReader(scope, "file", device)
+	lru_size := vql_subsystem.GetIntFromRow(scope, scope, constants.NTFS_CACHE_SIZE)
+	paged_reader := NewPagedReader(scope, "file", device, int(lru_size))
 	ntfs_ctx, err := ntfs.GetNTFSContext(paged_reader, 0)
 	if err != nil {
 		paged_reader.Close()

vql/readers/paged.go

@@ -104,6 +104,7 @@ type AccessorReader struct {
 
 	// How long to keep the file handle open
 	Lifetime time.Duration
+	lru_size int
 }
 
 func (self *AccessorReader) Size() int {
@@ -147,8 +148,13 @@ func (self *AccessorReader) ReadAt(buf []byte, offset int64) (int, error) {
 			return 0, err
 		}
 
+		lru_size := self.lru_size
+		if lru_size == 0 {
+			lru_size = 100
+		}
+
 		self.paged_reader, err = ntfs.NewPagedReader(
-			utils.ReaderAtter{self.reader}, 1024*8, 100)
+			utils.ReaderAtter{self.reader}, 1024*8, lru_size)
 		if err != nil {
 			return 0, err
 		}
@@ -214,7 +220,9 @@ func GetReaderPool(scope vfilter.Scope, lru_size int64) *ReaderPool {
 	return pool
 }
 
-func NewPagedReader(scope vfilter.Scope, accessor, filename string) *AccessorReader {
+func NewPagedReader(scope vfilter.Scope,
+	accessor, filename string,
+	lru_size int) *AccessorReader {
 
 	// Get the reader pool from the scope.
 	pool := GetReaderPool(scope, 50)
@@ -237,6 +245,7 @@ func NewPagedReader(scope vfilter.Scope, accessor, filename string) *AccessorRea
 
 		// By default close all files after a minute.
 		Lifetime: time.Minute,
+		lru_size: lru_size,
 	}
 
 	pool.lru.Set(key, result)

vql/windows/filesystems/readers/ntfs.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	ntfs "www.velocidex.com/golang/go-ntfs/parser"
+	"www.velocidex.com/golang/velociraptor/constants"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/readers"
 	"www.velocidex.com/golang/vfilter"
@@ -38,7 +39,8 @@ func GetNTFSContext(scope vfilter.Scope, device string) (*ntfs.NTFSContext, erro
 		return cache_ctx, nil
 	}
 
-	paged_reader := readers.NewPagedReader(scope, "file", device)
+	lru_size := vql_subsystem.GetIntFromRow(scope, scope, constants.NTFS_CACHE_SIZE)
+	paged_reader := readers.NewPagedReader(scope, "file", device, int(lru_size))
 	ntfs_ctx, err := ntfs.GetNTFSContext(paged_reader, 0)
 	if err != nil {
 		paged_reader.Close()

vql/windows/filesystems/readers/ntfs_windows.go

@@ -47,6 +47,7 @@ type NTFSCachedContext struct {
 	scope        vfilter.Scope
 	paged_reader *readers.AccessorReader
 	ntfs_ctx     *ntfs.NTFSContext
+	lru_size     int
 
 	// When this is closed we stop refreshing the cache. Normally
 	// only closed when the scope is destroyed.
@@ -111,7 +112,9 @@ func (self *NTFSCachedContext) GetNTFSContext() (*ntfs.NTFSContext, error) {
 		return self.ntfs_ctx, nil
 	}
 
-	self.paged_reader = readers.NewPagedReader(self.scope, "file", self.device)
+	lru_size := vql_subsystem.GetIntFromRow(self.scope, self.scope, constants.NTFS_CACHE_SIZE)
+	self.paged_reader = readers.NewPagedReader(
+		self.scope, "file", self.device, int(lru_size))
 	ntfs_ctx, err := ntfs.GetNTFSContext(self.paged_reader, 0)
 	if err != nil {
 		self._CloseWithLock()