Bugfix: Memory leak in diff plugin

Makefile

@@ -66,6 +66,9 @@ windows_bare:
 windowsx86:
 	go run make.go -v windowsx86
 
+windowsarm:
+	go run make.go -v windowsarm
+
 clean:
 	go run make.go -v clean
 

accessors/ext4/ext4_accessor.go

@@ -16,11 +16,6 @@ import (
 	"www.velocidex.com/golang/vfilter"
 )
 
-const (
-	// Scope cache tag for the FAT parser
-	Ext4FileSystemTag = "_EXT4"
-)
-
 type Ext4FileInfo struct {
 	*ext4.FileInfo
 	_full_path *accessors.OSPath

accessors/process/process_address_space_windows.go

@@ -1,3 +1,4 @@
+//go:build windows && amd64 && cgo
 // +build windows,amd64,cgo
 
 // An accessor for process address space.
@@ -233,6 +234,7 @@ func (self ProcessAccessor) New(scope vfilter.Scope) (
 			scope: scope,
 		}
 		result.lru.SetTTL(time.Second)
+		result.lru.SkipTTLExtensionOnHit(true)
 		result.lru.SetCheckExpirationCallback(func(key string, value interface{}) bool {
 			info, ok := value.(*ProcessReader)
 			if ok {

accessors/raw_registry/cache.go

@@ -0,0 +1,68 @@
+package raw_registry
+
+import (
+	"time"
+
+	"github.com/Velocidex/ttlcache/v2"
+	"www.velocidex.com/golang/velociraptor/constants"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/vfilter"
+)
+
+const (
+	RAW_CACHE_TAG = "_RAW_REG_CACHE"
+)
+
+type RawRegFileSystemAccessorCache struct {
+	lru         *ttlcache.Cache
+	readdir_lru *ttlcache.Cache
+}
+
+func (self *RawRegFileSystemAccessorCache) Close() {
+	self.lru.Close()
+	self.readdir_lru.Close()
+}
+
+func getRegFileSystemAccessorCache(scope vfilter.Scope) *RawRegFileSystemAccessorCache {
+	cache, ok := vql_subsystem.CacheGet(
+		scope, RAW_CACHE_TAG).(*RawRegFileSystemAccessorCache)
+	if ok {
+		return cache
+	}
+
+	cache = &RawRegFileSystemAccessorCache{
+		lru:         ttlcache.NewCache(),
+		readdir_lru: ttlcache.NewCache(),
+	}
+
+	cache_size := int(vql_subsystem.GetIntFromRow(
+		scope, scope, constants.RAW_REG_CACHE_SIZE))
+	if cache_size == 0 {
+		cache_size = 1000
+	}
+
+	cache_time := vql_subsystem.GetIntFromRow(
+		scope, scope, constants.RAW_REG_CACHE_TIME)
+	if cache_time == 0 {
+		cache_time = 10
+	}
+
+	cache.lru.SetCacheSizeLimit(cache_size)
+	cache.lru.SetTTL(time.Second * time.Duration(cache_time))
+	cache.lru.SkipTTLExtensionOnHit(true)
+
+	cache.readdir_lru.SetCacheSizeLimit(cache_size)
+	cache.readdir_lru.SetTTL(time.Second * time.Duration(cache_time))
+	cache.readdir_lru.SkipTTLExtensionOnHit(true)
+
+	// Add the cache to the root scope so it can be visible outside
+	// our scope. This should maximize cache hits
+	root_scope := vql_subsystem.GetRootScope(scope)
+
+	root_scope.AddDestructor(func() {
+		cache.Close()
+	})
+	vql_subsystem.CacheSet(root_scope, RAW_CACHE_TAG, cache)
+
+	return cache
+}

accessors/raw_registry/raw_registry.go

@@ -37,7 +37,6 @@ import (
 	"time"
 
 	"github.com/Velocidex/ordereddict"
-	"github.com/Velocidex/ttlcache/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -261,8 +260,7 @@ type RawRegFileSystemAccessor struct {
 	scope vfilter.Scope
 	root  *accessors.OSPath
 
-	lru         *ttlcache.Cache
-	readdir_lru *ttlcache.Cache
+	cache *RawRegFileSystemAccessorCache
 }
 
 func getRegHiveCache(scope vfilter.Scope) *rawHiveCache {
@@ -310,7 +308,7 @@ func getRegHive(scope vfilter.Scope,
 	paged_reader, err := readers.NewAccessorReader(
 		scope, pathspec.DelegateAccessor, delegate, int(lru_size))
 	if err != nil {
-		scope.Log("%v: did you provide a URL or Pathspec?", err)
+		scope.Log("%v: did you provide a Pathspec?", err)
 		return nil, err
 	}
 
@@ -339,30 +337,10 @@ const RawRegFileSystemTag = "_RawReg"
 func (self *RawRegFileSystemAccessor) New(scope vfilter.Scope) (
 	accessors.FileSystemAccessor, error) {
 
-	my_lru := self.lru
-	if my_lru == nil {
-		my_lru = ttlcache.NewCache()
-		my_lru.SetCacheSizeLimit(1000)
-		my_lru.SetTTL(time.Minute)
-	}
-
-	my_readdir_lru := self.readdir_lru
-	if my_readdir_lru == nil {
-		my_readdir_lru = ttlcache.NewCache()
-		my_readdir_lru.SetCacheSizeLimit(1000)
-		my_readdir_lru.SetTTL(time.Minute)
-	}
-	scope.AddDestructor(func() {
-		my_lru.Close()
-		my_readdir_lru.Close()
-	})
-
 	return &RawRegFileSystemAccessor{
 		scope: scope,
 		root:  self.root,
-
-		lru:         my_lru,
-		readdir_lru: my_readdir_lru,
+		cache: getRegFileSystemAccessorCache(scope),
 	}, nil
 }
 
@@ -467,7 +445,7 @@ func (self *RawRegFileSystemAccessor) _readDirWithOSPath(
 	full_path *accessors.OSPath) (result []accessors.FileInfo, key *regparser.CM_KEY_NODE, err error) {
 
 	cache_key := full_path.String()
-	cached, err := self.readdir_lru.Get(cache_key)
+	cached, err := self.cache.readdir_lru.Get(cache_key)
 	if err == nil {
 		cached_res, ok := cached.(*readDirLRUItem)
 		if ok {
@@ -479,7 +457,7 @@ func (self *RawRegFileSystemAccessor) _readDirWithOSPath(
 
 	// Cache the result of this function
 	defer func() {
-		self.readdir_lru.Set(cache_key, &readDirLRUItem{
+		self.cache.readdir_lru.Set(cache_key, &readDirLRUItem{
 			children: result,
 			err:      err,
 			key:      key,

accessors/registry/cache.go

@@ -0,0 +1,123 @@
+//go:build windows
+// +build windows
+
+package registry
+
+import (
+	"time"
+
+	"github.com/Velocidex/ttlcache/v2"
+	"www.velocidex.com/golang/velociraptor/constants"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/vfilter"
+)
+
+const (
+	CACHE_TAG = "_REG_CACHE"
+)
+
+type RegFileSystemAccessorCache struct {
+	lru         *ttlcache.Cache
+	readdir_lru *ttlcache.Cache
+}
+
+func (self *RegFileSystemAccessorCache) GetDir(key string) (*readDirLRUItem, bool) {
+	if self.readdir_lru == nil {
+		return nil, false
+	}
+
+	cached, err := self.readdir_lru.Get(key)
+	if err == nil {
+		cached_res, ok := cached.(*readDirLRUItem)
+		if ok {
+			metricsReadDirLruHit.Inc()
+			return cached_res, true
+		}
+	}
+	metricsReadDirLruMiss.Inc()
+
+	return nil, false
+}
+
+func (self *RegFileSystemAccessorCache) SetDir(key string, dir *readDirLRUItem) {
+	if self.readdir_lru != nil {
+		self.readdir_lru.Set(key, dir)
+	}
+}
+
+func (self *RegFileSystemAccessorCache) Get(key string) (*RegKeyInfo, bool) {
+	if self.lru == nil {
+		return nil, false
+	}
+
+	cached, err := self.lru.Get(key)
+	if err == nil {
+		res, ok := cached.(*RegKeyInfo)
+		if ok {
+			metricsLruHit.Inc()
+			return res, true
+		}
+	}
+
+	metricsLruMiss.Inc()
+	return nil, false
+}
+
+func (self *RegFileSystemAccessorCache) Set(key string, value *RegKeyInfo) {
+	if self.lru != nil {
+		self.lru.Set(key, value)
+	}
+}
+
+func (self *RegFileSystemAccessorCache) Close() {
+	self.lru.Close()
+	self.readdir_lru.Close()
+}
+
+func getRegFileSystemAccessorCache(scope vfilter.Scope) *RegFileSystemAccessorCache {
+	cache_size := int(vql_subsystem.GetIntFromRow(
+		scope, scope, constants.REG_CACHE_SIZE))
+	if cache_size == 0 {
+		cache_size = 1000
+	}
+
+	// Cache is disabled.
+	if cache_size < 0 {
+		return &RegFileSystemAccessorCache{}
+	}
+
+	cache_time := vql_subsystem.GetIntFromRow(
+		scope, scope, constants.REG_CACHE_TIME)
+	if cache_time == 0 {
+		cache_time = 10
+	}
+
+	cache, ok := vql_subsystem.CacheGet(scope, CACHE_TAG).(*RegFileSystemAccessorCache)
+	if ok {
+		return cache
+	}
+
+	cache = &RegFileSystemAccessorCache{
+		lru:         ttlcache.NewCache(),
+		readdir_lru: ttlcache.NewCache(),
+	}
+
+	cache.lru.SetCacheSizeLimit(cache_size)
+	cache.lru.SetTTL(time.Second * time.Duration(cache_time))
+	cache.lru.SkipTTLExtensionOnHit(true)
+
+	cache.readdir_lru.SetCacheSizeLimit(cache_size)
+	cache.readdir_lru.SetTTL(time.Second * time.Duration(cache_time))
+	cache.readdir_lru.SkipTTLExtensionOnHit(true)
+
+	// Add the cache to the root scope so it can be visible outside
+	// our scope. This should maximize cache hits
+	root_scope := vql_subsystem.GetRootScope(scope)
+
+	root_scope.AddDestructor(func() {
+		cache.Close()
+	})
+	vql_subsystem.CacheSet(root_scope, CACHE_TAG, cache)
+
+	return cache
+}

accessors/registry/lru.go

@@ -3,9 +3,14 @@
 
 package registry
 
-import "www.velocidex.com/golang/velociraptor/accessors"
+import (
+	"time"
+
+	"www.velocidex.com/golang/velociraptor/accessors"
+)
 
 type readDirLRUItem struct {
 	children []accessors.FileInfo
 	err      error
+	age      time.Time
 }