Added a Certutil metadata parser.

artifacts/definitions/Windows/Forensics/CertUtil.yaml

@@ -0,0 +1,82 @@
+name: Windows.Forensics.CertUtil
+description: |
+  The Windows Certutil binary is capable of downloading arbitrary
+  files. Attackers typically use it to fetch tools undetected using
+  Living off the Land (LOL) techniques.
+
+  Certutil maintains a cache of the downloaded files and this contains
+  valuable metadata. This artifact parses this metadata to establish
+  what was downloaded and when.
+
+reference:
+  - https://u0041.co/blog/post/3
+
+parameters:
+  - name: URLWhitelist
+    type: csv
+    default: |
+      URL
+      http://sf.symcd.com
+      http://oneocsp.microsoft.com
+      http://certificates.godaddy.com
+      http://ocsp.pki.goog
+      http://repository.certum.pl
+      http://www.microsoft.com
+      http://ocsp.verisign.com
+      http://ctldl.windowsupdate.com
+      http://ocsp.sectigo.com
+      http://ocsp.usertrust.com
+      http://ocsp.comodoca.com
+      http://cacerts.digicert.com
+      http://ocsp.digicert.com
+  - name: MetadataGlob
+    default: C:/Users/*/AppData/LocalLow/Microsoft/CryptnetUrlCache/MetaData/*
+  - name: AlsoUpload
+    type: bool
+
+sources:
+  - query: |
+      LET Profile = '[
+        ["Header", 0, [
+          ["UrlSize", 12, "uint32"],
+          ["HashSize", 100, "uint32"],
+          ["DownloadTime", 16, "uint64"],
+          ["FileSize", 112, "uint32"],
+          ["URL", 116, "String", {
+              "encoding": "utf16",
+              "length": "x=>x.UrlSize"
+          }],
+          ["Hash", "x=>x.UrlSize + 116", "String", {
+              "encoding": "utf16",
+              "length": "x=>x.HashSize"
+          }]
+        ]]
+      ]'
+
+      -- Build a whitelist regex
+      LET URLRegex <= "^" + join(array=URLWhitelist.URL, sep="|")
+      LET Files = SELECT FullPath,
+
+          -- Parse each metadata file.
+          parse_binary(filename=FullPath,
+                       profile=Profile,
+                       struct="Header") AS Header,
+
+                       -- The content is kept in the Content directory.
+                       regex_replace(re="MetaData",
+                                     replace="Content",
+                                     source=FullPath) AS Content
+      FROM glob(globs=MetadataGlob)
+
+      SELECT * FROM foreach(row=Files,
+      query={
+          SELECT FullPath AS _MetadataFile,
+               if(condition=AlsoUpload, then=upload(file=FullPath)) AS _MetdataUpload,
+               if(condition=AlsoUpload, then=upload(file=Content)) AS _Upload,
+               Header.URL AS URL,
+               Header.FileSize AS FileSize,
+               regex_replace(re='"', replace="", source=Header.Hash) AS Hash,
+               timestamp(winfiletime=Header.DownloadTime) AS DownloadTime
+          FROM scope()
+          WHERE NOT URL =~ URLRegex
+      })

bin/golden.go

@@ -187,10 +187,11 @@ func runTest(fixture *testFixture,
 	scope := manager.BuildScopeFromScratch(builder)
 	defer scope.Close()
 
-	scope.AddDestructor(func() {
+	err = scope.AddDestructor(func() {
 		container.Close()
 		os.Remove(tmpfile.Name()) // clean up
 	})
+	kingpin.FatalIfError(err, "AddDestructor")
 
 	result := ""
 	for _, query := range fixture.Queries {

go.mod

@@ -117,7 +117,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20210108163451-876ce85e7301
+	www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c
 	www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada
 )
 

go.sum

@@ -804,5 +804,9 @@ www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff h1:00WZKnVVX
 www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
 www.velocidex.com/golang/vfilter v0.0.0-20210108163451-876ce85e7301 h1:ffb5gMiTNREa9+YYxKxzLl3rPbH6FCpjyEu+dSWRxks=
 www.velocidex.com/golang/vfilter v0.0.0-20210108163451-876ce85e7301/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
+www.velocidex.com/golang/vfilter v0.0.0-20210110142356-d3f572a550df h1:Mq+PKDD3HXVzQy3uDv78oWhEgZZ87qNVkvPvqSMPsn0=
+www.velocidex.com/golang/vfilter v0.0.0-20210110142356-d3f572a550df/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
+www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c h1:soa7usa1BoStBC7NkqqRyZzFA0miIy9pdnaNMmcx6VI=
+www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
 www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada h1:Rtn8RTS/fTxHts5/PrBP7fei2eq4I6tboHyEm3xX0ew=
 www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada/go.mod h1:fkF6W4t1+qr1Nw51+EhAmCwQp2QtwcyNN1Ghwd1vLBM=

gui/velociraptor/Makefile

@@ -3,5 +3,6 @@ all:
 
 build: FORCE
 	npm run build
+	echo > build/static/.keep
 
 FORCE: ;

services/repository/scope.go

@@ -63,7 +63,7 @@ func _build(wg *sync.WaitGroup, self services.ScopeBuilder, from_scratch bool) v
 
 	env.Set(constants.SCOPE_ROOT, scope)
 
-	scope.AddDestructor(func() {
+	_ = scope.AddDestructor(func() {
 		scope.Log("Query Stats: %v", json.MustMarshalString(
 			scope.GetStats().Snapshot()))
 	})

vql/common/diff.go

@@ -139,9 +139,13 @@ func NewDiffCache(
 		done:         make(chan bool),
 	}
 
-	scope.AddDestructor(func() {
+	err := scope.AddDestructor(func() {
 		close(result.done)
 	})
+	if err != nil {
+		close(result.done)
+		scope.Log("AddDestructor: %s", err)
+	}
 
 	return result
 }

vql/common/fifo.go

@@ -126,9 +126,13 @@ func NewFIFOCache(
 	}
 
 	done := make(chan bool)
-	scope.AddDestructor(func() {
+	err := scope.AddDestructor(func() {
 		close(done)
 	})
+	if err != nil {
+		scope.Log("AddDestructor: %s", err)
+		close(done)
+	}
 
 	// Start the query and populate the _FIFOCache.
 	go func() {

vql/common/shell.go

@@ -92,7 +92,12 @@ func (self ShellPlugin) Call(
 
 		// Kill subprocess when the scope is destroyed.
 		sub_ctx, cancel := context.WithCancel(ctx)
-		scope.AddDestructor(cancel)
+		err = scope.AddDestructor(cancel)
+		if err != nil {
+			cancel()
+			scope.Log("shell: %v", err)
+			return
+		}
 
 		command := exec.CommandContext(sub_ctx, arg.Argv[0], arg.Argv[1:]...)
 		stdout_pipe, err := command.StdoutPipe()

vql/filesystem/raw_registry.go

@@ -260,15 +260,15 @@ func (self *RawRegFileSystemAccessor) New(scope vfilter.Scope) (
 		vql_subsystem.CacheSet(scope, RawRegFileSystemTag, result)
 
 		// When scope is destroyed, we close all the filehandles.
-		scope.AddDestructor(func() {
+		err := scope.AddDestructor(func() {
 			result.mu.Lock()
 			defer result.mu.Unlock()
 
 			for _, v := range result.fd_cache {
 				v.fd.Close()
 			}
 		})
-		return result, nil
+		return result, err
 	}
 
 	return result_any.(glob.FileSystemAccessor), nil

vql/filesystem/tempfile.go

@@ -109,9 +109,17 @@ func (self *TempfileFunction) Call(ctx context.Context,
 
 	if arg.RemoveLast {
 		scope.Log("Adding global destructor for %v", tmpfile.Name())
-		vql_subsystem.GetRootScope(scope).AddDestructor(removal)
+		err := vql_subsystem.GetRootScope(scope).AddDestructor(removal)
+		if err != nil {
+			removal()
+			scope.Log("tempfile: %v", err)
+		}
 	} else {
-		scope.AddDestructor(removal)
+		err := scope.AddDestructor(removal)
+		if err != nil {
+			removal()
+			scope.Log("tempfile: %v", err)
+		}
 	}
 	return tmpfile.Name()
 }
@@ -172,9 +180,19 @@ func (self *TempdirFunction) Call(ctx context.Context,
 
 	if arg.RemoveLast {
 		scope.Log("Adding global destructor for %v", dir)
-		vql_subsystem.GetRootScope(scope).AddDestructor(removal)
+		err := vql_subsystem.GetRootScope(scope).AddDestructor(removal)
+		if err != nil {
+			removal()
+			scope.Log("tempfile: %v", err)
+		}
+
 	} else {
-		scope.AddDestructor(removal)
+		err := scope.AddDestructor(removal)
+		if err != nil {
+			removal()
+			scope.Log("tempfile: %v", err)
+		}
+
 	}
 	return dir
 }

vql/filesystem/zip.go

@@ -446,7 +446,7 @@ func (self *ZipFileSystemAccessor) New(scope vfilter.Scope) (glob.FileSystemAcce
 		vql_subsystem.CacheSet(scope, ZipFileSystemAccessorTag, result)
 
 		// When scope is destroyed, we close all the filehandles.
-		scope.AddDestructor(func() {
+		err := scope.AddDestructor(func() {
 			// Decrement refs until we are allowed to
 			// close the file.
 			for _, v := range result.fd_cache {
@@ -456,7 +456,7 @@ func (self *ZipFileSystemAccessor) New(scope vfilter.Scope) (glob.FileSystemAcce
 				}
 			}
 		})
-		return result, nil
+		return result, err
 	}
 
 	return result_any.(glob.FileSystemAccessor), nil

vql/golang/profile.go

@@ -51,7 +51,13 @@ func writeProfile(scope vfilter.Scope,
 	}
 	defer tmpfile.Close()
 
-	scope.AddDestructor(func() { remove(scope, tmpfile.Name()) })
+	dest := func() { remove(scope, tmpfile.Name()) }
+	err = scope.AddDestructor(dest)
+	if err != nil {
+		dest()
+		scope.Log("profile: %s", err)
+		return
+	}
 
 	p := pprof.Lookup(name)
 	if p == nil {
@@ -81,7 +87,11 @@ func writeCPUProfile(
 	}
 	defer tmpfile.Close()
 
-	scope.AddDestructor(func() { remove(scope, tmpfile.Name()) })
+	err = scope.AddDestructor(func() { remove(scope, tmpfile.Name()) })
+	if err != nil {
+		scope.Log("profile: %s", err)
+		return
+	}
 
 	err = pprof.StartCPUProfile(tmpfile)
 	if err != nil {

vql/networking/http_client.go

@@ -335,13 +335,24 @@ func (self *_HttpPlugin) Call(
 				return
 			}
 
+			remove := func() {
+				remove_tmpfile(tmpfile.Name(), scope)
+			}
 			if arg.RemoveLast {
 				scope.Log("Adding global destructor for %v", tmpfile.Name())
-				vql_subsystem.GetRootScope(scope).AddDestructor(func() {
-					remove_tmpfile(tmpfile.Name(), scope)
-				})
+				err := vql_subsystem.GetRootScope(scope).AddDestructor(remove)
+				if err != nil {
+					remove()
+					scope.Log("http_client: %v", err)
+					return
+				}
 			} else {
-				scope.AddDestructor(func() { remove_tmpfile(tmpfile.Name(), scope) })
+				err := scope.AddDestructor(remove)
+				if err != nil {
+					remove()
+					scope.Log("http_client: %v", err)
+					return
+				}
 			}
 
 			scope.Log("http_client: Downloading %v into %v",

vql/parsers/ntfs.go

@@ -38,7 +38,12 @@ func GetNTFSContext(scope vfilter.Scope, device string) (*ntfs.NTFSContext, erro
 			return nil, err
 		}
 
-		scope.AddDestructor(func() { fd.Close() })
+		err = scope.AddDestructor(func() { fd.Close() })
+		if err != nil {
+			fd.Close()
+			return nil, err
+		}
+
 		paged_reader, err := ntfs.NewPagedReader(fd, 1024, 10000)
 		if err != nil {
 			return nil, err

vql/parsers/sqlite.go

@@ -196,9 +196,13 @@ func (self _SQLitePlugin) GetHandle(
 
 		// Add the destructor to the root scope to ensure we
 		// dont get closed too early.
-		vql_subsystem.GetRootScope(scope).AddDestructor(func() {
+		err = vql_subsystem.GetRootScope(scope).AddDestructor(func() {
 			handle.Close()
 		})
+		if err != nil {
+			handle.Close()
+			return nil, err
+		}
 	}
 	return handle, nil
 }
@@ -220,13 +224,18 @@ func (self _SQLitePlugin) _MakeTempfile(
 	defer tmpfile.Close()
 
 	// Make sure the file is removed when the query is done.
-	scope.AddDestructor(func() {
+	remove := func() {
 		scope.Log("sqlite: removing tempfile %v", tmpfile.Name())
 		err = os.Remove(tmpfile.Name())
 		if err != nil {
 			scope.Log("Error removing file: %v", err)
 		}
-	})
+	}
+	err = scope.AddDestructor(remove)
+	if err != nil {
+		remove()
+		return "", err
+	}
 
 	fs, err := glob.GetAccessor(arg.Accessor, scope)
 	if err != nil {

vql/readers/paged.go

@@ -141,7 +141,7 @@ func NewPagedReader(scope vfilter.Scope, accessor, filename string) *AccessorRea
 			lru: cache.NewLRUCache(50),
 		}
 		vql_subsystem.CacheSet(scope, READERS_CACHE, pool)
-		vql_subsystem.GetRootScope(scope).AddDestructor(pool.Close)
+		_ = vql_subsystem.GetRootScope(scope).AddDestructor(pool.Close)
 	} else {
 		pool = pool_any.(*ReaderPool)
 	}

vql/readers/paged_reader_test.go

@@ -30,7 +30,7 @@ func (self *TestSuite) TestPagedReader() {
 		lru: cache.NewLRUCache(5),
 	}
 	vql_subsystem.CacheSet(scope, READERS_CACHE, pool)
-	vql_subsystem.GetRootScope(scope).AddDestructor(pool.Close)
+	_ = vql_subsystem.GetRootScope(scope).AddDestructor(pool.Close)
 
 	tmp_dir, err := ioutil.TempDir("", "tmp")
 	assert.NoError(self.T(), err)

vql/tools/collector.go

@@ -174,12 +174,6 @@ func (self CollectPlugin) Call(
 			subscope := manager.BuildScope(builder)
 			subscope.AppendVars(env)
 
-			// Do not close the subscope until the root
-			// scope is called - this allows raw objects
-			// to be passed between the collector and the
-			// calling scope.
-			vql_subsystem.GetRootScope(scope).AddDestructor(subscope.Close)
-
 			// Run each query and store the results in the container
 			for _, query := range vql_request.Query {
 				// Useful to know what is going on with the collection.