Implemented a local hash database. (Velocidex#710)

artifacts/definitions/Windows/Detection/Usn.yaml

@@ -13,6 +13,8 @@ description: |
   prefetch files are not updated immediately - there could be a small
   delay between the execution and the prefetch being modified.
 
+type: CLIENT_EVENT
+
 parameters:
   - name: PathRegex
     description: A regex to match the entire path (you can watch a directory or a file type).

artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml

@@ -0,0 +1,59 @@
+name: Windows.Forensics.LocalHashes.Glob
+description: |
+  This artifact maintains a local (client side) database of file
+  hashes. It is then possible to query this database using the
+  Windows.Forensics.LocalHashes.Query artifact
+
+  Maintaining hashes client side allows Velociraptor to answer the
+  query - which machine has this hash on our network extremely
+  quickly. Velociraptor only needs to lookup the each client's local
+  database of file hashes.
+
+  Maintaining this database case be done using this artifact or using
+  the Windows.Forensics.LocalHashes.Usn artifact.
+
+  This artifact simply crawls the filesystem hashing files as
+  specified by the glob expression, and adds them to the local hash
+  database. You can rate limit this artifact using the ops/sec setting
+  to perform a slow update of the local file hash database.
+
+parameters:
+  - name: HashGlob
+    description: Search for files according to this glob and hash them.
+    default: C:/Users/**/*.exe
+
+  - name: HashDb
+    description: Name of the local hash database
+    default: hashdb.sqlite
+
+  - name: SuppressOutput
+    description: If this is set, the artifact does not return any rows to the server but will still update the local database.
+    type: bool
+
+precondition: SELECT OS from info() where OS = "windows"
+
+sources:
+  - query: |
+      LET hash_db <= SELECT HashDBPath
+      FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb)
+
+      LET path <= hash_db[0].HashDBPath
+
+      LET _ <= log(message="Will use local hash database " + path)
+
+      // Crawl the files and calculate their hashes
+      LET files = SELECT FullPath, Size, hash(path=FullPath).MD5 AS Hash
+      FROM glob(globs=HashGlob)
+      WHERE Mode.IsRegular
+
+      LET insertion = SELECT FullPath, Hash, Size, {
+         SELECT * FROM sqlite(file=path,
+            query="INSERT into hashes (path, md5, timestamp, size) values (?,?,?,?)",
+            args=[FullPath, Hash, now(), Size])
+      } AS Insert
+      FROM files
+      WHERE Insert OR TRUE
+
+      SELECT FullPath, Hash, Size
+      FROM insertion
+      WHERE SuppressOutput != "Y"

artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml

@@ -0,0 +1,78 @@
+name: Windows.Forensics.LocalHashes.Query
+description: |
+  This artifact maintains a local (client side) database of file
+  hashes. It is then possible to query this database using the
+  Windows.Forensics.LocalHashes.Query artifact.
+
+  NOTE: This artifact expects a CSV file with one hash per line. On
+  the command line you can encode carriage return using powershell
+  like this:
+
+  ```
+  .\velociraptor.exe -v artifacts collect Windows.Forensics.LocalHashes.Query --args "Hashes=Hash`ne6c1ce56e6729a0b077c0f2384726b30"
+  ```
+
+precondition: SELECT OS from info() where OS = "windows"
+
+parameters:
+  - name: Hashes
+    description: The hash to query for.
+    type: csv
+    default: |
+      Hash
+      XXX
+
+  - name: CommaDelimitedHashes
+    description: A set of comma delimited hashes
+    default:
+
+  - name: HashDb
+    description: Name of the local hash database
+    default: hashdb.sqlite
+
+sources:
+  - query: |
+      LET hash_db <= path_join(components=[dirname(path=tempfile()), HashDb])
+
+      LET _ <= log(message="Will use local hash database " + hash_db)
+
+      // SQL to create the initial database.
+      LET _ <= SELECT * FROM sqlite(file=hash_db,
+         query="CREATE table if not exists hashes(path text, md5 varchar(16), size bigint, timestamp bigint)")
+
+      LET _ <= SELECT * FROM sqlite(file=hash_db,
+         query="create index if not exists hashidx on hashes(md5)")
+
+      LET _ <= SELECT * FROM sqlite(file=hash_db,
+         query="create index if not exists pathidx on hashes(path)")
+
+      LET _ <= SELECT * FROM sqlite(file=hash_db,
+         query="create unique index if not exists uniqueidx on hashes(path, md5)")
+
+      LET lookup(Hash) = SELECT hash_db AS HashDBPath, path AS Path, md5 AS MD5, size AS Size,
+             timestamp(epoch=time) AS Timestamp
+      FROM sqlite(file=hash_db,
+         query="SELECT path, md5, size, timestamp AS time FROM hashes WHERE md5 = ?",
+         args=Hash)
+
+      -- Check hashes from the CSV or comma delimited input
+      LET hashes = SELECT Hash FROM chain(
+      a={
+        SELECT Hash FROM parse_csv(filename=Hashes, accessor="data")
+      }, b={
+        SELECT * FROM foreach(row=split(string=CommaDelimitedHashes, sep=","),
+        query={
+           SELECT _value AS Hash FROM scope()
+        })
+      })
+
+      SELECT * FROM switch(
+      a={
+          SELECT * FROM foreach(row=hashes,
+          query={
+              SELECT * FROM lookup(Hash=Hash)
+          })
+      },
+      b={
+         SELECT hash_db AS HashDBPath FROM scope()
+      })

artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml

@@ -0,0 +1,62 @@
+name: Windows.Forensics.LocalHashes.Usn
+description: |
+  This artifact maintains a local (client side) database of file
+  hashes. It is then possible to query this database using the
+  Windows.Forensics.LocalHashes.Query artifact
+
+type: CLIENT_EVENT
+
+parameters:
+  - name: PathRegex
+    description: A regex to match the entire path (you can watch a directory or a file type).
+    default: .exe$
+
+  - name: Device
+    description: The NTFS drive to watch
+    default: C:\\
+
+  - name: HashDb
+    description: Name of the local hash database
+    default: hashdb.sqlite
+
+  - name: SuppressOutput
+    description: If this is set, the artifact does not return any rows to the server but will still update the local database.
+    type: bool
+
+
+precondition: SELECT OS from info() where OS = "windows"
+
+sources:
+  - query: |
+      LET hash_db <= SELECT HashDBPath
+      FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb)
+
+      LET path <= hash_db[0].HashDBPath
+
+      LET _ <= log(message="Will use local hash database " + path)
+
+      LET file_overwrites = SELECT Device + FullPath AS FullPath
+      FROM watch_usn(device=Device)
+      WHERE FullPath =~ PathRegex AND "DATA_OVERWRITE" IN Reason
+
+      -- Stat each file that was changed to get its size and hash
+      LET files = SELECT * FROM foreach(row=file_overwrites,
+      query={
+         SELECT FullPath, Size, hash(path=FullPath).MD5 AS Hash, now() AS Time
+         FROM stat(filename=FullPath)
+         WHERE Mode.IsRegular
+      })
+
+      -- For each file hashed, insert to the local database
+      LET insertion = SELECT FullPath, Hash, Size, {
+         SELECT * FROM sqlite(file=path,
+            query="INSERT into hashes (path, md5, timestamp, size) values (?,?,?,?)",
+            args=[FullPath, Hash, Time, Size])
+      } AS Insert
+      FROM files
+      WHERE Insert OR TRUE
+
+      // If output is suppressed do not emit a row, but still update the local database.
+      SELECT FullPath, Hash, Size, Time
+      FROM insertion
+      WHERE SuppressOutput != "Y"

artifacts/testdata/windows/localhashes.in.yaml

@@ -0,0 +1,13 @@
+Queries:
+  # Populate the hash database
+  - SELECT basename(path=FullPath) AS Name,
+           Size, Hash FROM Artifact.Windows.Forensics.LocalHashes.Glob(
+           HashGlob=srcDir + '/artifacts/testdata/files/Security_1_record.evtx')
+
+  # Query the hash database
+  - SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query(
+      CommaDelimitedHashes="39985be74b8bb4ee716ab55b5f6dfbd4")
+
+  # Query the hash database using a CSV input
+  - SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query(
+      Hashes="Hash\n39985be74b8bb4ee716ab55b5f6dfbd4")

artifacts/testdata/windows/localhashes.out.yaml

@@ -0,0 +1,19 @@
+SELECT basename(path=FullPath) AS Name, Size, Hash FROM Artifact.Windows.Forensics.LocalHashes.Glob( HashGlob=srcDir + '/artifacts/testdata/files/Security_1_record.evtx')[
+ {
+  "Name": "Security_1_record.evtx",
+  "Size": 69632,
+  "Hash": "39985be74b8bb4ee716ab55b5f6dfbd4"
+ }
+]SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( CommaDelimitedHashes="39985be74b8bb4ee716ab55b5f6dfbd4")[
+ {
+  "Path": "D:\\a\\velociraptor\\velociraptor\\artifacts\\testdata\\files\\Security_1_record.evtx",
+  "MD5": "39985be74b8bb4ee716ab55b5f6dfbd4",
+  "Size": 69632
+ }
+]SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( Hashes="Hash\n39985be74b8bb4ee716ab55b5f6dfbd4")[
+ {
+  "Path": "D:\\a\\velociraptor\\velociraptor\\artifacts\\testdata\\files\\Security_1_record.evtx",
+  "MD5": "39985be74b8bb4ee716ab55b5f6dfbd4",
+  "Size": 69632
+ }
+]

vql/parsers/sqlite.go

@@ -34,6 +34,7 @@ import (
 	"github.com/Velocidex/ordereddict"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/glob"
 	utils "www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -148,8 +149,7 @@ func (self _SQLitePlugin) GetHandle(
 	filename := VFSPathToFilesystemPath(arg.Filename)
 
 	key := "sqlite_" + filename + arg.Accessor
-	handle, ok := vql_subsystem.CacheGet(
-		scope, key).(*sqlx.DB)
+	handle, ok := vql_subsystem.CacheGet(scope, key).(*sqlx.DB)
 	if !ok {
 		if arg.Accessor == "file" {
 			handle, err = sqlx.Connect("sqlite3", filename)
@@ -166,11 +166,15 @@ func (self _SQLitePlugin) GetHandle(
 				if !strings.Contains(err.Error(), "locked") {
 					return nil, err
 				}
+				scope.Log("Sqlite file %v is locked with %v, creating a local copy",
+					filename, err)
 				filename, err = self._MakeTempfile(ctx, arg, filename, scope)
 				if err != nil {
 					scope.Log("Unable to create temp file: %v", err)
 					return nil, err
 				}
+				scope.Log("Using local copy %v", filename)
+
 			}
 		} else {
 			filename, err = self._MakeTempfile(ctx, arg, filename, scope)
@@ -186,9 +190,18 @@ func (self _SQLitePlugin) GetHandle(
 		}
 
 		vql_subsystem.CacheSet(scope, key, handle)
-		scope.AddDestructor(func() {
-			handle.Close()
-		})
+
+		// Add the destructor to the root scope to ensure we
+		// dont get closed too early.
+		root_any, pres := scope.Resolve(constants.SCOPE_ROOT)
+		if pres {
+			root, ok := root_any.(*vfilter.Scope)
+			if ok {
+				root.AddDestructor(func() {
+					handle.Close()
+				})
+			}
+		}
 	}
 	return handle, nil
 }