From 44748662a7583710fa48131e9c97754fe86cc76a Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 2 Nov 2020 22:11:46 +1000 Subject: [PATCH] Implemented a local hash database. (#710) --- .../definitions/Windows/Detection/Usn.yaml | 2 + .../Windows/Forensics/LocalHashes/Glob.yaml | 59 ++++++++++++++ .../Windows/Forensics/LocalHashes/Query.yaml | 78 +++++++++++++++++++ .../Windows/Forensics/LocalHashes/Usn.yaml | 62 +++++++++++++++ .../testdata/windows/localhashes.in.yaml | 13 ++++ .../testdata/windows/localhashes.out.yaml | 19 +++++ vql/parsers/sqlite.go | 23 ++++-- 7 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml create mode 100644 artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml create mode 100644 artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml create mode 100644 artifacts/testdata/windows/localhashes.in.yaml create mode 100644 artifacts/testdata/windows/localhashes.out.yaml diff --git a/artifacts/definitions/Windows/Detection/Usn.yaml b/artifacts/definitions/Windows/Detection/Usn.yaml index 4c7872c50e0..83337f6cf33 100644 --- a/artifacts/definitions/Windows/Detection/Usn.yaml +++ b/artifacts/definitions/Windows/Detection/Usn.yaml @@ -13,6 +13,8 @@ description: | prefetch files are not updated immediately - there could be a small delay between the execution and the prefetch being modified. +type: CLIENT_EVENT + parameters: - name: PathRegex description: A regex to match the entire path (you can watch a directory or a file type). diff --git a/artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml b/artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml new file mode 100644 index 00000000000..ff38fdeb0d5 --- /dev/null +++ b/artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml @@ -0,0 +1,59 @@ +name: Windows.Forensics.LocalHashes.Glob +description: | + This artifact maintains a local (client side) database of file + hashes. It is then possible to query this database using the + Windows.Forensics.LocalHashes.Query artifact + + Maintaining hashes client side allows Velociraptor to answer the + query - which machine has this hash on our network extremely + quickly. Velociraptor only needs to lookup the each client's local + database of file hashes. + + Maintaining this database case be done using this artifact or using + the Windows.Forensics.LocalHashes.Usn artifact. + + This artifact simply crawls the filesystem hashing files as + specified by the glob expression, and adds them to the local hash + database. You can rate limit this artifact using the ops/sec setting + to perform a slow update of the local file hash database. + +parameters: + - name: HashGlob + description: Search for files according to this glob and hash them. + default: C:/Users/**/*.exe + + - name: HashDb + description: Name of the local hash database + default: hashdb.sqlite + + - name: SuppressOutput + description: If this is set, the artifact does not return any rows to the server but will still update the local database. + type: bool + +precondition: SELECT OS from info() where OS = "windows" + +sources: + - query: | + LET hash_db <= SELECT HashDBPath + FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb) + + LET path <= hash_db[0].HashDBPath + + LET _ <= log(message="Will use local hash database " + path) + + // Crawl the files and calculate their hashes + LET files = SELECT FullPath, Size, hash(path=FullPath).MD5 AS Hash + FROM glob(globs=HashGlob) + WHERE Mode.IsRegular + + LET insertion = SELECT FullPath, Hash, Size, { + SELECT * FROM sqlite(file=path, + query="INSERT into hashes (path, md5, timestamp, size) values (?,?,?,?)", + args=[FullPath, Hash, now(), Size]) + } AS Insert + FROM files + WHERE Insert OR TRUE + + SELECT FullPath, Hash, Size + FROM insertion + WHERE SuppressOutput != "Y" diff --git a/artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml b/artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml new file mode 100644 index 00000000000..3669c92d5a2 --- /dev/null +++ b/artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml @@ -0,0 +1,78 @@ +name: Windows.Forensics.LocalHashes.Query +description: | + This artifact maintains a local (client side) database of file + hashes. It is then possible to query this database using the + Windows.Forensics.LocalHashes.Query artifact. + + NOTE: This artifact expects a CSV file with one hash per line. On + the command line you can encode carriage return using powershell + like this: + + ``` + .\velociraptor.exe -v artifacts collect Windows.Forensics.LocalHashes.Query --args "Hashes=Hash`ne6c1ce56e6729a0b077c0f2384726b30" + ``` + +precondition: SELECT OS from info() where OS = "windows" + +parameters: + - name: Hashes + description: The hash to query for. + type: csv + default: | + Hash + XXX + + - name: CommaDelimitedHashes + description: A set of comma delimited hashes + default: + + - name: HashDb + description: Name of the local hash database + default: hashdb.sqlite + +sources: + - query: | + LET hash_db <= path_join(components=[dirname(path=tempfile()), HashDb]) + + LET _ <= log(message="Will use local hash database " + hash_db) + + // SQL to create the initial database. + LET _ <= SELECT * FROM sqlite(file=hash_db, + query="CREATE table if not exists hashes(path text, md5 varchar(16), size bigint, timestamp bigint)") + + LET _ <= SELECT * FROM sqlite(file=hash_db, + query="create index if not exists hashidx on hashes(md5)") + + LET _ <= SELECT * FROM sqlite(file=hash_db, + query="create index if not exists pathidx on hashes(path)") + + LET _ <= SELECT * FROM sqlite(file=hash_db, + query="create unique index if not exists uniqueidx on hashes(path, md5)") + + LET lookup(Hash) = SELECT hash_db AS HashDBPath, path AS Path, md5 AS MD5, size AS Size, + timestamp(epoch=time) AS Timestamp + FROM sqlite(file=hash_db, + query="SELECT path, md5, size, timestamp AS time FROM hashes WHERE md5 = ?", + args=Hash) + + -- Check hashes from the CSV or comma delimited input + LET hashes = SELECT Hash FROM chain( + a={ + SELECT Hash FROM parse_csv(filename=Hashes, accessor="data") + }, b={ + SELECT * FROM foreach(row=split(string=CommaDelimitedHashes, sep=","), + query={ + SELECT _value AS Hash FROM scope() + }) + }) + + SELECT * FROM switch( + a={ + SELECT * FROM foreach(row=hashes, + query={ + SELECT * FROM lookup(Hash=Hash) + }) + }, + b={ + SELECT hash_db AS HashDBPath FROM scope() + }) diff --git a/artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml b/artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml new file mode 100644 index 00000000000..1d2c569dc1f --- /dev/null +++ b/artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml @@ -0,0 +1,62 @@ +name: Windows.Forensics.LocalHashes.Usn +description: | + This artifact maintains a local (client side) database of file + hashes. It is then possible to query this database using the + Windows.Forensics.LocalHashes.Query artifact + +type: CLIENT_EVENT + +parameters: + - name: PathRegex + description: A regex to match the entire path (you can watch a directory or a file type). + default: .exe$ + + - name: Device + description: The NTFS drive to watch + default: C:\\ + + - name: HashDb + description: Name of the local hash database + default: hashdb.sqlite + + - name: SuppressOutput + description: If this is set, the artifact does not return any rows to the server but will still update the local database. + type: bool + + +precondition: SELECT OS from info() where OS = "windows" + +sources: + - query: | + LET hash_db <= SELECT HashDBPath + FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb) + + LET path <= hash_db[0].HashDBPath + + LET _ <= log(message="Will use local hash database " + path) + + LET file_overwrites = SELECT Device + FullPath AS FullPath + FROM watch_usn(device=Device) + WHERE FullPath =~ PathRegex AND "DATA_OVERWRITE" IN Reason + + -- Stat each file that was changed to get its size and hash + LET files = SELECT * FROM foreach(row=file_overwrites, + query={ + SELECT FullPath, Size, hash(path=FullPath).MD5 AS Hash, now() AS Time + FROM stat(filename=FullPath) + WHERE Mode.IsRegular + }) + + -- For each file hashed, insert to the local database + LET insertion = SELECT FullPath, Hash, Size, { + SELECT * FROM sqlite(file=path, + query="INSERT into hashes (path, md5, timestamp, size) values (?,?,?,?)", + args=[FullPath, Hash, Time, Size]) + } AS Insert + FROM files + WHERE Insert OR TRUE + + // If output is suppressed do not emit a row, but still update the local database. + SELECT FullPath, Hash, Size, Time + FROM insertion + WHERE SuppressOutput != "Y" diff --git a/artifacts/testdata/windows/localhashes.in.yaml b/artifacts/testdata/windows/localhashes.in.yaml new file mode 100644 index 00000000000..ee1086847f1 --- /dev/null +++ b/artifacts/testdata/windows/localhashes.in.yaml @@ -0,0 +1,13 @@ +Queries: + # Populate the hash database + - SELECT basename(path=FullPath) AS Name, + Size, Hash FROM Artifact.Windows.Forensics.LocalHashes.Glob( + HashGlob=srcDir + '/artifacts/testdata/files/Security_1_record.evtx') + + # Query the hash database + - SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( + CommaDelimitedHashes="39985be74b8bb4ee716ab55b5f6dfbd4") + + # Query the hash database using a CSV input + - SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( + Hashes="Hash\n39985be74b8bb4ee716ab55b5f6dfbd4") diff --git a/artifacts/testdata/windows/localhashes.out.yaml b/artifacts/testdata/windows/localhashes.out.yaml new file mode 100644 index 00000000000..8aa6a2f2d31 --- /dev/null +++ b/artifacts/testdata/windows/localhashes.out.yaml @@ -0,0 +1,19 @@ +SELECT basename(path=FullPath) AS Name, Size, Hash FROM Artifact.Windows.Forensics.LocalHashes.Glob( HashGlob=srcDir + '/artifacts/testdata/files/Security_1_record.evtx')[ + { + "Name": "Security_1_record.evtx", + "Size": 69632, + "Hash": "39985be74b8bb4ee716ab55b5f6dfbd4" + } +]SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( CommaDelimitedHashes="39985be74b8bb4ee716ab55b5f6dfbd4")[ + { + "Path": "D:\\a\\velociraptor\\velociraptor\\artifacts\\testdata\\files\\Security_1_record.evtx", + "MD5": "39985be74b8bb4ee716ab55b5f6dfbd4", + "Size": 69632 + } +]SELECT Path, MD5, Size FROM Artifact.Windows.Forensics.LocalHashes.Query( Hashes="Hash\n39985be74b8bb4ee716ab55b5f6dfbd4")[ + { + "Path": "D:\\a\\velociraptor\\velociraptor\\artifacts\\testdata\\files\\Security_1_record.evtx", + "MD5": "39985be74b8bb4ee716ab55b5f6dfbd4", + "Size": 69632 + } +] \ No newline at end of file diff --git a/vql/parsers/sqlite.go b/vql/parsers/sqlite.go index 6fab7c9afe9..c416673dd9d 100644 --- a/vql/parsers/sqlite.go +++ b/vql/parsers/sqlite.go @@ -34,6 +34,7 @@ import ( "github.com/Velocidex/ordereddict" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/glob" utils "www.velocidex.com/golang/velociraptor/utils" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -148,8 +149,7 @@ func (self _SQLitePlugin) GetHandle( filename := VFSPathToFilesystemPath(arg.Filename) key := "sqlite_" + filename + arg.Accessor - handle, ok := vql_subsystem.CacheGet( - scope, key).(*sqlx.DB) + handle, ok := vql_subsystem.CacheGet(scope, key).(*sqlx.DB) if !ok { if arg.Accessor == "file" { handle, err = sqlx.Connect("sqlite3", filename) @@ -166,11 +166,15 @@ func (self _SQLitePlugin) GetHandle( if !strings.Contains(err.Error(), "locked") { return nil, err } + scope.Log("Sqlite file %v is locked with %v, creating a local copy", + filename, err) filename, err = self._MakeTempfile(ctx, arg, filename, scope) if err != nil { scope.Log("Unable to create temp file: %v", err) return nil, err } + scope.Log("Using local copy %v", filename) + } } else { filename, err = self._MakeTempfile(ctx, arg, filename, scope) @@ -186,9 +190,18 @@ func (self _SQLitePlugin) GetHandle( } vql_subsystem.CacheSet(scope, key, handle) - scope.AddDestructor(func() { - handle.Close() - }) + + // Add the destructor to the root scope to ensure we + // dont get closed too early. + root_any, pres := scope.Resolve(constants.SCOPE_ROOT) + if pres { + root, ok := root_any.(*vfilter.Scope) + if ok { + root.AddDestructor(func() { + handle.Close() + }) + } + } } return handle, nil }