Implement file based merge sort for large row sets. (Velocidex#882)

artifacts/definitions/Windows/Detection/CryptnetUrlCache.yaml

@@ -1,15 +1,15 @@
 name: Windows.Detection.CryptnetUrlCache
 description: |
    This artifact will hunt for evidence of Certutil use as a download cradle.
-   
-   The CryptnetUrlCache contains both content and metadata of files downloaded by 
-   CertUtil and other Windows Crypto components. The artifact will first look for 
-   content larger than a specified size, then check headers against a whitelist 
-   of common content types. Additional options include a UrlWhitelist and search 
+
+   The CryptnetUrlCache contains both content and metadata of files downloaded by
+   CertUtil and other Windows Crypto components. The artifact will first look for
+   content larger than a specified size, then check headers against a whitelist
+   of common content types. Additional options include a UrlWhitelist and search
    of VSS.
-   
-   NOTE: Expect some false positives and build a whitelist of Urls to add for 
-   regular hunts. Alternatively target specific headers such as PE files by 
+
+   NOTE: Expect some false positives and build a whitelist of Urls to add for
+   regular hunts. Alternatively target specific headers such as PE files by
    adding '^MZ' to the HeaderRegex field.
 
 author: "@mgreen27 - Matt Green"
@@ -65,12 +65,16 @@ sources:
       LET files <= SELECT * FROM foreach(row=CryptnetUrlCache,
             query={
                 SELECT * FROM if(condition=SearchVSS,
-                    then= vsspaths(path=FileGlob),
-                    else= fspaths(path=FileGlob))
+                    then= {
+                       SELECT * FROM vsspaths(path=FileGlob)
+                    },
+                    else= {
+                       SELECT * FROM fspaths(path=FileGlob)
+                    })
             })
 
 
-      -- extract metadata lines 
+      -- extract metadata lines
       LET metadata = SELECT * FROM foreach(row=files,
         query={
             SELECT
@@ -85,25 +89,25 @@ sources:
             FROM parse_lines(filename=FullPath)
             WHERE MetaPath =~ '\\\\Microsoft\\\\CryptnetUrlCache\\\\metadata\\\\'
             GROUP BY MetaPath
-        }) 
+        })
 
 
       -- find suspicious content files and extract headers
-      LET hits = SELECT 
+      LET hits = SELECT
             FullPath,Name,Size,
             Mtime, Atime, Ctime,
             hash(path=FullPath) as Hash,
             read_file(length=4,filename=FullPath) as Header
         FROM files
-        WHERE 
+        WHERE
             FullPath =~ '\\\\Microsoft\\\\CryptnetUrlCache\\\\Content\\\\'
             AND Size > int(int=SusSize)
 
 
       -- output rows
       SELECT * FROM foreach(row=hits,
         query={
-            SELECT 
+            SELECT
                 FullPath,Name,Size,Header,
                 Mtime, Atime, Ctime,
                 Url, Hash,
@@ -116,7 +120,7 @@ sources:
                 MetaPath,
                 MetaMtime,MetaAtime,MetaCtime
             FROM metadata
-            WHERE 
+            WHERE
                 MetaPath =~ Name
                 AND Header =~ HeaderRegex
                 AND NOT Header =~ HeaderWhitelist
@@ -127,4 +131,4 @@ sources:
                     string=FullPath,
                         sep='CryptnetUrlCache')[0] = split(string=MetaPath,
                             sep='CryptnetUrlCache')[0]
-        })
+        })

artifacts/definitions/Windows/NTFS/MFT.yaml

@@ -107,4 +107,6 @@ sources:
       -- return rows
       SELECT * FROM if(condition=AllDrives,
         then= all_drives,
-        else= { SELECT * FROM mftsearch(MFTPath=MFTFilename) })
+        else= {
+           SELECT * FROM mftsearch(MFTPath=MFTFilename)
+        })

artifacts/definitions/Windows/Search/VSS.yaml

@@ -29,7 +29,7 @@ sources:
          regex="^(?P<Device>(\\\\\\\\.\\\\GLOBALROOT\\\\Device\\\\HarddiskVolumeShadowCopy[^\\\\]+\\\\|\\\\\\\\.\\\\.:\\\\))")
 
       -- Build a SearchGlob for all logical disks and VSS
-      LET globs = SELECT
+      LET globs = SELECT FullPath,
             FullPath + '/' + extract_path(FullPath=SearchFilesGlob).Path as SearchGlob
         FROM glob(globs='/*', accessor='ntfs')
         ORDER BY FullPath  DESC

artifacts/definitions/Windows/Timeline/Registry/RunMRU.yaml

@@ -104,7 +104,7 @@ sources:
                         SELECT mru_grouped
                         FROM usercommands
                         WHERE user = Username
-                        ORDER BY mru_order
+                        ORDER BY mru_grouped
                     } as mru_grouped
                 FROM results
                 ORDER BY mru_grouped

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d
 	github.com/russross/blackfriday/v2 v2.0.1
 	github.com/sebdah/goldie v1.0.0
+	github.com/sebdah/goldie/v2 v2.5.3
 	github.com/sergi/go-diff v1.1.0
 	github.com/shirou/gopsutil v3.20.10+incompatible
 	github.com/sirupsen/logrus v1.6.0
@@ -117,7 +118,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20210115155733-3088911a7a29
+	www.velocidex.com/golang/vfilter v0.0.0-20210120155859-69f51f06d7d2
 	www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8
 )
 

go.sum

@@ -804,7 +804,7 @@ www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiA
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns=
 www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff h1:00WZKnVVXwsfaV18UNn7iGQMCqQUf0cCFQ2RbbprYns=
 www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
-www.velocidex.com/golang/vfilter v0.0.0-20210115155733-3088911a7a29 h1:uYAnMC5EjW9aIkA0pHIEoSsWTyMT4LHtOSXI6BNs5ok=
-www.velocidex.com/golang/vfilter v0.0.0-20210115155733-3088911a7a29/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
+www.velocidex.com/golang/vfilter v0.0.0-20210120155859-69f51f06d7d2 h1:JiPeM1oNm1oU/SmR1DvP/MVz2KU96Nm5AWAHHUfItrk=
+www.velocidex.com/golang/vfilter v0.0.0-20210120155859-69f51f06d7d2/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
 www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8 h1:QLV9zSccnI0IcfU0eIswx0G7g+umEGoCtaS+Hr9zRzk=
 www.velocidex.com/golang/vtypes v0.0.0-20210116160458-a505b35bdfb8/go.mod h1:34AZRfhNvJ1QAwPpYrDxjCyOFys+NbSmH6LLVSjsAEg=

services/repository/scope.go

@@ -8,6 +8,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/velociraptor/vql/sorter"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -53,6 +54,10 @@ func _build(wg *sync.WaitGroup, self services.ScopeBuilder, from_scratch bool) v
 	} else {
 		scope = vql_subsystem.MakeScope()
 	}
+
+	// Use our own sorter
+	scope.SetSorter(sorter.MergeSorter{ChunkSize: 10000})
+
 	artifact_plugin := NewArtifactRepositoryPlugin(wg, self.Repository.(*Repository))
 	env.Set("Artifact", artifact_plugin)