Added lazy objects to pe parser. (Velocidex#857)

The parse_pe() function can spend a lot of time in export/import
tables and if the caller does not care about it then it makes no sense
to do it.

Implementing lazy semantics allows functions and plugins to avoid
unnecessary work and increases performance significantly.

.gitignore

@@ -17,4 +17,5 @@ bin/rsrc.syso
 
 artifacts/assets/ab0x.go
 gui/assets/ab0x.go
-config/ab0x.go
+config/ab0x.go
+.eslintcache

Makefile

@@ -8,7 +8,7 @@ test:
 	go test ./... --tags server_vql
 
 golden:
-	./output/velociraptor -v --config artifacts/testdata/windows/test.config.yaml golden artifacts/testdata/server/testcases/ --env srcDir=`pwd`
+	./output/velociraptor -v --config artifacts/testdata/windows/test.config.yaml golden artifacts/testdata/server/testcases/ --env srcDir=`pwd` --filter=
 
 references:
 	./output/velociraptor vql export docs/references/vql.yaml > docs/references/vql.yaml.tmp

README.md

@@ -99,6 +99,17 @@ need to prepare a PR for a new feature or modify an existing feature
 you can use this to build your own binaries for testing on all
 architectures before send us the PR.
 
+## Supported platforms
+
+Velociraptor is written in Golang and so is available for all the
+platforms [supported by Go](https://github.com/golang/go/wiki/MinimumRequirements). This means that Windows XP and Windows server 2003 are **not** supported but anything after Windows 7/Vista is.
+
+We build our releases on Centos 6 (x64) for Linux and Sierra for MacOS
+so earlier platforms may not be supported by our release pipeline. If
+you need 32 bit builds you will need to build from source. You can do
+this easily by forking the project on GitHub, enabling GitHub Actions
+in your fork and editing the `Linux Build All Arches` pipeline.
+
 ## Getting help
 
 Questions and feedback are welcome at velociraptor-discuss@googlegroups.com

artifacts/definitions/Windows/Applications/TeamViewer/Incoming.yaml

@@ -34,19 +34,16 @@ parameters:
   - name: SearchVSS
     description: "Add VSS into query."
     type: bool
-
 
-sources:
-  - precondition:
-      SELECT OS From info() where OS = 'windows'
 
-    query: |
+sources:
+  - query: |
       -- Build time bounds
       LET DateAfterTime <= if(condition=DateAfter,
         then=DateAfter, else="1600-01-01")
       LET DateBeforeTime <= if(condition=DateBefore,
         then=DateBefore, else="2200-01-01")
-      
+
       -- expand provided glob into a list of paths on the file system (fs)
       LET fspaths <= SELECT FullPath FROM glob(globs=expand(path=FileGlob))
 
@@ -77,11 +74,11 @@ sources:
                     AND Record.SourceHost =~ SourceHostRegex
                     AND Record.User =~ UserRegex
                     AND ( timestamp(epoch=Record.StartTime) < DateBeforeTime
-                        AND timestamp(epoch=Record.StartTime) > DateAfterTime) 
+                        AND timestamp(epoch=Record.StartTime) > DateAfterTime)
                     AND ( timestamp(epoch=Record.EndTime) < DateBeforeTime
                         AND timestamp(epoch=Record.EndTime) > DateAfterTime )
             })
-          
+
       -- include VSS in calculation and deduplicate with GROUP BY by file
       LET include_vss = SELECT * FROM foreach(row=fspaths,
             query={
@@ -97,7 +94,7 @@ sources:
 
 
       -- return rows
-      SELECT 
+      SELECT
         Record.TeamViewerID as TeamViewerID,
         Record.SourceHost as SourceHost,
         timestamp(epoch=Record.StartTime) as StartTime,
@@ -108,4 +105,4 @@ sources:
         FullPath
       FROM if(condition=SearchVSS,
             then=include_vss,
-            else=exclude_vss)
+            else=exclude_vss)

artifacts/definitions/Windows/Remediation/Sinkhole.yaml

@@ -84,7 +84,7 @@ sources:
         FROM execve(argv=['cmd.exe', '/c','ipconfig','/flushdns'])
 
       -- Find existing entries to modify
-      LET existing = SELECT
+      LET existing <= SELECT
             parse_string_with_regex(
             string=Line,
             regex=[
@@ -110,7 +110,7 @@ sources:
             })
 
       -- Set existing entries to sinkholed values
-      LET find_modline = SELECT * FROM foreach(row=changes,
+      LET find_modline <= SELECT * FROM foreach(row=changes,
             query={
                 SELECT
                     format(format='\t%v\t\t%v\t\t# %v',
@@ -125,7 +125,7 @@ sources:
             })
 
       -- Add new hostsfile entries
-      LET find_newline = SELECT * FROM foreach(row=changes,
+      LET find_newline <= SELECT * FROM foreach(row=changes,
             query={
                 SELECT
                     format(format='\t%v\t\t%v\t\t# %v',
@@ -139,7 +139,7 @@ sources:
             })
 
       -- Determine which lines should stay the same
-      LET find_line= SELECT
+      LET find_line <= SELECT
                 Line,
                 Record.Hostname as Domain,
                 'old entry' as Type
@@ -149,7 +149,7 @@ sources:
                 AND NOT Domain in find_newline.Domain
 
       -- Add all lines to staging object
-      LET build_lines = SELECT Line FROM chain(
+      LET build_lines <= SELECT Line FROM chain(
             a=find_modline,
             b=find_newline,
             c=find_line
@@ -158,7 +158,6 @@ sources:
       -- Join lines from staging object
       LET HostsData = join(array=build_lines.Line,sep='\r\n')
 
-
       -- Force start of backup or restore if applicable
       LET backup_restore <= if(condition= RestoreBackup,
                 then= if(condition= check_backup,

artifacts/testdata/server/testcases/file_finder.in.yaml

@@ -1,4 +1,7 @@
 Queries:
+  - SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] )
+    FROM scope()
+
 # Just find a zip file.
   - SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords
     FROM Artifact.Windows.Search.FileFinder(

artifacts/testdata/server/testcases/file_finder.out.yaml

@@ -1,4 +1,8 @@
-SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifact.Windows.Search.FileFinder( SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[
+SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] ) FROM scope()[
+ {
+  "mock(plugin='info', results= [dict(OS='windows'), dict(OS='windows')])": null
+ }
+]SELECT basename(path=FullPath) AS File, Hash, Size, Upload, Keywords FROM Artifact.Windows.Search.FileFinder( SearchFilesGlob=srcDir + "/artifacts/testdata/files/*.zip")[
  {
   "File": "test.zip",
   "Hash": null,

artifacts/testdata/server/testcases/pe.in.yaml

@@ -1,3 +1,33 @@
 Queries:
-  - SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(
-        globs=srcDir + "/artifacts/**10/*.exe")
+  # parse_pe() is a lazy function providing a lot of functionality by
+  # setting callables in the result set. These tests make sure that
+  # the lazy functions are materialized when needed in a transparent
+  # way.
+
+  # PEInfo is a lazy object - PEInfo.ImpHash is a callable - this
+  # tests a callable regex works.
+  - SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo
+    FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")
+    WHERE PEInfo.ImpHash =~ "f34d5f2d4577"
+
+  # Check that Imports (which are lazy) can be passed to function args
+  - SELECT filter(list=parse_pe(file=FullPath).Imports, regex='MmGetPhysicalMemoryRanges')
+    FROM glob(globs=srcDir + "/artifacts/**10/*.sys")
+
+  - LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo
+    FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")
+
+  # Test Associative protocol.
+  - SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X
+    WHERE CompanyName =~ "Microsoft"
+
+  # Test membership protocol
+  - SELECT Name
+    FROM glob(globs=srcDir + "/artifacts/**10/*.{sys,exe}")
+    WHERE "ntoskrnl.exe!MmGetSystemRoutineAddress" IN parse_pe(file=FullPath).Imports
+
+  # Test Iterate protocol on callables - Imports should iterate over
+  # each string in the array.
+  - SELECT * FROM foreach(
+       row=parse_pe(file=srcDir + "/artifacts/testdata/files/winpmem_x64.sys").Imports)
+    WHERE _value =~ "Physical"

artifacts/testdata/server/testcases/pe.out.yaml

@@ -1,9 +1,13 @@
-SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob( globs=srcDir + "/artifacts/**10/*.exe")[
+SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}") WHERE PEInfo.ImpHash =~ "f34d5f2d4577"[
  {
   "Name": "3DBuilder.ResourceResolver.exe",
   "PEInfo": {
-   "Machine": "IMAGE_FILE_MACHINE_I386",
-   "TimeDateStamp": "2019-03-15T09:32:39Z",
+   "FileHeader": {
+    "Machine": "IMAGE_FILE_MACHINE_I386",
+    "TimeDateStamp": "2019-03-15T09:32:39Z",
+    "TimeDateStampRaw": 1552642359,
+    "Characteristics": 34
+   },
    "GUIDAge": "84068848695B4DFDA86ECCE04021A1A91",
    "PDB": "C:\\BA\\2821\\i\\obj\\resourceresolver.csproj__1458271508\\Release\\x64\\3DBuilder.ResourceResolver.pdb",
    "Sections": [
@@ -36,12 +40,36 @@ SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM g
     "InternalName": "3DBuilder.ResourceResolver",
     "LegalCopyright": "©Microsoft Corporation.  All rights reserved.",
     "OriginalFilename": "3DBuilder.ResourceResolver.exe",
-    "ProductName": "3D Builder",
-    "ProductVersion": "16.1.190315001-3DBuilderR16.1"
+    "ProductVersion": "16.1.190315001-3DBuilderR16.1",
+    "ProductName": "3D Builder"
    },
    "Imports": [
     "mscoree.dll!_CorExeMain"
-   ]
+   ],
+   "Exports": [],
+   "Forwards": [],
+   "ImpHash": "f34d5f2d4577ed6d9ceec516c1f5a744"
   }
  }
+]SELECT filter(list=parse_pe(file=FullPath).Imports, regex='MmGetPhysicalMemoryRanges') FROM glob(globs=srcDir + "/artifacts/**10/*.sys")[
+ {
+  "filter(list=parse_pe(file=FullPath).Imports, regex='MmGetPhysicalMemoryRanges')": [
+   "ntoskrnl.exe!MmGetPhysicalMemoryRanges"
+  ]
+ }
+]LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")[]SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X WHERE CompanyName =~ "Microsoft"[
+ {
+  "CompanyName": "Microsoft Corporation"
+ }
+]SELECT Name FROM glob(globs=srcDir + "/artifacts/**10/*.{sys,exe}") WHERE "ntoskrnl.exe!MmGetSystemRoutineAddress" IN parse_pe(file=FullPath).Imports[
+ {
+  "Name": "winpmem_x64.sys"
+ }
+]SELECT * FROM foreach( row=parse_pe(file=srcDir + "/artifacts/testdata/files/winpmem_x64.sys").Imports) WHERE _value =~ "Physical"[
+ {
+  "_value": "ntoskrnl.exe!MmGetVirtualForPhysical"
+ },
+ {
+  "_value": "ntoskrnl.exe!MmGetPhysicalMemoryRanges"
+ }
 ]

artifacts/testdata/server/testcases/raw_registry.in.yaml

@@ -1,4 +1,7 @@
 Queries:
+  - SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] )
+    FROM scope()
+
   # This artifact uses the raw registry parser.
   - SELECT LastModified, Binary, Name, Size, ProductName, Publisher, BinFileVersion
     FROM Artifact.Windows.System.Amcache(

artifacts/testdata/server/testcases/raw_registry.out.yaml

@@ -1,4 +1,8 @@
-SELECT LastModified, Binary, Name, Size, ProductName, Publisher, BinFileVersion FROM Artifact.Windows.System.Amcache( amCacheGlob=srcDir+"/artifacts/testdata/files/Amcache.hve") LIMIT 5[
+SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] ) FROM scope()[
+ {
+  "mock(plugin='info', results= [dict(OS='windows'), dict(OS='windows')])": null
+ }
+]SELECT LastModified, Binary, Name, Size, ProductName, Publisher, BinFileVersion FROM Artifact.Windows.System.Amcache( amCacheGlob=srcDir+"/artifacts/testdata/files/Amcache.hve") LIMIT 5[
  {
   "LastModified": "2019-03-02T08:21:12Z",
   "Binary": "c:\\windows\\softwaredistribution\\download\\install\\am_base.exe",

artifacts/testdata/server/testcases/yara.in.yaml

@@ -11,13 +11,22 @@ Parameters:
        condition: uint32(0) == 0x66676572 and $a
     }
 
+  imphashSig: |
+    import "pe"
+    rule Hive {
+       condition: pe.imphash() == "f34d5f2d4577ed6d9ceec516c1f5a744"
+    }
+
   findX: |
     rule X {
        strings: $a = "X"
        condition: any of them
     }
 
 Queries:
+  - SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] )
+    FROM scope()
+
   # Check the yara plugin works. For large block size the entire file
   # will be in memory.
   - |
@@ -78,6 +87,13 @@ Queries:
     FROM yara(rules=hiveSig, context=10, accessor="file",
     files=srcDir+"/artifacts/testdata/files/yara_test.txt")
 
+
+  # Imphash should also work
+  - |
+    SELECT "imphash", basename(path=FileName)
+    FROM yara(rules=imphashSig, accessor="file",
+    files=srcDir+"/artifacts/testdata/files/3DBuilder.ResourceResolver.exe")
+
   # Setup our mocks
   - |
     LET _ <= SELECT

artifacts/testdata/server/testcases/yara.out.yaml

@@ -1,4 +1,8 @@
-SELECT String.Offset, str(str=String.Data) AS Hit
+SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] ) FROM scope()[
+ {
+  "mock(plugin='info', results= [dict(OS='windows'), dict(OS='windows')])": null
+ }
+]SELECT String.Offset, str(str=String.Data) AS Hit
 FROM yara(rules=hiveSig, blocksize=10,
 files="regfXXXXXXXXXXXXXXXXXXXXXXprogram", accessor="data")
 []SELECT String.Offset, str(str=String.Data) AS Hit
@@ -89,6 +93,14 @@ files=srcDir+"/artifacts/testdata/files/yara_test.txt")
   "String.Offset": 12,
   "Hit": "gfXXXXXXXXprogram\n"
  }
+]SELECT "imphash", basename(path=FileName)
+FROM yara(rules=imphashSig, accessor="file",
+files=srcDir+"/artifacts/testdata/files/3DBuilder.ResourceResolver.exe")
+[
+ {
+  "\"imphash\"": "imphash",
+  "basename(path=FileName)": "3DBuilder.ResourceResolver.exe"
+ }
 ]LET _ <= SELECT
       mock(plugin='http_client', results=[
          dict(Url='http://remote',

artifacts/testdata/windows/users.out.yaml

@@ -3,7 +3,7 @@ Select Name, Uid, Gid, Directory, Type  from Artifact.Windows.Sys.Users() WHERE
   "Name": "Guest",
   "Uid": 501,
   "Gid": 513,
-  "Directory": null,
+  "Directory": [],
   "Type": "local"
  },
  {

bin/query.go

@@ -84,7 +84,7 @@ func outputJSONL(ctx context.Context,
 		vql_subsystem.MarshalJsonl(scope),
 		10, *max_wait) {
 		_, err := out.Write(result.Payload)
-		kingpin.FatalIfError(err, "outputCSV")
+		kingpin.FatalIfError(err, "outputJSONL")
 	}
 }