Add new embedded pe in data section parse (Velocidex#1943)

Co-authored-by: Mike Cohen <mike@velocidex.com>

artifacts/definitions/Windows/Carving/CobaltStrike.yaml

@@ -376,6 +376,23 @@ export: |
         ["__LicenceBytes", 0, "Value",{"value":"x=>read_file(accessor='data',filename=x.__Position, offset=len(list=x.Server) + 1 ,length=4)"}],
         ["Licence", 0, "Value",{"value":"x=>parse_binary(accessor='data', filename=x.__LicenceBytes,struct='uint32b')"}],
         ["Strings", 0, "Value",{"value":"x=>find_strings(data=_Data,length=5,filter='.').Strings"}],
+    ]],
+
+    ["EmbeddedPE", 0, [
+        ["__PayloadType", 0, "uint32"],
+        ["PayloadType", 0, "Value",{"value":"x=>format(format='0x%08x',args=x.__PayloadType)"}],
+        ["__PayloadSize", 4, "uint32"],
+        #["PayloadSize", 4, "Value",{"value":"x=>format(format='0x%08x',args=x.__PayloadSize)"}],
+        ["__XorKey", 8, "uint32b"],
+        ["XorKey", 8, "Value",{"value":"x=>format(format='0x%08x',args=x.__XorKey)"}],
+        ["__Id2", 12, "uint32"],
+        ["Id2", 12, "Value",{"value":"x=>format(format='0x%08x',args=x.__Id2)"}],
+        ["__Payload", 16, "Value",{"value":"x=>read_file(accessor='data',filename=embedded_section(path=TargetBytes || FullPath,
+                type=if(condition=TargetBytes,then='data',else='auto'))[0].Data,offset=16,length=x.__PayloadSize)"}],
+        #["__Payload", 16, "String",{"term_hex":"",length=x.__PayloadSize)"}],
+        ["DecodedPayload", 16, "Value",{"value":"x=>xor(string=x.__Payload,key=unhex(string=x.XorKey))"}],
+        ["PayloadHash", 16, "Value",{"value":"x=>hash(path=xor(string=x.__Payload,key=unhex(string=x.XorKey)),accessor='data')"}],
+        ["OriginalFileHash", 16, "Value",{"value":"x=>hash(path=FullPath)"}],
     ]]]'''
 
 
@@ -420,11 +437,29 @@ sources:
                 replace=join(array=YaraStrings.Line, sep=" $$"))
 
 
+      -- function to extract potential additional encoded PE in data section
+      LET embedded_section(path,type) = SELECT *,
+            read_file(filename=path,accessor=type,offset=FileOffset,length=Size) as Data
+        FROM foreach(row= parse_pe(file=path,accessor=type).Sections,
+            query={
+                    SELECT
+                        path as OriginalFileName,
+                        _value.Name as Name,
+                        _value.Size as Size,
+                        _value.FileOffset as FileOffset,
+                        _value.VMA as VMA,
+                        _value.RVA as RVA,
+                        _value.Perm as Perm
+                    FROM scope() WHERE Name = '.data' AND Size > 15
+                })
+
+
+
       -- scan DataBytes for CobaltStrike config
       LET ByteConfiguration = SELECT Rule,
                 len(list=TargetBytes) as Size,
                 hash(path=TargetBytes,accessor='data') as Hash,
-                String.Offset as Offset,
+                --String.Offset as Offset,
                 Xor,_Data,
                 Rule  as _Group
             FROM switch( -- switchcase will find beacon as priority, then search for shellcode
@@ -445,6 +480,15 @@ sources:
                             then= 'Sleep mask 32-bit 4.2 deobfuscation routine found.',
                             else= 'Sleep mask 64-bit 4.2 deobfuscation routine found.') as _Data
                     FROM yara(accessor='data',files=TargetBytes, rules=FindSleepFunction, number=1)
+                },
+                section_encoded_pe = {
+                    SELECT
+                        FullPath,Size,
+                        'Embedded data section: ' + Rule as Rule,
+                        substr(start=0,end=1,str=String.Data) as Xor,
+                        --parse_binary(accessor='data',filename= embedded_section(path=FullPath)[0].Data,profile=Profile,struct="EmbeddedPE") as ParsedSection,
+                        read_file(accessor='data',filename=File.FullPath,offset=String.Offset,length=ExtractBytes) as _Data
+                    FROM yara(files=parse_binary(accessor='data',filename= embedded_section(path=TargetBytes,type='data')[0].Data,profile=Profile,struct="EmbeddedPE").DecodedPayload, accessor='data', rules=FindConfig, number=99)
                 })
 
       -- find target files
@@ -456,9 +500,9 @@ sources:
       LET FileConfiguration = SELECT * FROM foreach(row=TargetFiles,
             query={
                 SELECT
-                    Rule,FullPath, Size,
+                    Rule,
+                    FullPath, Size,
                     hash(path=FullPath) as Hash,
-                    String.Offset as Offset,
                     Xor,_Data,
                     Rule + '|' + FullPath as _Group
                 FROM switch( -- switchcase will find beacon as priority, then search for shellcode
@@ -479,11 +523,23 @@ sources:
                                 then= 'Sleep mask 32-bit 4.2 deobfuscation routine found.',
                                 else= 'Sleep mask 64-bit 4.2 deobfuscation routine found.') as _Data
                         FROM yara(files=FullPath, rules=FindSleepFunction, number=1)
+                    },
+                    section_encoded_pe = {
+                        SELECT
+                            FullPath,Size,
+                            'Embedded data section: ' + Rule as Rule,
+                            substr(start=0,end=1,str=String.Data) as Xor,
+                            --parse_binary(accessor='data',filename= embedded_section(path=FullPath)[0].Data,profile=Profile,struct="EmbeddedPE") as ParsedSection,
+                            read_file(accessor='data',filename=File.FullPath,offset=String.Offset,length=ExtractBytes) as _Data
+                        FROM yara(files=parse_binary(accessor='data',filename= embedded_section(path=FullPath,type='auto')[0].Data,profile=Profile,struct="EmbeddedPE").DecodedPayload, accessor='data', rules=FindConfig, number=99)
                     })
             })
-                
+
+
       -- find velociraptor process
-      LET me <= SELECT Pid FROM pslist(pid=getpid())
+      LET me <= SELECT * FROM if(condition= NOT ( TargetFileGlob OR TargetBytes ),
+                    then = { SELECT Pid FROM pslist(pid=getpid()) })
+
 
       -- find all processes and add filters
       LET processes = SELECT Name as ProcessName, CommandLine, Pid
@@ -499,7 +555,7 @@ sources:
         query={
             SELECT Rule,
                 Pid, ProcessName, CommandLine,
-                String.Offset as Offset,
+                --String.Offset as Offset,
                 Xor,_Data,_Group
             FROM switch( -- switchcase will find beacon as priority, then search for shellcode
                 beacon = {
@@ -516,7 +572,7 @@ sources:
                     FROM yara(accessor='process',files=str(str=Pid), rules=FindShellcode, number=99)
                 },
                 sleepfunction = {
-                    SELECT *, '' as Xor, 
+                    SELECT *, '' as Xor,
                         if(condition= String.Name= '$x86',
                             then= 'Sleep mask 32-bit 4.2 deobfuscation routine found.',
                             else= 'Sleep mask 64-bit 4.2 deobfuscation routine found.') as _Data,
@@ -538,11 +594,11 @@ sources:
         LIMIT 150
 
 
-      -- generate results remove any without Server to remove FPs
+      -- generate results remove any FPs
       LET results <= SELECT *,
-            if(condition= Rule='cobalt_strike_beacon',
+            if(condition= Rule=~'cobalt_strike_beacon$',
                 then= format(format='0x%x',args=Xor),else='0x00') as Xor,
-            if(condition= Rule='cobalt_strike_beacon',
+            if(condition= Rule=~'cobalt_strike_beacon',
                 then= parse_binary(accessor='data',filename= xor(string=_Data,key=Xor),
                     profile = Profile,struct  = "CobaltConfig"),
                 else= if(condition= Rule='cobalt_strike_shellcode',
@@ -555,9 +611,10 @@ sources:
                 then= FileConfiguration,
                 else= ProcessConfiguration))
         WHERE _Data
-            AND 
+            AND
                 ( DecodedConfig.C2Server =~ '^[ -~]+$'
-            OR ( DecodedConfig.Server =~ '^[ -~]+$' AND DecodedConfig.TargetUri ) 
+            OR ( DecodedConfig.Server =~ '^[ -~]+$' AND DecodedConfig.TargetUri )
+            OR ( DecodedConfig.Pipename AND DecodedConfig.BeaconType )
             OR Rule='cobalt_strike_sleepfunction')
 
       -- add decoded data seperate to keep pretty output
@@ -570,7 +627,7 @@ sources:
 
 
       -- output rows, standard config priority, exclude _Data
-      SELECT * FROM column_filter(
+      LET y = SELECT * FROM column_filter(
         query={
             SELECT * ,
                  -- NOTE: some junk strings for shellcode _Group are removed in GROUP BY
@@ -582,3 +639,4 @@ sources:
                 else= results)
             GROUP BY _Group
         }, exclude=["_Data","_Group"])
+      SELECT * FROM y --FileConfiguration

artifacts/testdata/server/testcases/cobalt.in.yaml

@@ -1,17 +1,18 @@
 Queries:
+  #prefix: srcDir + "/artifacts/testdata/files/CSSectionPE.bin"
   # tests for file parsing
-  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Offset,Xor,DecodedConfig
+  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Xor,DecodedConfig
     FROM Artifact.Windows.Carving.CobaltStrike(TargetFileGlob=srcDir + "/artifacts/testdata/files/CSDump.bin")
 
   # Test for data stream parsing, DecodedData output and bruteforce xor
   - LET Encoded = SELECT Data FROM read_file(filenames=srcDir + "/artifacts/testdata/files/CSDump.bin")
-  - SELECT Rule,Hash,Offset,Xor,DecodedConfig,DecodedData
+  - SELECT Rule,Hash,Xor,DecodedConfig,DecodedData
     FROM Artifact.Windows.Carving.CobaltStrike(TargetBytes=Encoded.Data[0],BruteXor="T", IncludeDecodedData="Y")
 
   # tests shellcode parsing
-  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Offset,Xor,DecodedConfig
+  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Xor,DecodedConfig
     FROM Artifact.Windows.Carving.CobaltStrike(TargetFileGlob=srcDir + "/artifacts/testdata/files/CSShellcode.bin")
-  
-  # tests sleepfunctino parsing
-  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Offset,Xor,DecodedConfig
+
+  # tests sleepfunction parsing
+  - SELECT Rule, relpath(path=FullPath, base=srcDir, sep="/") as FullPath,Hash,Xor,DecodedConfig
     FROM Artifact.Windows.Carving.CobaltStrike(TargetFileGlob=srcDir + "/artifacts/testdata/files/CSx86sleep.bin")

artifacts/testdata/server/testcases/cobalt2.in.yaml

@@ -0,0 +1,30 @@
+Parameters:
+  RemappingTemplate: |
+    remappings:
+      - type: mount
+        from:
+          accessor: zip
+          path_type: linux
+          prefix: |
+            {
+              "DelegateAccessor": "scope",
+              "DelegatePath": "Content",
+              "Path": "/"
+            }
+        on:
+          accessor: "auto"
+          prefix: "/"
+          path_type: "windows"
+
+Queries:
+  # tests embedded pe in data section parsing
+  - LET Content <= xor(key='infected',string=read_file(filename=srcDir+"/artifacts/testdata/files/CSSectionPE.zip.xor"))
+  - LET _ <= remap(config=RemappingTemplate, copy=["zip", "data", "scope"], clear=TRUE)
+
+  # Scan all the files in the container.
+  - SELECT Rule,FullPath,Hash,Xor,DecodedConfig
+    FROM Artifact.Windows.Carving.CobaltStrike(TargetFileGlob="/**")
+
+  # Use byte scanning of the raw file.
+  - SELECT Rule,Hash,Xor,DecodedConfig
+    FROM Artifact.Windows.Carving.CobaltStrike(TargetBytes=read_file(filename="C:/CSSectionPE.bin"))

artifacts/testdata/server/testcases/cobalt2.out.yaml

@@ -0,0 +1,130 @@
+LET Content <= xor(key='infected',string=read_file(filename=srcDir+"/artifacts/testdata/files/CSSectionPE.zip.xor"))[]LET _ <= remap(config=RemappingTemplate, copy=["zip", "data", "scope"], clear=TRUE)[]SELECT Rule,FullPath,Hash,Xor,DecodedConfig FROM Artifact.Windows.Carving.CobaltStrike(TargetFileGlob="/**")[
+ {
+  "Rule": "Embedded data section: cobalt_strike_beacon",
+  "FullPath": "C:\\CSSectionPE.bin",
+  "Hash": {
+   "MD5": "6e1ee07d95a9b169567856313ff324eb",
+   "SHA1": "3727f7592d7f4d00285800e4d855431801dc4ceb",
+   "SHA256": "fb4fc487ca159700b0af9fb54f6010c78709c9cc2b95c7cde017f49d7bb833c8"
+  },
+  "Xor": "0x69",
+  "DecodedConfig": {
+   "BeaconType": "windows-beacon_smb-bind_pipe",
+   "Port": 4444,
+   "Sleeptime": 10000,
+   "Maxgetsize": 1048576,
+   "Jitter": 0,
+   "MaxDns": 0,
+   "PublicKey": "30 81 9f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 aa ed 4a fd 72 4f 7d ae c2 10 f6 99 ba db 4e 4d d1 3f 16 05 74 09 33 5d 10 b5 03 fc 0b 87 59 a9 15 3f 88 4f ec f5 13 e6 b8 32 30 f9 54 ae 12 98 6d b8 a3 ff c8 fa 1e 10 7e e7 12 e1 ee da f4 63 36 db 4b d4 3c 64 1a 80 c6 c6 53 1a f4 de eb c0 38 c1 db 59 5b 5b 4a de 5d 63 09 39 29 5c 61 8c c0 43 50 94 dd 31 2f 91 1a e1 e4 7c c2 ae 0c 3a 50 15 3f 4c d4 24 71 d6 6e 06 79 2c e1 07 8c cf 02 03 01 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00",
+   "C2Server": "",
+   "UserAgent": "",
+   "PostURI": "",
+   "MalleableC2Instructions": "",
+   "HttpGetHeader": "",
+   "HttpPostHeader": "",
+   "SpawnTo": "",
+   "Pipename": "\\\\%s\\pipe\\fhsvc-%x",
+   "KillDateYear": 0,
+   "KillDateMonth": 0,
+   "KillDateDay": 0,
+   "DNSIdle": "0.0.0.0",
+   "DNSSleep": 0,
+   "SSH_1": "",
+   "SSH_2": "",
+   "SSH_3": "",
+   "SSH_4": "",
+   "SSH_5": "",
+   "GetVerb": "",
+   "PostVerb": "",
+   "HttpPostChunk": 0,
+   "SpawnTox86": "%windir%\\syswow64\\rundll32.exe",
+   "SpawnTox64": "%windir%\\sysnative\\rundll32.exe",
+   "CryptoScheme": 0,
+   "Proxy": "",
+   "ProxyUsername": "",
+   "ProxyPassword": "",
+   "ProxyType": "0x0",
+   "Deprecated": 3,
+   "LicenseId": 331083543,
+   "bStageCleanup": 0,
+   "bCFGCaution": 0,
+   "KillDate": 0,
+   "TextSectionEnd": 0,
+   "ObfuscateSectionsInfo": "",
+   "ProcessInjectStartRWX": "0x0",
+   "ProcessInjectUseRWX": "0x0",
+   "ProcessInjectMinAlloc": 0,
+   "ProcessInjectTransformx86": "",
+   "ProcessInjectTransformx64": "",
+   "UsesCookies": 0,
+   "ProcessInjectExecute": "",
+   "ProcessInjectAllocationMethod": 0,
+   "ProcessInjectStub": "",
+   "HostHeader": ""
+  }
+ }
+]SELECT Rule,Hash,Xor,DecodedConfig FROM Artifact.Windows.Carving.CobaltStrike(TargetBytes=read_file(filename="C:/CSSectionPE.bin"))[
+ {
+  "Rule": "Embedded data section: cobalt_strike_beacon",
+  "Hash": {
+   "MD5": "6e1ee07d95a9b169567856313ff324eb",
+   "SHA1": "3727f7592d7f4d00285800e4d855431801dc4ceb",
+   "SHA256": "fb4fc487ca159700b0af9fb54f6010c78709c9cc2b95c7cde017f49d7bb833c8"
+  },
+  "Xor": "0x69",
+  "DecodedConfig": {
+   "BeaconType": "windows-beacon_smb-bind_pipe",
+   "Port": 4444,
+   "Sleeptime": 10000,
+   "Maxgetsize": 1048576,
+   "Jitter": 0,
+   "MaxDns": 0,
+   "PublicKey": "30 81 9f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 aa ed 4a fd 72 4f 7d ae c2 10 f6 99 ba db 4e 4d d1 3f 16 05 74 09 33 5d 10 b5 03 fc 0b 87 59 a9 15 3f 88 4f ec f5 13 e6 b8 32 30 f9 54 ae 12 98 6d b8 a3 ff c8 fa 1e 10 7e e7 12 e1 ee da f4 63 36 db 4b d4 3c 64 1a 80 c6 c6 53 1a f4 de eb c0 38 c1 db 59 5b 5b 4a de 5d 63 09 39 29 5c 61 8c c0 43 50 94 dd 31 2f 91 1a e1 e4 7c c2 ae 0c 3a 50 15 3f 4c d4 24 71 d6 6e 06 79 2c e1 07 8c cf 02 03 01 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00",
+   "C2Server": "",
+   "UserAgent": "",
+   "PostURI": "",
+   "MalleableC2Instructions": "",
+   "HttpGetHeader": "",
+   "HttpPostHeader": "",
+   "SpawnTo": "",
+   "Pipename": "\\\\%s\\pipe\\fhsvc-%x",
+   "KillDateYear": 0,
+   "KillDateMonth": 0,
+   "KillDateDay": 0,
+   "DNSIdle": "0.0.0.0",
+   "DNSSleep": 0,
+   "SSH_1": "",
+   "SSH_2": "",
+   "SSH_3": "",
+   "SSH_4": "",
+   "SSH_5": "",
+   "GetVerb": "",
+   "PostVerb": "",
+   "HttpPostChunk": 0,
+   "SpawnTox86": "%windir%\\syswow64\\rundll32.exe",
+   "SpawnTox64": "%windir%\\sysnative\\rundll32.exe",
+   "CryptoScheme": 0,
+   "Proxy": "",
+   "ProxyUsername": "",
+   "ProxyPassword": "",
+   "ProxyType": "0x0",
+   "Deprecated": 3,
+   "LicenseId": 331083543,
+   "bStageCleanup": 0,
+   "bCFGCaution": 0,
+   "KillDate": 0,
+   "TextSectionEnd": 0,
+   "ObfuscateSectionsInfo": "",
+   "ProcessInjectStartRWX": "0x0",
+   "ProcessInjectUseRWX": "0x0",
+   "ProcessInjectMinAlloc": 0,
+   "ProcessInjectTransformx86": "",
+   "ProcessInjectTransformx64": "",
+   "UsesCookies": 0,
+   "ProcessInjectExecute": "",
+   "ProcessInjectAllocationMethod": 0,
+   "ProcessInjectStub": "",
+   "HostHeader": ""
+  }
+ }
+]

artifacts/testdata/server/testcases/remapping.in.yaml

@@ -153,8 +153,8 @@ Queries:
 
   # Make sure we can parse the mft as normal.
   - SELECT * FROM parse_mft(filename="C:/$MFT", accessor="ntfs")
-    WHERE FullPath =~ 'Hello world'
-    ORDER BY FullPath
+    WHERE OSPath =~ 'Hello world'
+    ORDER BY OSPath
 
   - SELECT parse_ntfs(device="C:/$MFT", inode="46-128-0", accessor="ntfs")
     FROM scope()
@@ -165,7 +165,7 @@ Queries:
   - SELECT * FROM parse_ntfs_ranges(accessor='ntfs', device='c:/$MFT',
            inode="46-128-5")
 
-  - SELECT FullPath FROM glob(accessor='registry', globs="/HKLM/*/xbox*")
+  - SELECT OSPath FROM glob(accessor='registry', globs="/HKLM/*/xbox*")
 
   # Test data remapping
   - SELECT read_file(filename="hello world", accessor="data")