Allow parse_ntfs() to operate on an image file. (Velocidex#1610)

Previously it could only operate on the raw device in live mode.

accessors/ntfs/ntfs_accessor.go

@@ -591,7 +591,25 @@ func Open(scope vfilter.Scope, self *ntfs.MFT_ENTRY,
 
 func init() {
 	accessors.Register("raw_ntfs", &NTFSFileSystemAccessor{},
-		`Access the NTFS filesystem by parsing NTFS structures.`)
+		`Access the NTFS filesystem inside an image by parsing NTFS.
+
+This accessor is designed to operate on images directly. It requires a
+delegate accessor to get the raw image and will open files using the
+NTFS full path rooted at the top of the filesystem.
+
+## Example
+
+The following query will open the $MFT file from the raw image file
+that will be accessed using the file accessor.
+
+SELECT * FROM parse_mft(
+  filename=pathspec(
+    Path="$MFT",
+    DelegateAccessor="file",
+    DelegatePath='ntfs.dd'),
+  accessor="raw_ntfs")
+
+`)
 
 	json.RegisterCustomEncoder(&NTFSFileInfo{}, accessors.MarshalGlobFileInfo)
 }

artifacts/testdata/server/testcases/ntfs.in.yaml

@@ -0,0 +1,16 @@
+Queries:
+  # parse_ntfs can use an image file.
+  - SELECT parse_ntfs(
+      filename=srcDir+'/artifacts/testdata/files/test.ntfs.dd',
+      inode="46-128-0").FullPath AS FullPath
+    FROM scope()
+
+  # Parsing the MFT from a raw image requires extracting it using the
+  # raw_ntfs accessor because parse_mft() expect an $MFT file to read.
+  - SELECT * FROM parse_mft(
+      filename=pathspec(
+         Path="$MFT",
+         DelegateAccessor="file",
+         DelegatePath=srcDir+'/artifacts/testdata/files/test.ntfs.dd'),
+      accessor="raw_ntfs")
+    WHERE FullPath =~ "document.txt:goodbye.txt"

artifacts/testdata/server/testcases/ntfs.out.yaml

@@ -0,0 +1,35 @@
+SELECT parse_ntfs( filename=srcDir+'/artifacts/testdata/files/test.ntfs.dd', inode="46-128-0").FullPath AS FullPath FROM scope()[
+ {
+  "FullPath": "/Folder A/Folder B/Hello world text document.txt"
+ }
+]SELECT * FROM parse_mft( filename=pathspec( Path="$MFT", DelegateAccessor="file", DelegatePath=srcDir+'/artifacts/testdata/files/test.ntfs.dd'), accessor="raw_ntfs") WHERE FullPath =~ "document.txt:goodbye.txt"[
+ {
+  "EntryNumber": 46,
+  "SequenceNumber": 1,
+  "InUse": true,
+  "ParentEntryNumber": 45,
+  "ParentSequenceNumber": 1,
+  "FullPath": "/Folder A/Folder B/Hello world text document.txt:goodbye.txt",
+  "FileName": "Hello world text document.txt",
+  "FileNames": [
+   "Hello world text document.txt:goodbye.txt"
+  ],
+  "FileNameTypes": "POSIX",
+  "FileSize": 12,
+  "ReferenceCount": 1,
+  "IsDir": false,
+  "HasADS": true,
+  "SI_Lt_FN": false,
+  "Copied": false,
+  "SIFlags": "2080 (ARCHIVE,COMPRESSED)",
+  "Created0x10": "2018-09-24T07:55:29.7664719Z",
+  "Created0x30": "2018-09-24T07:55:29.7664719Z",
+  "LastModified0x10": "2018-09-24T07:56:35.3135567Z",
+  "LastModified0x30": "2018-09-24T07:55:29.7664719Z",
+  "LastRecordChange0x10": "2018-09-24T07:56:35.3135567Z",
+  "LastRecordChange0x30": "2018-09-24T07:55:29.7664719Z",
+  "LastAccess0x10": "2018-09-24T07:56:35.3135567Z",
+  "LastAccess0x30": "2018-09-24T07:55:29.7664719Z",
+  "LogFileSeqNum": 1096672
+ }
+]

artifacts/testdata/server/testcases/remapping.out.yaml

@@ -240,7 +240,7 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
      "Name": "goodbye.txt"
     }
    ],
-   "Device": "\\\\.\\C:\\$MFT"
+   "Device": "\\\\.\\C:"
   }
  }
 ]SELECT * FROM parse_ntfs_i30( accessor='ntfs', device='c:/$MFT', inode="41-144-1")[

docs/references/vql.yaml

@@ -3152,26 +3152,69 @@
     description: Maximum size of line buffer.
   category: parsers
 - name: parse_mft
-  description: Scan the $MFT from an NTFS volume.
+  description: |
+    Scan the $MFT from an NTFS volume.
+
+    This plugin expect an $MFT file to operate on. For example, it is
+    commonly used with the 'ntfs' accessor which opens the local raw
+    device to provide access to the $MFT
+
+    ```vql
+    SELECT * FROM parse_mft(filename="C:/$MFT", accessor="ntfs")
+    ```
+
+    For parsing from an image file, you can extract the $MFT file
+    using the raw_ntfs accessor (which operates on images).
+
+    ```vql
+    SELECT * FROM parse_mft(
+         filename=pathspec(
+           Path="$MFT",
+           DelegateAccessor="file",
+           DelegatePath='ntfs_image.dd'),
+         accessor="raw_ntfs")
+    ```
   type: Plugin
   args:
   - name: filename
     type: string
-    description: A list of event log files to parse.
+    description: The MFT file.
     required: true
   - name: accessor
     type: string
     description: The accessor to use.
   category: parsers
 - name: parse_ntfs
-  description: Parse an NTFS image file.
+  description: |
+    Parse specific inodes from an NTFS image file or the raw device.
+
+    This function retrieves more information about a specific MFT
+    entry including listing all its attributes.
+
+    It can either operate on an image file or the raw device (on
+    windows).
+
+    ## Example:
+
+    ```vql
+    SELECT parse_ntfs(
+        filename='ntfs_image.dd',
+        inode="46-128-0")
+    FROM scope()
+    ```
+
+    You can get the MFT entry number from `parse_mft()` or from the
+    Data attribute of a `glob()` using the `ntfs` accessor.
   type: Function
   args:
   - name: device
+    type: string
+    description: The device file to open. This may be a full path for example C:\Windows
+      - we will figure out the device automatically.
+  - name: filename
     type: accessors.OSPath
-    description: The device file to open. This may be a full path - we will figure
-      out the device automatically.
-    required: true
+    description: A raw image to open. You can also provide the accessor if using a
+      raw image file.
   - name: accessor
     type: string
     description: The accessor to use.
@@ -3186,14 +3229,21 @@
     description: The offset to the MFT entry to parse.
   category: parsers
 - name: parse_ntfs_i30
-  description: Scan the $I30 stream from an NTFS MFT entry.
+  description: |
+    Scan the $I30 stream from an NTFS MFT entry.
+
+    This is similar in use to the parse_ntfs() function but parses the
+    $I30 stream.
   type: Plugin
   args:
   - name: device
+    type: string
+    description: The device file to open. This may be a full path for example C:\Windows
+      - we will figure out the device automatically.
+  - name: filename
     type: accessors.OSPath
-    description: The device file to open. This may be a full path - we will figure
-      out the device automatically.
-    required: true
+    description: A raw image to open. You can also provide the accessor if using a
+      raw image file.
   - name: accessor
     type: string
     description: The accessor to use.
@@ -3212,10 +3262,13 @@
   type: Plugin
   args:
   - name: device
+    type: string
+    description: The device file to open. This may be a full path for example C:\Windows
+      - we will figure out the device automatically.
+  - name: filename
     type: accessors.OSPath
-    description: The device file to open. This may be a full path - we will figure
-      out the device automatically.
-    required: true
+    description: A raw image to open. You can also provide the accessor if using a
+      raw image file.
   - name: accessor
     type: string
     description: The accessor to use.

http_comms/e2e_test.go

@@ -218,7 +218,7 @@ func (self *TestSuite) TestServerRotateKeyE2E() {
 	logging.ClearMemoryLogs()
 
 	// Sending another one will produce an error.
-	vtesting.WaitUntil(2*time.Second, self.T(), func() bool {
+	vtesting.WaitUntil(5*time.Second, self.T(), func() bool {
 		err := comm.sender.sendToURL(client_ctx, [][]byte{}, false)
 		if err == nil {
 			// No error yet - retry

vql/parsers/ntfs.go

@@ -19,6 +19,8 @@ package parsers
 
 import (
 	"context"
+	"errors"
+	"strings"
 
 	"github.com/Velocidex/ordereddict"
 	ntfs "www.velocidex.com/golang/go-ntfs/parser"
@@ -31,7 +33,8 @@ import (
 )
 
 type NTFSFunctionArgs struct {
-	Device    *accessors.OSPath `vfilter:"required,field=device,doc=The device file to open. This may be a full path - we will figure out the device automatically."`
+	Device    string            `vfilter:"optional,field=device,doc=The device file to open. This may be a full path for example C:\\Windows - we will figure out the device automatically."`
+	Filename  *accessors.OSPath `vfilter:"optional,field=filename,doc=A raw image to open. You can also provide the accessor if using a raw image file."`
 	Accessor  string            `vfilter:"optional,field=accessor,doc=The accessor to use."`
 	Inode     string            `vfilter:"optional,field=inode,doc=The MFT entry to parse in inode notation (5-144-1)."`
 	MFT       int64             `vfilter:"optional,field=mft,doc=The MFT entry to parse."`
@@ -49,7 +52,7 @@ type NTFSFunction struct{}
 func (self NTFSFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:    "parse_ntfs",
-		Doc:     "Parse an NTFS image file.",
+		Doc:     "Parse specific inodes from an NTFS image file or the raw device.",
 		ArgType: type_map.AddType(scope, &NTFSFunctionArgs{}),
 	}
 }
@@ -67,6 +70,13 @@ func (self NTFSFunction) Call(
 		return &vfilter.Null{}
 	}
 
+	arg.Filename, arg.Accessor, err = getOSPathAndAccessor(arg.Device,
+		arg.Filename, arg.Accessor)
+	if err != nil {
+		scope.Log("parse_ntfs: %v", err)
+		return &vfilter.Null{}
+	}
+
 	if arg.Inode != "" {
 		mft_idx, _, _, err := ntfs.ParseMFTId(arg.Inode)
 		if err != nil {
@@ -76,14 +86,7 @@ func (self NTFSFunction) Call(
 		arg.MFT = mft_idx
 	}
 
-	device, accessor, err := readers.GetRawDeviceAndAccessor(
-		scope, arg.Device, arg.Accessor)
-	if err != nil {
-		scope.Log("parse_ntfs: %v", err)
-		return &vfilter.Null{}
-	}
-
-	ntfs_ctx, err := readers.GetNTFSContext(scope, device, accessor)
+	ntfs_ctx, err := readers.GetNTFSContext(scope, arg.Filename, arg.Accessor)
 	if err != nil {
 		scope.Log("parse_ntfs: GetNTFSContext %v", err)
 		return &vfilter.Null{}
@@ -111,11 +114,11 @@ func (self NTFSFunction) Call(
 		return &vfilter.Null{}
 	}
 
-	return &NTFSModel{NTFSFileInformation: result, Device: arg.Device}
+	return &NTFSModel{NTFSFileInformation: result, Device: arg.Filename}
 }
 
 type MFTScanPluginArgs struct {
-	Filename string `vfilter:"required,field=filename,doc=A list of event log files to parse."`
+	Filename string `vfilter:"required,field=filename,doc=The MFT file."`
 	Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use."`
 }
 
@@ -205,6 +208,13 @@ func (self NTFSI30ScanPlugin) Call(
 			return
 		}
 
+		arg.Filename, arg.Accessor, err = getOSPathAndAccessor(arg.Device,
+			arg.Filename, arg.Accessor)
+		if err != nil {
+			scope.Log("parse_ntfs_i30: %v", err)
+			return
+		}
+
 		if arg.Inode != "" {
 			mft_idx, _, _, err := ntfs.ParseMFTId(arg.Inode)
 			if err != nil {
@@ -214,14 +224,7 @@ func (self NTFSI30ScanPlugin) Call(
 			arg.MFT = mft_idx
 		}
 
-		device, accessor, err := readers.GetRawDeviceAndAccessor(
-			scope, arg.Device, arg.Accessor)
-		if err != nil {
-			scope.Log("parse_ntfs_i30: %v", err)
-			return
-		}
-
-		ntfs_ctx, err := readers.GetNTFSContext(scope, device, accessor)
+		ntfs_ctx, err := readers.GetNTFSContext(scope, arg.Filename, arg.Accessor)
 		if err != nil {
 			scope.Log("parse_ntfs_i30: %v", err)
 			return
@@ -278,6 +281,13 @@ func (self NTFSRangesPlugin) Call(
 			return
 		}
 
+		arg.Filename, arg.Accessor, err = getOSPathAndAccessor(arg.Device,
+			arg.Filename, arg.Accessor)
+		if err != nil {
+			scope.Log("parse_ntfs_ranges: %v", err)
+			return
+		}
+
 		attr_type := int64(0)
 		attr_id := int64(0)
 		mft_idx := int64(arg.MFT)
@@ -292,14 +302,7 @@ func (self NTFSRangesPlugin) Call(
 			attr_type = 128
 		}
 
-		device, accessor, err := readers.GetRawDeviceAndAccessor(
-			scope, arg.Device, arg.Accessor)
-		if err != nil {
-			scope.Log("parse_ntfs_ranges: %v", err)
-			return
-		}
-
-		ntfs_ctx, err := readers.GetNTFSContext(scope, device, accessor)
+		ntfs_ctx, err := readers.GetNTFSContext(scope, arg.Filename, arg.Accessor)
 		if err != nil {
 			scope.Log("parse_ntfs_ranges: %v", err)
 			return
@@ -344,6 +347,40 @@ func (self NTFSRangesPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap
 	}
 }
 
+func getOSPathAndAccessor(
+	device string,
+	filename *accessors.OSPath,
+	accessor string) (*accessors.OSPath, string, error) {
+
+	// Extract the device from the device string
+	if device != "" {
+		filename, err := accessors.NewWindowsNTFSPath(device)
+		if err != nil {
+			return nil, "", err
+		}
+
+		if filename == nil || len(filename.Components) == 0 {
+			return nil, "", errors.New("Invalid device")
+		}
+
+		if !strings.HasPrefix(filename.Components[0], "\\\\.\\") {
+			return nil, "", errors.New("Device should begin with \\\\.\\")
+		}
+
+		// The device is the first component (e.g. \\.\C:) so make a
+		// new OSPath for it to be accessed using "ntfs".
+		filename, err = accessors.NewWindowsNTFSPath(filename.Components[0])
+		accessor = "ntfs"
+		return filename, accessor, nil
+	}
+
+	if filename == nil {
+		return nil, "", errors.New("either filename or device must be provided")
+	}
+
+	return filename, accessor, nil
+}
+
 func init() {
 	vql_subsystem.RegisterFunction(&NTFSFunction{})
 	vql_subsystem.RegisterPlugin(&NTFSI30ScanPlugin{})