Bugfixes: Artifact bugs due to FullPath->OSPath refactor (#2923)

artifacts/definitions/Admin/Client/Upgrade.yaml

@@ -30,7 +30,7 @@ sources:
       // Force the file to be copied to the real temp directory since
       // we are just about to remove the Tools directory.
       LET bin <= SELECT copy(filename=OSPath,
-          dest=expand(path="%SYSTEMROOT%\\Temp\\") + OSPath.Basename) AS Dest
+          dest=expand(path="%SYSTEMROOT%\\Temp\\") + basename(path=OSPath)) AS Dest
       FROM Artifact.Generic.Utils.FetchBinary(
          ToolName="WindowsMSI", IsExecutable=FALSE,
          SleepDuration=SleepDuration)

artifacts/definitions/Windows/NTFS/Recover.yaml

@@ -26,7 +26,7 @@ sources:
        LET Parsed <= parse_ntfs(device=Drive, inode=MFTId)
 
        SELECT *, upload(accessor="mft", file=Drive + Inode,
-                        name=pathspec(Path=Parsed.OSPath + "/" + Inode)) AS IndexUpload
+                        name=Parsed.OSPath + Inode) AS IndexUpload
        FROM foreach(
             row=Parsed.Attributes,
             query={

artifacts/testdata/server/testcases/mft.out.yaml

@@ -252,7 +252,8 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
    "Hardlinks": [
     "just_a_file.txt"
    ],
-   "Device": "\\\\.\\C:"
+   "Device": "\\\\.\\C:",
+   "OSPath": "\\\\.\\C:\\just_a_file.txt"
   },
   "_Source": "Windows.NTFS.MFT"
  }

artifacts/testdata/server/testcases/ntfs.in.yaml

@@ -1,8 +1,13 @@
 Queries:
   # parse_ntfs can use an image file.
-  - SELECT parse_ntfs(
+  - LET NTFSInfoFromImage <= parse_ntfs(
       filename=srcDir+'/artifacts/testdata/files/test.ntfs.dd',
-      inode="46-128-0").OSPath AS OSPath
+      inode="46-128-0")
+
+  # Prove we can read that data from the OSPath we received
+  - SELECT NTFSInfoFromImage.OSPath.Path AS OSPath_Path,
+      read_file(accessor="raw_ntfs",
+                filename= NTFSInfoFromImage.OSPath) AS Data
     FROM scope()
 
   # Parsing the MFT from a raw image requires extracting it using the

artifacts/testdata/server/testcases/ntfs.out.yaml

@@ -1,6 +1,7 @@
-SELECT parse_ntfs( filename=srcDir+'/artifacts/testdata/files/test.ntfs.dd', inode="46-128-0").OSPath AS OSPath FROM scope()[
+LET NTFSInfoFromImage <= parse_ntfs( filename=srcDir+'/artifacts/testdata/files/test.ntfs.dd', inode="46-128-0")[]SELECT NTFSInfoFromImage.OSPath.Path AS OSPath_Path, read_file(accessor="raw_ntfs", filename= NTFSInfoFromImage.OSPath) AS Data FROM scope()[
  {
-  "OSPath": null
+  "OSPath_Path": "Folder A\\Folder B\\Hello world text document.txt",
+  "Data": "Hello world!"
  }
 ]SELECT * FROM parse_mft( filename=pathspec( Path="$MFT", DelegateAccessor="file", DelegatePath=srcDir+'/artifacts/testdata/files/test.ntfs.dd'), accessor="raw_ntfs") WHERE OSPath =~ "document.txt:goodbye.txt"[
  {

artifacts/testdata/server/testcases/ntfs_ea.out.yaml

@@ -73,7 +73,8 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
    "Hardlinks": [
     "another_file.txt"
    ],
-   "Device": "\\\\.\\C:"
+   "Device": "\\\\.\\C:",
+   "OSPath": "\\\\.\\C:\\another_file.txt"
   },
   "EA": {
    "Name": "SECOND",
@@ -165,7 +166,8 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
    "Hardlinks": [
     "another_file.txt"
    ],
-   "Device": "\\\\.\\C:"
+   "Device": "\\\\.\\C:",
+   "OSPath": "\\\\.\\C:\\another_file.txt"
   },
   "EA": {
    "Name": "THIRD",
@@ -258,7 +260,8 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
    "Hardlinks": [
     "just_a_file.txt"
    ],
-   "Device": "\\\\.\\C:"
+   "Device": "\\\\.\\C:",
+   "OSPath": "\\\\.\\C:\\just_a_file.txt"
   },
   "EA": {
    "Name": "NOTHINGTOSEEHERE",

artifacts/testdata/server/testcases/remapping.out.yaml

@@ -165,7 +165,8 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
    "Hardlinks": [
     "Folder A\\Folder B\\Hello world text document.txt"
    ],
-   "Device": "\\\\.\\C:"
+   "Device": "\\\\.\\C:",
+   "OSPath": "\\\\.\\C:\\Folder A\\Folder B\\Hello world text document.txt"
   }
  }
 ]SELECT * FROM parse_ntfs_i30( accessor='ntfs', device='c:/$MFT', inode="41-144-1")[

services/labels/labels_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
+	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/test_utils"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/services/labels"
@@ -44,6 +45,55 @@ func (self *LabelsTestSuite) SetupTest() {
 	labeler.(*labels.Labeler).Clock = self.Clock
 }
 
+// Check how labels interact with the indexing service
+func (self *LabelsTestSuite) TestLabelsAndIndexing() {
+
+	indexer, err := services.GetIndexer(self.ConfigObj)
+	assert.NoError(self.T(), err)
+
+	// Search for clients with label
+	resp, err := indexer.SearchClients(self.Ctx, self.ConfigObj,
+		&api_proto.SearchClientsRequest{
+			Offset: 0, Limit: 10,
+			Query: "label:Label1",
+		}, "admin")
+	assert.NoError(self.T(), err)
+
+	// No clients have this label yet
+	assert.Equal(self.T(), 0, len(resp.Items))
+
+	labeler := services.GetLabeler(self.ConfigObj)
+	err = labeler.SetClientLabel(
+		self.Ctx, self.ConfigObj, self.client_id, "Label1")
+	assert.NoError(self.T(), err)
+
+	resp, err = indexer.SearchClients(self.Ctx, self.ConfigObj,
+		&api_proto.SearchClientsRequest{
+			Offset: 0, Limit: 10,
+			Query: "label:Label1",
+		}, "admin")
+	assert.NoError(self.T(), err)
+
+	// Client should have label now.
+	assert.Equal(self.T(), 1, len(resp.Items))
+	assert.Equal(self.T(), self.client_id, resp.Items[0].ClientId)
+
+	// Now remove the label.
+	err = labeler.RemoveClientLabel(
+		self.Ctx, self.ConfigObj, self.client_id, "Label1")
+	assert.NoError(self.T(), err)
+
+	resp, err = indexer.SearchClients(self.Ctx, self.ConfigObj,
+		&api_proto.SearchClientsRequest{
+			Offset: 0, Limit: 10,
+			Query: "label:Label1",
+		}, "admin")
+	assert.NoError(self.T(), err)
+
+	// No client should not match the label
+	assert.Equal(self.T(), 0, len(resp.Items))
+}
+
 func (self *LabelsTestSuite) TestAddLabel() {
 	now := uint64(self.Clock.Now().UnixNano())
 

vql/parsers/ntfs.go

@@ -47,6 +47,7 @@ type NTFSModel struct {
 	*ntfs.NTFSFileInformation
 
 	Device *accessors.OSPath
+	OSPath *accessors.OSPath
 }
 
 type NTFSFunction struct{}
@@ -116,7 +117,34 @@ func (self NTFSFunction) Call(
 		return &vfilter.Null{}
 	}
 
-	return &NTFSModel{NTFSFileInformation: result, Device: arg.Filename}
+	var ospath *accessors.OSPath
+
+	// A Device was given the OSPath should be relative to the device
+	// so it can be opened by the 'ntfs' accessor
+	if arg.Device != "" && arg.Filename != nil {
+		if len(result.Hardlinks) > 0 {
+			ospath = arg.Filename.Append(strings.Split(result.Hardlinks[0], "\\")...)
+		}
+	} else {
+
+		// A filename was given - we just return the OSPath relative
+		// to the root of the filesystem. This can be used to open the
+		// file with the 'raw_ntfs' accessor.
+		if len(result.Hardlinks) > 0 {
+			ospath, _ = accessors.NewWindowsNTFSPath("")
+			ospath.SetPathSpec(&accessors.PathSpec{
+				DelegateAccessor: arg.Accessor,
+				DelegatePath:     arg.Filename.Path(),
+				Path:             result.Hardlinks[0],
+			})
+		}
+	}
+
+	return &NTFSModel{
+		NTFSFileInformation: result,
+		Device:              arg.Filename,
+		OSPath:              ospath,
+	}
 }
 
 type MFTScanPluginArgs struct {