Fixed the Windows.KapeFiles.Extract artifact (Velocidex#2275)

Refactor code to pass OSPath into VQL functions.
Fixed bug in escaping the paths in the collector accessor.

.github/workflows/windows.yml

@@ -87,7 +87,8 @@ jobs:
         CGO_ENABLED: "0"
 
       run: |
-        go test -v ./... --tags server_vql -p 2
+        # Only run short tests on CI because the CI machines are slow
+        go test -short -v ./... --tags server_vql -p 2
 
     - name: Test Golden Generic
       shell: cmd

accessors/api.go

@@ -247,6 +247,8 @@ type FileSystemAccessor interface {
 	Open(path string) (ReadSeekCloser, error)
 	Lstat(filename string) (FileInfo, error)
 
+	// Converts from a string path to an OSPath suitable for this
+	// accessor.
 	ParsePath(filename string) (*OSPath, error)
 
 	// The new more efficient API

accessors/collector/collector.go

@@ -11,8 +11,10 @@ import (
 	"www.velocidex.com/golang/velociraptor/accessors/zip"
 	"www.velocidex.com/golang/velociraptor/acls"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	"www.velocidex.com/golang/velociraptor/utils"
 
 	crypto_utils "www.velocidex.com/golang/velociraptor/crypto/utils"
@@ -95,6 +97,7 @@ type CollectorAccessor struct {
 func (self *CollectorAccessor) New(scope vfilter.Scope) (accessors.FileSystemAccessor, error) {
 	delegate, err := (&zip.ZipFileSystemAccessor{}).New(scope)
 	return &CollectorAccessor{
+		expandSparse:          self.expandSparse,
 		ZipFileSystemAccessor: delegate.(*zip.ZipFileSystemAccessor),
 		scope:                 scope,
 	}, err
@@ -261,6 +264,12 @@ func (self *CollectorAccessor) maybeSetZipPassword(
 	return full_path, nil
 }
 
+// Zip files typically use standard / path separators.
+func (self *CollectorAccessor) ParsePath(path string) (
+	*accessors.OSPath, error) {
+	return accessors.NewZipFilePath(path)
+}
+
 func (self *CollectorAccessor) Open(
 	filename string) (accessors.ReadSeekCloser, error) {
 
@@ -325,6 +334,16 @@ func (self *CollectorAccessor) OpenWithOSPath(
 	if self.expandSparse {
 		index, err := self.getIndex(updated_full_path)
 		if err == nil {
+			config_obj, ok := vql_subsystem.GetServerConfig(self.scope)
+			if !ok {
+				config_obj = &config_proto.Config{}
+			}
+
+			if !uploads.ShouldPadFile(config_obj, index) {
+				self.scope.Log("Error: File %v is too sparse - unable to expand it.", full_path)
+				return reader, nil
+			}
+
 			return &rangedReader{
 				delegate: &utils.RangedReader{
 					ReaderAt: utils.MakeReaderAtter(reader),

accessors/manipulators.go

@@ -678,6 +678,7 @@ func (self ZipFileManipulator) PathJoin(path *OSPath) string {
 
 func (self ZipFileManipulator) PathParse(
 	path string, result *OSPath) error {
+	osPathUnserializations.Inc()
 
 	err := maybeParsePathSpec(path, result)
 	if err != nil {
@@ -696,10 +697,11 @@ func (self ZipFileManipulator) PathParse(
 	return nil
 }
 
-func NewZipFilePath(path string) *OSPath {
+func NewZipFilePath(path string) (*OSPath, error) {
 	manipulator := &ZipFileManipulator{}
-	return &OSPath{
-		Components:  []string{path},
+	result := &OSPath{
 		Manipulator: manipulator,
 	}
+	err := manipulator.PathParse(path, result)
+	return result, err
 }

accessors/protocols.go

@@ -2,6 +2,7 @@ package accessors
 
 import (
 	"context"
+	"reflect"
 
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -86,7 +87,12 @@ func (self _AddOSPath) Applicable(a vfilter.Any, b vfilter.Any) bool {
 	}
 
 	switch b.(type) {
-	case *OSPath, string, []vfilter.Any:
+	case *OSPath, string:
+		return true
+	}
+
+	a_value := reflect.Indirect(reflect.ValueOf(b))
+	if a_value.Type().Kind() == reflect.Slice {
 		return true
 	}
 	return false
@@ -98,23 +104,23 @@ func (self _AddOSPath) Add(scope vfilter.Scope, a vfilter.Any, b vfilter.Any) vf
 		return false
 	}
 
-	var b_os_path *OSPath
-
 	switch t := b.(type) {
 	case *OSPath:
-		b_os_path = t
+		return a_os_path.Append(t.Components...)
 
 	case string:
 		parsed, err := ParsePath(t, "")
 		if err != nil {
 			return vfilter.Null{}
 		}
-		b_os_path = parsed
+		return a_os_path.Append(parsed.Components...)
+	}
 
-		// Support OSPath("/root") + ["foo", "bar"] -> OSPath("/root/foo/bar")
-	case []vfilter.Any:
-		components := make([]string, 0, len(t))
-		for _, item := range t {
+	a_value := reflect.Indirect(reflect.ValueOf(b))
+	if a_value.Type().Kind() == reflect.Slice {
+		components := []string{}
+		for idx := 0; idx < a_value.Len(); idx++ {
+			item := a_value.Index(int(idx)).Interface()
 			str_item, ok := item.(string)
 			if ok {
 				components = append(components, str_item)
@@ -123,7 +129,8 @@ func (self _AddOSPath) Add(scope vfilter.Scope, a vfilter.Any, b vfilter.Any) vf
 
 		return a_os_path.Append(components...)
 	}
-	return a_os_path.Append(b_os_path.Components...)
+
+	return a_os_path
 }
 
 type _AssociativeOSPath struct{}

accessors/utils.go

@@ -16,6 +16,10 @@ func ParsePath(path, path_type string) (res *OSPath, err error) {
 		res, err = NewGenericOSPath(path)
 	case "pathspec":
 		res, err = NewPathspecOSPath(path)
+
+	case "zip":
+		res, err = NewZipFilePath(path)
+
 	default:
 		err = fmt.Errorf("Unknown path type: %v (should be one of windows,linux,generic)", path_type)
 	}

accessors/vql_arg_parser.go

@@ -3,6 +3,7 @@ package accessors
 import (
 	"context"
 	"fmt"
+	"reflect"
 
 	"github.com/Velocidex/ordereddict"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
@@ -134,6 +135,42 @@ func ParseOSPath(ctx context.Context,
 	}
 }
 
+func parseOSPathArray(ctx context.Context,
+	scope types.Scope, args *ordereddict.Dict,
+	value interface{}) (interface{}, error) {
+
+	result := []*OSPath{}
+
+	a_value := reflect.Indirect(reflect.ValueOf(value))
+	if a_value.Type().Kind() == reflect.Slice {
+		for idx := 0; idx < a_value.Len(); idx++ {
+			item, err := parseOSPath(ctx, scope, args,
+				a_value.Index(int(idx)).Interface())
+			if err != nil {
+				continue
+			}
+			item_os_path, ok := item.(*OSPath)
+			if ok {
+				result = append(result, item_os_path)
+			}
+		}
+		return result, nil
+	}
+
+	// If the arg is not a slice then treat it as a single ospath.
+	item, err := parseOSPath(ctx, scope, args, value)
+	if err != nil {
+		return nil, err
+	}
+
+	item_os_path, ok := item.(*OSPath)
+	if ok {
+		result = append(result, item_os_path)
+	}
+	return result, nil
+}
+
 func init() {
 	arg_parser.RegisterParser(&OSPath{}, parseOSPath)
+	arg_parser.RegisterParser([]*OSPath{}, parseOSPathArray)
 }

accessors/zip/accessor.go

@@ -194,22 +194,40 @@ func _GetZipFile(self *ZipFileSystemAccessor,
 	}
 
 	for _, i := range zip_file.File {
+		if strings.HasPrefix(i.Name, "{") {
+			continue
+		}
+
 		// Ignore directories which are signified by a
 		// trailing / and have no content.
 		if strings.HasSuffix(i.Name, "/") && i.UncompressedSize64 == 0 {
 			continue
 		}
 
 		// Prepare the pathspec for each zip member. In order to
-		// access the members, we need to open the currect zipfile (in
+		// access the members, we need to open the current zipfile (in
 		// full_path) and open i.Name as the path.
-		next_pathspec := full_path.PathSpec()
-		next_pathspec.Path = i.Name
 
-		next_path, err := full_path.Parse(next_pathspec.String())
+		// So if the zip has has a pathspec like:
+		// {DelegateAccessor: "auto", DelegatePath: "path/to/zip"}
+
+		// We need to parse the zip members (unescaping as needed into
+		// components), and append those components to the zip
+		// pathspec to get at the member pathspec.
+		//
+		// {DelegateAccessor: "auto", DelegatePath: "path/to/zip",
+		//  Path: "zip_escaped_component_list"}
+		next_path := full_path.Copy()
+
+		// Parse the i.Name as an encoded zip file. Some Zip accessors
+		// (e.g. the collector accessor) use a special path
+		// manipulator that allows any path to be represented in a zip
+		// file by encoding unrepresentable characters.
+		zip_path, err := full_path.Parse(i.Name)
 		if err != nil {
 			continue
 		}
+		next_path.Components = zip_path.Components
 
 		next_item := _CDLookup{
 			full_path:   next_path,

artifacts/definitions/MacOS/System/Users.yaml

@@ -12,23 +12,27 @@ parameters:
 
 sources:
   - query: |
-      LET user_plist = SELECT FullPath FROM glob(globs=UserPlistGlob)
-      LET UserDetails =
+      LET user_plist = SELECT OSPath FROM glob(globs=UserPlistGlob)
+      LET UserDetails(OSPath) =
               SELECT get(member="name.0", default="") AS Name,
                      get(member="realname.0", default="") AS RealName,
                      get(member="shell.0", default="") AS UserShell,
                      get(member="home.0", default="") AS HomeDir,
-                     plist(file=get(member="LinkedIdentity.0", default=""),
-                           accessor='data') as AppleId,
-                     plist(file=get(member="accountPolicyData.0", default=""),
-                           accessor='data') AS AccountPolicyData
-              FROM plist(file=FullPath)
+                     if(condition=LinkedIdentity,
+                        then=plist(file=LinkedIdentity[0],
+                                   accessor='data')) as AppleId,
+                     if(condition=accountPolicyData,
+                        then=plist(file=accountPolicyData[0],
+                                   accessor='data')) AS AccountPolicyData
+              FROM plist(file=OSPath)
 
       SELECT Name, RealName, UserShell, HomeDir,
                get(item=AppleId, field="appleid.apple.com") AS AppleId,
                timestamp(epoch=AccountPolicyData.creationTime) AS CreationTime,
                AccountPolicyData.failedLoginCount AS FailedLoginCount,
                timestamp(epoch=AccountPolicyData.failedLoginTimestamp) AS FailedLoginTimestamp,
                timestamp(epoch=AccountPolicyData.passwordLastSetTime) AS PasswordLastSetTime
-      FROM foreach(row=user_plist, query=UserDetails)
+      FROM foreach(row=user_plist, query={
+         SELECT * FROM UserDetails(OSPath= OSPath)
+      })
       WHERE NOT OnlyShowRealUsers OR NOT UserShell =~ 'false'

artifacts/definitions/Windows/KapeFiles/Extract.yaml

@@ -25,52 +25,52 @@ description: |
   ## Example - command line invocation
 
   ```
-  velociraptor-v0.6.1-rc1-linux-amd64 artifacts collect Windows.KapeFiles.Extract --args ContainerPath=Collection-DESKTOP-2OR51GL-2021-07-16_06_56_50_-0700_PDT.zip --args OutputDirectory=/tmp/MyOutput/
+  velociraptor-v0.6.7-linux-amd64 artifacts collect Windows.KapeFiles.Extract --args ContainerPath=Collection-DESKTOP-2OR51GL-2021-07-16_06_56_50_-0700_PDT.zip --args OutputDirectory=/tmp/MyOutput/
   ```
 
 parameters:
-  - name: MetadataFile
-    default: "Windows.KapeFiles.Targets/All File Metadata.json"
-    description: Name of the KapeFile.Targets metadata file.
-  - name: UploadsFile
-    default: "Windows.KapeFiles.Targets/Uploads.json"
-    description: Name of the KapeFile.Targets uploads file.
   - name: OutputDirectory
     description: Directory to write on (must be set).
   - name: ContainerPath
     description: Path to container (zip file) to unpack.
 
 sources:
   - query: |
-      // Read the uploads source into memory so we can translate
-      // between the original path to the zip path quickly.
-      LET Uploads <= memoize(key="SourceFile", query={
-        SELECT SourceFile, DestinationFile
-        FROM parse_jsonl(accessor="zip",
-                         filename=pathspec(DelegatePath=ContainerPath, Path=UploadsFile))
-      })
+      LET MetadataFile = ("results", "Windows.KapeFiles.Targets/All File Metadata.json")
+      LET UploadsFile = "uploads.json"
 
-      // Walk over the Metadata source and resolve each file's container name.
-      LET FileStats = SELECT *, get(item=Uploads, field=SourceFile).DestinationFile AS ZipPath
-         FROM parse_jsonl(
-           accessor="zip",
-           filename=pathspec(DelegatePath=ContainerPath, Path=MetadataFile))
+      // Path to the root of the container
+      LET RootPathSpec = pathspec(DelegateAccessor="auto",
+                                  path_type="zip",
+                                  DelegatePath=ContainerPath)
 
-      // Copy the file from the container to the output directory
-      // preserving timestamps.
-      LET doit = SELECT ZipPath, Created, LastAccessed, Modified,
-         upload_directory(
-           output=OutputDirectory, name=ZipPath,
-           mtime=Modified, atime=LastAccessed, ctime=Created,
-           accessor='zip',
-           file=pathspec(
-             DelegatePath=ContainerPath,
-             Path=ZipPath)) AS CreatedFile
-      FROM FileStats
+      // The pathspec for where to store the file
+      LET OutputPathSpec = pathspec()
 
-      // Check that the user gave us all the require args.
-      SELECT * FROM if(condition= OutputDirectory AND ContainerPath, then=doit,
-         else={
-           SELECT * FROM scope() WHERE
-           log(message="<red>ERROR</>: Both OutputDirectory and ContainerPath must be specified.") AND FALSE
-      })
+      // Memoize the metadata stored in the container file so we can
+      // quickly extract the file times.
+      LET AllFileMetadata <= memoize(
+          key="SourceFile",
+          query={
+            SELECT *
+            FROM parse_jsonl(accessor="collector",
+                             filename=RootPathSpec + MetadataFile)
+          })
+
+      LET ALLUploads = SELECT *, RootPathSpec + _Components AS FileUpload,
+                              OutputPathSpec + _Components[3:] AS Dest,
+                              get(item=AllFileMetadata,
+                                  field=vfs_path) AS Metadata
+        FROM parse_jsonl(accessor="collector",
+                         filename=RootPathSpec + UploadsFile)
+        WHERE Type != "idx"
+
+      SELECT *, upload_directory(
+          accessor="collector",
+          output=OutputDirectory,
+          mtime=Metadata.Modified,
+          atime=Metadata.LastAccessed,
+          ctime=Metadata.Created,
+          name=Dest,
+          file=RootPathSpec + _Components) AS UploadedFile
+      FROM ALLUploads