Disable compatibility with URL style paths (#2912)

In the distant past we used to use url encoding to encode recursive
pathspecs but this is problematic because url encoding is just not
reliable enough. We have since moved to an OSPath object that properly
encodes paths as JSON objects but there is still some backwards
compatibility code in there that tries to detect a URL style path by
looking for # (the fragment component of the url).

This is probably a good time to remove this compatibility code because
we dont use urls for probably about 4-5 releases now.

Also sync latest KapeFiles Targets and fixed tests

Fixes: #2906

accessors/manipulators.go

@@ -2,7 +2,6 @@ package accessors
 
 import (
 	"fmt"
-	"net/url"
 	"regexp"
 	"strings"
 
@@ -516,23 +515,6 @@ func maybeParsePathSpec(path string, result *OSPath) error {
 		return nil
 	}
 
-	// This is a hack to support old URL based pathspecs.
-
-	// TODO: deprecate them completely in future.
-	if strings.Contains(path, "#") {
-		parsed_url, err := url.Parse(path)
-		if err == nil {
-			// Support urls for backwards compatibility.
-			result.pathspec = &PathSpec{
-				DelegateAccessor: parsed_url.Scheme,
-				DelegatePath:     parsed_url.Path,
-				Path:             parsed_url.Fragment,
-				url_based:        true,
-			}
-			return nil
-		}
-	}
-
 	result.pathspec = &PathSpec{
 		Path: path,
 	}

accessors/manipulators_test.go

@@ -101,12 +101,6 @@ var registry_testcases = []testcase{
 	{"HKLM\\\"http://www.google.com\"\\Foo",
 		[]string{"HKEY_LOCAL_MACHINE", "http://www.google.com", "Foo"},
 		"HKEY_LOCAL_MACHINE\\\"http://www.google.com\"\\Foo"},
-
-	// Support backwards compatible paths based on URLs.
-	{"/C:/Users/yolo/NTUSER.DAT#%5CSoftware%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5CMRUList",
-		[]string{"Software", "Microsoft", "Windows", "CurrentVersion", "Explorer",
-			"RunMRU", "MRUList"},
-		"/C:/Users/yolo/NTUSER.DAT#Software%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5CMRUList"},
 }
 
 func TestRegistryManipulators(t *testing.T) {

artifacts/definitions/Windows/Timeline/Registry/RunMRU.yaml

@@ -42,6 +42,7 @@ parameters:
 sources:
  - query: |
         LET hostname_lu <= SELECT Fqdn FROM info()
+        LET HKEY_USERS <= pathspec(parse="HKEY_USERS", path_type="registry")
 
         // First we need to extract populated RunMRU
         LET MRUList <= SELECT OSPath,
@@ -60,9 +61,7 @@ sources:
                 Username,
                 Mtime as reg_mtime,
                 OSPath.Basename as reg_name,
-                path_join(components=[
-                   "HKEY_USERS", UUID, OSPath.Dirname.Path
-                ], path_type="registry") as reg_key,
+                HKEY_USERS + UUID + OSPath.Dirname.Path as reg_key,
 
                 -- Value data is similar to 'cmd.exe\1' so we just need the bit before the \
                 regex_replace(source=Data.value, re="\\\\1$", replace="") as reg_value,

artifacts/testdata/server/testcases/mru.in.yaml

@@ -1,7 +1,7 @@
 Parameters:
   NTUserRunMRU: |
     [{
-      "OSPath": "ntfs:///C:/Users/yolo/NTUSER.DAT#%5CSoftware%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5CMRUList",
+      "OSPath": "{\"Path\":\"/Software/Microsoft/Windows/CurrentVersion/Explorer/RunMRU/MRUList\", \"DelegateAccessor\":\"ntfs\", \"DelegatePath\":\"C:/Users/yolo/NTUSER.DAT\"}",
       "Data": {
         "type": "REG_SZ",
         "data_len": 6,
@@ -26,7 +26,7 @@ Parameters:
        "data_len": 6,
        "value": "ab"
       },
-      "OSPath": "/C:/Users/yolo/NTUSER.DAT#%5CSoftware%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5CMRUList",
+      "OSPath": "{\"Path\":\"/Software/Microsoft/Windows/CurrentVersion/Explorer/RunMRU/MRUList\", \"DelegatePath\":\"C:/Users/yolo/NTUSER.DAT\"}",
       "GetLink": "",
       "IsDir": false,
       "IsLink": false,
@@ -44,7 +44,7 @@ Parameters:
        "data_len": 22,
        "value": "powershell"
       },
-      "OSPath": "/C:/Users/yolo/NTUSER.DAT#%5CSoftware%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5Ca",
+      "OSPath": "{\"Path\":\"/Software/Microsoft/Windows/CurrentVersion/Explorer/RunMRU/a\", \"DelegatePath\":\"C:/Users/yolo/NTUSER.DAT\"}",
       "GetLink": "",
       "IsDir": false,
       "IsLink": false,
@@ -62,7 +62,7 @@ Parameters:
        "data_len": 44,
        "value": "\\\\criticalserver\\c$\\1"
       },
-      "OSPath": "/C:/Users/yolo/NTUSER.DAT#%5CSoftware%5CMicrosoft%5CWindows%5CCurrentVersion%5CExplorer%5CRunMRU%5Cb",
+      "OSPath": "{\"Path\":\"/Software/Microsoft/Windows/CurrentVersion/Explorer/RunMRU/b\", \"DelegatePath\":\"C:/Users/yolo/NTUSER.DAT\"}",
       "GetLink": "",
       "IsDir": false,
       "IsLink": false,

artifacts/testdata/server/testcases/mru.out.yaml

@@ -57,7 +57,7 @@ SELECT LastWriteTime, Type, MruEntries, Key, basename(path=HiveName) as HiveName
   "hostname": "DESKTOP-687T2NR",
   "parser": "RunMRU",
   "message": "RunMRU evidence user: yolo, order: 0, command: powershell'",
-  "source": "/C:/Users/yolo/NTUSER.DAT",
+  "source": "C:/Users/yolo/NTUSER.DAT",
   "user": "yolo",
   "reg_key": "HKEY_USERS\\S-1-5-21-2201333269-445358460-1233016754-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU",
   "reg_mtime": "2020-04-30T01:46:57Z",
@@ -71,7 +71,7 @@ SELECT LastWriteTime, Type, MruEntries, Key, basename(path=HiveName) as HiveName
   "hostname": "DESKTOP-687T2NR",
   "parser": "RunMRU",
   "message": "RunMRU evidence user: yolo, order: 1, command: \\\\criticalserver\\c$'",
-  "source": "/C:/Users/yolo/NTUSER.DAT",
+  "source": "C:/Users/yolo/NTUSER.DAT",
   "user": "yolo",
   "reg_key": "HKEY_USERS\\S-1-5-21-2201333269-445358460-1233016754-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU",
   "reg_mtime": "2020-04-30T01:46:57Z",
@@ -86,7 +86,7 @@ SELECT LastWriteTime, Type, MruEntries, Key, basename(path=HiveName) as HiveName
   "hostname": "DESKTOP-687T2NR",
   "parser": "RunMRU",
   "message": "RunMRU evidence user: yolo, MRU0: powershell | MRU1: \\\\criticalserver\\c$'",
-  "source": "/C:/Users/yolo/NTUSER.DAT",
+  "source": "C:/Users/yolo/NTUSER.DAT",
   "user": "yolo",
   "_Source": "Windows.Timeline.Registry.RunMRU"
  }

artifacts/testdata/server/testcases/paths.in.yaml

@@ -40,15 +40,6 @@ Queries:
 
        -- Parse a registry path (Expand hive)
        pathspec(parse="HKU/Software", path_type='registry'),
-
-       -- Parse a URL based path - serialize back to URL
-       pathspec(parse="file:///usr/lib#*", path_type='generic'),
-
-       -- Access a field from a URL based path
-       pathspec(parse="file:///usr/lib#*", path_type='generic').DelegatePath,
-
-       -- Add subpaths to a complex pathspec
-       pathspec(parse="file:///usr/lib#foo", path_type='generic') + "bar",
        pathspec(parse="HKU/Software", path_type='registry') + "Microsoft",
 
        -- Access components from an OSPath

artifacts/testdata/server/testcases/paths.out.yaml

@@ -60,15 +60,6 @@ SELECT basename(path="/foo/bar/baz", path_type='linux') FROM scope()[
 
    -- Parse a registry path (Expand hive)
    pathspec(parse="HKU/Software", path_type='registry'),
-
-   -- Parse a URL based path - serialize back to URL
-   pathspec(parse="file:///usr/lib#*", path_type='generic'),
-
-   -- Access a field from a URL based path
-   pathspec(parse="file:///usr/lib#*", path_type='generic').DelegatePath,
-
-   -- Add subpaths to a complex pathspec
-   pathspec(parse="file:///usr/lib#foo", path_type='generic') + "bar",
    pathspec(parse="HKU/Software", path_type='registry') + "Microsoft",
 
    -- Access components from an OSPath
@@ -91,9 +82,6 @@ FROM scope()
   "pathspec(parse=\"C:\\\\Windows\", path_type='windows')": "C:\\Windows",
   "pathspec(parse=\"/usr/lib\", path_type='linux')": "/usr/lib",
   "pathspec(parse=\"HKU/Software\", path_type='registry')": "HKEY_USERS\\Software",
-  "pathspec(parse=\"file:///usr/lib#*\", path_type='generic')": "file:///usr/lib#/*",
-  "pathspec(parse=\"file:///usr/lib#*\", path_type='generic').DelegatePath": "/usr/lib",
-  "pathspec(parse=\"file:///usr/lib#foo\", path_type='generic') + \"bar\"": "file:///usr/lib#/foo/bar",
   "pathspec(parse=\"HKU/Software\", path_type='registry') + \"Microsoft\"": "HKEY_USERS\\Software\\Microsoft",
   "pathspec(parse=\"HKU/Software\", path_type='registry').Components": [
    "HKEY_USERS",

bin/artifacts.go

@@ -204,12 +204,11 @@ func doArtifactCollect() error {
 		return err
 	}
 
-	logger := log.New(&LogWriter{config_obj}, "", 0)
-
+	logger := &LogWriter{config_obj: config_obj}
 	scope := manager.BuildScope(services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NullACLManager{},
-		Logger:     logger,
+		Logger:     log.New(&LogWriter{config_obj: config_obj}, "", 0),
 		Env: ordereddict.NewDict().
 			Set("Artifacts", *artifact_command_collect_names).
 			Set("Output", *artifact_command_collect_output).
@@ -266,9 +265,14 @@ func doArtifactCollect() error {
                         timeout=Timeout, progress_timeout=ProgressTimeout,
                         cpu_limit=CpuLimit,
                         password=Password, args=Args, format=Format)`
-	return eval_local_query(
+	err = eval_local_query(
 		sm.Ctx, config_obj,
 		*artifact_command_collect_format, query, scope)
+	if err != nil {
+		return err
+	}
+
+	return logger.Error
 }
 
 func getFilterRegEx(pattern string) (*regexp.Regexp, error) {

bin/binary_test.go

@@ -375,7 +375,7 @@ func TestGenerateConfigWithMerge(t *testing.T) {
 	// Repack the config in the binary.
 	cmd = exec.Command(binary, "config", "repack", config_file.Name(), exe.Name())
 	out, err = cmd.CombinedOutput()
-	require.NoError(t, err)
+	require.NoError(t, err, string(out))
 
 	os.Chmod(exe.Name(), 0755)
 

bin/csv.go

@@ -42,10 +42,11 @@ func doCSV() error {
 		return err
 	}
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NullACLManager{},
-		Logger:     log.New(&LogWriter{config_obj}, "", 0),
+		Logger:     log.New(logger, "", 0),
 		Env: ordereddict.NewDict().
 			Set(vql_subsystem.ACL_MANAGER_VAR,
 				acl_managers.NewRoleACLManager(config_obj, "administrator")).
@@ -77,12 +78,19 @@ func doCSV() error {
 		table.Render()
 
 	case "jsonl":
-		return outputJSONL(ctx, scope, vql, os.Stdout)
+		err = outputJSONL(ctx, scope, vql, os.Stdout)
+		if err != nil {
+			return err
+		}
 
 	case "json":
-		return outputJSON(ctx, scope, vql, os.Stdout)
+		err = outputJSON(ctx, scope, vql, os.Stdout)
+		if err != nil {
+			return err
+		}
+
 	}
-	return nil
+	return logger.Error
 }
 
 func init() {

bin/deaddisk.go

@@ -56,10 +56,11 @@ func addWindowsDirectory(
 	directory_path string, config_obj *config_proto.Config) error {
 	addCommonPermissions(config_obj)
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NullACLManager{},
-		Logger:     log.New(&LogWriter{config_obj}, "", 0),
+		Logger:     log.New(logger, "", 0),
 	}
 
 	manager, err := services.GetRepositoryManager(config_obj)
@@ -158,16 +159,17 @@ func addWindowsDirectory(
 				},
 			})
 	}
-	return nil
+	return logger.Error
 }
 
 func addWindowsHardDisk(
 	image string, config_obj *config_proto.Config) error {
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NullACLManager{},
-		Logger:     log.New(&LogWriter{config_obj}, "", 0),
+		Logger:     log.New(logger, "", 0),
 		Env: ordereddict.NewDict().
 			Set(vql_subsystem.ACL_MANAGER_VAR,
 				acl_managers.NewRoleACLManager(config_obj, "administrator")).
@@ -204,7 +206,7 @@ func addWindowsHardDisk(
 
 	addCommonShadowAccessors(config_obj)
 
-	return nil
+	return logger.Error
 }
 
 func getPartitionOffsets(

bin/fs.go

@@ -150,10 +150,11 @@ func doLS(path, accessor string) error {
 		path += "*"
 	}
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NullACLManager{},
-		Logger:     log.New(&LogWriter{config_obj}, "", 0),
+		Logger:     log.New(logger, "", 0),
 		Env: ordereddict.NewDict().
 			Set(vql_subsystem.ACL_MANAGER_VAR,
 				acl_managers.NewRoleACLManager(config_obj, "administrator")).
@@ -179,8 +180,13 @@ func doLS(path, accessor string) error {
 		query += " WHERE Sys.name_type != 'DOS' "
 	}
 
-	return eval_query(sm.Ctx, config_obj,
+	err = eval_query(sm.Ctx, config_obj,
 		*fs_command_format, query, scope, builder.Env)
+	if err != nil {
+		return err
+	}
+
+	return logger.Error
 }
 
 func doRM(path, accessor string) error {
@@ -216,10 +222,11 @@ func doRM(path, accessor string) error {
 		return fmt.Errorf("Only fs:// URLs support removal")
 	}
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: acl_managers.NewRoleACLManager(config_obj, "administrator"),
-		Logger:     log.New(&LogWriter{config_obj}, "", 0),
+		Logger:     log.New(logger, "", 0),
 		Env: ordereddict.NewDict().
 			Set("accessor", accessor).
 			Set("path", path),
@@ -235,8 +242,13 @@ func doRM(path, accessor string) error {
 		"file_store_delete(path=FullPath) AS Deletion " +
 		"FROM glob(globs=path, accessor=accessor) "
 
-	return eval_query(sm.Ctx,
+	err = eval_query(sm.Ctx,
 		config_obj, *fs_command_format, query, scope, builder.Env)
+	if err != nil {
+		return err
+	}
+
+	return logger.Error
 }
 
 func doCp(path, accessor string, dump_dir string) error {
@@ -282,9 +294,10 @@ func doCp(path, accessor string, dump_dir string) error {
 		output_path = matches[2]
 	}
 
+	logger := &LogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config: config_obj,
-		Logger: log.New(&LogWriter{config_obj}, "", 0),
+		Logger: log.New(logger, "", 0),
 		Env: ordereddict.NewDict().
 			Set("accessor", accessor).
 			Set("path", path),
@@ -318,7 +331,7 @@ func doCp(path, accessor string, dump_dir string) error {
 	scope.Log("Copy from %v (%v) to %v (%v)",
 		path, accessor, output_path, output_accessor)
 
-	return eval_query(sm.Ctx, config_obj, *fs_command_format, `
+	err = eval_query(sm.Ctx, config_obj, *fs_command_format, `
 SELECT * from foreach(
   row={
     SELECT Name, Size, Mode.String AS Mode,
@@ -329,6 +342,11 @@ SELECT * from foreach(
      upload(file=FullPath, accessor=accessor, name=Name) AS Upload
      FROM scope()
   })`, scope, builder.Env)
+	if err != nil {
+		return err
+	}
+
+	return logger.Error
 }
 
 func doCat(path, accessor_name string) error {

bin/hunts.go

@@ -47,10 +47,11 @@ func doHuntReconstruct() error {
 		return err
 	}
 
+	logger := &StdoutLogWriter{}
 	builder := services.ScopeBuilder{
 		Config:     sm.Config,
 		ACLManager: acl_managers.NewRoleACLManager(sm.Config, "administrator"),
-		Logger:     log.New(&StdoutLogWriter{}, "", 0),
+		Logger:     log.New(logger, "", 0),
 		Env:        ordereddict.NewDict(),
 	}
 
@@ -99,7 +100,7 @@ func doHuntReconstruct() error {
 		}
 	}
 
-	return nil
+	return logger.Error
 }
 
 func init() {