Introduced lazy_ntfs accessor. (Velocidex#83)

This experimental accessor is optimized for searching globs. It only
scans the I30 indexes when listing a directory and returns a lazy
object. When the object is accessed for its metadata - only then does
it read the corresponding MFT entry.

The downside of this approach is that the lazy_ntfs accessor is not
aware of ADS stream since it does not inspect the attribute name at
all. However it is much faster than the regular ntfs accessor.

artifacts/definitions/Windows/Collectors/File.yaml

@@ -17,7 +17,7 @@ parameters:
     description: The device to apply all the glob on.
     default: "C:"
   - name: Accessor
-    default: ntfs
+    default: lazy_ntfs
 
 sources:
    - name: All Matches Metadata
@@ -35,7 +35,7 @@ sources:
                timestamp(epoch=Mtime.Sec) AS Modified,
                timestamp(epoch=Atime.Sec) AS LastAccessed
         FROM glob(globs=specs.Glob, accessor=Accessor)
-        WHERE NOT IsDir
+        WHERE NOT IsDir AND log(message="Found " + SourceFile)
 
       # Create a unique key to group by - modification time and path name.
       # Order by device name so we get C:\ above the VSS device.

artifacts/definitions/Windows/Collectors/VSS.yaml

@@ -29,15 +29,15 @@ parameters:
     description: The device to apply all the glob on.
     default: "C:"
   - name: Accessor
-    default: ntfs
+    default: lazy_ntfs
   - name: VSSDateRegex
     default: .
 
 sources:
    - name: All Matches Metadata
      queries:
       - LET originating_machine <= SELECT Data.SystemName AS System
-            FROM glob(globs="/*", accessor="ntfs")
+            FROM glob(globs="/*", accessor=Accessor)
             WHERE Name = "\\\\.\\" + RootDevice
 
       # Generate the collection globs for each device
@@ -60,7 +60,7 @@ sources:
       # Get all volume shadows on this system.
       - LET volume_shadows = SELECT Data.InstallDate AS InstallDate,
                Data.DeviceObject + "\\" AS Device
-        FROM glob(globs='/*', accessor='ntfs')
+        FROM glob(globs='/*', accessor=Accessor)
         WHERE Device =~ 'VolumeShadowCopy' AND
               Data.OriginatingMachine = originating_machine.System[0] AND
               InstallDate =~ VSSDateRegex

artifacts/definitions/Windows/Kape/Targets.yaml

@@ -752,7 +752,7 @@ parameters:
   - name: Device
     default: "C:"
   - name: Accessor
-    default: ntfs
+    default: lazy_ntfs
   - name: VSSAnalysis
     type: bool
     default:
@@ -772,7 +772,8 @@ sources:
       - |
         LET rule_specs <= SELECT Id, Glob
         FROM parse_csv(filename=KapeRules, accessor="data")
-        WHERE Id in array(array=targets.RuleIds)
+        WHERE Id in array(array=targets.RuleIds) AND log(
+           message="Searching for glob " + Glob)
 
       # Call the generic VSS file collector with the globs we want in a new CSV file.
       - |
@@ -789,6 +790,7 @@ sources:
                 collectionSpec=serialize(item=rule_specs, format="csv"))
            })
       - SELECT * FROM all_results WHERE _Source =~ "Metadata"
+
   - name: Uploads
     queries:
       - SELECT * FROM all_results WHERE _Source =~ "Uploads"

artifacts/testdata/windows/ntfs.in.yaml

@@ -10,3 +10,15 @@ Queries:
     # Expect to see an ADS and an alternative short file name.
   - SELECT FullPath FROM glob(globs="c:\\*", accessor="ntfs")
     WHERE Name =~ "txt"
+
+  - SELECT FullPath FROM glob(globs="/*", accessor="lazy_ntfs")
+  - SELECT FullPath FROM glob(globs="\\*", accessor="lazy_ntfs")
+  - SELECT FullPath FROM glob(globs="\\\\.\\c:\\*", accessor="lazy_ntfs")
+    WHERE Name = "pagefile.sys"
+  - SELECT FullPath FROM glob(globs="\\\\.\\c:\\Windows\\*", accessor="lazy_ntfs")
+    WHERE Name = "notepad.exe"
+  - SELECT FullPath FROM glob(globs="c:/Windows/*", accessor="lazy_ntfs")
+    WHERE Name = "notepad.exe"
+    # Expect to see an ADS and an alternative short file name.
+  - SELECT FullPath FROM glob(globs="c:\\*", accessor="lazy_ntfs")
+    WHERE Name =~ "txt"

artifacts/testdata/windows/ntfs.out.yaml

@@ -31,4 +31,28 @@ SELECT FullPath FROM glob(globs="/*", accessor="ntfs")[
  {
   "FullPath": "\\\\.\\c:\\hello.txt:myads"
  }
+]SELECT FullPath FROM glob(globs="/*", accessor="lazy_ntfs")[
+ {
+  "FullPath": "\\\\.\\C:"
+ }
+]SELECT FullPath FROM glob(globs="\\*", accessor="lazy_ntfs")[
+ {
+  "FullPath": "\\\\.\\C:"
+ }
+]SELECT FullPath FROM glob(globs="\\\\.\\c:\\*", accessor="lazy_ntfs") WHERE Name = "pagefile.sys"[
+ {
+  "FullPath": "\\\\.\\c:\\pagefile.sys"
+ }
+]SELECT FullPath FROM glob(globs="\\\\.\\c:\\Windows\\*", accessor="lazy_ntfs") WHERE Name = "notepad.exe"[
+ {
+  "FullPath": "\\\\.\\c:\\Windows\\notepad.exe"
+ }
+]SELECT FullPath FROM glob(globs="c:/Windows/*", accessor="lazy_ntfs") WHERE Name = "notepad.exe"[
+ {
+  "FullPath": "\\\\.\\c:\\Windows\\notepad.exe"
+ }
+]SELECT FullPath FROM glob(globs="c:\\*", accessor="lazy_ntfs") WHERE Name =~ "txt"[
+ {
+  "FullPath": "\\\\.\\c:\\hello.txt"
+ }
 ]

bin/artifacts.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Velocidex/yaml"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -255,6 +256,14 @@ func doArtifactCollect() {
 	config_obj := get_config_or_default()
 	repository := getRepository(config_obj)
 
+	now := time.Now()
+	defer func() {
+		logging.GetLogger(config_obj, &logging.ToolComponent).
+			Info("Collection completed in %v Secondgs",
+				time.Now().Unix()-now.Unix())
+
+	}()
+
 	var container *reporting.Container
 	var err error
 

bin/fs.go

@@ -33,7 +33,7 @@ var (
 	fs_command          = app.Command("fs", "Run filesystem commands.")
 	fs_command_accessor = fs_command.Flag(
 		"accessor", "The FS accessor to use").Default("file").Enum(
-		"file", "ntfs", "reg", "zip", "raw_reg")
+		"file", "ntfs", "reg", "zip", "raw_reg", "lazy_ntfs")
 	fs_command_verbose = fs_command.Flag(
 		"details", "Show more verbose info").Short('d').
 		Default("false").Bool()

glob/glob.go

@@ -88,13 +88,16 @@ func (self _RecursiveComponent) Match(f FileInfo) bool {
 }
 
 type _RegexComponent struct {
-	regexp string
+	regexp   string
+	compiled *regexp.Regexp
 }
 
-func (self _RegexComponent) Match(f FileInfo) bool {
-	re := regexp.MustCompile("^(?msi)" + self.regexp)
+func (self *_RegexComponent) Match(f FileInfo) bool {
+	if self.compiled == nil {
+		self.compiled = regexp.MustCompile("^(?msi)" + self.regexp)
+	}
 
-	return re.MatchString(f.Name())
+	return self.compiled.MatchString(f.Name())
 }
 
 func (self _RegexComponent) String() string {
@@ -382,7 +385,7 @@ func (self Globber) _expand_path_components(filter []_PathFilterer, depth int) e
 						return err
 					}
 				}
-				middle = append(middle, _RegexComponent{
+				middle = append(middle, &_RegexComponent{
 					regexp: fnmatch_translate("*"),
 				})
 			}
@@ -452,7 +455,7 @@ func convert_glob_into_path_components(pattern string, path_sep func(path string
 			})
 
 		} else if m := _GLOB_MAGIC_CHECK.FindString(path_component); len(m) > 0 {
-			result = append(result, _RegexComponent{
+			result = append(result, &_RegexComponent{
 				regexp: fnmatch_translate(path_component),
 			})
 		} else {

glob/glob_test.go

@@ -48,7 +48,7 @@ var pathComponentsTestFixture = []pathComponentsTestFixtureType{
 		_RecursiveComponent{`foo.*\z(?ms)`, 5},
 	}},
 	{"*.exe", []_PathFilterer{
-		_RegexComponent{`.*\.exe\z(?ms)`},
+		&_RegexComponent{regexp: `.*\.exe\z(?ms)`},
 	}},
 	{"/bin/ls", []_PathFilterer{
 		_LiteralComponent{"bin"},

go.mod

@@ -90,7 +90,7 @@ require (
 	gopkg.in/olivere/elastic.v5 v5.0.81
 	gopkg.in/russross/blackfriday.v2 v2.0.1
 	www.velocidex.com/golang/evtx v0.0.0-20190210013513-b45fe1505163
-	www.velocidex.com/golang/go-ntfs v0.0.0-20190923111259-525352093fe8
+	www.velocidex.com/golang/go-ntfs v0.0.0-20190924105901-ddba091fd8d5
 	www.velocidex.com/golang/go-pe v0.1.0
 	www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196

go.sum

@@ -409,6 +409,8 @@ www.velocidex.com/golang/go-ntfs v0.0.0-20190921133542-301e100035d0 h1:nlUya8Eah
 www.velocidex.com/golang/go-ntfs v0.0.0-20190921133542-301e100035d0/go.mod h1:PqtikStGr+veDXyoha9wqB/VImxtJDljK2+raRNURzg=
 www.velocidex.com/golang/go-ntfs v0.0.0-20190923111259-525352093fe8 h1:HDaWf54SK8Z2djHpnibBZBy18RUmpJtQRSlXDilhRWc=
 www.velocidex.com/golang/go-ntfs v0.0.0-20190923111259-525352093fe8/go.mod h1:PqtikStGr+veDXyoha9wqB/VImxtJDljK2+raRNURzg=
+www.velocidex.com/golang/go-ntfs v0.0.0-20190924105901-ddba091fd8d5 h1:jrYx+gMeZ6mS0bvuDpEdtToaG26XKXSmexzFWYoxeuQ=
+www.velocidex.com/golang/go-ntfs v0.0.0-20190924105901-ddba091fd8d5/go.mod h1:PqtikStGr+veDXyoha9wqB/VImxtJDljK2+raRNURzg=
 www.velocidex.com/golang/go-pe v0.1.0 h1:W5oYsEMz1GiBv84CyZzOzTHjLaGLv8oTaDfrRkcDB6E=
 www.velocidex.com/golang/go-pe v0.1.0/go.mod h1:BBNxoooAx/LA/F9rn3QliOCR2FtiDccxX3MVsJ4OX2g=
 www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf h1:MhCevuCZJLBXoiYjL2W7qy7OugVZm/LBB11D6iKYUMY=

magefile.go

@@ -96,6 +96,9 @@ func Linux() error {
 	}
 
 	env := make(map[string]string)
+	env["GOOS"] = "linux"
+	env["GOARCH"] = "amd64"
+
 	err = sh.RunWith(
 		env,
 		mg.GoCmd(), "build",
@@ -164,7 +167,12 @@ func Appveyor() error {
 
 	os.Chdir(cwd)
 
-	return Windows()
+	err = Windows()
+	if err != nil {
+		return err
+	}
+
+	return Linux()
 }
 
 // Cross compile the windows binary using mingw. Note that this does

vql/functions/log.go

@@ -45,7 +45,7 @@ func (self *LogFunction) Call(ctx context.Context,
 	}
 
 	if arg.Message != last_log {
-		scope.Log(arg.Message)
+		scope.Log("%v", arg.Message)
 		last_log = arg.Message
 	}
 

vql/windows/filesystems/mft_windows.go

@@ -59,11 +59,13 @@ func (self *MFTFileSystemAccessor) Open(path string) (glob.ReadSeekCloser, error
 		return nil, err
 	}
 
-	ntfs_ctx, err := self.getNTFSContext(device)
+	accessor_ctx, err := self.getNTFSContext(device)
 	if err != nil {
 		return nil, err
 	}
 
+	ntfs_ctx := accessor_ctx.ntfs_ctx
+
 	mft_entry, err := ntfs_ctx.GetMFT(mft_idx)
 	if err != nil {
 		return nil, err
@@ -112,11 +114,12 @@ func (self *MFTFileSystemAccessor) Lstat(path string) (glob.FileInfo, error) {
 		return nil, err
 	}
 
-	ntfs_ctx, err := self.getNTFSContext(device)
+	accessor_ctx, err := self.getNTFSContext(device)
 	if err != nil {
 		return nil, err
 	}
 
+	ntfs_ctx := accessor_ctx.ntfs_ctx
 	mft_entry, err := ntfs_ctx.GetMFT(mft_idx)
 	if err != nil {
 		return nil, err