Refactored client index (Velocidex#1214)

On EFS the old client index was very non performant as EFS is very
slow and unable to handle large directories.

The new index has uses balanced b tree to keep directory size
small. We alos use a local in memory LRU to reduce number of EFS
operations.

api/notebooks.go

@@ -1044,14 +1044,15 @@ func getAvailableTimelines(
 
 	result := []string{}
 	db, err := datastore.GetDB(config_obj)
-	files, err := db.ListChildren(
-		config_obj, path_manager.SuperTimelineDir(), 0, 1000)
+	files, err := db.ListChildren(config_obj, path_manager.SuperTimelineDir())
 	if err != nil {
 		return nil
 	}
 
 	for _, f := range files {
-		result = append(result, f.Base())
+		if !f.IsDir() {
+			result = append(result, f.Base())
+		}
 	}
 	return result
 }

api/proto/clients.proto

@@ -177,3 +177,12 @@ message Uname {
       "running on a 64 bit system)",
     }];
 };
+
+
+message IndexRecord {
+    // Usually client id
+    string entity = 1;
+
+    // The term under which we indexed the entity.
+    string term = 2;
+}

api/vfs.go

@@ -117,8 +117,7 @@ func renderDBVFS(
 
 	// Figure out where the download info files are.
 	download_info_path := path_manager.VFSDownloadInfoPath(components)
-	downloaded_files, _ := db.ListChildren(
-		config_obj, download_info_path, 0, 1000)
+	downloaded_files, _ := db.ListChildren(config_obj, download_info_path)
 
 	result := &api_proto.VFSListResponse{}
 

artifacts/definitions/Windows/Search/FileFinder.yaml

@@ -75,10 +75,10 @@ sources:
       LET file_search = SELECT FullPath,
                get(item=Sys, field="mft") as Inode,
                Mode.String AS Mode, Size,
-               Mtime AS Modified,
+               Mtime AS MTime,
                Atime AS ATime,
-               Mtime AS MTime, "" AS Keywords,
-               Ctime AS CTime, IsDir
+               Btime AS BTime,
+               Ctime AS CTime, "" AS Keywords, IsDir
         FROM glob(globs=SearchFilesGlobTable.Glob + SearchFilesGlob,
                   accessor=Accessor)
 
@@ -106,7 +106,7 @@ sources:
             },
             query={
                SELECT FullPath, Inode, Mode,
-                      Size, Modified, ATime, MTime, CTime,
+                      Size, MTime, ATime, CTime, BTime,
                       str(str=String.Data) As Keywords, IsDir
 
                FROM yara(files=FullPath,
@@ -116,8 +116,8 @@ sources:
             })
         }, else=modified_before)
 
-      SELECT FullPath, Inode, Mode, Size, Modified, ATime,
-             MTime, CTime, Keywords, IsDir,
+      SELECT FullPath, Inode, Mode, Size, MTime, ATime,
+             CTime, BTime, Keywords, IsDir,
                if(condition=Upload_File and NOT IsDir,
                   then=upload(file=FullPath, accessor=Accessor)) AS Upload,
                if(condition=Calculate_Hash and NOT IsDir,

artifacts/definitions/Windows/Sys/Interfaces.yaml

@@ -25,7 +25,7 @@ sources:
           "Physical Address[^:]+: (?P<MAC>.+)\r\n",
           "IPv4 Address[^:]+: (?P<IP>[0-9.]+)",
           "Default Gateway[^:]+: (?P<Gateway>.+)\r\n",
-          "DNS Servers[^:]+: (?P<DNS>.+)\r\n",
+          "DNS Servers[^:]+: (?P<DNS>.+)\r\n   [^ ]",
           "DHCP Server[^:]+: (?P<DHCP>.+)\r\n"
         ]
      ) As Details FROM interfaces

artifacts/testdata/server/client_idx/all/c.4/f5e/52a/df0/a33/7a9/C.4f5e52adf0a337a9.db

@@ -0,0 +1,2 @@
+
+C.4f5e52adf0a337a9all

artifacts/testdata/server/client_idx/des/kto/p-6/cbj/8mj/c.4/f5e/52a/df0/a33/7a9/C.4f5e52adf0a337a9.db

@@ -0,0 +1,2 @@
+
+C.4f5e52adf0a337a9desktop-6cbj8mj

artifacts/testdata/server/client_idx/hos/t%3Ad/esk/top/-6c/bj8/mjc/%2E4f/5e5/2ad/f0a/337/a9/C.4f5e52adf0a337a9.db

@@ -0,0 +1,2 @@
+
+C.4f5e52adf0a337a9host:desktop-6cbj8mj

datastore/datastore.go

@@ -32,6 +32,8 @@ import (
 
 var (
 	mu sync.Mutex
+
+	StopIteration = errors.New("StopIteration")
 )
 
 type SortingSense int
@@ -47,8 +49,8 @@ type DatastoreInfo struct {
 	Modified time.Time
 }
 
+// When WalkFunc return StopIteration we exit the walk.
 type WalkFunc func(urn api.DSPathSpec) error
-type ComponentWalkFunc func(components api.DSPathSpec) error
 
 type DataStore interface {
 	// Retrieve all the client's tasks.
@@ -87,8 +89,7 @@ type DataStore interface {
 	// Lists all the children of a URN.
 	ListChildren(
 		config_obj *config_proto.Config,
-		urn api.DSPathSpec,
-		offset uint64, length uint64) ([]api.DSPathSpec, error)
+		urn api.DSPathSpec) ([]api.DSPathSpec, error)
 
 	Walk(config_obj *config_proto.Config,
 		root api.DSPathSpec, walkFn WalkFunc) error