Fix bug in uploading of sparse files. (Velocidex#2481)

GUI did not present the option of viewing the padded version of the file.

api/proto/users.proto

@@ -145,6 +145,7 @@ message ApiUser {
 message GUICustomizations {
     bool disable_server_events = 1;
     bool disable_user_management = 2;
+    bool disable_quarantine_button = 3;
 }
 
 message SetGUIOptionsRequest {
@@ -167,7 +168,8 @@ message SetGUIOptionsRequest {
 
     repeated GUILink links = 9;
 
-    // Optional features of the UI to disable.
+    // Optional features of the UI to disable:
+    // TODO: Move to GUICustomizations.
     bool disable_server_events = 26;
     string auth_redirect_template = 27;
     bool disable_quarantine_button = 28;

artifacts/testdata/manual/Test.Sparse.yaml

@@ -0,0 +1,13 @@
+name: Test.Sparse
+description: Test that sparse uploads are properly stored and processed.
+sources:
+  - query: |
+      -- Create a sparse file in memory.
+      LET FileName = pathspec(
+         DelegateAccessor="data",
+         DelegatePath="The quick red fox jumped oved the lazy",
+         Path=[dict(Offset=0,Length=5), dict(Offset=10,Length=5)])
+
+      -- Upload to the server
+      SELECT upload(accessor="sparse", file=FileName, name="X.txt")
+      FROM scope()

config/proto/config.proto

@@ -940,6 +940,7 @@ message Defaults {
     bool disable_server_events = 26;
     string auth_redirect_template = 27;
     bool disable_quarantine_button = 28;
+    string default_theme = 29;
 }
 
 // Configures crypto preferences

flows/client_flow_runner.go

@@ -245,6 +245,8 @@ func (self *ClientFlowRunner) FileBuffer(
 
 	// Keep track of all the files we uploaded.
 	if file_buffer.Offset == 0 {
+
+		// Truncate the file on first buffer received.
 		err = fd.Truncate()
 		if err != nil {
 			return err
@@ -259,8 +261,6 @@ func (self *ClientFlowRunner) FileBuffer(
 		if err != nil {
 			return err
 		}
-		defer rs_writer.Close()
-
 		rs_writer.Write(ordereddict.NewDict().
 			Set("Timestamp", utils.GetTime().Now().UTC().Unix()).
 			Set("started", utils.GetTime().Now().UTC().String()).
@@ -274,18 +274,31 @@ func (self *ClientFlowRunner) FileBuffer(
 			Set("_client_components", file_buffer.Pathspec.Components).
 			Set("uploaded_size", file_buffer.StoredSize))
 
-		// Additional row for sparse files
-		if file_buffer.Index != nil {
-			rs_writer.Write(ordereddict.NewDict().
-				Set("Timestamp", time.Now().UTC().Unix()).
-				Set("started", time.Now().UTC().String()).
-				Set("vfs_path", file_path_manager.VisibleVFSPath()+".idx").
-				Set("_Components", file_path_manager.Path().Components()).
-				Set("_accessor", file_buffer.Pathspec.Accessor).
-				Set("_client_components", file_buffer.Pathspec.Components).
-				Set("file_size", file_buffer.Size).
-				Set("uploaded_size", file_buffer.StoredSize))
+		rs_writer.Close()
+	}
+
+	// Additional row for sparse files
+	if file_buffer.Index != nil {
+		rs_writer, err := result_sets.NewResultSetWriter(
+			file_store_factory, flow_path_manager.UploadMetadata(),
+			json.DefaultEncOpts(),
+			self.completer.GetCompletionFunc(),
+			result_sets.AppendMode)
+		if err != nil {
+			return err
 		}
+
+		rs_writer.Write(ordereddict.NewDict().
+			Set("Timestamp", time.Now().UTC().Unix()).
+			Set("started", time.Now().UTC().String()).
+			Set("vfs_path", file_path_manager.VisibleVFSPath()+".idx").
+			Set("_Components", file_path_manager.Path().Components()).
+			Set("_accessor", file_buffer.Pathspec.Accessor).
+			Set("_client_components", file_buffer.Pathspec.Components).
+			Set("file_size", file_buffer.Size).
+			Set("uploaded_size", file_buffer.StoredSize))
+
+		rs_writer.Close()
 	}
 
 	_, err = fd.Write(file_buffer.Data)

gui/velociraptor/src/components/clients/client_info.jsx

@@ -51,14 +51,13 @@ class ClientSetterFromRoute extends Component {
                 client_id: client_id,
             });
 
-            api.get('/v1/SearchClients', {
-                query: client_id,
-            }, this.source.token).then(resp => {
-                if (resp.data && resp.data.items) {
-                    window.globals.client = resp.data.items[0];
-                    this.props.setClient(resp.data.items[0]);
-                }
-            });
+            api.get('v1/GetClient/' + client_id, {},
+                    this.source.token).then(resp => {
+                        if (resp.data && resp.data.items) {
+                            window.globals.client = resp.data;
+                            this.props.setClient(resp.data);
+                        }
+                    });
         }
     }
 

gui/velociraptor/src/components/flows/flow-uploads.jsx

@@ -117,10 +117,12 @@ export default class FlowUploads extends React.Component {
                 let flow_id = this.props.flow && this.props.flow.session_id;
                 let components = normalizeComponentList(
                     row._Components, client_id, flow_id);
+                let padding = row.vfs_path && row.vfs_path.endsWith(".idx");
                 return <PreviewUpload
                          env={{client_id: client_id, flow_id: flow_id}}
                          upload={{Path: row.vfs_path,
                                   Timestamp: row.started,
+                                  Padding: padding,
                                   Size: row.uploaded_size || row.file_size,
                                   Components: components}} />;
             },

gui/velociraptor/src/components/widgets/preview_uploads.jsx

@@ -308,6 +308,7 @@ export default class PreviewUpload extends Component {
         let params = {
             offset: 0,
             length: 100,
+            padding: this.props.upload.Padding,
             fs_components: normalizeComponentList(
                 components, client_id, flow_id, accessor),
             client_id: client_id,

services/inventory/inventory.go

@@ -44,6 +44,8 @@ import (
 	"time"
 
 	"github.com/go-errors/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"google.golang.org/protobuf/proto"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
@@ -56,6 +58,13 @@ import (
 	"www.velocidex.com/golang/velociraptor/vql/networking"
 )
 
+var (
+	inventoryTotalLoad = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "inventory_service_total_file_load",
+		Help: "Total number of times we synced from the filestore.",
+	})
+)
+
 type HTTPClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
@@ -445,6 +454,11 @@ func (self *InventoryService) LoadFromFile(config_obj *config_proto.Config) erro
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	inventoryTotalLoad.Inc()
+
+	logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
+	logger.Info("InventoryService: Reloading inventory from file")
+
 	inventory := &artifacts_proto.ThirdParty{}
 
 	db, err := datastore.GetDB(config_obj)
@@ -523,15 +537,18 @@ func NewInventoryService(
 
 			select {
 			case <-ctx.Done():
+				cancel()
 				return
 
 			case <-notification:
+				cancel()
 				err := inventory_service.LoadFromFile(config_obj)
 				if err != nil {
 					logger.Error("StartInventoryService: %v", err)
 				}
 
 			case <-time.After(600 * time.Second):
+				cancel()
 				err := inventory_service.LoadFromFile(config_obj)
 				if err != nil {
 					logger.Error("StartInventoryService: %v", err)

services/users/users.go

@@ -369,6 +369,11 @@ func (self UserManager) GetUserOptions(ctx context.Context, username string) (
 	options.DisableServerEvents = defaults.DisableServerEvents
 	options.DisableQuarantineButton = defaults.DisableQuarantineButton
 
+	// Specify a default theme if specified in the config file.
+	if options.Theme == "" {
+		options.Theme = defaults.DefaultTheme
+	}
+
 	return options, nil
 }
 

uploads/client_uploader.go

@@ -317,6 +317,7 @@ func (self *VelociraptorUploader) maybeUploadSparse(
 				IsSparse:     is_sparse,
 				Mtime:        mtime.UnixNano(),
 				Data:         data,
+				DataLength:   uint64(len(data)),
 				UploadNumber: upload_id,
 			}
 

uploads/fixtures/ClientUploaderSparse.golden

@@ -13,7 +13,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "SGVsbG8g"
+   "data": "SGVsbG8g",
+   "data_length": 6
   }
  },
  {
@@ -31,7 +32,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "aGVsbG8g"
+   "data": "aGVsbG8g",
+   "data_length": 6
   }
  },
  {

uploads/fixtures/ClientUploaderSparseMultiBuffer.golden

@@ -12,7 +12,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "SGU="
+   "data": "SGU=",
+   "data_length": 2
   }
  },
  {
@@ -29,7 +30,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "bGw="
+   "data": "bGw=",
+   "data_length": 2
   }
  },
  {
@@ -46,7 +48,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "byA="
+   "data": "byA=",
+   "data_length": 2
   }
  },
  {
@@ -63,7 +66,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "aGU="
+   "data": "aGU=",
+   "data_length": 2
   }
  },
  {
@@ -80,7 +84,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "bGw="
+   "data": "bGw=",
+   "data_length": 2
   }
  },
  {
@@ -97,7 +102,8 @@
    "size": 18,
    "stored_size": 12,
    "is_sparse": true,
-   "data": "byA="
+   "data": "byA=",
+   "data_length": 2
   }
  },
  {