Update tool definitions to support expected_hash and version (Velocid…

…ex#2629)

Previously it was impossible to have multiple versions of the same
tool because tools were indexed by name. This made it hard to upgrade
versions automatically because the existing tool was considered to
satisfy the dependency.

This PR adds a version field to the tool definitions making the
combination of name and version unique. This also allows an
artifact to declare a specific version of a tool to use.

Additionally the tool definition now supports an expected hash field
which declares what the hash the artifact writer expects for the
tool. Velociraptor will download the tool and ensure the hash is the
same as the expected hash to avoid the tool being changed upstream.

accessors/manipulators.go

@@ -130,6 +130,8 @@ func (self LinuxPathManipulator) AsPathSpec(path *OSPath) *PathSpec {
 	if result == nil {
 		result = &PathSpec{}
 		path.pathspec = result
+	} else {
+		result = result.Copy()
 	}
 	result.Path = "/" + strings.Join(path.Components, "/")
 	return result
@@ -296,6 +298,8 @@ func (self WindowsNTFSManipulator) AsPathSpec(path *OSPath) *PathSpec {
 	if result == nil {
 		result = &PathSpec{}
 		path.pathspec = result
+	} else {
+		result = result.Copy()
 	}
 
 	// The first component is usually the drive letter or device and
@@ -463,6 +467,8 @@ func (self PathSpecPathManipulator) AsPathSpec(path *OSPath) *PathSpec {
 	if result == nil {
 		result = &PathSpec{}
 		path.pathspec = result
+	} else {
+		result = result.Copy()
 	}
 	return result
 }
@@ -542,6 +548,8 @@ func (self FileStorePathManipulator) AsPathSpec(path *OSPath) *PathSpec {
 	if result == nil {
 		result = &PathSpec{}
 		path.pathspec = result
+	} else {
+		result = result.Copy()
 	}
 
 	// The first component is usually the drive letter or device and

actions/events_test.go

@@ -54,7 +54,7 @@ type EventsTestSuite struct {
 
 func (self *EventsTestSuite) SetupTest() {
 	self.ConfigObj = self.LoadConfig()
-	self.LoadArtifacts(artifact_definitions)
+	self.LoadArtifactsIntoConfig(artifact_definitions)
 
 	// Set a tempfile for the writeback we need to check that the
 	// new event query is written there.

api/tools.go

@@ -29,10 +29,10 @@ func (self *ApiServer) GetToolInfo(ctx context.Context,
 		return nil, Status(self.verbose, err)
 	}
 	if in.Materialize {
-		return inventory.GetToolInfo(ctx, org_config_obj, in.Name)
+		return inventory.GetToolInfo(ctx, org_config_obj, in.Name, in.Version)
 	}
 
-	return inventory.ProbeToolInfo(ctx, org_config_obj, in.Name)
+	return inventory.ProbeToolInfo(ctx, org_config_obj, in.Name, in.Version)
 }
 
 func (self *ApiServer) SetToolInfo(ctx context.Context,
@@ -62,6 +62,12 @@ func (self *ApiServer) SetToolInfo(ctx context.Context,
 		return nil, Status(self.verbose, err)
 	}
 
+	// Clear internally managed tools the user should not be allowed
+	// to set.
+	in.Versions = nil
+	in.ServeUrl = ""
+	in.InvalidHash = ""
+
 	err = inventory.AddTool(ctx, org_config_obj, in,
 		services.ToolOptions{
 			AdminOverride: true,
@@ -73,7 +79,7 @@ func (self *ApiServer) SetToolInfo(ctx context.Context,
 	// If materialized we re-fetch the tool and send back the full
 	// record.
 	if materialize {
-		return inventory.GetToolInfo(ctx, org_config_obj, in.Name)
+		return inventory.GetToolInfo(ctx, org_config_obj, in.Name, in.Version)
 	}
 
 	return in, nil

api/upload.go

@@ -138,7 +138,7 @@ func toolUploadHandler() http.Handler {
 
 		// Now materialize the tool
 		tool, err = inventory.GetToolInfo(
-			r.Context(), org_config_obj, tool.Name)
+			r.Context(), org_config_obj, tool.Name, tool.Version)
 		if err != nil {
 			returnError(w, http.StatusInternalServerError,
 				fmt.Sprintf("Error: %v", err))

artifacts/proto/artifact.proto

@@ -244,6 +244,30 @@ message Tool {
     // server versions.
     bool admin_override = 7;
 
+    // Hex encoded sha256 hash of the expected hash of the file. When
+    // the server downloads the file it will check against the
+    // expected hash and refuse to download an incorrect hash.  This
+    // is a way to pin the dependency into the artifact definition to
+    // protect against supply side attacks.
+    string expected_hash = 15;
+
+    // Specify a version for the tool. This allows multiple versions
+    // of the same tool to be used at the same time.
+    string version = 16;
+
+    // If set on a request we refresh the hash by downloading the file again.
+    bool materialize = 11;
+
+    // The artifact this definition came from.
+    string artifact = 13;
+
+    // A filestore path where the file can be downloaded from - if
+    // served locally. This can be provided by the caller to AddTool()
+    // after they uploaded the file to the filestore through some
+    // other mechanism. Note this path is relative to the /public/
+    // directory.
+    string filestore_path = 4;
+
 
     // Once the tool is added with the above fields, the following
     // fields are used to keep state on it.
@@ -256,26 +280,22 @@ message Tool {
     // Only valid for local dummy inventory.
     string serve_path = 12;
 
-    // A filestore path where the file can be downloaded from - if
-    // served locally.
-    string filestore_path = 4;
-
     // The name of the cached file on the endpoint. This file will
     // persist and can be accessed again if this tool is needed in
     // future. If the file is missing (or has the wrong hash), then it
     // will be downloaded again.
     string filename = 5;
 
     // Hex encoded sha256 hash of the file. Endpoints will check this
-    // hash against their fetch file to ensure it was correctly
+    // hash against their fetched file to ensure it was correctly
     // transferred.
     string hash = 6;
 
-    // If set on a request we refresh the hash.
-    bool materialize = 11;
-
-    // The artifact this definition came from.
-    string artifact = 13;
+    // When we attempted to download the file, and the hash did not
+    // match the expected hash, we record the actual hash here. The
+    // user can then opt for trusting this hash in the GUI which will
+    // copy it into the expected hash and clear this hash.
+    string invalid_hash = 17;
 
     // Additional versions of this tool.
     repeated Tool versions = 14;

artifacts/testdata/server/testcases/binary_blobs.out.yaml

@@ -11,11 +11,14 @@ SELECT * FROM switch( b={SELECT Complete FROM execve(argv=["rm", "-f", "/tmp/aut
    "github_asset_regex": "",
    "serve_locally": false,
    "admin_override": true,
+   "expected_hash": "",
+   "version": "",
    "serve_url": "",
    "serve_path": "",
    "filestore_path": "",
    "filename": "winpmem_v3.3.rc3.exe",
    "hash": "",
+   "invalid_hash": "",
    "materialize": false,
    "artifact": "",
    "versions": []
@@ -29,11 +32,14 @@ SELECT * FROM switch( b={SELECT Complete FROM execve(argv=["rm", "-f", "/tmp/aut
   "github_asset_regex": "",
   "serve_locally": false,
   "admin_override": true,
+  "expected_hash": "",
+  "version": "",
   "serve_url": "",
   "serve_path": "",
   "filestore_path": "351b4f6d59a4266cc7a2eab9cedf959eb6a4c924746044e6edeabdd1a477643e",
   "filename": "winpmem_v3.3.rc3.exe",
   "hash": "",
+  "invalid_hash": "",
   "materialize": false,
   "artifact": "",
   "versions": []

artifacts/testdata/server/testcases/tools.out.yaml

@@ -7,11 +7,14 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g
    "github_asset_regex": "",
    "serve_locally": false,
    "admin_override": true,
+   "expected_hash": "",
+   "version": "",
    "serve_url": "",
    "serve_path": "",
    "filestore_path": "",
    "filename": "autorunsc_x64.exe",
    "hash": "083d7eee4ed40a3e5a35675503b0b6be0cb627b4cb1009d185a558a805f64153",
+   "invalid_hash": "",
    "materialize": false,
    "artifact": "",
    "versions": []
@@ -50,11 +53,14 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g
    "github_asset_regex": "",
    "serve_locally": true,
    "admin_override": true,
+   "expected_hash": "",
+   "version": "",
    "serve_url": "",
    "serve_path": "",
    "filestore_path": "",
    "filename": "autorunsc_x64.exe",
    "hash": "083d7eee4ed40a3e5a35675503b0b6be0cb627b4cb1009d185a558a805f64153",
+   "invalid_hash": "",
    "materialize": false,
    "artifact": "",
    "versions": []
@@ -95,11 +101,14 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g
    "github_asset_regex": "",
    "serve_locally": true,
    "admin_override": true,
+   "expected_hash": "",
+   "version": "",
    "serve_url": "",
    "serve_path": "",
    "filestore_path": "1c21ee4d8609f81482dc0a78c641e4586488a9fd562ee28eec25e448a9d0b2e1",
    "filename": "yara_test.txt",
    "hash": "f03278c10a41adcc97f24a612a680e7aa43efb461b337fef3d2a3d47b51e77bb",
+   "invalid_hash": "",
    "materialize": false,
    "artifact": "",
    "versions": []