Refactored LocalHash database artifacts.

api/csv.go

@@ -53,7 +53,7 @@ func getTable(
 	rs_reader, err := result_sets.NewResultSetReader(
 		file_store_factory, path_manager)
 	if err != nil {
-		return nil, err
+		return result, nil
 	}
 	defer rs_reader.Close()
 

artifacts/definitions/Windows/Forensics/LocalHashes/Glob.yaml

@@ -34,10 +34,10 @@ precondition: SELECT OS from info() where OS = "windows"
 
 sources:
   - query: |
-      LET hash_db <= SELECT HashDBPath
-      FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb)
+      LET hash_db <= SELECT FullPath
+      FROM Artifact.Windows.Forensics.LocalHashes.Init(HashDb=HashDb)
 
-      LET path <= hash_db[0].HashDBPath
+      LET path <= hash_db[0].FullPath
 
       LET _ <= log(message="Will use local hash database " + path)
 

artifacts/definitions/Windows/Forensics/LocalHashes/Init.yaml

@@ -0,0 +1,35 @@
+name: Windows.Forensics.LocalHashes.Init
+description: |
+   This artifact creates an SQLite database on the endpoint to hold
+   local file hashes. These hashes can then be queried quickly.
+
+precondition: SELECT OS from info() where OS = "windows"
+
+parameters:
+  - name: HashDb
+    description: Name of the local hash database
+    default: hashdb.sqlite
+
+sources:
+  - query: |
+      LET SQL = "
+        CREATE table if not exists hashes(path text, md5 varchar(16), size bigint, timestamp bigint)
+        create index if not exists hashidx on hashes(md5)
+        create index if not exists pathidx on hashes(path)
+        create unique index if not exists uniqueidx on hashes(path, md5)
+        "
+
+      LET hash_db <= path_join(components=[dirname(path=tempfile()), HashDb])
+
+      LET _ <= log(message="Will use local hash database " + hash_db)
+
+      // SQL to create the initial database.
+      LET _ <= SELECT * FROM foreach(
+      row={
+          SELECT Line FROM parse_lines(filename=SQL, accessor="data")
+          WHERE Line
+      }, query={
+         SELECT * FROM sqlite(file=hash_db, query=Line)
+      })
+
+      SELECT hash_db AS FullPath FROM scope()

artifacts/definitions/Windows/Forensics/LocalHashes/Query.yaml

@@ -32,47 +32,26 @@ parameters:
 
 sources:
   - query: |
-      LET hash_db <= path_join(components=[dirname(path=tempfile()), HashDb])
-
-      LET _ <= log(message="Will use local hash database " + hash_db)
-
-      // SQL to create the initial database.
-      LET _ <= SELECT * FROM sqlite(file=hash_db,
-         query="CREATE table if not exists hashes(path text, md5 varchar(16), size bigint, timestamp bigint)")
-
-      LET _ <= SELECT * FROM sqlite(file=hash_db,
-         query="create index if not exists hashidx on hashes(md5)")
-
-      LET _ <= SELECT * FROM sqlite(file=hash_db,
-         query="create index if not exists pathidx on hashes(path)")
-
-      LET _ <= SELECT * FROM sqlite(file=hash_db,
-         query="create unique index if not exists uniqueidx on hashes(path, md5)")
-
-      LET lookup(Hash) = SELECT hash_db AS HashDBPath, path AS Path, md5 AS MD5, size AS Size,
-             timestamp(epoch=time) AS Timestamp
-      FROM sqlite(file=hash_db,
-         query="SELECT path, md5, size, timestamp AS time FROM hashes WHERE md5 = ?",
-         args=Hash)
+      LET hash_db <= SELECT FullPath
+      FROM Artifact.Windows.Forensics.LocalHashes.Init(HashDb=HashDb)
 
       -- Check hashes from the CSV or comma delimited input
       LET hashes = SELECT Hash FROM chain(
       a={
-        SELECT Hash FROM parse_csv(filename=Hashes, accessor="data")
+        SELECT strip(string=Hash) AS Hash
+        FROM parse_csv(filename=Hashes, accessor="data")
       }, b={
         SELECT * FROM foreach(row=split(string=CommaDelimitedHashes, sep=","),
         query={
-           SELECT _value AS Hash FROM scope()
+           SELECT strip(string=_value) AS Hash FROM scope()
         })
       })
 
-      SELECT * FROM switch(
-      a={
-          SELECT * FROM foreach(row=hashes,
-          query={
-              SELECT * FROM lookup(Hash=Hash)
-          })
-      },
-      b={
-         SELECT hash_db AS HashDBPath FROM scope()
+      SELECT * FROM foreach(row=hashes,
+      query={
+         SELECT path AS Path, md5 AS MD5, size AS Size,
+                timestamp(epoch=time) AS Timestamp
+         FROM sqlite(file=hash_db[0].FullPath,
+                     query="SELECT path, md5, size, timestamp AS time FROM hashes WHERE md5 = ?",
+                     args=Hash)
       })

artifacts/definitions/Windows/Forensics/LocalHashes/Usn.yaml

@@ -31,10 +31,10 @@ precondition: SELECT OS from info() where OS = "windows"
 
 sources:
   - query: |
-      LET hash_db <= SELECT HashDBPath
-      FROM Artifact.Windows.Forensics.LocalHashes.Query(HashDb=HashDb)
+      LET hash_db <= SELECT FullPath
+      FROM Artifact.Windows.Forensics.LocalHashes.Init(HashDb=HashDb)
 
-      LET path <= hash_db[0].HashDBPath
+      LET path <= hash_db[0].FullPath
 
       LET _ <= log(message="Will use local hash database " + path)
 

datastore/filebased.go

@@ -37,7 +37,6 @@
 package datastore
 
 import (
-	"compress/gzip"
 	"io"
 	"io/ioutil"
 	"os"
@@ -177,13 +176,11 @@ func (self *FileBaseDataStore) Walk(config_obj *config_proto.Config,
 			}
 
 			// We are only interested in filenames that end with .db
-			basename := strings.TrimSuffix(info.Name(), ".gz")
+			basename := info.Name()
 			if !strings.HasSuffix(basename, ".db") {
 				return nil
 			}
 
-			path = strings.TrimSuffix(path, ".gz")
-
 			urn, err := FilenameToURN(config_obj, path)
 			if err != nil {
 				return err
@@ -232,14 +229,6 @@ func (self *FileBaseDataStore) DeleteSubject(
 		return errors.WithStack(err)
 	}
 
-	filename += ".gz"
-	err = os.Remove(filename)
-
-	// It is ok to remove a file that does not exist.
-	if err != nil && os.IsExist(err) {
-		return errors.WithStack(err)
-	}
-
 	// Note: We do not currently remove empty intermediate
 	// directories.
 	return nil
@@ -284,7 +273,6 @@ func (self *FileBaseDataStore) ListChildren(
 		}
 
 		name := UnsanitizeComponent(children[i].Name())
-		name = strings.TrimSuffix(name, ".gz")
 		if !strings.HasSuffix(name, ".db") {
 			continue
 		}
@@ -383,7 +371,6 @@ func (self *FileBaseDataStore) SearchClients(
 
 		for _, child_urn := range children {
 			name := UnsanitizeComponent(child_urn.Name())
-			name = strings.TrimSuffix(name, ".gz")
 			name = strings.TrimSuffix(name, ".db")
 			_, pres := seen[name]
 			if !pres {
@@ -412,7 +399,6 @@ func (self *FileBaseDataStore) SearchClients(
 
 		for _, set := range sets {
 			name := UnsanitizeComponent(set.Name())
-			name = strings.TrimSuffix(name, ".gz")
 			name = strings.TrimSuffix(name, ".db")
 			matched, err := path.Match(query, name)
 			if err != nil {
@@ -631,20 +617,6 @@ func readContentFromFile(
 		return result, errors.WithStack(err)
 	}
 
-	// File does not exist - try the gzip version
-	if os.IsNotExist(err) {
-		file, err = os.Open(filename + ".gz")
-		if err == nil {
-			zr, err := gzip.NewReader(file)
-			if err != nil {
-				return nil, errors.WithStack(err)
-			}
-			result, err := ioutil.ReadAll(
-				io.LimitReader(zr, constants.MAX_MEMORY))
-			return result, errors.WithStack(err)
-		}
-	}
-
 	// Its ok if the file does not exist - no error.
 	if !must_exist && os.IsNotExist(err) {
 		return []byte{}, nil
@@ -665,7 +637,6 @@ func FilenameToURN(config_obj *config_proto.Config, filename string) (string, er
 	for _, component := range strings.Split(
 		filename,
 		string(os.PathSeparator)) {
-		component = strings.TrimSuffix(component, ".gz")
 		component = strings.TrimSuffix(component, ".db")
 		components = append(components,
 			string(UnsanitizeComponent(component)))

file_store/directory/directory.go

@@ -29,7 +29,6 @@ package directory
 */
 
 import (
-	"compress/gzip"
 	"os"
 	"path"
 	"path/filepath"
@@ -118,32 +117,10 @@ func (self *DirectoryFileStore) ListDirectory(dirname string) (
 	return result, nil
 }
 
-func getCompressed(filename string) (api.FileReader, error) {
-	fd, err := os.Open(filename)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	zr, err := gzip.NewReader(fd)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return &GzipReader{zr, fd, filename}, nil
-}
-
 func (self *DirectoryFileStore) ReadFile(filename string) (api.FileReader, error) {
 	file_path := self.FilenameToFileStorePath(filename)
-	if strings.HasSuffix(".gz", file_path) {
-		return getCompressed(file_path)
-	}
-
 	openCounter.Inc()
 	file, err := os.Open(file_path)
-	if os.IsNotExist(err) {
-		return getCompressed(file_path + ".gz")
-	}
-
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}

flows/artifacts.go

@@ -671,8 +671,6 @@ func (self *FlowRunner) ProcessSingleMessage(
 		return
 	}
 
-	json.Dump(job)
-
 	if job.SessionId == constants.MONITORING_WELL_KNOWN_FLOW {
 		err := MonitoringProcessMessage(self.config_obj, collection_context, job)
 		if err != nil {

flows/artifacts_test.go

@@ -136,8 +136,6 @@ func (self *TestSuite) TestRetransmission() {
 		self.client_id, flow_id)
 	assert.NoError(self.T(), err)
 
-	// json.Dump(collection_context)
-
 	// The flow should have only a single row though.
 	assert.Equal(self.T(), collection_context.TotalCollectedRows, uint64(1))
 

go.mod

@@ -114,7 +114,7 @@ require (
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb
 	www.velocidex.com/golang/evtx v0.0.2-0.20201104041743-4b6cdb206c95
 	www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321
-	www.velocidex.com/golang/go-ntfs v0.1.2-0.20201103060907-e4d6cbc89067
+	www.velocidex.com/golang/go-ntfs v0.1.2-0.20201111050421-bbba6f6a13d3
 	www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196

go.sum

@@ -777,6 +777,8 @@ www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321 h1:0FELGb4G9d
 www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321/go.mod h1:d3PHzQhyhe+AO9RYBnDKZ40As15T+38zr++Dnv4ufuc=
 www.velocidex.com/golang/go-ntfs v0.1.2-0.20201103060907-e4d6cbc89067 h1:HMRGdsX06YmFig5ambwqxhFg7gcIUKVAZLETmwgW2LY=
 www.velocidex.com/golang/go-ntfs v0.1.2-0.20201103060907-e4d6cbc89067/go.mod h1:ep0kkFXbnXff4YS0Xpcp2CTzh4dZlPju305HKeH+jKo=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20201111050421-bbba6f6a13d3 h1:MkxoKsZRp15/y/cxD0WogLaty6NHsgwSsxVzYxWQUus=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20201111050421-bbba6f6a13d3/go.mod h1:ep0kkFXbnXff4YS0Xpcp2CTzh4dZlPju305HKeH+jKo=
 www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6 h1:t+4cuel1gs9H854UvXSVwL/Ph7wAhGUq8xNEiEuG7x4=
 www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6/go.mod h1:vpdQNvh2GfQT5NNhtpyGirOSYGTy2IJheePUBpP/zHs=
 www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca h1:wQlRPcnpblrt/xc5rSe6a0K/Rh7yDEfwD0/qMrnOTrI=

gui/velociraptor/src/components/notebooks/notebook-cell-renderer.js

@@ -153,12 +153,16 @@ export default class NotebookCellRenderer extends React.Component {
     componentDidUpdate = (prevProps, prevState, rootNode) => {
         // Do not update the editor if we are currently editing it -
         // otherwise it will wipe the text.
-        if (this.state.currently_editing) {
+        let selected = this.state.cell.cell_id === this.props.selected_cell_id;
+        if (!selected && this.state.currently_editing) {
+            // We are not currently editing if this cell is not selected.
+            this.setState({currently_editing: false});
             return;
         }
 
         let this_timestamp = this.props.cell_metadata && this.props.cell_metadata.timestamp;
         if (this_timestamp !== this.state.cell_timestamp) {
+            // Prevent further updates to this cell.
             this.setState({cell_timestamp: this_timestamp});
             this.fetchCellContents();
         }