Implemented multithreaded Zip offline collector (Velocidex#846)

The offline collector uses zip for storing the results, but in the
default Golang implementation the zip writer is single threaded and
can not utilize more than one CPU. This change allows the writer to
run multiple workers and compress multiple files at once.

It seems to lead to acquisition times reducing by a factor of 3-4
approx.

Makefile

@@ -4,7 +4,7 @@ all:
 auto:
 	go run make.go -v auto
 
-test: golden
+test:
 	go test ./... --tags server_vql
 
 golden:

artifacts/definitions/Windows/Collectors/File.yaml

@@ -50,12 +50,16 @@ sources:
    - name: Uploads
      queries:
       # Upload the files
-      - LET uploaded_tiles = SELECT Created, LastAccessed, Modified, SourceFile, Size,
+      - LET uploaded_files = SELECT * FROM foreach(row=all_results,
+        workers=30,
+        query={
+            SELECT Created, LastAccessed, Modified, SourceFile, Size,
                upload(file=SourceFile, accessor=Accessor, name=SourceFile) AS Upload
-        FROM all_results
+            FROM scope()
+        })
 
       # Separate the hashes into their own column.
       - SELECT now() AS CopiedOnTimestamp, SourceFile, Upload.Path AS DestinationFile,
                Size AS FileSize, Upload.sha256 AS SourceFileSha256,
                Created, Modified, LastAccessed
-        FROM uploaded_tiles
+        FROM uploaded_files

artifacts/definitions/Windows/Collectors/VSS.yaml

@@ -90,12 +90,16 @@ sources:
       - LET unique_mtimes = SELECT * FROM all_results GROUP BY Key
 
       # Upload the files using the MFT accessor.
-      - LET uploaded_tiles = SELECT Created, LastAccessed, Modified, MFT, SourceFile, Size,
+      - LET uploaded_files = SELECT * FROM foreach(row=unique_mtimes,
+        workers=30,
+        query={
+            SELECT Created, LastAccessed, Modified, MFT, SourceFile, Size,
                upload(file=Device+MFT, name=SourceFile, accessor="mft") AS Upload
-        FROM unique_mtimes
+            FROM scope()
+        })
 
       # Seperate the hashes into their own column.
       - SELECT now() AS CopiedOnTimestamp, SourceFile, Upload.Path AS DestinationFile,
                Size AS FileSize, Upload.sha256 AS SourceFileSha256,
                Created, Modified, LastAccessed, MFT
-        FROM uploaded_tiles
+        FROM uploaded_files

bin/golden.go

@@ -157,7 +157,7 @@ func runTest(fixture *testFixture,
 	}
 	defer os.Remove(tmpfile.Name())
 
-	container, err := reporting.NewContainer(tmpfile.Name())
+	container, err := reporting.NewContainer(tmpfile.Name(), "")
 	kingpin.FatalIfError(err, "Can not create output container")
 
 	builder := services.ScopeBuilder{

go.mod

@@ -17,11 +17,11 @@ require (
 	github.com/Velocidex/ordereddict v0.0.0-20200723153557-9460a6764ab8
 	github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49
 	github.com/Velocidex/yaml/v2 v2.2.5
+	github.com/Velocidex/zip v0.0.0-20201230073523-29e172503d03
 	github.com/ZachtimusPrime/Go-Splunk-HTTP v0.0.0-20200420213219-094ff9e8d788
 	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
 	github.com/alecthomas/chroma v0.7.2
 	github.com/alecthomas/participle v0.7.1
-	github.com/alecthomas/repr v0.0.0-20201120212035-bb82daffcca2 // indirect
 	github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0
 	github.com/aws/aws-sdk-go v1.26.7
 	github.com/bi-zone/etw v0.0.0-20200916105032-b215904fae4f
@@ -48,6 +48,7 @@ require (
 	github.com/gorilla/schema v1.1.0
 	github.com/grpc-ecosystem/grpc-gateway v1.14.7
 	github.com/hanwen/go-fuse v1.0.1-0.20190726130028-2f298055551b
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hillu/go-ntdll v0.0.0-20190226223014-dd4204aa705e
 	github.com/hillu/go-yara v1.2.2 // indirect
 	github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c // indirect
@@ -97,7 +98,6 @@ require (
 	golang.org/x/net v0.0.0-20201031054903-ff519b6c9102
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20201101102859-da207088b7d1
-	golang.org/x/text v0.3.4 // indirect
 	golang.org/x/tools v0.0.0-20200828161849-5deb26317202 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.30.0
@@ -120,7 +120,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20201229033732-028282d980f6
+	www.velocidex.com/golang/vfilter v0.0.0-20201230064643-0a996eda7d99
 	www.velocidex.com/golang/vtypes v0.0.0-20201229070616-0b57670f1d00
 )
 
@@ -133,6 +133,7 @@ require (
 // replace github.com/Velocidex/json => /home/mic/projects/json
 // replace github.com/russross/blackfriday/v2 => /home/mic/projects/blackfriday
 // replace www.velocidex.com/golang/vtypes => /home/mic/projects/vtypes
+// replace github.com/Velocidex/zip => /home/mic/projects/zip
 
 replace github.com/russross/blackfriday/v2 => github.com/Velocidex/blackfriday/v2 v2.0.2-0.20200811050547-4f26a09e2b3b
 

go.sum

@@ -79,6 +79,8 @@ github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49 h1:TJVN1zYl5sKJ
 github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49/go.mod h1:kfPUQ2gP0xtIydiR52dirNYt4OvCr+iZuepL4XaIk58=
 github.com/Velocidex/yaml/v2 v2.2.5 h1:8XZwR8tmm5UWAXotI3fL17s9fjKEMqZ293fgqUiMhBw=
 github.com/Velocidex/yaml/v2 v2.2.5/go.mod h1:VBjrsTMc/b1h0ankOOnJPYoCbJNwhpGYpnDgICEs2mk=
+github.com/Velocidex/zip v0.0.0-20201230073523-29e172503d03 h1:ifFF7cfMvLoF3e9vxyPsyRBLUMaJ1PLhxK+Ef+gicik=
+github.com/Velocidex/zip v0.0.0-20201230073523-29e172503d03/go.mod h1:1p8CU2cp64BG4334sKzhuyH/vm3k1OXEdeBCwYTssAs=
 github.com/ZachtimusPrime/Go-Splunk-HTTP v0.0.0-20200420213219-094ff9e8d788 h1:QYyFO87/l5IdxKVBCMXW5YlwqGR3nHCN3RfAdjIYnVk=
 github.com/ZachtimusPrime/Go-Splunk-HTTP v0.0.0-20200420213219-094ff9e8d788/go.mod h1:RsO+aYhoc9OyzUJwKHWWXXsjLN/85QaaEHx/PWa2+/I=
 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
@@ -260,6 +262,12 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.7 h1:Nk5kuHrnWUTf/0GL1a/vchH/om9Ap2
 github.com/grpc-ecosystem/grpc-gateway v1.14.7/go.mod h1:oYZKL012gGh6LMyg/xA7Q2yq6j8bu0wa+9w14EEthWU=
 github.com/hanwen/go-fuse v1.0.1-0.20190726130028-2f298055551b h1:ek6pftAVF7gXlt1SgPMIotytc+sSx5TpXgKa/kBcIy4=
 github.com/hanwen/go-fuse v1.0.1-0.20190726130028-2f298055551b/go.mod h1:PHVWttMW0DYH6ESFXdZ8S+STmGjwEuGX6gsCPi605mg=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
@@ -794,13 +802,8 @@ www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 h1:3oYZ7hPN
 www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196/go.mod h1:i7M+d4Vxir8nmDACh+c6CsUU1r1Wcj00aRgNp8mXcPQ=
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiAbjPIsTZcEPbqqqABS/ZV5SB7j33eczNsqD60=
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns=
-www.velocidex.com/golang/vfilter v0.0.0-20201215135433-62fce9f56d6d h1:KrLCmEWeT+NrU70Hd7/XRxvuz8r0PuM1uZGo9Q6IVvg=
-www.velocidex.com/golang/vfilter v0.0.0-20201215135433-62fce9f56d6d/go.mod h1:XlUeViBwZxeefhxbkxW2oGUVcB/oQfxtBgnxL9jLryg=
-www.velocidex.com/golang/vfilter v0.0.0-20201229033732-028282d980f6 h1:Panir8Bu3BVI1R/MNjS+bPl/++Gg4vl0f3ZOMV9i+/A=
 www.velocidex.com/golang/vfilter v0.0.0-20201229033732-028282d980f6/go.mod h1:XlUeViBwZxeefhxbkxW2oGUVcB/oQfxtBgnxL9jLryg=
-www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b h1:z5v5o1dhtzaxvlWm6qSTYZ4OTr56Ol2JpM1Y5Wu9zQE=
-www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b/go.mod h1:tXxIx8UJuI81Hoxcv0DTq2a1Pi1H6l1uCf4dhqUSUkw=
-www.velocidex.com/golang/vtypes v0.0.0-20201229051546-6d7b229e4372 h1:XLrFhprsy06yGNZrhYeF2OhQOZ8h0/4jDZ+xeMg8rfA=
-www.velocidex.com/golang/vtypes v0.0.0-20201229051546-6d7b229e4372/go.mod h1:l1FULlPMESSjMrkimRrX0EzriJ+YIKgQAN13YicEI6E=
+www.velocidex.com/golang/vfilter v0.0.0-20201230064643-0a996eda7d99 h1:UhxTxA3KSyHbmGxMALq/g+U1psH/vUFaA7hNsiWqYF8=
+www.velocidex.com/golang/vfilter v0.0.0-20201230064643-0a996eda7d99/go.mod h1:XlUeViBwZxeefhxbkxW2oGUVcB/oQfxtBgnxL9jLryg=
 www.velocidex.com/golang/vtypes v0.0.0-20201229070616-0b57670f1d00 h1:9E9t+4d4uMWIdVqMSM13CxOK2828r+xQKqg2GdETfis=
 www.velocidex.com/golang/vtypes v0.0.0-20201229070616-0b57670f1d00/go.mod h1:l1FULlPMESSjMrkimRrX0EzriJ+YIKgQAN13YicEI6E=

reporting/archive.go

@@ -95,6 +95,10 @@ func (self *Archive) ListArtifacts() []string {
 	return result
 }
 
+func (self *Archive) Close() {
+	self.fd.Close()
+}
+
 func NewArchiveReader(path string) (*Archive, error) {
 	fd, err := os.Open(path)
 	if err != nil {