From 9f3b18f0741cb275954da38c503d6e2b453247b8 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 30 Jul 2020 13:58:46 +1000 Subject: [PATCH] Fixed bug in uploading sparse files. (#519) * Fixed bug in uploading sparse files. This affected files fetched with NTFS which spanned multiple runs. * Added test. --- .../client/shell-viewer-directive.js | 5 +- json/wrappers.go | 8 ++ responder/responder.go | 4 +- responder/testutils.go | 20 +++ uploads/api.go | 5 +- uploads/client_uploader.go | 18 ++- uploads/client_uploader_test.go | 131 ++++++++++++++++++ uploads/fixtures/ClientUploaderSparse.golden | 40 ++++++ .../ClientUploaderSparseMultiBuffer.golden | 92 ++++++++++++ vql/windows/process/common.go | 5 + vql/windows/process/dump.go | 4 +- vql/windows/process/handles.go | 6 +- vql/windows/process/vad.go | 8 +- 13 files changed, 319 insertions(+), 27 deletions(-) create mode 100644 responder/testutils.go create mode 100644 uploads/client_uploader_test.go create mode 100644 uploads/fixtures/ClientUploaderSparse.golden create mode 100644 uploads/fixtures/ClientUploaderSparseMultiBuffer.golden create mode 100644 vql/windows/process/common.go diff --git a/gui/static/angular-components/client/shell-viewer-directive.js b/gui/static/angular-components/client/shell-viewer-directive.js index 2dc4fad4fb2..d5e7554030e 100644 --- a/gui/static/angular-components/client/shell-viewer-directive.js +++ b/gui/static/angular-components/client/shell-viewer-directive.js @@ -44,7 +44,7 @@ ShellViewerController.prototype.launchCommand = function() { artifact = "Windows.System.PowerShell"; } else if(this.type == "Cmd") { artifact = "Windows.System.CmdShell"; - } else if(this.type == "Base") { + } else if(this.type == "Bash") { artifact = "Linux.Sys.BashShell"; } else { return; @@ -91,7 +91,8 @@ ShellViewerController.prototype.fetchLastShellCollections = function() { for (var j=0; j 0 { + to_read_buf := to_read + // Ensure there is a fresh allocation for every // iteration to prevent overwriting in-flight buffers. - if to_read > BUFF_SIZE { - to_read = BUFF_SIZE + if to_read_buf > BUFF_SIZE { + to_read_buf = BUFF_SIZE } - buffer := make([]byte, BUFF_SIZE) + buffer := make([]byte, to_read_buf) read_bytes, err := range_reader.Read(buffer) // Hard read error - give up. if err != nil && err != io.EOF { return nil, err } - // End of range - go to the next range if read_bytes == 0 { - break + continue } data := buffer[:read_bytes] - sha_sum.Write(data) md5_sum.Write(data) @@ -246,7 +246,6 @@ func (self *VelociraptorUploader) maybeUploadSparse( to_read -= int64(read_bytes) write_offset += int64(read_bytes) read_offset += int64(read_bytes) - } } @@ -256,7 +255,6 @@ func (self *VelociraptorUploader) maybeUploadSparse( if err != nil { return nil, err } - self.Responder.AddResponse(&crypto_proto.GrrMessage{ RequestId: constants.TransferWellKnownFlowId, FileBuffer: &actions_proto.FileBuffer{ diff --git a/uploads/client_uploader_test.go b/uploads/client_uploader_test.go new file mode 100644 index 00000000000..b89d2e928ae --- /dev/null +++ b/uploads/client_uploader_test.go @@ -0,0 +1,131 @@ +package uploads + +import ( + "bytes" + "context" + "testing" + + "github.com/alecthomas/assert" + "github.com/sebdah/goldie" + crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" + "www.velocidex.com/golang/velociraptor/json" + "www.velocidex.com/golang/velociraptor/responder" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" +) + +type TestRangeReader struct { + *bytes.Reader + ranges []Range +} + +func (self *TestRangeReader) Ranges() []Range { + return self.ranges +} + +// Combine the output of all fragments into a strings +func CombineOutput(name string, responses []*crypto_proto.GrrMessage) string { + result := []byte{} + + for _, item := range responses { + if item.FileBuffer.Pathspec.Path == name { + result = append(result, item.FileBuffer.Data...) + } + } + + return string(result) +} + +func TestClientUploaderSparse(t *testing.T) { + resp := responder.TestResponder() + uploader := &VelociraptorUploader{ + Responder: resp, + } + + BUFF_SIZE = 10000 + + reader := &TestRangeReader{ + Reader: bytes.NewReader([]byte( + "Hello world hello world")), + ranges: []Range{ + {Offset: 0, Length: 6, IsSparse: false}, + {Offset: 6, Length: 6, IsSparse: true}, + {Offset: 12, Length: 6, IsSparse: false}, + }, + } + range_reader, ok := interface{}(reader).(RangeReader) + assert.Equal(t, ok, true) + ctx := context.Background() + scope := vql_subsystem.MakeScope() + uploader.maybeUploadSparse(ctx, scope, + "foo", "ntfs", "", 1000, range_reader) + responses := responder.GetTestResponses(resp) + + // Expected size is the combined sum of all ranges with data + // in them + assert.Equal(t, responses[0].FileBuffer.Size, uint64(12)) + + assert.Equal(t, CombineOutput("foo", responses), + "Hello hello ") + goldie.Assert(t, "ClientUploaderSparse", + json.MustMarshalIndent(responses)) + assert.NotEqual(t, CombineOutput("foo.idx", responses), "") +} + +func TestClientUploaderSparseMultiBuffer(t *testing.T) { + resp := responder.TestResponder() + uploader := &VelociraptorUploader{ + Responder: resp, + } + + // 2 bytes per message + BUFF_SIZE = 2 + reader := &TestRangeReader{ + Reader: bytes.NewReader([]byte( + "Hello world hello world")), + ranges: []Range{ + {Offset: 0, Length: 6, IsSparse: false}, + {Offset: 6, Length: 6, IsSparse: true}, + {Offset: 12, Length: 6, IsSparse: false}, + }, + } + range_reader, ok := interface{}(reader).(RangeReader) + assert.Equal(t, ok, true) + ctx := context.Background() + scope := vql_subsystem.MakeScope() + uploader.maybeUploadSparse(ctx, scope, + "foo", "ntfs", "", 1000, range_reader) + responses := responder.GetTestResponses(resp) + assert.Equal(t, CombineOutput("foo", responses), "Hello hello ") + goldie.Assert(t, "ClientUploaderSparseMultiBuffer", + json.MustMarshalIndent(responses)) + assert.NotEqual(t, CombineOutput("foo.idx", responses), "") +} + +func TestClientUploaderNoIndexIfNotSparse(t *testing.T) { + resp := responder.TestResponder() + uploader := &VelociraptorUploader{ + Responder: resp, + } + + // 2 bytes per message + BUFF_SIZE = 2 + reader := &TestRangeReader{ + Reader: bytes.NewReader([]byte( + "Hello world hello world")), + ranges: []Range{ + {Offset: 0, Length: 6, IsSparse: false}, + {Offset: 12, Length: 6, IsSparse: false}, + }, + } + range_reader, ok := interface{}(reader).(RangeReader) + assert.Equal(t, ok, true) + ctx := context.Background() + scope := vql_subsystem.MakeScope() + uploader.maybeUploadSparse(ctx, scope, + "foo", "ntfs", "", 1000, range_reader) + responses := responder.GetTestResponses(resp) + assert.Equal(t, CombineOutput("foo", responses), "Hello hello ") + + // No idx written when there are no sparse ranges. + assert.Equal(t, CombineOutput("foo.idx", responses), "") +} diff --git a/uploads/fixtures/ClientUploaderSparse.golden b/uploads/fixtures/ClientUploaderSparse.golden new file mode 100644 index 00000000000..4ab78d4d35b --- /dev/null +++ b/uploads/fixtures/ClientUploaderSparse.golden @@ -0,0 +1,40 @@ +[ + { + "request_id": 5, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "size": 12, + "data": "SGVsbG8g" + } + }, + { + "request_id": 5, + "response_id": 1, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 6, + "size": 12, + "data": "aGVsbG8g", + "eof": true + } + }, + { + "request_id": 5, + "response_id": 2, + "FileBuffer": { + "pathspec": { + "path": "foo.idx", + "accessor": "ntfs" + }, + "size": 196, + "data": "eyJmaWxlX29mZnNldCI6MCwib3JpZ2luYWxfb2Zmc2V0IjowLCJmaWxlX2xlbmd0aCI6NiwibGVuZ3RoIjo2fQp7ImZpbGVfb2Zmc2V0Ijo2LCJvcmlnaW5hbF9vZmZzZXQiOjYsImZpbGVfbGVuZ3RoIjowLCJsZW5ndGgiOjZ9CnsiZmlsZV9vZmZzZXQiOjYsIm9yaWdpbmFsX29mZnNldCI6MTIsImZpbGVfbGVuZ3RoIjo2LCJsZW5ndGgiOjZ9Cg==", + "eof": true + } + } +] \ No newline at end of file diff --git a/uploads/fixtures/ClientUploaderSparseMultiBuffer.golden b/uploads/fixtures/ClientUploaderSparseMultiBuffer.golden new file mode 100644 index 00000000000..72c8457490f --- /dev/null +++ b/uploads/fixtures/ClientUploaderSparseMultiBuffer.golden @@ -0,0 +1,92 @@ +[ + { + "request_id": 5, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "size": 12, + "data": "SGU=" + } + }, + { + "request_id": 5, + "response_id": 1, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 2, + "size": 12, + "data": "bGw=" + } + }, + { + "request_id": 5, + "response_id": 2, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 4, + "size": 12, + "data": "byA=" + } + }, + { + "request_id": 5, + "response_id": 3, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 6, + "size": 12, + "data": "aGU=" + } + }, + { + "request_id": 5, + "response_id": 4, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 8, + "size": 12, + "data": "bGw=" + } + }, + { + "request_id": 5, + "response_id": 5, + "FileBuffer": { + "pathspec": { + "path": "foo", + "accessor": "ntfs" + }, + "offset": 10, + "size": 12, + "data": "byA=", + "eof": true + } + }, + { + "request_id": 5, + "response_id": 6, + "FileBuffer": { + "pathspec": { + "path": "foo.idx", + "accessor": "ntfs" + }, + "size": 196, + "data": "eyJmaWxlX29mZnNldCI6MCwib3JpZ2luYWxfb2Zmc2V0IjowLCJmaWxlX2xlbmd0aCI6NiwibGVuZ3RoIjo2fQp7ImZpbGVfb2Zmc2V0Ijo2LCJvcmlnaW5hbF9vZmZzZXQiOjYsImZpbGVfbGVuZ3RoIjowLCJsZW5ndGgiOjZ9CnsiZmlsZV9vZmZzZXQiOjYsIm9yaWdpbmFsX29mZnNldCI6MTIsImZpbGVfbGVuZ3RoIjo2LCJsZW5ndGgiOjZ9Cg==", + "eof": true + } + } +] \ No newline at end of file diff --git a/vql/windows/process/common.go b/vql/windows/process/common.go new file mode 100644 index 00000000000..a47edc4431d --- /dev/null +++ b/vql/windows/process/common.go @@ -0,0 +1,5 @@ +package process + +type PidArgs struct { + Pid int64 `vfilter:"required,field=pid,doc=The PID to dump out."` +} diff --git a/vql/windows/process/dump.go b/vql/windows/process/dump.go index 9cc3f62a3d5..9da5332cc2a 100644 --- a/vql/windows/process/dump.go +++ b/vql/windows/process/dump.go @@ -47,7 +47,7 @@ func (self ProcDumpPlugin) Call( scope *vfilter.Scope, args *ordereddict.Dict) <-chan vfilter.Row { output_chan := make(chan vfilter.Row) - arg := &ProcDumpArgs{} + arg := &PidArgs{} go func() { defer close(output_chan) @@ -107,7 +107,7 @@ func (self ProcDumpPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) return &vfilter.PluginInfo{ Name: "proc_dump", Doc: "Dumps process memory.", - ArgType: type_map.AddType(scope, &ProcDumpArgs{}), + ArgType: type_map.AddType(scope, &PidArgs{}), } } diff --git a/vql/windows/process/handles.go b/vql/windows/process/handles.go index 99f560c758e..202cce1f20f 100644 --- a/vql/windows/process/handles.go +++ b/vql/windows/process/handles.go @@ -22,10 +22,6 @@ import ( "www.velocidex.com/golang/vfilter" ) -type ProcDumpArgs struct { - Pid int64 `vfilter:"required,field=pid,doc=The PID to dump out."` -} - type ThreadHandleInfo struct { ThreadId uint64 ProcessId uint64 @@ -110,7 +106,7 @@ func (self HandlesPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) return &vfilter.PluginInfo{ Name: "handles", Doc: "Enumerate process handles.", - ArgType: type_map.AddType(scope, &ProcDumpArgs{}), + ArgType: type_map.AddType(scope, &PidArgs{}), } } diff --git a/vql/windows/process/vad.go b/vql/windows/process/vad.go index 3a726ba40ed..03e0fa34284 100644 --- a/vql/windows/process/vad.go +++ b/vql/windows/process/vad.go @@ -40,7 +40,7 @@ func (self ModulesPlugin) Call( scope *vfilter.Scope, args *ordereddict.Dict) <-chan vfilter.Row { output_chan := make(chan vfilter.Row) - arg := &ProcDumpArgs{} + arg := &PidArgs{} go func() { defer close(output_chan) @@ -81,7 +81,7 @@ func (self ModulesPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) return &vfilter.PluginInfo{ Name: "modules", Doc: "Enumerate Loaded DLLs.", - ArgType: type_map.AddType(scope, &ProcDumpArgs{}), + ArgType: type_map.AddType(scope, &PidArgs{}), } } @@ -92,7 +92,7 @@ func (self VADPlugin) Call( scope *vfilter.Scope, args *ordereddict.Dict) <-chan vfilter.Row { output_chan := make(chan vfilter.Row) - arg := &ProcDumpArgs{} + arg := &PidArgs{} go func() { defer close(output_chan) @@ -125,7 +125,7 @@ func (self VADPlugin) Info(scope *vfilter.Scope, type_map *vfilter.TypeMap) *vfi return &vfilter.PluginInfo{ Name: "vad", Doc: "Enumerate process memory regions.", - ArgType: type_map.AddType(scope, &ProcDumpArgs{}), + ArgType: type_map.AddType(scope, &PidArgs{}), } }