Client is now conservative in memory use.

Makefile

@@ -21,31 +21,33 @@ ifneq ("$(MINGW_EXISTS)", "")
 	CGO_ENABLED = 1
 endif
 
-ASSETS := artifacts/b0x.yaml config/b0x.yaml
+required_assets:
+	fileb0x artifacts/b0x.yaml
+	fileb0x config/b0x.yaml
+
+gui_assets:
+	fileb0x gui/b0x.yaml
 
 # Just regular binaries for local testing. The GUI will be serving
 # files from the filesystem.
-build:
-	fileb0x $(ASSETS)
+build: required_assets
 	GOOS=linux GOARCH=amd64 \
             go build \
             -tags devel \
             -ldflags "$(LDFLAGS)" \
 	    -o output/velociraptor ./bin/
 
-windows:
+windows: required_assets
 ifeq ("$(MINGW_EXISTS)", "")
 	@echo Disabling cgo modules. To enable install $(MINGW_CC)
 endif
-	fileb0x $(ASSETS)
 	GOOS=windows GOARCH=amd64 \
         CC=$(CC) CGO_ENABLED=$(CGO_ENABLED) \
             go build \
             -ldflags "$(LDFLAGS)" \
 	    -o output/velociraptor.exe ./bin/
 
-darwin:
-	fileb0x gui/b0x.yaml $(ASSETS)
+darwin: required_assets gui_assets
 	GOOS=darwin GOARCH=amd64 \
             go build \
             -tags release \
@@ -54,8 +56,7 @@ darwin:
 
 # Build release binaries. The GUI will embed assets and ship with
 # everything in it.
-release:
-	fileb0x gui/b0x.yaml $(ASSETS)
+release: required_assets gui_assets
 	GOOS=linux GOARCH=amd64 \
             go build \
             -ldflags "$(LDFLAGS)" \
@@ -69,7 +70,8 @@ install: release
 
 clean:
 	rm -f gui/assets/ab0x.go \
-              $(ASSETS)
+        artifacts/assets/ab0x.go \
+        config/ab0x.go
 
 
 generate:

README.md

@@ -66,22 +66,22 @@ To create a windows executable:
 1. Embed the client config in the binary. This makes the binary self
    contained for your particular installation. It is therefore very
    easy to install:
-   ```bash
-   $ velociraptor config repack --exe velociraptor_windows.exe \
+    ```bash
+    $ velociraptor config repack --exe velociraptor_windows.exe \
       client.config.yaml my_velociraptor.exe
-   ```
+    ```
 
    Where velociraptor_windows.exe is the Windows binary release for
    Velociraptor.
 
 2. On a windows system you can now install the service:
-   ```bash
-   $ my_velociraptor.exe service install
-   INFO:2018/08/28 00:18:19 Stopped service Velociraptor
-   INFO:2018/08/28 00:18:20 Copied binary to C:\Program Files\Velociraptor\Velociraptor.exe
-   INFO:2018/08/28 00:18:20 Installed service Velociraptor
-   INFO:2018/08/28 00:18:21 Started service Velociraptor
-   ```
+    ```bash
+    $ my_velociraptor.exe service install
+    INFO:2018/08/28 00:18:19 Stopped service Velociraptor
+    INFO:2018/08/28 00:18:20 Copied binary to C:\Program Files\Velociraptor\Velociraptor.exe
+    INFO:2018/08/28 00:18:20 Installed service Velociraptor
+    INFO:2018/08/28 00:18:21 Started service Velociraptor
+    ```
 
 This will copy the binary into the install_dir specified in the config
 file, create and start the service.
@@ -93,22 +93,22 @@ might find Velociraptor's artifacts especially useful for quickly
 capturing important information about a running system. You can
 collect artifacts by using the "artifacts collect" command:
 
-   ```bash
-   $ velociraptor artifacts list
-   INFO:2018/08/20 22:28:56 Loaded 18 built in artifacts
-   INFO:2018/08/20 22:28:56 Loaded 18 artifacts from artifacts/definitions/
-   Linux.Applications.Chrome.Extensions
-   Linux.Applications.Chrome.Extensions.Upload
-   Linux.Applications.Docker.Info
-   Linux.Applications.Docker.Version
-   Linux.Debian.AptSources
-
-   $ velociraptor artifacts list -v Linux.Debian.AptSources
-   .... displays the artifacts
-
-   $ velociraptor artifacts collect Linux.Debian.AptSources
-   ... Collects all the named artifacts
-   ```
+    ```bash
+    $ velociraptor artifacts list
+    INFO:2018/08/20 22:28:56 Loaded 18 built in artifacts
+    INFO:2018/08/20 22:28:56 Loaded 18 artifacts from artifacts/definitions/
+    Linux.Applications.Chrome.Extensions
+    Linux.Applications.Chrome.Extensions.Upload
+    Linux.Applications.Docker.Info
+    Linux.Applications.Docker.Version
+    Linux.Debian.AptSources
+
+    $ velociraptor artifacts list -v Linux.Debian.AptSources
+    .... displays the artifacts
+
+    $ velociraptor artifacts collect Linux.Debian.AptSources
+    ... Collects all the named artifacts
+    ```
 
 Explore more of Velociraptor's options using the -h flag.
 

bin/client.go

@@ -17,9 +17,8 @@ var (
 func RunClient(config_path *string) {
 	ctx := context.Background()
 	config_obj, err := config.LoadClientConfig(*config_path)
-	if err != nil {
-		kingpin.FatalIfError(err, "Unable to load config file")
-	}
+	kingpin.FatalIfError(err, "Unable to load config file")
+
 	ctx.Config = config_obj
 
 	// Make sure the config is ok.

bin/debug.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"gopkg.in/alecthomas/kingpin.v2"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	config "www.velocidex.com/golang/velociraptor/config"
+	logging "www.velocidex.com/golang/velociraptor/logging"
+)
+
+var (
+	debug_flag = app.Flag("debug", "Enables debug and profile server.").Bool()
+)
+
+func doDebug() {
+	config_obj, err := config.LoadClientConfig(*config_path)
+	kingpin.FatalIfError(err, "Unable to load config file")
+
+	logger := logging.NewLogger(config_obj)
+	logger.Info("Starting debug server on port 6060")
+
+	go func() {
+		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+	}()
+}
+
+func init() {
+	// Add this first to ensure it always runs.
+	command_handlers = append([]CommandHandler{
+		func(command string) bool {
+			if *debug_flag {
+				doDebug()
+			}
+			return false
+		},
+	}, command_handlers...)
+}

bin/main.go

@@ -10,6 +10,8 @@ import (
 	_ "www.velocidex.com/golang/velociraptor/vql_plugins"
 )
 
+type CommandHandler func(command string) bool
+
 var (
 	app = kingpin.New("velociraptor",
 		"An advanced incident response and monitoring agent.")
@@ -19,7 +21,7 @@ var (
 	artifact_definitions_dir = app.Flag(
 		"definitions", "A directory containing artifact definitions").String()
 
-	command_handlers []func(command string) bool
+	command_handlers []CommandHandler
 )
 
 func validateServerConfig(configuration *config.Config) error {

http_comms/comms.go

@@ -2,13 +2,13 @@ package http_comms
 
 import (
 	"bytes"
+	"github.com/golang/protobuf/proto"
 	errors "github.com/pkg/errors"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"runtime/debug"
 	"time"
-
-	"github.com/golang/protobuf/proto"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/context"
@@ -173,6 +173,11 @@ func (self *HTTPCommunicator) sendToURL(
 	url string,
 	message_list *crypto_proto.MessageList) error {
 
+	// We need to make sure our memory footprint is as small as
+	// possible. The Velociraptor client prioritizes low memory
+	// footprint over performance.
+	defer debug.FreeOSMemory()
+
 	if self.server_name == "" {
 		// Try to fetch the server pem.
 		resp, err := self.client.Get(url + "server.pem")

vql/filesystem/filesystem.go

@@ -163,7 +163,7 @@ func (self ReadFilePlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 }
 
 type StatArgs struct {
-	Filename string `vfilter:"required,field=filename"`
+	Filename []string `vfilter:"required,field=filename"`
 }
 
 func init() {
@@ -203,9 +203,11 @@ func init() {
 				}
 
 				accessor := &glob.OSFileSystemAccessor{}
-				f, err := accessor.Lstat(arg.Filename)
-				if err == nil {
-					result = append(result, f)
+				for _, filename := range arg.Filename {
+					f, err := accessor.Lstat(filename)
+					if err == nil {
+						result = append(result, f)
+					}
 				}
 				return result
 			},

vql/networking/http_client.go

@@ -151,7 +151,7 @@ func (self _HttpPlugin) Name() string {
 
 func (self _HttpPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
 	return &vfilter.PluginInfo{
-		Name:    "http_client",
+		Name:    self.Name(),
 		Doc:     "Make a http request.",
 		RowType: type_map.AddType(&_HttpPluginResponse{}),
 		ArgType: type_map.AddType(&_HttpPluginRequest{}),

vql/networking/upload.go

@@ -82,6 +82,69 @@ func (self UploadFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo
 	}
 }
 
+type UploadPluginArgs struct {
+	Files    []string `vfilter:"required,field=files"`
+	Accessor string   `vfilter:"optional,field=accessor"`
+}
+
+type UploadPlugin struct{}
+
+func (self *UploadPlugin) Call(
+	ctx context.Context,
+	scope *vfilter.Scope,
+	args *vfilter.Dict) <-chan vfilter.Row {
+	output_chan := make(chan vfilter.Row)
+	arg := &UploadPluginArgs{}
+	err := vfilter.ExtractArgs(scope, args, arg)
+	if err != nil {
+		scope.Log("upload: %s", err.Error())
+		close(output_chan)
+		return output_chan
+	}
+
+	uploader_obj, _ := scope.Resolve("$uploader")
+	uploader, ok := uploader_obj.(Uploader)
+	if !ok {
+		scope.Log("upload: Uploader not configured.")
+		close(output_chan)
+		return output_chan
+	}
+
+	go func() {
+		defer close(output_chan)
+
+		accessor := glob.GetAccessor(arg.Accessor)
+		for _, filename := range arg.Files {
+			file, err := accessor.Open(filename)
+			if err != nil {
+				scope.Log("upload: Unable to open %s: %s",
+					filename, err.Error())
+				continue
+			}
+
+			upload_response, err := uploader.Upload(
+				scope, filename, file)
+			if err != nil {
+				scope.Log("upload: Failed to upload %s: %s",
+					filename, err.Error())
+				continue
+			}
+			output_chan <- upload_response
+		}
+	}()
+	return output_chan
+}
+
+func (self UploadPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo {
+	return &vfilter.PluginInfo{
+		Name:    "upload",
+		Doc:     "Upload files to the server.",
+		RowType: type_map.AddType(&UploadResponse{}),
+		ArgType: type_map.AddType(&UploadPluginArgs{}),
+	}
+}
+
 func init() {
 	vql_subsystem.RegisterFunction(&UploadFunction{})
+	vql_subsystem.RegisterPlugin(&UploadPlugin{})
 }