Implemented file upload through the VQL upload() plugin.

File uploads are sent through the responsible flow which ensures they
are properly accounted for and upload quota is enforced.

actions/vql.go

@@ -1,6 +1,8 @@
 package actions
 
 import (
+	"log"
+	"os"
 	"time"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	"www.velocidex.com/golang/velociraptor/context"
@@ -32,13 +34,16 @@ func (self *VQLClientAction) Run(
 	// objects in there. VQL plugins may then use the environment
 	// to communicate with the server.
 	env := vfilter.NewDict().
-		Set("responder", responder).
+		Set("$responder", responder).
+		Set("$uploader", &vql_subsystem.VelociraptorUploader{responder}).
 		Set("config", ctx.Config)
 
 	for _, env_spec := range arg.Env {
 		env.Set(env_spec.Key, env_spec.Value)
 	}
+
 	scope := vql_subsystem.MakeScope().AppendVars(env)
+	scope.Logger = log.New(os.Stderr, "vql: ", log.Lshortfile)
 
 	// All the queries will use the same scope. This allows one
 	// query to define functions for the next query in order.

bin/vraptor.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/olekukonko/tablewriter"
 	"gopkg.in/alecthomas/kingpin.v2"
+	"log"
 	"os"
 	"strings"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -15,8 +16,12 @@ import (
 
 var (
 	query   = kingpin.Command("query", "Run a VQL query")
-	queries = query.Arg("query", "The VQL Query to run.").Required().Strings()
-	format  = query.Flag("format", "Output format to use.").Default("json").Enum("text", "json")
+	queries = query.Arg("query", "The VQL Query to run.").
+		Required().Strings()
+	format = query.Flag("format", "Output format to use.").
+		Default("json").Enum("text", "json")
+	dump_dir = query.Flag("dump_dir", "Directory to dump output files.").
+			Default(".").String()
 
 	explain        = kingpin.Command("explain", "Explain the output from a plugin")
 	explain_plugin = explain.Arg("plugin", "Plugin to explain").Required().String()
@@ -26,8 +31,7 @@ var (
 	show_config = client.Flag("show_config", "Display the client's configuration").Bool()
 )
 
-func outputJSON(vql *vfilter.VQL) {
-	scope := vql_subsystem.MakeScope()
+func outputJSON(scope *vfilter.Scope, vql *vfilter.VQL) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -51,8 +55,7 @@ func hard_wrap(text string, colBreak int) string {
 	return wrapped
 }
 
-func evalQuery(vql *vfilter.VQL) {
-	scope := vql_subsystem.MakeScope()
+func evalQuery(scope *vfilter.Scope, vql *vfilter.VQL) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -125,6 +128,11 @@ func main() {
 	case "explain":
 		doExplain(*explain_plugin)
 	case "query":
+		env := vfilter.NewDict().
+			Set("$uploader", &vql_subsystem.FileBasedUploader{*dump_dir})
+		scope := vql_subsystem.MakeScope().AppendVars(env)
+
+		scope.Logger = log.New(os.Stderr, "vraptor: ", log.Lshortfile)
 		for _, query := range *queries {
 			vql, err := vfilter.Parse(query)
 			if err != nil {
@@ -133,9 +141,9 @@ func main() {
 
 			switch *format {
 			case "text":
-				evalQuery(vql)
+				evalQuery(scope, vql)
 			case "json":
-				outputJSON(vql)
+				outputJSON(scope, vql)
 			}
 		}
 	case "repack":

config/config.go

@@ -96,6 +96,9 @@ type Config struct {
 
 	// The Admin UI
 	AdminUI_document_root *string `yaml:"AdminUI.document_root,omitempty"`
+
+	// File Store
+	FileStore_directory *string `yaml:"FileStore.directory,omitempty"`
 }
 
 func GetDefaultConfig() *Config {

file_store/file_store.go

@@ -0,0 +1,51 @@
+package file_store
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path"
+	config "www.velocidex.com/golang/velociraptor/config"
+	logging "www.velocidex.com/golang/velociraptor/logging"
+)
+
+type WriteSeekCloser interface {
+	io.WriteSeeker
+	io.Closer
+}
+
+type FileStore interface {
+	WriteFile(filename string) (WriteSeekCloser, error)
+}
+
+type DirectoryFileStore struct {
+	config_obj *config.Config
+}
+
+func (self *DirectoryFileStore) WriteFile(filename string) (WriteSeekCloser, error) {
+	if self.config_obj.FileStore_directory == nil {
+		return nil, errors.New("No configured file store directory.")
+	}
+
+	file_path := path.Join(*self.config_obj.FileStore_directory, filename)
+	err := os.MkdirAll(path.Dir(file_path), 0700)
+	if err != nil {
+		logging.NewLogger(self.config_obj).Error(
+			"Can not create dir: %s", err.Error())
+		return nil, err
+	}
+
+	file, err := os.OpenFile(file_path, os.O_RDWR|os.O_CREATE, 0700)
+	if err != nil {
+		logging.NewLogger(self.config_obj).Error(
+			"Unable to open file %s: %s", file_path, err.Error())
+		return nil, err
+	}
+
+	return file, nil
+}
+
+// Currently we only support a DirectoryFileStore.
+func GetFileStore(config_obj *config.Config) FileStore {
+	return &DirectoryFileStore{config_obj}
+}

flows/flows.go

@@ -59,7 +59,13 @@ func (self *FlowRunner) ProcessMessages(messages []*crypto_proto.GrrMessage) {
 		err = cached_flow.impl.ProcessMessage(
 			self.config, cached_flow, message)
 		if err != nil {
+			cached_flow.FlowContext.State = flows_proto.FlowContext_ERROR
+			cached_flow.FlowContext.Status = err.Error()
+			cached_flow.FlowContext.Backtrace = ""
+			cached_flow.dirty = true
+
 			utils.Debug(err)
+			return
 		}
 	}
 }

flows/generic.go

@@ -4,12 +4,15 @@ import (
 	"errors"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
+	"path"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	config "www.velocidex.com/golang/velociraptor/config"
 	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
 	datastore "www.velocidex.com/golang/velociraptor/datastore"
+	"www.velocidex.com/golang/velociraptor/file_store"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
-	utils "www.velocidex.com/golang/velociraptor/testing"
+	"www.velocidex.com/golang/velociraptor/responder"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
 
 const (
@@ -63,13 +66,36 @@ func (self *VQLCollector) ProcessMessage(
 			return nil
 		}
 
-		utils.Debug(message)
 		err = StoreResultInFlow(config_obj, flow_obj, message)
 		if err != nil {
 			return err
 		}
-	}
 
+		// Receive any file upload the client sent.
+	case vql_subsystem.TransferWellKnownFlowId:
+		payload := responder.ExtractGrrMessagePayload(message)
+		if payload != nil {
+			file_buffer, ok := payload.(*actions_proto.FileBuffer)
+			if ok {
+				file_store_factory := file_store.GetFileStore(
+					config_obj)
+				file_path := path.Join(
+					flow_obj.RunnerArgs.ClientId,
+					path.Base(message.SessionId),
+					file_buffer.Pathspec.Path)
+				fd, err := file_store_factory.WriteFile(
+					file_path)
+				if err != nil {
+					return err
+				}
+
+				defer fd.Close()
+
+				fd.Seek(int64(file_buffer.Offset), 0)
+				fd.Write(file_buffer.Data)
+			}
+		}
+	}
 	return nil
 }
 

logging/logging.go

@@ -14,18 +14,23 @@ type Logger struct {
 
 func NewLogger(config *config.Config) *Logger {
 	result := Logger{
-		config:    config,
-		error_log: log.New(os.Stderr, "ERR:", log.LstdFlags),
-		info_log:  log.New(os.Stderr, "INFO:", log.LstdFlags),
+		config: config,
 	}
 
 	return &result
 }
 
 func (self *Logger) Error(format string, v ...interface{}) {
+	if self.error_log == nil {
+		self.error_log = log.New(os.Stderr, "ERR:", log.LstdFlags)
+	}
+
 	self.error_log.Printf(format, v...)
 }
 
 func (self *Logger) Info(format string, v ...interface{}) {
+	if self.info_log == nil {
+		self.info_log = log.New(os.Stderr, "INFO:", log.LstdFlags)
+	}
 	self.info_log.Printf(format, v...)
 }