Updated the fs command to be able to use the filestore. (Velocidex#301)

It is essential to be able to easily copy files into and out of the
filestore. With the new filestore being inside mysql it is otherwise
hard to administer it.

This change adds the ability to see inside the filestore to the `fs ls`,
`fs cp` and `fs cat` command.

It is now possible to add items to the public directory like:

```
velociraptor -v fs cp '*.exe'  fs:///public/
```

actions/vql.go

@@ -33,8 +33,8 @@ import (
 	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/responder"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -96,7 +96,7 @@ func (self VQLClientAction) StartQuery(
 	// Create a new query environment and store some useful
 	// objects in there. VQL plugins may then use the environment
 	// to communicate with the server.
-	uploader := &vql_networking.VelociraptorUploader{
+	uploader := &uploads.VelociraptorUploader{
 		Responder: responder,
 	}
 

bin/artifacts_acquire.go

@@ -33,8 +33,8 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
 	logging "www.velocidex.com/golang/velociraptor/logging"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	vfilter "www.velocidex.com/golang/vfilter"
 )
 
@@ -75,7 +75,7 @@ func acquireArtifact(ctx context.Context, config_obj *config_proto.Config,
 	env := ordereddict.NewDict().
 		Set("config", config_obj.Client).
 		Set("server_config", config_obj).
-		Set("$uploader", &vql_networking.FileBasedUploader{
+		Set("$uploader", &uploads.FileBasedUploader{
 			UploadDir: filepath.Join(subdir, "files"),
 		}).
 		Set(vql_subsystem.ACL_MANAGER_VAR, acl_manager).

bin/console.go

@@ -35,9 +35,9 @@ import (
 	artifacts "www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	vfilter "www.velocidex.com/golang/vfilter"
 )
 
@@ -601,7 +601,7 @@ func doConsole() {
 	env := ordereddict.NewDict().
 		Set("config", config_obj.Client).
 		Set("server_config", config_obj).
-		Set("$uploader", &vql_networking.FileBasedUploader{
+		Set("$uploader", &uploads.FileBasedUploader{
 			UploadDir: *console_dump_dir,
 		}).
 		Set(vql_subsystem.ACL_MANAGER_VAR, acl_manager).

bin/fs.go

@@ -18,38 +18,51 @@
 package main
 
 import (
+	"context"
+	"io"
 	"os"
+	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/Velocidex/ordereddict"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"www.velocidex.com/golang/velociraptor/file_store"
+	"www.velocidex.com/golang/velociraptor/glob"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	vfilter "www.velocidex.com/golang/vfilter"
 )
 
 var (
+	accessor_reg = regexp.MustCompile(
+		"^(file|ntft|reg|registry|zip|raw_reg|lazy_ntfs|file_links|fs)://(.+)$")
+
 	fs_command          = app.Command("fs", "Run filesystem commands.")
 	fs_command_accessor = fs_command.Flag(
-		"accessor", "The FS accessor to use").Default("file").Enum(
-		"file", "ntfs", "reg", "zip", "raw_reg", "lazy_ntfs", "file_links")
+		"accessor", "The FS accessor to use").Default("file").String()
 	fs_command_verbose = fs_command.Flag(
-		"details", "Show more verbose info").Short('d').
+		"details", "Show more verbose info").Short('l').
 		Default("false").Bool()
-	fs_command_format = fs_command.Flag("format", "Output format to use  (text,json,jsonl).").
-				Default("text").Enum("text", "json", "jsonl")
+	fs_command_format = fs_command.Flag("format", "Output format to use  (text,json,jsonl,csv).").
+				Default("jsonl").Enum("text", "json", "jsonl", "csv")
 
 	fs_command_ls      = fs_command.Command("ls", "List files")
 	fs_command_ls_path = fs_command_ls.Arg(
-		"path", "The path to list").Default("/").String()
+		"path", "The path or glob to list").Default("/").String()
 
 	fs_command_cp      = fs_command.Command("cp", "Copy files to a directory.")
 	fs_command_cp_path = fs_command_cp.Arg(
-		"path", "The path to list").Default("/").String()
+		"path", "The path or glob to list").Required().String()
+
 	fs_command_cp_outdir = fs_command_cp.Arg(
-		"dumpdir", "The directory to store files at.").Default(".").
-		ExistingDir()
+		"dumpdir", "The directory to store files at.").Required().
+		String()
+
+	fs_command_cat      = fs_command.Command("cat", "Dump a file to the terminal")
+	fs_command_cat_path = fs_command_cat.Arg(
+		"path", "The path to cat").Required().String()
 )
 
 func eval_query(query string, scope *vfilter.Scope) {
@@ -73,7 +86,15 @@ func eval_query(query string, scope *vfilter.Scope) {
 	}
 }
 
-func doLS(path string) {
+func doLS(path, accessor string) {
+	initFilestoreAccessor()
+
+	matches := accessor_reg.FindStringSubmatch(path)
+	if matches != nil {
+		accessor = matches[1]
+		path = matches[2]
+	}
+
 	if len(path) > 0 && (path[len(path)-1] == '/' ||
 		path[len(path)-1] == '\\') {
 		path += "*"
@@ -82,69 +103,139 @@ func doLS(path string) {
 	env := ordereddict.NewDict().
 		Set(vql_subsystem.ACL_MANAGER_VAR,
 			vql_subsystem.NewRoleACLManager("administrator")).
-		Set("accessor", *fs_command_accessor).
+		Set("accessor", accessor).
 		Set("path", path)
 
 	scope := vql_subsystem.MakeScope().AppendVars(env)
 	defer scope.Close()
 
 	AddLogger(scope, get_config_or_default())
 
-	query := "SELECT Name, Size, Mode.String AS Mode, " +
-		"timestamp(epoch=Mtime.Sec) as mtime, Data " +
+	query := "SELECT Name, Size, Mode.String AS Mode, Mtime, Data " +
 		"FROM glob(globs=path, accessor=accessor) "
 	if *fs_command_verbose {
 		query = strings.Replace(query, "Name", "FullPath", 1)
 	}
+
 	// Special handling for ntfs.
-	if !*fs_command_verbose && *fs_command_accessor == "ntfs" {
+	if !*fs_command_verbose && accessor == "ntfs" {
 		query += " WHERE Sys.name_type != 'DOS' "
 	}
 
 	eval_query(query, scope)
 }
 
-func doCp(path string, dump_dir string) {
+func doCp(path, accessor string, dump_dir string) {
+	initFilestoreAccessor()
+	config_obj := get_config_or_default()
+
+	matches := accessor_reg.FindStringSubmatch(path)
+	if matches != nil {
+		accessor = matches[1]
+		path = matches[2]
+	}
+
 	if len(path) > 0 && (path[len(path)-1] == '/' ||
 		path[len(path)-1] == '\\') {
 		path += "*"
 	}
 
+	if accessor == "file" {
+		path, _ = filepath.Abs(path)
+	}
+
+	output_accessor := ""
+	output_path := dump_dir
+
+	matches = accessor_reg.FindStringSubmatch(dump_dir)
+	if matches != nil {
+		output_accessor = matches[1]
+		output_path = matches[2]
+	}
+
 	env := ordereddict.NewDict().
-		Set("accessor", *fs_command_accessor).
+		Set("accessor", accessor).
 		Set("path", path).
-		Set("$uploader", &vql_networking.FileBasedUploader{
-			UploadDir: dump_dir,
-		}).
 		Set(vql_subsystem.ACL_MANAGER_VAR,
 			vql_subsystem.NewRoleACLManager("administrator")).
 		Set(vql_subsystem.CACHE_VAR, vql_subsystem.NewScopeCache())
 
+	switch output_accessor {
+	case "", "file":
+		env.Set("$uploader", &uploads.FileBasedUploader{
+			UploadDir: output_path,
+		})
+
+	case "fs":
+		uploader := file_store.NewFileStoreUploader(
+			config_obj, output_path)
+		env.Set("$uploader", uploader)
+
+	default:
+		kingpin.Fatalf("Can not write to accessor %v\n", output_accessor)
+	}
+
 	scope := vql_subsystem.MakeScope().AppendVars(env)
 	defer scope.Close()
 
 	AddLogger(scope, get_config_or_default())
 
-	eval_query(`SELECT * from foreach(
+	scope.Log("Copy from %v (%v) to %v (%v)",
+		path, accessor, output_path, output_accessor)
+
+	eval_query(`
+SELECT * from foreach(
   row={
     SELECT Name, Size, Mode.String AS Mode,
-       timestamp(epoch=Mtime.Sec) as mtime, Data, FullPath
+       Mtime, Data, FullPath
     FROM glob(globs=path, accessor=accessor)
-    WHERE Sys.name_type != 'DOS'
   }, query={
-     SELECT Name, Size, Mode, mtime, Data,
-     upload(file=FullPath, accessor=accessor) AS Upload
+     SELECT Name, Size, Mode, Mtime, Data,
+     upload(file=FullPath, accessor=accessor, name=Name) AS Upload
      FROM scope()
   })`, scope)
 }
 
+func initFilestoreAccessor() {
+	config_obj, err := get_server_config(*config_path)
+	if err != nil {
+		return
+	}
+
+	accessor, err := file_store.GetFileStoreFileSystemAccessor(config_obj)
+	kingpin.FatalIfError(err, "GetFileStoreFileSystemAccessor")
+	glob.Register("fs", accessor)
+}
+
+func doCat(path, accessor_name string) {
+	initFilestoreAccessor()
+	matches := accessor_reg.FindStringSubmatch(path)
+	if matches != nil {
+		accessor_name = matches[1]
+		path = matches[2]
+	}
+
+	ctx := context.Background()
+	accessor, err := glob.GetAccessor(accessor_name, ctx)
+	kingpin.FatalIfError(err, "GetAccessor")
+
+	fd, err := accessor.Open(path)
+	kingpin.FatalIfError(err, "ReadFile")
+
+	io.Copy(os.Stdout, fd)
+}
+
 func init() {
 	command_handlers = append(command_handlers, func(command string) bool {
 		switch command {
-		case "fs ls":
-			doLS(*fs_command_ls_path)
-		case "fs cp":
-			doCp(*fs_command_cp_path, *fs_command_cp_outdir)
+		case fs_command_ls.FullCommand():
+			doLS(*fs_command_ls_path, *fs_command_accessor)
+
+		case fs_command_cp.FullCommand():
+			doCp(*fs_command_cp_path, *fs_command_accessor, *fs_command_cp_outdir)
+
+		case fs_command_cat.FullCommand():
+			doCat(*fs_command_cat_path, *fs_command_accessor)
 
 		default:
 			return false

bin/query.go

@@ -29,8 +29,8 @@ import (
 	artifacts "www.velocidex.com/golang/velociraptor/artifacts"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -158,7 +158,7 @@ func doQuery() {
 	env := ordereddict.NewDict().
 		Set("config", config_obj.Client).
 		Set("server_config", config_obj).
-		Set("$uploader", &vql_networking.FileBasedUploader{
+		Set("$uploader", &uploads.FileBasedUploader{
 			UploadDir: *dump_dir,
 		}).
 

bin/unzip.go

@@ -7,8 +7,8 @@ import (
 	"github.com/Velocidex/ordereddict"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	vql_networking "www.velocidex.com/golang/velociraptor/vql/networking"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -78,7 +78,7 @@ func doUnzip() {
 		}
 
 	} else {
-		env.Set("$uploader", &vql_networking.FileBasedUploader{
+		env.Set("$uploader", &uploads.FileBasedUploader{
 			UploadDir: *unzip_path,
 		})