Converted import command to artifact (Velocidex#872)

Server.Utils.ImportCollection can now be used to import an offline
collector zip file into the server. The artifact will also generate a
flow completion event which may trigger other listeners (e.g. elastic
upload).

actions/proto/vql.proto

@@ -30,6 +30,12 @@ message VQLCollectorArgs {
     option (flow_metadata) = {
         category: "Generic";
     };
+
+    // The principal that created this request - only sent when
+    // scheduling the server so the server may keep track on who ran
+    // each query.
+    string principal = 28;
+
     repeated VQLEnv env = 3 [(sem_type) = {
             description: "Environment variables to be provided for the query.",
         }];

api/api.go

@@ -482,9 +482,15 @@ func (self *ApiServer) NotifyClients(
 			"User is not allowed to launch flows.")
 	}
 
+	notifier := services.GetNotifier()
+	if notifier == nil {
+		return nil, errors.New("Notifier not ready")
+	}
+
 	if in.NotifyAll {
 		self.server_obj.Info("sending notification to everyone")
-		err = services.GetNotifier().NotifyAllListeners(self.config)
+		err = notifier.NotifyAllListeners(self.config)
+
 	} else if in.ClientId != "" {
 		self.server_obj.Info("sending notification to %s", in.ClientId)
 		err = services.GetNotifier().NotifyListener(self.config, in.ClientId)

artifacts/definitions/Server/Utils/ImportCollection.yaml

@@ -0,0 +1,49 @@
+name: Server.Utils.ImportCollection
+description: |
+  The Velociraptor offline collector is an automated, preconfigured
+  collection tool. Users can use the collector to automatically
+  collect any artifacts on endpoints that do not have the Velociraptor
+  client (offline endpoints).
+
+  The collector creates a ZIP archive with the results of the
+  collection in JSON files (and any uploaded files).
+
+  This artifact allows for these offline collections to be imported
+  back into the Velociraptor GUI. The collected data can then treated
+  exactly the same as if it was collected by the regular Velociraptor
+  client (i.e. post processed through the notebook interface), except
+  it was collected via the Sneakernet.
+
+  NOTE: This artifact reads the collection ZIP from the server's
+  filesystem. It is up to you to arrange for the file to be stored on
+  the server (e.g. scp it over).
+
+  NOTE: This artifact is still experimental - please provide feedback
+  on our issue board.
+
+type: SERVER
+
+parameters:
+  - name: ClientId
+    default: auto
+    description: |
+      The client id to upload this collection into. The
+      default is "auto" which will create a new client id.
+  - name: Hostname
+    description: If creating a new client, this must contain the hostname.
+  - name: Path
+    description: A path on the server containing the zip file to upload.
+
+sources:
+  - query: |
+      LET result = SELECT import_collection(
+               client_id=ClientId, hostname=Hostname,
+               filename=Path) AS Import
+        FROM scope()
+
+      SELECT Import.client_id AS ClientId, Import.session_id AS FlowId,
+             Import.total_collected_rows AS TotalRows,
+             Import.total_uploaded_files AS UploadedFiles,
+             Import.total_uploaded_bytes AS UploadedBytes,
+             Import.artifacts_with_results AS Artifacts
+      FROM result

bin/config_interactive.go

@@ -65,7 +65,7 @@ What OS will the server be deployed on?
 	}
 
 	// https://docs.microsoft.com/en-us/troubleshoot/windows-server/identity/naming-conventions-for-computer-domain-site-ou#dns-host-names
-	url_validator = regexValidator("^[a-z0-9.A-Z]+$")
+	url_validator = regexValidator("^[a-z0-9.A-Z\\-]+$")
 	port_question = &survey.Input{
 		Message: "Enter the frontend port to listen on.",
 		Default: "8000",
@@ -397,8 +397,11 @@ func dynDNSConfig(config_obj *config_proto.Config) error {
 }
 
 func configAutocert(config_obj *config_proto.Config) error {
-	err := survey.Ask([]*survey.Question{
-		{Name: "Hostname", Prompt: url_question},
+	err := survey.Ask([]*survey.Question{{
+		Name:     "Hostname",
+		Validate: url_validator,
+		Prompt:   url_question,
+	},
 	}, config_obj.Frontend)
 	if err != nil {
 		return err

flows/hunts.go

@@ -164,9 +164,14 @@ func CreateHunt(
 		// set it started.
 	} else if hunt.State == api_proto.Hunt_RUNNING {
 		hunt.StartTime = hunt.CreateTime
-		err = services.GetNotifier().NotifyAllListeners(config_obj)
-		if err != nil {
-			return "", err
+
+		// Notify all the clients.
+		notifier := services.GetNotifier()
+		if notifier != nil {
+			err = notifier.NotifyByRegex(config_obj, "^[Cc]\\.")
+			if err != nil {
+				return "", err
+			}
 		}
 	}
 
@@ -353,5 +358,9 @@ func ModifyHunt(
 	// Notify all the clients about the new hunt. New hunts are
 	// not that common so notifying all the clients at once is
 	// probably ok.
-	return services.GetNotifier().NotifyAllListeners(config_obj)
+	notifier := services.GetNotifier()
+	if notifier == nil {
+		return errors.New("Notifier not ready")
+	}
+	return notifier.NotifyByRegex(config_obj, "^[Cc]\\.")
 }

go.mod

@@ -117,8 +117,8 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c
-	www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada
+	www.velocidex.com/golang/vfilter v0.0.0-20210114050946-1e02dba2abec
+	www.velocidex.com/golang/vtypes v0.0.0-20210114034841-b184d28b61b7
 )
 
 // replace www.velocidex.com/golang/go-pe => /home/mic/projects/go-pe

go.sum

@@ -77,6 +77,8 @@ github.com/Velocidex/ordereddict v0.0.0-20200723153557-9460a6764ab8 h1:wRt5CLsj5
 github.com/Velocidex/ordereddict v0.0.0-20200723153557-9460a6764ab8/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo=
 github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49 h1:TJVN1zYl5sKJUq571gYqU/5VqFIap9MUo2KNujc0fcg=
 github.com/Velocidex/survey v1.8.7-0.20190926071832-2ff99cc7aa49/go.mod h1:kfPUQ2gP0xtIydiR52dirNYt4OvCr+iZuepL4XaIk58=
+github.com/Velocidex/yaml v2.1.0+incompatible h1:XtpJSzJ95+UbheJ5ZwHZOUH8sHhYpypGaL3zmaL2klc=
+github.com/Velocidex/yaml v2.1.0+incompatible/go.mod h1:CzH+PG+PA8o7RiH4/ZkHx5rHDWFZlzZVc3R6pe/gJMg=
 github.com/Velocidex/yaml/v2 v2.2.5 h1:8XZwR8tmm5UWAXotI3fL17s9fjKEMqZ293fgqUiMhBw=
 github.com/Velocidex/yaml/v2 v2.2.5/go.mod h1:VBjrsTMc/b1h0ankOOnJPYoCbJNwhpGYpnDgICEs2mk=
 github.com/Velocidex/zip v0.0.0-20210101070220-e7ecefb7aad7 h1:IAry9WUMrVYA+XPvMF5UMN56ya5II/hoUOtqaHKOHrs=
@@ -802,11 +804,7 @@ www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiA
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns=
 www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff h1:00WZKnVVXwsfaV18UNn7iGQMCqQUf0cCFQ2RbbprYns=
 www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
-www.velocidex.com/golang/vfilter v0.0.0-20210108163451-876ce85e7301 h1:ffb5gMiTNREa9+YYxKxzLl3rPbH6FCpjyEu+dSWRxks=
-www.velocidex.com/golang/vfilter v0.0.0-20210108163451-876ce85e7301/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
-www.velocidex.com/golang/vfilter v0.0.0-20210110142356-d3f572a550df h1:Mq+PKDD3HXVzQy3uDv78oWhEgZZ87qNVkvPvqSMPsn0=
-www.velocidex.com/golang/vfilter v0.0.0-20210110142356-d3f572a550df/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
-www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c h1:soa7usa1BoStBC7NkqqRyZzFA0miIy9pdnaNMmcx6VI=
-www.velocidex.com/golang/vfilter v0.0.0-20210111083751-13cae4a7330c/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
-www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada h1:Rtn8RTS/fTxHts5/PrBP7fei2eq4I6tboHyEm3xX0ew=
-www.velocidex.com/golang/vtypes v0.0.0-20210108052555-8a27f80edada/go.mod h1:fkF6W4t1+qr1Nw51+EhAmCwQp2QtwcyNN1Ghwd1vLBM=
+www.velocidex.com/golang/vfilter v0.0.0-20210114050946-1e02dba2abec h1:xAkbNZQvivJkCd+HoqFE0fY5aAPjp5UwVjSIiCDuN6E=
+www.velocidex.com/golang/vfilter v0.0.0-20210114050946-1e02dba2abec/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA=
+www.velocidex.com/golang/vtypes v0.0.0-20210114034841-b184d28b61b7 h1:mzduH30kLBExDyBkMDdwYMH7nFZG9Pm6kBpIv4nGJnA=
+www.velocidex.com/golang/vtypes v0.0.0-20210114034841-b184d28b61b7/go.mod h1:34AZRfhNvJ1QAwPpYrDxjCyOFys+NbSmH6LLVSjsAEg=

gui/velociraptor/src/components/events/events.js

@@ -307,6 +307,7 @@ class EventMonitoring extends React.Component {
 
         let renderers = {
             "_ts": timestamp_renderer,
+            "Timestamp": timestamp_renderer,
             "client_time": timestamp_renderer,
         };
 

notifications/notifications.go

@@ -1,6 +1,7 @@
 package notifications
 
 import (
+	"regexp"
 	"sync"
 )
 
@@ -67,6 +68,18 @@ func (self *NotificationPool) Notify(client_id string) {
 	}
 }
 
+func (self *NotificationPool) NotifyByRegex(re *regexp.Regexp) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	for key, ch := range self.clients {
+		if re.MatchString(key) {
+			close(ch)
+			delete(self.clients, key)
+		}
+	}
+}
+
 func (self *NotificationPool) Shutdown() {
 	self.mu.Lock()
 	defer self.mu.Unlock()

services/client_monitoring/client_monitoring.go

@@ -75,6 +75,7 @@ func (self *ClientEventTable) CheckClientEventsVersion(
 	if labeler == nil {
 		return false
 	}
+
 	if client_version < self.state.Version {
 		return true
 	}

services/launcher/artifacts_test.go

@@ -22,7 +22,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/services/repository"
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
-	_ "www.velocidex.com/golang/velociraptor/vql_plugins"
 )
 
 var (

services/launcher/launcher.go

@@ -332,11 +332,17 @@ func ScheduleArtifactCollectionFromCollectorArgs(
 	tasks := []*crypto_proto.GrrMessage{}
 
 	for _, arg := range vql_collector_args {
+		// If sending to the server record who actually launched this.
+		if client_id == "server" {
+			arg.Principal = collection_context.Request.Creator
+		}
+
 		// The task we will schedule for the client.
 		task := &crypto_proto.GrrMessage{
 			SessionId:       collection_context.SessionId,
 			RequestId:       constants.ProcessVQLResponses,
-			VQLClientAction: arg}
+			VQLClientAction: arg,
+		}
 
 		// Send an urgent request to the client.
 		if collector_request.Urgent {

services/launcher/launcher_test.go

@@ -35,6 +35,10 @@ import (
 	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
+
+	// Load plugins (timestamp, parse_csv)
+	_ "www.velocidex.com/golang/velociraptor/vql/functions"
+	_ "www.velocidex.com/golang/velociraptor/vql/parsers/csv"
 )
 
 const (

services/notifications.go

@@ -50,9 +50,14 @@ type Notifier interface {
 	ListenForNotification(id string) (chan bool, func())
 
 	// Send a notification to everyone - this is a global event
-	// directed at everyone.
+	// directed at everyone. Only used in server shutdown - use
+	// NotifyByRegex instead.
 	NotifyAllListeners(config_obj *config_proto.Config) error
 
+	// Send a notification to all listeners with an id matching
+	// this regex.
+	NotifyByRegex(config_obj *config_proto.Config, regex string) error
+
 	// Send a notification to a specific listener based on its id
 	// that was registered above.
 	NotifyListener(config_obj *config_proto.Config, id string) error

services/notifications/notifications.go

@@ -16,6 +16,7 @@ package notifications
 
 import (
 	"context"
+	"regexp"
 	"sync"
 
 	"github.com/Velocidex/ordereddict"
@@ -90,7 +91,20 @@ func StartNotificationService(
 				}
 
 				self.pool_mu.Lock()
-				if target == "All" {
+				if target == "Regex" {
+					regex_str, ok := event.GetString("Regex")
+					if ok {
+						regex, err := regexp.Compile(regex_str)
+						if err != nil {
+							logger.Error("Notification service: "+
+								"Unable to compiler regex '%v': %v\n",
+								regex_str, err)
+							continue
+						}
+						self.notification_pool.NotifyByRegex(regex)
+					}
+
+				} else if target == "All" {
 					self.notification_pool.NotifyAll()
 				} else {
 					self.notification_pool.Notify(target)
@@ -128,6 +142,19 @@ func (self *Notifier) NotifyAllListeners(config_obj *config_proto.Config) error
 	)
 }
 
+func (self *Notifier) NotifyByRegex(config_obj *config_proto.Config, regex string) error {
+	journal, err := services.GetJournal()
+	if err != nil {
+		return err
+	}
+
+	return journal.PushRowsToArtifact(config_obj,
+		[]*ordereddict.Dict{ordereddict.NewDict().Set("Target", "Regex").
+			Set("Regex", regex)},
+		"Server.Internal.Notifications", "server", "",
+	)
+}
+
 func (self *Notifier) NotifyListener(config_obj *config_proto.Config, id string) error {
 	journal, err := services.GetJournal()
 	if err != nil {

services/server_artifacts/server_artifacts.go

@@ -261,6 +261,11 @@ func (self *ServerArtifactsRunner) runQuery(
 		return err
 	}
 
+	principal := arg.Principal
+	if principal == "" {
+		principal = "administrator"
+	}
+
 	scope := manager.BuildScope(services.ScopeBuilder{
 		Config: self.config_obj,
 
@@ -270,14 +275,19 @@ func (self *ServerArtifactsRunner) runQuery(
 		// files in the filestore using VQL artifacts.
 		Uploader: NewServerUploader(self.config_obj,
 			path_manager, collection_context),
-		ACLManager: vql_subsystem.NewRoleACLManager("administrator"),
+
+		// Run this query on behalf of the caller so they are
+		// subject to ACL checks
+		ACLManager: vql_subsystem.NewServerACLManager(self.config_obj, principal),
 		Logger: log.New(&serverLogger{
 			self.config_obj,
 			path_manager.Log(),
 		}, "", 0),
 	})
 	defer scope.Close()
 
+	scope.Log("Running query on behalf of user %v", principal)
+
 	env := ordereddict.NewDict()
 	for _, env_spec := range arg.Env {
 		env.Set(env_spec.Key, env_spec.Value)

vql/server/hunts/create.go

@@ -91,6 +91,7 @@ func (self *ScheduleHuntFunction) Call(ctx context.Context,
 
 	hunt_request := &api_proto.Hunt{
 		HuntDescription: arg.Description,
+		Creator:         vql_subsystem.GetPrincipal(scope),
 		StartRequest:    request,
 		Expires:         arg.Expires,
 		State:           api_proto.Hunt_RUNNING,
@@ -105,7 +106,9 @@ func (self *ScheduleHuntFunction) Call(ctx context.Context,
 		return vfilter.Null{}
 	}
 
-	return ordereddict.NewDict().Set("HuntId", hunt_id)
+	return ordereddict.NewDict().
+		Set("HuntId", hunt_id).
+		Set("Request", hunt_request)
 }
 
 func (self ScheduleHuntFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {

vql/server/whoami.go

@@ -0,0 +1,31 @@
+package server
+
+import (
+	"context"
+
+	"github.com/Velocidex/ordereddict"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/vfilter"
+)
+
+type WhoAmIFunctionArgs struct{}
+type WhoAmIFunction struct{}
+
+func (self *WhoAmIFunction) Call(ctx context.Context,
+	scope vfilter.Scope,
+	args *ordereddict.Dict) vfilter.Any {
+
+	return vql_subsystem.GetPrincipal(scope)
+}
+
+func (self WhoAmIFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+	return &vfilter.FunctionInfo{
+		Name:    "whoami",
+		Doc:     "Returns the username that is running the query.",
+		ArgType: type_map.AddType(scope, &WhoAmIFunctionArgs{}),
+	}
+}
+
+func init() {
+	vql_subsystem.RegisterFunction(&WhoAmIFunction{})
+}