From 7f488ec84c2d0732502a305dca41e7618cbb376d Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 8 Nov 2020 01:35:00 +1000 Subject: [PATCH] Mock info in pool client. (#727) Make pool clients look a little different by mocking the info() plugin to return different hostnames. --- api/api.go | 2 +- bin/pool.go | 180 +++++++++++++----- flows/api.go | 7 + flows/hunts.go | 8 +- .../src/components/flows/new-collection.js | 9 +- .../src/components/hunts/hunt-results.js | 2 - .../{hunts/hunt.css => utils/spinner.css} | 0 .../src/components/utils/spinner.js | 2 + server/comms.go | 7 +- .../client_monitoring/client_monitoring.go | 12 +- services/frontend/frontend.go | 4 + services/hunt_manager/hunt_manager.go | 11 +- services/interrogation/interrogation.go | 11 +- vql/vql.go | 5 + 14 files changed, 186 insertions(+), 74 deletions(-) rename gui/velociraptor/src/components/{hunts/hunt.css => utils/spinner.css} (100%) diff --git a/api/api.go b/api/api.go index 65a6ef529d4..eb55dc9348e 100644 --- a/api/api.go +++ b/api/api.go @@ -436,7 +436,7 @@ func (self *ApiServer) ListClients( result.Items = append(result.Items, api_client) - if uint64(len(result.Items)) > limit { + if uint64(len(result.Items)) >= in.Limit { break } } diff --git a/bin/pool.go b/bin/pool.go index f143bf776dd..8be50b6f793 100644 --- a/bin/pool.go +++ b/bin/pool.go @@ -21,17 +21,25 @@ import ( "context" "fmt" "io/ioutil" + "os" "path" + "runtime" + "github.com/Velocidex/ordereddict" "github.com/Velocidex/yaml/v2" + "github.com/shirou/gopsutil/host" kingpin "gopkg.in/alecthomas/kingpin.v2" + "www.velocidex.com/golang/velociraptor/artifacts" config "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/crypto" "www.velocidex.com/golang/velociraptor/executor" "www.velocidex.com/golang/velociraptor/http_comms" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/server" "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/vfilter" ) var ( @@ -47,6 +55,16 @@ var ( ) func doPoolClient() { + registerMockInfo() + + number_of_clients := *pool_client_number + if number_of_clients <= 0 { + number_of_clients = 2 + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client_config, err := DefaultConfigLoader. WithRequiredClient(). WithVerbose(true). @@ -59,62 +77,68 @@ func doPoolClient() { server.IncreaseLimits(client_config) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // Make a copy of all the configs for each client. + configs := make([]*config_proto.Config, 0, number_of_clients) + serialized, _ := json.Marshal(client_config) - number_of_clients := *pool_client_number - if number_of_clients <= 0 { - number_of_clients = 2 + for i := 0; i < number_of_clients; i++ { + client_config := &config_proto.Config{} + err := json.Unmarshal(serialized, &client_config) + kingpin.FatalIfError(err, "Copying configs.") + configs = append(configs, client_config) } for i := 0; i < number_of_clients; i++ { - filename := fmt.Sprintf("pool_client.yaml.%d", i) - client_config.Client.WritebackLinux = path.Join( - *pool_client_writeback_dir, filename) - - client_config.Client.WritebackWindows = client_config.Client.WritebackLinux - - existing_writeback := &config_proto.Writeback{} - writeback, err := config.WritebackLocation(client_config) - kingpin.FatalIfError(err, "Unable to load writeback file") - - data, err := ioutil.ReadFile(writeback) - - // Failing to read the file is not an error - the file may not - // exist yet. - if err == nil { - err = yaml.Unmarshal(data, existing_writeback) - kingpin.FatalIfError(err, "Unable to load config file "+filename) - } - - // Merge the writeback with the config. - client_config.Writeback = existing_writeback - - // Make sure the config is ok. - err = crypto.VerifyConfig(client_config) - if err != nil { - kingpin.FatalIfError(err, "Invalid config") - } - - manager, err := crypto.NewClientCryptoManager( - client_config, []byte(client_config.Writeback.PrivateKey)) - kingpin.FatalIfError(err, "Unable to parse config file") - - exe, err := executor.NewClientExecutor(ctx, client_config) - kingpin.FatalIfError(err, "Can not create executor.") - - comm, err := http_comms.NewHTTPCommunicator( - client_config, - manager, - exe, - client_config.Client.ServerUrls, - nil, - utils.RealClock{}, - ) - kingpin.FatalIfError(err, "Can not create HTTPCommunicator.") - - // Run the client in the background. - go comm.Run(ctx) + go func(i int) { + client_config := configs[i] + filename := fmt.Sprintf("pool_client.yaml.%d", i) + client_config.Client.WritebackLinux = path.Join( + *pool_client_writeback_dir, filename) + + client_config.Client.WritebackWindows = client_config.Client.WritebackLinux + + existing_writeback := &config_proto.Writeback{} + writeback, err := config.WritebackLocation(client_config) + kingpin.FatalIfError(err, "Unable to load writeback file") + + data, err := ioutil.ReadFile(writeback) + + // Failing to read the file is not an error - the file may not + // exist yet. + if err == nil { + err = yaml.Unmarshal(data, existing_writeback) + kingpin.FatalIfError(err, "Unable to load config file "+filename) + } + + // Merge the writeback with the config. + client_config.Writeback = existing_writeback + + // Make sure the config is ok. + err = crypto.VerifyConfig(client_config) + if err != nil { + kingpin.FatalIfError(err, "Invalid config") + } + + manager, err := crypto.NewClientCryptoManager( + client_config, []byte(client_config.Writeback.PrivateKey)) + kingpin.FatalIfError(err, "Unable to parse config file") + + exe, err := executor.NewClientExecutor(ctx, client_config) + kingpin.FatalIfError(err, "Can not create executor.") + + comm, err := http_comms.NewHTTPCommunicator( + client_config, + manager, + exe, + client_config.Client.ServerUrls, + nil, + utils.RealClock{}, + ) + kingpin.FatalIfError(err, "Can not create HTTPCommunicator.") + + // Run the client in the background. + comm.Run(ctx) + }(i) } // Block forever. @@ -132,3 +156,55 @@ func init() { return true }) } + +func registerMockInfo() { + vql.OverridePlugin( + vfilter.GenericListPlugin{ + PluginName: "info", + Function: func( + scope *vfilter.Scope, + args *ordereddict.Dict) []vfilter.Row { + var result []vfilter.Row + + config_obj, ok := artifacts.GetServerConfig(scope) + if !ok { + return result + } + + key, err := crypto.ParseRsaPrivateKeyFromPemStr( + []byte(config_obj.Writeback.PrivateKey)) + if err != nil { + scope.Log("info: %s", err) + return result + } + + client_id := crypto.ClientIDFromPublicKey(&key.PublicKey) + + me, _ := os.Executable() + info, err := host.Info() + if err != nil { + scope.Log("info: %s", err) + return result + } + + fqdn := fmt.Sprintf("%s.%s", info.Hostname, client_id) + item := ordereddict.NewDict(). + Set("Hostname", fqdn). + Set("Uptime", info.Uptime). + Set("BootTime", info.BootTime). + Set("Procs", info.Procs). + Set("OS", info.OS). + Set("Platform", info.Platform). + Set("PlatformFamily", info.PlatformFamily). + Set("PlatformVersion", info.PlatformVersion). + Set("KernelVersion", info.KernelVersion). + Set("VirtualizationSystem", info.VirtualizationSystem). + Set("VirtualizationRole", info.VirtualizationRole). + Set("Fqdn", fqdn). + Set("Architecture", runtime.GOARCH). + Set("Exe", me) + result = append(result, item) + return result + }, + }) +} diff --git a/flows/api.go b/flows/api.go index 777a74df4df..5b15ac88f82 100644 --- a/flows/api.go +++ b/flows/api.go @@ -20,6 +20,7 @@ package flows import ( "context" "path" + "sort" "strings" "time" @@ -55,6 +56,12 @@ func GetFlows( return nil, err } + // Flow IDs represent timestamp so they are sortable. The UI + // relies on more recent flows being at the top. + sort.Slice(flow_urns, func(i, j int) bool { + return flow_urns[i] > flow_urns[j] + }) + for _, urn := range flow_urns { // Hide the monitoring flow since it is not a real flow. if strings.HasSuffix(urn, constants.MONITORING_WELL_KNOWN_FLOW) { diff --git a/flows/hunts.go b/flows/hunts.go index 7272b5c6274..cc02ea0eddb 100644 --- a/flows/hunts.go +++ b/flows/hunts.go @@ -224,6 +224,9 @@ func GetHunt(config_obj *config_proto.Config, in *api_proto.GetHuntRequest) ( err = services.GetHuntDispatcher().ModifyHunt( in.HuntId, func(hunt_obj *api_proto.Hunt) error { + // Make a copy + result = proto.Clone(hunt_obj).(*api_proto.Hunt) + // HACK: Velociraptor only knows how to // collect artifacts now. Eventually the whole // concept of a flow will go away but for now @@ -231,10 +234,7 @@ func GetHunt(config_obj *config_proto.Config, in *api_proto.GetHuntRequest) ( // are actually collecting - there are not // many possibilities since we have reduced // the number of possible flows significantly. - FindCollectedArtifacts(config_obj, hunt_obj) - - // Make a copy - result = proto.Clone(hunt_obj).(*api_proto.Hunt) + FindCollectedArtifacts(config_obj, result) return nil }) diff --git a/gui/velociraptor/src/components/flows/new-collection.js b/gui/velociraptor/src/components/flows/new-collection.js index 15c0b0ac86c..4a1242ee1fa 100644 --- a/gui/velociraptor/src/components/flows/new-collection.js +++ b/gui/velociraptor/src/components/flows/new-collection.js @@ -597,10 +597,11 @@ class NewCollectionWizard extends React.Component { response.data.items && response.data.items.length) { let parameters = {}; - _.each(request.parameters.env, param=>{ - parameters[param.key] = param.value; - }); - + if (request.parameters) { + _.each(request.parameters.env, param=>{ + parameters[param.key] = param.value; + }); + } this.setState({ artifacts: [...response.data.items], parameters: parameters, diff --git a/gui/velociraptor/src/components/hunts/hunt-results.js b/gui/velociraptor/src/components/hunts/hunt-results.js index 30be7f62c5d..09c9af5111a 100644 --- a/gui/velociraptor/src/components/hunts/hunt-results.js +++ b/gui/velociraptor/src/components/hunts/hunt-results.js @@ -1,5 +1,3 @@ -import './hunt.css'; - import React from 'react'; import PropTypes from 'prop-types'; diff --git a/gui/velociraptor/src/components/hunts/hunt.css b/gui/velociraptor/src/components/utils/spinner.css similarity index 100% rename from gui/velociraptor/src/components/hunts/hunt.css rename to gui/velociraptor/src/components/utils/spinner.css diff --git a/gui/velociraptor/src/components/utils/spinner.js b/gui/velociraptor/src/components/utils/spinner.js index c42c429d6b2..1a03789f21c 100644 --- a/gui/velociraptor/src/components/utils/spinner.js +++ b/gui/velociraptor/src/components/utils/spinner.js @@ -1,3 +1,5 @@ +import './spinner.css'; + import React from 'react'; import PropTypes from 'prop-types'; diff --git a/server/comms.go b/server/comms.go index 3988a2a69e7..01642379d53 100644 --- a/server/comms.go +++ b/server/comms.go @@ -19,6 +19,7 @@ package server import ( "errors" + "fmt" "io" "io/ioutil" "math/rand" @@ -342,15 +343,17 @@ func reader(config_obj *config_proto.Config, server_obj *Server) http.Handler { // Must be before the Process() call to prevent race. source := message_info.Source - if services.GetNotifier().IsClientConnected(source) { + notifier := services.GetNotifier() + if notifier != nil && notifier.IsClientConnected(source) { http.Error(w, "Another Client connection exists. "+ "Only a single instance of the client is "+ "allowed to connect at the same time.", http.StatusConflict) + fmt.Printf("Source %v Conflict\n", source) return } - notification, cancel := services.GetNotifier().ListenForNotification(source) + notification, cancel := notifier.ListenForNotification(source) defer cancel() // Deadlines are designed to ensure that connections diff --git a/services/client_monitoring/client_monitoring.go b/services/client_monitoring/client_monitoring.go index 64fb2a4735d..2da21783552 100644 --- a/services/client_monitoring/client_monitoring.go +++ b/services/client_monitoring/client_monitoring.go @@ -13,6 +13,7 @@ import ( "github.com/Velocidex/ordereddict" "github.com/google/uuid" + "google.golang.org/protobuf/proto" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/constants" @@ -225,15 +226,18 @@ func (self *ClientEventTable) GetClientUpdateEventTableMessage( self.state.Artifacts = &flows_proto.ArtifactCollectorArgs{} } - result.Event = append(result.Event, - self.state.Artifacts.CompiledCollectorArgs...) + for _, event := range self.state.Artifacts.CompiledCollectorArgs { + result.Event = append(result.Event, proto.Clone(event).(*actions_proto.VQLCollectorArgs)) + } // Now apply any event queries that belong to this client based on labels. labeler := services.GetLabeler() for _, table := range self.state.LabelEvents { if labeler.IsLabelSet(config_obj, client_id, table.Label) { - result.Event = append( - result.Event, table.Artifacts.CompiledCollectorArgs...) + for _, event := range table.Artifacts.CompiledCollectorArgs { + result.Event = append(result.Event, + proto.Clone(event).(*actions_proto.VQLCollectorArgs)) + } } } diff --git a/services/frontend/frontend.go b/services/frontend/frontend.go index 77930b3776c..7e810ad550d 100644 --- a/services/frontend/frontend.go +++ b/services/frontend/frontend.go @@ -395,6 +395,10 @@ func StartFrontendService(ctx context.Context, wg *sync.WaitGroup, } } }() + notifier := services.GetNotifier() + if notifier == nil { + return errors.New("Notifier not ready") + } return services.GetNotifier().NotifyListener(config_obj, "Frontend") } diff --git a/services/hunt_manager/hunt_manager.go b/services/hunt_manager/hunt_manager.go index c789f8b5145..6828d9eea3e 100644 --- a/services/hunt_manager/hunt_manager.go +++ b/services/hunt_manager/hunt_manager.go @@ -357,10 +357,13 @@ func (self *HuntManager) ProcessRow( } // Notify the client - err = services.GetNotifier().NotifyListener( - config_obj, participation_row.ClientId) - if err != nil { - scope.Log("hunt manager: %v", err) + notifier := services.GetNotifier() + if notifier != nil { + err = services.GetNotifier().NotifyListener( + config_obj, participation_row.ClientId) + if err != nil { + scope.Log("hunt manager: %v", err) + } } } diff --git a/services/interrogation/interrogation.go b/services/interrogation/interrogation.go index 94e3c546c42..47a72324ebf 100644 --- a/services/interrogation/interrogation.go +++ b/services/interrogation/interrogation.go @@ -141,8 +141,17 @@ func (self *EnrollmentService) ProcessRow( client_info.ClientId = client_id client_info.LastInterrogateFlowId = flow_id err = db.SetSubject(config_obj, client_path_manager.Path(), client_info) + if err != nil { + return err + } - return err + // Notify the client + notifier := services.GetNotifier() + if notifier != nil { + err = services.GetNotifier().NotifyListener(config_obj, client_id) + return err + } + return nil } // Watch the system's flow completion log for interrogate artifacts. diff --git a/vql/vql.go b/vql/vql.go index 0b12482cc4e..e6a7cac22d6 100755 --- a/vql/vql.go +++ b/vql/vql.go @@ -45,6 +45,11 @@ var ( }) ) +func OverridePlugin(plugin vfilter.PluginGeneratorInterface) { + name := plugin.Info(nil, nil).Name + exportedPlugins[name] = plugin +} + func RegisterPlugin(plugin vfilter.PluginGeneratorInterface) { name := plugin.Info(nil, nil).Name _, pres := exportedPlugins[name]