Refactored the artifact repository into its own service. (Velocidex#577)

* Moved code between the repository and launcher.
* Made the notification service into its own service.
* Adding tools is now a noop - all tools are always added every time
  when artifacts are compiled.

actions/vql.go

@@ -28,11 +28,11 @@ import (
 	"github.com/Velocidex/ordereddict"
 	humanize "github.com/dustin/go-humanize"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
-	artifacts "www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/responder"
+	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/uploads"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
@@ -95,7 +95,7 @@ func (self VQLClientAction) StartQuery(
 
 	// Clients do not have a copy of artifacts so they need to be
 	// sent all artifacts from the server.
-	repository := artifacts.NewRepository()
+	repository := services.GetRepositoryManager().NewRepository()
 	for _, artifact := range arg.Artifacts {
 		_, err := repository.LoadProto(artifact, false /* validate */)
 		if err != nil {
@@ -109,7 +109,7 @@ func (self VQLClientAction) StartQuery(
 		Responder: responder,
 	}
 
-	builder := artifacts.ScopeBuilder{
+	builder := services.ScopeBuilder{
 		Config: config_obj,
 		// Disable ACLs on the client.
 		ACLManager: vql_subsystem.NullACLManager{},
@@ -123,7 +123,7 @@ func (self VQLClientAction) StartQuery(
 		builder.Env.Set(env_spec.Key, env_spec.Value)
 	}
 
-	scope := builder.Build()
+	scope := services.GetRepositoryManager().BuildScope(builder)
 	defer scope.Close()
 
 	scope.Log("Starting query execution.")

api/api.go

@@ -42,7 +42,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/acls"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	"www.velocidex.com/golang/velociraptor/artifacts"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
@@ -51,7 +50,6 @@ import (
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
 	"www.velocidex.com/golang/velociraptor/grpc_client"
 	"www.velocidex.com/golang/velociraptor/logging"
-	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/server"
 	"www.velocidex.com/golang/velociraptor/services"
 	users "www.velocidex.com/golang/velociraptor/users"
@@ -151,7 +149,7 @@ func (self *ApiServer) GetReport(
 
 	acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name)
 
-	global_repo, err := artifacts.GetGlobalRepository(self.config)
+	global_repo, err := services.GetRepositoryManager().GetGlobalRepository(self.config)
 	if err != nil {
 		return nil, err
 	}
@@ -165,6 +163,8 @@ func (self *ApiServer) CollectArtifact(
 	result := &flows_proto.ArtifactCollectorResponse{Request: in}
 	creator := GetGRPCUserInfo(self.config, ctx).Name
 
+	var acl_manager vql_subsystem.ACLManager = vql_subsystem.NullACLManager{}
+
 	// Internal calls from the frontend can set the creator.
 	if creator != self.config.Client.PinnedServerName {
 		in.Creator = creator
@@ -174,27 +174,30 @@ func (self *ApiServer) CollectArtifact(
 			permissions = acls.COLLECT_SERVER
 		}
 
-		perm, err := acls.CheckAccess(self.config, creator, permissions)
+		acl_manager = vql_subsystem.NewServerACLManager(self.config,
+			creator)
+
+		perm, err := acl_manager.CheckAccess(permissions)
 		if !perm || err != nil {
 			return nil, status.Error(codes.PermissionDenied,
 				"User is not allowed to launch flows.")
 		}
 	}
 
-	repository, err := artifacts.GetGlobalRepository(self.config)
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(self.config)
 	if err != nil {
 		return nil, err
 	}
 
 	flow_id, err := services.GetLauncher().ScheduleArtifactCollection(
-		ctx, self.config, in.Creator, repository, in)
+		ctx, acl_manager, repository, in)
 	if err != nil {
 		return nil, err
 	}
 
 	result.FlowId = flow_id
 
-	err = services.NotifyListener(self.config, in.ClientId)
+	err = services.GetNotifier().NotifyListener(self.config, in.ClientId)
 	if err != nil {
 		return nil, err
 	}
@@ -219,6 +222,8 @@ func (self *ApiServer) CreateHunt(
 	in.Creator = GetGRPCUserInfo(self.config, ctx).Name
 	in.HuntId = flows.GetNewHuntId()
 
+	acl_manager := vql_subsystem.NewServerACLManager(self.config, in.Creator)
+
 	permissions := acls.COLLECT_CLIENT
 	perm, err := acls.CheckAccess(self.config, in.Creator, permissions)
 	if !perm || err != nil {
@@ -235,7 +240,7 @@ func (self *ApiServer) CreateHunt(
 
 	result := &api_proto.StartFlowResponse{}
 	hunt_id, err := flows.CreateHunt(
-		ctx, self.config, in.Creator, in)
+		ctx, self.config, acl_manager, in)
 	if err != nil {
 		return nil, err
 	}
@@ -406,10 +411,10 @@ func (self *ApiServer) NotifyClients(
 
 	if in.NotifyAll {
 		self.server_obj.Info("sending notification to everyone")
-		services.NotifyAllListeners(self.config)
+		services.GetNotifier().NotifyAllListeners(self.config)
 	} else if in.ClientId != "" {
 		self.server_obj.Info("sending notification to %s", in.ClientId)
-		services.NotifyListener(self.config, in.ClientId)
+		services.GetNotifier().NotifyListener(self.config, in.ClientId)
 	} else {
 		return nil, status.Error(codes.InvalidArgument,
 			"client id should be specified")
@@ -659,7 +664,7 @@ func (self *ApiServer) GetArtifacts(
 
 	if len(in.Names) > 0 {
 		result := &artifacts_proto.ArtifactDescriptors{}
-		repository, err := artifacts.GetGlobalRepository(self.config)
+		repository, err := services.GetRepositoryManager().GetGlobalRepository(self.config)
 		if err != nil {
 			return nil, err
 		}
@@ -717,7 +722,7 @@ func (self *ApiServer) SetArtifactFile(
 	permissions := acls.ARTIFACT_WRITER
 
 	// First ensure that the artifact is correct.
-	tmp_repository := artifacts.NewRepository()
+	tmp_repository := services.GetRepositoryManager().NewRepository()
 	artifact_definition, err := tmp_repository.LoadYaml(
 		in.Artifact, true /* validate */)
 	if err != nil {
@@ -807,11 +812,8 @@ func (self *ApiServer) WriteEvent(
 			return nil, err
 		}
 
-		path_manager := result_sets.NewArtifactPathManager(self.config,
-			peer_name, "", in.Query.Name)
-
-		return &empty.Empty{}, services.GetJournal().PushRows(
-			path_manager, rows)
+		return &empty.Empty{}, services.GetJournal().PushRowsToArtifact(
+			rows, in.Query.Name, peer_name, "")
 	}
 
 	return nil, status.Error(codes.InvalidArgument, "no peer certs?")
@@ -991,12 +993,13 @@ func (self *ApiServer) CreateDownloadFile(ctx context.Context,
 
 	}
 
-	scope := artifacts.ScopeBuilder{
-		Config:     self.config,
-		Env:        env,
-		ACLManager: vql_subsystem.NewServerACLManager(self.config, user_name),
-		Logger:     logging.NewPlainLogger(self.config, &logging.FrontendComponent),
-	}.Build()
+	scope := services.GetRepositoryManager().BuildScope(
+		services.ScopeBuilder{
+			Config:     self.config,
+			Env:        env,
+			ACLManager: vql_subsystem.NewServerACLManager(self.config, user_name),
+			Logger:     logging.NewPlainLogger(self.config, &logging.FrontendComponent),
+		})
 	defer scope.Close()
 
 	vql, err := vfilter.Parse(query)

api/artifacts.go

@@ -36,7 +36,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/acls"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	"www.velocidex.com/golang/velociraptor/artifacts"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
@@ -80,7 +79,7 @@ func getArtifactFile(
 	config_obj *config_proto.Config,
 	name string) (string, error) {
 
-	repository, err := artifacts.GetGlobalRepository(config_obj)
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(config_obj)
 	if err != nil {
 		return "", err
 	}
@@ -123,7 +122,7 @@ func setArtifactFile(config_obj *config_proto.Config,
 	case api_proto.SetArtifactRequest_DELETE:
 
 		// First ensure that the artifact is correct.
-		tmp_repository := artifacts.NewRepository()
+		tmp_repository := services.GetRepositoryManager().NewRepository()
 		artifact_definition, err := tmp_repository.LoadYaml(
 			in.Artifact, true /* validate */)
 		if err != nil {
@@ -156,7 +155,7 @@ func getReportArtifacts(
 		number_of_results = 100
 	}
 
-	repository, err := artifacts.GetGlobalRepository(config_obj)
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(config_obj)
 	if err != nil {
 		return nil, err
 	}
@@ -226,7 +225,7 @@ func searchArtifact(
 		return true
 	}
 
-	repository, err := artifacts.GetGlobalRepository(config_obj)
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(config_obj)
 	if err != nil {
 		return nil, err
 	}

api/builder.go

@@ -294,7 +294,7 @@ func StartFrontendHttps(
 		defer cancel()
 
 		server.SetKeepAlivesEnabled(false)
-		services.NotifyAllListeners(config_obj)
+		services.GetNotifier().NotifyAllListeners(config_obj)
 		err := server.Shutdown(time_ctx)
 		if err != nil {
 			server_obj.Error("Frontend server error", err)
@@ -356,7 +356,7 @@ func StartFrontendPlainHttp(
 		defer cancel()
 
 		server.SetKeepAlivesEnabled(false)
-		services.NotifyAllListeners(config_obj)
+		services.GetNotifier().NotifyAllListeners(config_obj)
 		err := server.Shutdown(time_ctx)
 		if err != nil {
 			server_obj.Error("Frontend server error", err)
@@ -429,7 +429,7 @@ func StartFrontendWithAutocert(
 		defer cancel()
 
 		server.SetKeepAlivesEnabled(false)
-		services.NotifyAllListeners(config_obj)
+		services.GetNotifier().NotifyAllListeners(config_obj)
 		err := server.Shutdown(timeout_ctx)
 		if err != nil {
 			logger.Error("Frontend shutdown error ", err)

api/clients.go

@@ -125,7 +125,7 @@ func GetApiClient(
 	// Update the time to now if the client is currently actually
 	// connected.
 	if server_obj != nil &&
-		services.IsClientConnected(client_id) {
+		services.GetNotifier().IsClientConnected(client_id) {
 		result.LastSeenAt = uint64(time.Now().UnixNano() / 1000)
 	}
 

api/notebooks.go

@@ -19,7 +19,6 @@ import (
 	"google.golang.org/grpc/status"
 	"www.velocidex.com/golang/velociraptor/acls"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	"www.velocidex.com/golang/velociraptor/artifacts"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/datastore"
@@ -484,7 +483,7 @@ func (self *ApiServer) UpdateNotebookCell(
 
 	acl_manager := vql_subsystem.NewServerACLManager(self.config, user_name)
 
-	global_repo, err := artifacts.GetGlobalRepository(self.config)
+	global_repo, err := services.GetRepositoryManager().GetGlobalRepository(self.config)
 	if err != nil {
 		return nil, err
 	}
@@ -515,9 +514,10 @@ func (self *ApiServer) UpdateNotebookCell(
 		}
 
 		// Update the artifact plugin in the template.
+		/* FIXME
 		artifact_plugin := artifacts.NewArtifactRepositoryPlugin(repository)
 		tmpl.Env.Set("Artifact", artifact_plugin)
-
+		*/
 		input = fmt.Sprintf(`{{ Query "SELECT * FROM Artifact.%v()" | Table}}`,
 			artifact_obj.Name)
 	}
@@ -537,7 +537,8 @@ func (self *ApiServer) UpdateNotebookCell(
 	go func() {
 		defer query_cancel()
 
-		cancel_notify, remove_notification := services.ListenForNotification(in.CellId)
+		cancel_notify, remove_notification := services.GetNotifier().
+			ListenForNotification(in.CellId)
 		defer remove_notification()
 
 		select {
@@ -615,7 +616,7 @@ func (self *ApiServer) CancelNotebookCell(
 			"User is not allowed to edit notebooks.")
 	}
 
-	return &empty.Empty{}, services.NotifyListener(self.config, in.CellId)
+	return &empty.Empty{}, services.GetNotifier().NotifyListener(self.config, in.CellId)
 }
 
 func (self *ApiServer) UploadNotebookAttachment(

api/query.go

@@ -29,9 +29,9 @@ import (
 	"github.com/sirupsen/logrus"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/logging"
+	"www.velocidex.com/golang/velociraptor/services"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -72,28 +72,26 @@ func streamQuery(
 	response_channel := make(chan *actions_proto.VQLResponse)
 	scope_logger := MakeLogger(response_channel)
 
-	builder := artifacts.ScopeBuilder{
+	// Add extra artifacts to the query from the global repository.
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(config_obj)
+	if err != nil {
+		return err
+	}
+
+	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: vql_subsystem.NewServerACLManager(config_obj, peer_name),
 		Logger:     scope_logger,
+		Repository: repository,
 		Env:        ordereddict.NewDict(),
 	}
 
 	for _, env_spec := range arg.Env {
 		builder.Env.Set(env_spec.Key, env_spec.Value)
 	}
 
-	repository, err := artifacts.GetGlobalRepository(config_obj)
-	if err != nil {
-		return err
-	}
-
-	err = repository.PopulateArtifactsVQLCollectorArgs(arg)
-	if err != nil {
-		return err
-	}
-
-	scope := builder.Build()
+	// Now execute the query.
+	scope := services.GetRepositoryManager().BuildScope(builder)
 	defer scope.Close()
 
 	go func() {

api/reflect.go

@@ -24,7 +24,7 @@ import (
 	"github.com/golang/protobuf/ptypes/empty"
 	context "golang.org/x/net/context"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
-	"www.velocidex.com/golang/velociraptor/artifacts"
+	"www.velocidex.com/golang/velociraptor/services"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -73,7 +73,7 @@ func (self *ApiServer) GetKeywordCompletions(
 		})
 	}
 
-	repository, err := artifacts.GetGlobalRepository(self.config)
+	repository, err := services.GetRepositoryManager().GetGlobalRepository(self.config)
 	if err != nil {
 		return nil, err
 	}
@@ -125,7 +125,7 @@ func getArgDescriptors(arg_type string, type_map *vfilter.TypeMap,
 }
 
 func getArtifactParamDescriptors(name string, type_map *vfilter.TypeMap,
-	repository *artifacts.Repository) []*api_proto.ArgDescriptor {
+	repository services.Repository) []*api_proto.ArgDescriptor {
 	args := []*api_proto.ArgDescriptor{}
 	artifact, pres := repository.Get(name)
 	if !pres {