Initial commit of multitenant support (Velocidex#1917)

This large refactor introduces the concept of organizations within Velociraptor. The server uses an org manager to select between a set of org specific configuration files - all derived from the main configuration.

The root org is the org with an empty org id and is what is running when using a standard deployment.

A new org can be created using the command velociraptor orgs create orgname This will also generate a new orgs id.

Orgs store all their files in the filestore under their own org directory (/orgs// ) this keeps all the data from one org separate from other orgs.

A user may belong to multiple orgs and have different ACLs on different orgs (add a user to an org using the vql function user_create() with the orgid arg.

When a user belongs to multiple orgs they receive a select box in the user preferences. This allows them to switch between orgs.

switching between orgs gives a completely different view - since orgs have completely different filestore directories none of the data is shared - so it feels like logging into a completely different server. Searching will give only the org clients, labels are only present into the org, notebooks and hunts are separate etc. Users can only target clients within the org.

On the client's side, the client's config contains the org id and a separate nonce that allows to connect to this org. This means that we need to create new client packages to each org and the clients will automatically connect to their own org.

This is the initial PR and some work is still needed.

actions/events_test.go

@@ -19,7 +19,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/responder"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/services/client_monitoring"
-	"www.velocidex.com/golang/velociraptor/services/indexing"
 	"www.velocidex.com/golang/velociraptor/services/labels"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/velociraptor/vtesting"
@@ -51,33 +50,31 @@ type EventsTestSuite struct {
 }
 
 func (self *EventsTestSuite) SetupTest() {
-	self.TestSuite.SetupTest()
-
-	assert.NoError(self.T(),
-		self.Sm.Start(client_monitoring.StartClientMonitoringService))
-	assert.NoError(self.T(), self.Sm.Start(indexing.StartIndexingService))
-
-	self.client_id = "C.2232"
-	self.Clock = &utils.IncClock{}
+	self.ConfigObj = self.LoadConfig()
+	self.LoadArtifacts(artifact_definitions)
 
+	// Set a tempfile for the writeback we need to check that the
+	// new event query is written there.
 	tmpfile, err := ioutil.TempFile("", "")
 	assert.NoError(self.T(), err)
 	tmpfile.Close()
 
-	// Set a tempfile for the writeback we need to check that the
-	// new event query is written there.
 	self.writeback = tmpfile.Name()
 	self.ConfigObj.Client.WritebackLinux = self.writeback
 	self.ConfigObj.Client.WritebackWindows = self.writeback
 	self.ConfigObj.Client.WritebackDarwin = self.writeback
+	self.ConfigObj.Frontend.ServerServices.ClientMonitoring = true
+	self.ConfigObj.Frontend.ServerServices.IndexServer = true
+	self.TestSuite.SetupTest()
+
+	self.client_id = "C.2232"
+	self.Clock = &utils.IncClock{}
 
 	self.responder = responder.TestResponder()
 
 	actions.GlobalEventTable = actions.NewEventTable(
 		self.ConfigObj, self.responder,
 		&actions_proto.VQLEventTable{})
-
-	self.LoadArtifacts(artifact_definitions)
 }
 
 func (self *EventsTestSuite) TearDownTest() {
@@ -103,7 +100,7 @@ var server_state = &flows_proto.ClientEventTable{
 }
 
 func (self *EventsTestSuite) TestEventTableUpdate() {
-	client_manager := services.ClientEventManager()
+	client_manager, err := services.ClientEventManager(self.ConfigObj)
 	client_manager.(*client_monitoring.ClientEventTable).Clock = self.Clock
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
@@ -162,7 +159,7 @@ func (self *EventsTestSuite) TestEventTableUpdate() {
 	// actually change the label groups, the new event table will
 	// be the same as the old one, except the version will be
 	// advanced.
-	label_manager := services.GetLabeler()
+	label_manager := services.GetLabeler(self.ConfigObj)
 	label_manager.(*labels.Labeler).Clock = self.Clock
 
 	require.NoError(self.T(),

actions/vql.go

@@ -112,7 +112,7 @@ func (self VQLClientAction) StartQuery(
 
 	// Clients do not have a copy of artifacts so they need to be
 	// sent all artifacts from the server.
-	manager, err := services.GetRepositoryManager()
+	manager, err := services.GetRepositoryManager(config_obj)
 	if err != nil {
 		responder.RaiseError(ctx, fmt.Sprintf("%v", err))
 		return