Backups should also include custom artifacts. (Velocidex#3819)

* Root org will create a backup for all other orgs. The data for each
org can be stored in the same zip file.
* It is now possible to restore only some providers selectively.

docs/references/vql.yaml

@@ -435,6 +435,13 @@
     type: string
     description: The name of the backup file.
     required: true
+  - name: prefix
+    type: string
+    description: Restore the backup from under this prefix in the zip file (defaults
+      to org id).
+  - name: providers
+    type: string
+    description: If provided only restore providers matching this regex.
   platforms:
   - darwin_amd64_cgo
   - darwin_arm64_cgo
@@ -4405,7 +4412,6 @@
            link_to(upload=Upload) AS Download
     FROM scope()
     ```
-
   type: Function
   args:
   - name: type
@@ -10807,3 +10813,4 @@
   platforms:
   - linux_amd64_cgo
   - windows_amd64_cgo
+

services/acl_manager/backup.go

@@ -26,8 +26,8 @@ func (self ACLBackupProvider) Name() []string {
 }
 
 func (self ACLBackupProvider) BackupResults(
-	ctx context.Context, wg *sync.WaitGroup) (
-	<-chan vfilter.Row, error) {
+	ctx context.Context, wg *sync.WaitGroup,
+	container services.BackupContainerWriter) (<-chan vfilter.Row, error) {
 
 	users_manager := services.GetUserManager()
 	user_list, err := users_manager.ListUsers(ctx,
@@ -70,6 +70,7 @@ func (self ACLBackupProvider) BackupResults(
 // because this may represent a security compromise but we want to
 // allow users to see the ACL permissions that were backed up.
 func (self ACLBackupProvider) Restore(ctx context.Context,
+	container services.BackupContainerReader,
 	in <-chan vfilter.Row) (stat services.BackupStat, err error) {
 
 	count := 0

services/backup.go

@@ -2,13 +2,32 @@ package services
 
 import (
 	"context"
+	"io"
+	"io/fs"
+	"regexp"
 	"sync"
+	"time"
 
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/vfilter"
 )
 
+// For writing.
+type BackupContainerWriter interface {
+	Create(name string, mtime time.Time) (io.WriteCloser, error)
+
+	WriteResultSet(
+		ctx context.Context,
+		config_obj *config_proto.Config,
+		dest string, in <-chan vfilter.Row) (total_rows int, err error)
+}
+
+// For reading.
+type BackupContainerReader interface {
+	Open(name string) (fs.File, error)
+}
+
 // Callers may register a backup provider to be included in the backup
 type BackupProvider interface {
 	// The name of this provider
@@ -22,13 +41,16 @@ type BackupProvider interface {
 	// can write the backup file (named in Name() above).
 	BackupResults(
 		ctx context.Context,
-		wg *sync.WaitGroup) (<-chan vfilter.Row, error)
+		wg *sync.WaitGroup,
+		container BackupContainerWriter) (<-chan vfilter.Row, error)
 
 	// This is the opposite of backup - it allows a provider to
 	// recover from an existing backup. Typcially providers need to
 	// clear their data and read new data from this channel. The
 	// provider may return stats about its operation.
-	Restore(ctx context.Context, in <-chan vfilter.Row) (BackupStat, error)
+	Restore(ctx context.Context,
+		container BackupContainerReader,
+		in <-chan vfilter.Row) (BackupStat, error)
 }
 
 // Alows each provider to report the stats of the most recent
@@ -38,11 +60,24 @@ type BackupStat struct {
 	Name    string
 	Error   error
 	Message string
+
+	// Which org owns this backup service.
+	OrgId string
+}
+
+type BackupRestoreOptions struct {
+	// By default the prefix in the zip is calculated based on the
+	// current org id. This allows this to be overriden.
+	Prefix string
+
+	// Only restore matching providers
+	ProviderRegex *regexp.Regexp
 }
 
 type BackupService interface {
 	Register(provider BackupProvider)
-	RestoreBackup(export_path api.FSPathSpec) ([]BackupStat, error)
+	RestoreBackup(export_path api.FSPathSpec,
+		opts BackupRestoreOptions) ([]BackupStat, error)
 	CreateBackup(export_path api.FSPathSpec) ([]BackupStat, error)
 }
 

services/backup/backup.go

@@ -17,7 +17,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/reporting"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/utils"
-	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
 
@@ -78,16 +77,57 @@ func (self *BackupService) CreateBackup(
 
 	defer container.Close()
 
+	org_container := &containerDelegate{
+		Container: container,
+	}
+
+	// The root org is responsible for dumping all child orgs as well.
+	if utils.IsRootOrg(self.config_obj.OrgId) {
+		org_manager, err := services.GetOrgManager()
+		if err != nil {
+			return stats, err
+		}
+
+		for _, org := range org_manager.ListOrgs() {
+			backup, err := org_manager.Services(org.Id).BackupService()
+			if err != nil {
+				continue
+			}
+
+			prefix := fmt.Sprintf("orgs/%v", org.Id)
+			org_container := &containerDelegate{
+				Container: container,
+				prefix:    prefix,
+			}
+			org_stats, _ := backup.(*BackupService).writeBackups(
+				org_container, prefix)
+			stats = append(stats, org_stats...)
+		}
+
+		// Not the root org, just backup this org only.
+	} else if !utils.IsRootOrg(self.config_obj.OrgId) {
+		prefix := fmt.Sprintf("orgs/%v", utils.GetOrgId(self.config_obj))
+		org_stats, _ := self.writeBackups(org_container, prefix)
+		stats = append(stats, org_stats...)
+	}
+
+	return stats, err
+}
+
+func (self *BackupService) writeBackups(
+	container services.BackupContainerWriter,
+	prefix string) (stats []services.BackupStat, err error) {
+
 	// Now we can dump all providers into the file.
-	scope := vql_subsystem.MakeScope()
+	logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent)
 
 	for _, provider := range self.registrations {
-		dest := strings.Join(provider.Name(), "/")
+		dest := strings.Join(append([]string{prefix}, provider.Name()...), "/")
 		stat := services.BackupStat{
 			Name: provider.ProviderName(),
 		}
 
-		rows, err := provider.BackupResults(self.ctx, self.wg)
+		rows, err := provider.BackupResults(self.ctx, self.wg, container)
 		if err != nil {
 			logger.Info("BackupService: <red>Error writing to %v: %v",
 				dest, err)
@@ -98,8 +138,8 @@ func (self *BackupService) CreateBackup(
 		}
 
 		// Write the results to the container now
-		total_rows, err := container.WriteResultSet(self.ctx, self.config_obj,
-			scope, reporting.ContainerFormatJson, dest, rows)
+		total_rows, err := container.WriteResultSet(
+			self.ctx, self.config_obj, dest, rows)
 		if err != nil {
 			logger.Info("BackupService: <red>Error writing to %v: %v",
 				dest, err)
@@ -117,7 +157,8 @@ func (self *BackupService) CreateBackup(
 
 // Opens a backup file and recovers all the data in it.
 func (self *BackupService) RestoreBackup(
-	export_path api.FSPathSpec) (stats []services.BackupStat, err error) {
+	export_path api.FSPathSpec,
+	opts services.BackupRestoreOptions) (stats []services.BackupStat, err error) {
 	// Create a container to hold the backup
 	file_store_factory := file_store.GetFileStore(self.config_obj)
 
@@ -141,8 +182,21 @@ func (self *BackupService) RestoreBackup(
 
 	logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent)
 
+	prefix := opts.Prefix
+	if prefix == "" {
+		prefix = fmt.Sprintf("orgs/%v", utils.GetOrgId(self.config_obj))
+	}
+
 	for _, provider := range self.registrations {
-		stat, err := self.feedProvider(provider, zip_reader)
+		if opts.ProviderRegex != nil && !opts.ProviderRegex.MatchString(
+			provider.ProviderName()) {
+			continue
+		}
+
+		stat, err := self.feedProvider(provider, zipDelegate{
+			Reader: zip_reader,
+			prefix: prefix,
+		})
 		if err != nil {
 			dest := strings.Join(provider.Name(), "/")
 			logger.Info("BackupService: <red>Error restoring to %v: %v",
@@ -158,7 +212,7 @@ func (self *BackupService) RestoreBackup(
 
 func (self *BackupService) feedProvider(
 	provider services.BackupProvider,
-	container *zip.Reader) (stat services.BackupStat, err error) {
+	container services.BackupContainerReader) (stat services.BackupStat, err error) {
 
 	dest := strings.Join(provider.Name(), "/")
 	member, err := container.Open(dest)
@@ -180,6 +234,8 @@ func (self *BackupService) feedProvider(
 		// Wait here until the provider is done.
 		stat = <-results
 		stat.Name = provider.ProviderName()
+		stat.OrgId = utils.GetOrgId(self.config_obj)
+
 		if stat.Error != nil {
 			err = stat.Error
 		}
@@ -198,7 +254,7 @@ func (self *BackupService) feedProvider(
 		defer close(results)
 
 		// Preserve the provider error as our return
-		stat, err := provider.Restore(sub_ctx, output)
+		stat, err := provider.Restore(sub_ctx, container, output)
 		if err != nil {
 			stat.Error = err
 		}

services/backup/backup_test.go

@@ -44,8 +44,8 @@ func (self TestBackupProvider) Name() []string {
 }
 
 func (self TestBackupProvider) BackupResults(
-	ctx context.Context, wg *sync.WaitGroup) (
-	<-chan vfilter.Row, error) {
+	ctx context.Context, wg *sync.WaitGroup,
+	container services.BackupContainerWriter) (<-chan vfilter.Row, error) {
 
 	output := make(chan vfilter.Row)
 
@@ -63,7 +63,9 @@ func (self TestBackupProvider) BackupResults(
 }
 
 func (self *TestBackupProvider) Restore(
-	ctx context.Context, in <-chan vfilter.Row) (services.BackupStat, error) {
+	ctx context.Context,
+	container services.BackupContainerReader,
+	in <-chan vfilter.Row) (services.BackupStat, error) {
 
 	if self.restored_error != nil {
 		return services.BackupStat{
@@ -124,17 +126,23 @@ func (self *BackupTestSuite) TestBackups() {
 		CreateBackup(export_path)
 	assert.NoError(self.T(), err)
 
+	// test_utils.GetMemoryFileStore(self.T(), self.ConfigObj).Debug()
+
 	// Backup file should be dependend on the mocked time.
 	result := self.readBackupFile(export_path)
-
-	test_provider, _ := result.Get("TestProvider.json")
+	prefix := "orgs/root/"
+	test_provider, _ := result.Get(prefix + "TestProvider.json")
 	golden := ordereddict.NewDict().
 		Set("TestProvider.json", test_provider).
 		Set("TestProvider Stats", filterStats(stats))
 
-	// Now restore the data from backup
+	opts := services.BackupRestoreOptions{}
+
+	// Now restore the data from backup. NOTE: Each org restores only
+	// its own data from the zip file. This allows the same zip file
+	// to be shared between all the orgs.
 	stats, err = backup_service.(*backup.BackupService).
-		RestoreBackup(export_path)
+		RestoreBackup(export_path, opts)
 	assert.NoError(self.T(), err)
 
 	golden.Set("RestoredTestProvider", provider.restored).
@@ -145,7 +153,7 @@ func (self *BackupTestSuite) TestBackups() {
 	provider.restored = nil
 
 	stats, err = backup_service.(*backup.BackupService).
-		RestoreBackup(export_path)
+		RestoreBackup(export_path, opts)
 	assert.NoError(self.T(), err)
 
 	golden.Set("RestoredTestProvider With Error", provider.restored).
@@ -164,7 +172,8 @@ func filterStats(stats []services.BackupStat) (res []services.BackupStat) {
 	return res
 }
 
-func (self *BackupTestSuite) readBackupFile(export_path api.FSPathSpec) *ordereddict.Dict {
+func (self *BackupTestSuite) readBackupFile(
+	export_path api.FSPathSpec) *ordereddict.Dict {
 	file_store_factory := file_store.GetFileStore(self.ConfigObj)
 	fd, err := file_store_factory.ReadFile(export_path)
 	assert.NoError(self.T(), err)

services/backup/delegates.go

@@ -0,0 +1,44 @@
+package backup
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"io/fs"
+	"time"
+
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/reporting"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/vfilter"
+)
+
+type containerDelegate struct {
+	*reporting.Container
+	prefix string
+}
+
+func (self *containerDelegate) Create(name string, mtime time.Time) (
+	io.WriteCloser, error) {
+	return self.Container.Create(self.prefix+"/"+name, mtime)
+}
+
+func (self *containerDelegate) WriteResultSet(
+	ctx context.Context,
+	config_obj *config_proto.Config,
+	dest string, in <-chan vfilter.Row) (total_rows int, err error) {
+
+	scope := vql_subsystem.MakeScope()
+
+	return self.Container.WriteResultSet(
+		ctx, config_obj, scope, reporting.ContainerFormatJson, dest, in)
+}
+
+type zipDelegate struct {
+	*zip.Reader
+	prefix string
+}
+
+func (self zipDelegate) Open(name string) (fs.File, error) {
+	return self.Reader.Open(self.prefix + "/" + name)
+}