Org manager maintains services lifetime (Velocidex#2045)

artifacts/definitions/Windows/NTFS/MFT.yaml

@@ -14,23 +14,23 @@ description: |
   MFT artifact. Due to the multi-drive features, the MFTPath will output the MFT
   path of the entry.
 
-  Available filters include:  
-  
-  - PathRegex (OSPath): e.g `^C:\\folder\\file\.ext$` or partial `\\folder\\folder2\\` or `string|string2|string3`    
-  - Fileregex: `^filename.ext$` or partial `string1|string2`  
-  - Time bounds to select files with a timestamp within time ranges  
-  - FileSize bounds  
-  - MFTDrive: drive to target collection and show as source in results during offline pricessing.  
-  - MFTPath: optional filter for offline MFT processing.  
-  
-  NOTE: Generally more efficient to filter on filename. 
+  Available filters include:
+
+  - PathRegex (OSPath): e.g `^C:\\folder\\file\.ext$` or partial `\\folder\\folder2\\` or `string|string2|string3`
+  - Fileregex: `^filename.ext$` or partial `string1|string2`
+  - Time bounds to select files with a timestamp within time ranges
+  - FileSize bounds
+  - MFTDrive: drive to target collection and show as source in results during offline pricessing.
+  - MFTPath: optional filter for offline MFT processing.
+
+  NOTE: Generally more efficient to filter on filename.
   Multiple filters are cumulative.
   OSPath output now uses expected Windows backslash "`\`".
-  
+
 parameters:
   - name: MFTDrive
     description: |
-      The path to to the drive that holds the MFT file (can be a pathspec). This 
+      The path to to the drive that holds the MFT file (can be a pathspec). This
       drive is also used for results for offline processing.
     default: "C:"
   - name: MFTPath
@@ -67,7 +67,7 @@ parameters:
 
 sources:
   - query: |
-      -- The path to to the drive that holds the MFT file (can be a pathspec)
+      -- The path to to the drive that holds the M}FT file (can be a pathspec)
       LET Drive <= pathspec(parse=MFTDrive, path_type="windows")
 
       -- time testing
@@ -96,9 +96,9 @@ sources:
         SELECT EntryNumber, InUse, ParentEntryNumber,
             Drive + OSPath AS OSPath,
             FileName, FileSize, ReferenceCount, IsDir,
-            Created0x10, Created0x30, 
+            Created0x10, Created0x30,
             LastModified0x10, LastModified0x30,
-            LastRecordChange0x10, LastRecordChange0x30, 
+            LastRecordChange0x10, LastRecordChange0x30,
             LastAccess0x10,LastAccess0x30,
             HasADS, SI_Lt_FN, uSecZeros, Copied,
             FileNames, FileNameTypes
@@ -108,13 +108,13 @@ sources:
 
       -- Check only one date bound
       LET mftsearch_after_date(Drive, MFTPath) =
-        SELECT 
+        SELECT
             EntryNumber, InUse, ParentEntryNumber,
             Drive + OSPath AS OSPath,
             FileName, FileSize, ReferenceCount, IsDir,
-            Created0x10, Created0x30, 
+            Created0x10, Created0x30,
             LastModified0x10, LastModified0x30,
-            LastRecordChange0x10, LastRecordChange0x30, 
+            LastRecordChange0x10, LastRecordChange0x30,
             LastAccess0x10,LastAccess0x30,
             HasADS, SI_Lt_FN, uSecZeros, Copied,
             FileNames, FileNameTypes
@@ -133,9 +133,9 @@ sources:
         SELECT EntryNumber, InUse, ParentEntryNumber,
             Drive + OSPath AS OSPath,
             FileName, FileSize, ReferenceCount, IsDir,
-            Created0x10, Created0x30, 
+            Created0x10, Created0x30,
             LastModified0x10, LastModified0x30,
-            LastRecordChange0x10, LastRecordChange0x30, 
+            LastRecordChange0x10, LastRecordChange0x30,
             LastAccess0x10,LastAccess0x30,
             HasADS, SI_Lt_FN, uSecZeros, Copied,
             FileNames, FileNameTypes
@@ -155,9 +155,9 @@ sources:
         SELECT EntryNumber, InUse, ParentEntryNumber,
             Drive + OSPath AS OSPath,
             FileName, FileSize, ReferenceCount, IsDir,
-            Created0x10, Created0x30, 
+            Created0x10, Created0x30,
             LastModified0x10, LastModified0x30,
-            LastRecordChange0x10, LastRecordChange0x30, 
+            LastRecordChange0x10, LastRecordChange0x30,
             LastAccess0x10,LastAccess0x30,
             HasADS, SI_Lt_FN, uSecZeros, Copied,
             FileNames, FileNameTypes
@@ -221,7 +221,7 @@ sources:
               SELECT * FROM all_drives
             },
             else={
-              SELECT * FROM mftsearch(Drive=Drive, 
+              SELECT * FROM mftsearch(Drive=Drive,
                                 MFTPath=if(condition= MFTPath ,
                                             then= MFTPath,
                                             else= Drive + "$MFT"))
@@ -230,8 +230,8 @@ sources:
       LET enriched_results = SELECT *,
             parse_ntfs(mft=EntryNumber, device=Drive ) as NtfsMetadata
         FROM results
-        
+
       -- return rows
       SELECT * FROM if(condition=AllNtfs,
         then= enriched_results,
-        else= results)
+        else= results)

artifacts/testdata/server/testcases/orgs.out.yaml

@@ -1,20 +1,20 @@
 SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
  {
-  "Name": "\u003croot org\u003e",
-  "OrgId": ""
+  "Name": "\u003croot\u003e",
+  "OrgId": "root"
  }
 ]LET _ <= user_create(user="FailedUser", roles="reader", password="X", orgs=["ORGID"])[]SELECT * FROM test_read_logs() WHERE Log =~ "Org not found" AND NOT Log =~ "SELECT"[
  {
   "Log": "Velociraptor: user_create: Org not found\n"
  }
 ]SELECT * FROM gui_users(all_orgs=TRUE) ORDER BY name[]LET _ <= org_create(name="MyOrg", org_id="ORGID")[]SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
- {
-  "Name": "\u003croot org\u003e",
-  "OrgId": ""
- },
  {
   "Name": "MyOrg",
   "OrgId": "ORGID"
+ },
+ {
+  "Name": "\u003croot\u003e",
+  "OrgId": "root"
  }
 ]SELECT user_create(user="ReaderUser", roles="reader", password="X", orgs=["root", "ORGID"]), user_create(user="OrgUser", roles="administrator", password="X", orgs=["ORGID"]), user_create(user="OrgAdmin", roles="administrator", password="X") FROM scope()[
  {
@@ -25,8 +25,8 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
 ]SELECT * FROM gui_users(all_orgs=TRUE) ORDER BY name[
  {
   "name": "OrgAdmin",
-  "org_id": "",
-  "org_name": "\u003croot org\u003e",
+  "org_id": "root",
+  "org_name": "\u003croot\u003e",
   "picture": "",
   "email": false,
   "roles": [
@@ -82,8 +82,8 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
  },
  {
   "name": "ReaderUser",
-  "org_id": "",
-  "org_name": "\u003croot org\u003e",
+  "org_id": "root",
+  "org_name": "\u003croot\u003e",
   "picture": "",
   "email": false,
   "roles": [
@@ -109,8 +109,8 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
 ]SELECT * FROM gui_users() ORDER BY name[
  {
   "name": "OrgAdmin",
-  "org_id": "",
-  "org_name": "\u003croot org\u003e",
+  "org_id": "root",
+  "org_name": "\u003croot\u003e",
   "picture": "",
   "email": false,
   "roles": [
@@ -138,8 +138,8 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
  },
  {
   "name": "ReaderUser",
-  "org_id": "",
-  "org_name": "\u003croot org\u003e",
+  "org_id": "root",
+  "org_name": "\u003croot\u003e",
   "picture": "",
   "email": false,
   "roles": [
@@ -197,10 +197,10 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
  }
 ]SELECT * FROM query(query={ SELECT OrgId FROM orgs() ORDER BY OrgId }, runas="OrgAdmin", org_id="root")[
  {
-  "OrgId": ""
+  "OrgId": "ORGID"
  },
  {
-  "OrgId": "ORGID"
+  "OrgId": "root"
  }
 ]SELECT * FROM query(query={ SELECT OrgId FROM orgs() ORDER BY OrgId }, runas="OrgUser", org_id="ORGID")[]SELECT * FROM test_read_logs() WHERE Log =~ "Permission denied.+ORG_ADMIN" AND NOT Log =~ "SELECT"[
  {
@@ -302,7 +302,7 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
  }
 ]LET _ <= org_delete(org="ORGID2")[]SELECT OrgId FROM orgs()[
  {
-  "OrgId": ""
+  "OrgId": "root"
  },
  {
   "OrgId": "ORGID"

datastore/memcache_file.go

@@ -114,9 +114,12 @@ type MemcacheFileDataStore struct {
 	mu    sync.Mutex
 	cache *MemcacheDatastore
 
-	writer chan *Mutation
-	ctx    context.Context
-	cancel func()
+	writer     chan *Mutation
+	ctx        context.Context
+	cancel     func()
+	config_obj *config_proto.Config
+
+	started bool
 }
 
 func (self *MemcacheFileDataStore) Stats() *MemcacheStats {
@@ -148,6 +151,20 @@ func (self *MemcacheFileDataStore) ExpirationPolicy(
 	return true
 }
 
+func (self *MemcacheFileDataStore) Flush() {
+	for {
+		select {
+		case mutation, ok := <-self.writer:
+			if !ok {
+				return
+			}
+			self.processMutation(self.config_obj, mutation)
+		default:
+			return
+		}
+	}
+}
+
 // Starts the writer loop.
 func (self *MemcacheFileDataStore) StartWriter(
 	ctx context.Context, wg *sync.WaitGroup,
@@ -173,6 +190,7 @@ func (self *MemcacheFileDataStore) StartWriter(
 	self.mu.Lock()
 	self.writer = make(chan *Mutation, buffer_size)
 	self.ctx = ctx
+	self.started = true
 	self.mu.Unlock()
 
 	if writers == 0 {
@@ -200,40 +218,44 @@ func (self *MemcacheFileDataStore) StartWriter(
 					if !ok {
 						return
 					}
-
-					metricIdleWriters.Dec()
-					switch mutation.op {
-					case MUTATION_OP_SET_SUBJECT:
-						writeContentToFile(config_obj, mutation.urn, mutation.data)
-						self.invalidateDirCache(config_obj, mutation.urn)
-
-						// Call the completion function once we hit
-						// the directory datastore.
-						if mutation.completion != nil {
-							mutation.completion()
-						}
-
-					case MUTATION_OP_DEL_SUBJECT:
-						file_based_imp.DeleteSubject(config_obj, mutation.urn)
-						self.invalidateDirCache(config_obj, mutation.urn.Dir())
-
-						// Call the completion function once we hit
-						// the directory datastore.
-						if mutation.completion != nil {
-							mutation.completion()
-						}
-					}
-
-					metricIdleWriters.Inc()
-					if mutation.wg != nil {
-						mutation.wg.Done()
-					}
+					self.processMutation(config_obj, mutation)
 				}
 			}
 		}()
 	}
 }
 
+func (self *MemcacheFileDataStore) processMutation(
+	config_obj *config_proto.Config, mutation *Mutation) {
+	metricIdleWriters.Dec()
+	switch mutation.op {
+	case MUTATION_OP_SET_SUBJECT:
+		writeContentToFile(config_obj, mutation.urn, mutation.data)
+		self.invalidateDirCache(config_obj, mutation.urn)
+
+		// Call the completion function once we hit
+		// the directory datastore.
+		if mutation.completion != nil {
+			mutation.completion()
+		}
+
+	case MUTATION_OP_DEL_SUBJECT:
+		file_based_imp.DeleteSubject(config_obj, mutation.urn)
+		self.invalidateDirCache(config_obj, mutation.urn.Dir())
+
+		// Call the completion function once we hit
+		// the directory datastore.
+		if mutation.completion != nil {
+			mutation.completion()
+		}
+	}
+
+	metricIdleWriters.Inc()
+	if mutation.wg != nil {
+		mutation.wg.Done()
+	}
+}
+
 func (self *MemcacheFileDataStore) GetSubject(
 	config_obj *config_proto.Config,
 	urn api.DSPathSpec,
@@ -621,6 +643,7 @@ func NewMemcacheFileDataStore(config_obj *config_proto.Config) *MemcacheFileData
 	}
 
 	result := &MemcacheFileDataStore{
+		config_obj: config_obj,
 		cache: &MemcacheDatastore{
 			data_cache: NewDataLRUCache(config_obj,
 				data_max_size, data_max_item_size),
@@ -637,15 +660,26 @@ func StartMemcacheFileService(
 	ctx context.Context, wg *sync.WaitGroup,
 	config_obj *config_proto.Config) error {
 
-	_, err := GetDB(config_obj)
+	if config_obj.Datastore == nil {
+		return nil
+	}
+
+	db, err := GetDB(config_obj)
 	if err != nil {
 		return err
 	}
 
-	if memcache_file_imp != nil {
+	memcache_file_db, ok := db.(*MemcacheFileDataStore)
+	if !ok {
+		// If it not a MemcacheFileDataStore so we dont need to do
+		// anything to it.
+		return nil
+	}
+
+	if !memcache_file_db.started {
 		logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
 		logger.Info("<green>Starting</> memcache service")
-		memcache_file_imp.StartWriter(ctx, wg, config_obj)
+		memcache_file_db.StartWriter(ctx, wg, config_obj)
 	}
 
 	return nil

datastore/readonly.go

@@ -7,6 +7,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
+	"www.velocidex.com/golang/velociraptor/utils"
 )
 
 var (
@@ -34,7 +35,8 @@ func (self *ReadOnlyDataStore) SetSubjectWithCompletion(
 	completion func()) error {
 
 	err := self.cache.SetSubject(config_obj, urn, message)
-	if completion != nil {
+	if completion != nil &&
+		!utils.CompareFuncs(completion, utils.SyncCompleter) {
 		completion()
 	}
 	return err