From 2169f99eb2dfedc486230015cd27dcc505e2a629 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 7 Sep 2024 16:46:19 +1000 Subject: [PATCH] Allows arbitrary length filenames in filestore. (#3736) On Linux file names must be less then 255 bytes. The filestore attempts to preserve and sanitize the original filename of uploaded files, which means that in some cases the filestore filename can easily exceed the hard limit imposed by the OS. This PR switches to a hash based compression when the filename exceeds 250 chars. This replaces the filestore filename with a hash, and the full component is stored in the datastore separately. This allows us to store arbitrary length filenames in the filestore. --- .../Server/Utils/StartHuntExample.yaml | 6 +- bin/hunts.go | 2 +- datastore/datastore_test.go | 39 ++++ datastore/filebased.go | 62 +++--- datastore/filebased_generic.go | 1 + datastore/filebased_test.go | 27 ++- datastore/filebased_utils.go | 184 ++++++++++++++++++ datastore/memcache.go | 51 +++-- datastore/memcache_file.go | 20 +- datastore/utils.go | 2 +- file_store/api/paths.go | 11 +- file_store/directory/directory.go | 33 +++- file_store/memory/memory.go | 35 ++-- file_store/path_specs/fs_path_spec.go | 20 -- file_store/path_specs/path_specs.go | 74 +------ file_store/tests/testsuite.go | 3 +- paths/artifacts/paths_test.go | 7 +- paths/paths_test.go | 2 +- server/server_test.go | 7 +- services/repository/manager_test.go | 6 +- services/repository/plugin_test.go | 31 +-- utils/sanitize.go | 6 + vql/server/file_store.go | 29 +-- 23 files changed, 443 insertions(+), 215 deletions(-) create mode 100644 datastore/filebased_utils.go diff --git a/artifacts/definitions/Server/Utils/StartHuntExample.yaml b/artifacts/definitions/Server/Utils/StartHuntExample.yaml index 33c64adbddd..97753f779f1 100644 --- a/artifacts/definitions/Server/Utils/StartHuntExample.yaml +++ b/artifacts/definitions/Server/Utils/StartHuntExample.yaml @@ -28,7 +28,7 @@ description: | ``` This will allow users with the `COLLECT_BASIC` permission to also - collect it. Once collected the artifact specifies an impersonation + collect it. Once collected the artifact specifies the impersonate field to `admin` which will cause it to run under the `admin` user's permissions. @@ -40,9 +40,9 @@ description: | type: SERVER # Collect this artifact under the admin user permissions. -impersonation: admin +impersonate: admin -source: +sources: - query: | -- This query will run with admin ACLs. SELECT hunt( diff --git a/bin/hunts.go b/bin/hunts.go index 696a02c5e7a..a3ada1fb176 100644 --- a/bin/hunts.go +++ b/bin/hunts.go @@ -95,7 +95,7 @@ func doHuntReconstruct() error { return err } fmt.Printf("Rebuilding %v to %v\n", hunt.HuntId, - target.AsDatastoreFilename(config_obj)) + target.String()) } } } diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go index da1c4f562ea..b55c9f9c335 100644 --- a/datastore/datastore_test.go +++ b/datastore/datastore_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "sort" + "strings" "sync" "testing" "time" @@ -328,6 +329,44 @@ func (self BaseTestSuite) TestListChildrenSubdirs() { "/Root/item"}, asStrings(children)) } +// Make sure all the other data stores handle very long filenames +func (self BaseTestSuite) TestVeryLongFilename() { + message := &crypto_proto.VeloMessage{Source: "Server"} + very_long_filename := strings.Repeat("Very Long Filename", 100) + assert.Equal(self.T(), len(very_long_filename), 1800) + + path := path_specs.NewUnsafeDatastorePath("longfiles", very_long_filename) + filename := datastore.AsDatastoreFilename( + self.datastore, self.config_obj, path) + + // Filename should be smaller than the read filename because it is + // compressed into a hash. + assert.True(self.T(), len(filename) < 250) + err := self.datastore.SetSubject( + self.config_obj, path, message) + assert.NoError(self.T(), err) + + read_message := &crypto_proto.VeloMessage{} + err = self.datastore.GetSubject(self.config_obj, + path, read_message) + assert.NoError(self.T(), err) + + assert.Equal(self.T(), message.Source, read_message.Source) + + // Now test that ListChildren works properly. + children, err := self.datastore.ListChildren( + self.config_obj, path_specs.NewUnsafeDatastorePath("longfiles")) + assert.NoError(self.T(), err) + + results := []string{} + for _, i := range children { + results = append(results, i.Base()) + + // Make sure the resulting filename is very long + assert.Equal(self.T(), i.Base(), very_long_filename) + } +} + func benchmarkSearchClient(b *testing.B, data_store datastore.DataStore, config_obj *config_proto.Config) { diff --git a/datastore/filebased.go b/datastore/filebased.go index ea5e3aa56ff..34f941a5f6b 100644 --- a/datastore/filebased.go +++ b/datastore/filebased.go @@ -82,8 +82,8 @@ func (self *FileBaseDataStore) GetSubject( defer InstrumentWithDelay("read", "FileBaseDataStore", urn)() - Trace(config_obj, "GetSubject", urn) - serialized_content, err := readContentFromFile(config_obj, urn) + Trace(self, config_obj, "GetSubject", urn) + serialized_content, err := readContentFromFile(self, config_obj, urn) if err != nil { return fmt.Errorf("While opening %v: %w", urn.AsClientPath(), os.ErrNotExist) @@ -150,7 +150,7 @@ func (self *FileBaseDataStore) SetSubjectWithCompletion( } }() - Trace(config_obj, "SetSubject", urn) + Trace(self, config_obj, "SetSubject", urn) // Encode as JSON if urn.Type() == api.PATH_TYPE_DATASTORE_JSON { @@ -158,14 +158,14 @@ func (self *FileBaseDataStore) SetSubjectWithCompletion( if err != nil { return err } - return writeContentToFile(config_obj, urn, serialized_content) + return writeContentToFile(self, config_obj, urn, serialized_content) } serialized_content, err := proto.Marshal(message) if err != nil { return errors.Wrap(err, 0) } - return writeContentToFile(config_obj, urn, serialized_content) + return writeContentToFile(self, config_obj, urn, serialized_content) } func (self *FileBaseDataStore) DeleteSubjectWithCompletion( @@ -187,9 +187,9 @@ func (self *FileBaseDataStore) DeleteSubject( defer InstrumentWithDelay("delete", "FileBaseDataStore", urn)() - Trace(config_obj, "DeleteSubject", urn) + Trace(self, config_obj, "DeleteSubject", urn) - err := os.Remove(urn.AsDatastoreFilename(config_obj)) + err := os.Remove(AsDatastoreFilename(self, config_obj, urn)) // It is ok to remove a file that does not exist. if err != nil && os.IsExist(err) { @@ -201,13 +201,13 @@ func (self *FileBaseDataStore) DeleteSubject( return nil } -func listChildren(config_obj *config_proto.Config, +func (self *FileBaseDataStore) listChildren(config_obj *config_proto.Config, urn api.DSPathSpec) ([]os.FileInfo, error) { defer InstrumentWithDelay("list", "FileBaseDataStore", urn)() children, err := utils.ReadDirUnsorted( - urn.AsDatastoreDirectory(config_obj)) + AsDatastoreDirectory(self, config_obj, urn)) if err != nil { if os.IsNotExist(err) { return []os.FileInfo{}, nil @@ -237,9 +237,9 @@ func (self *FileBaseDataStore) ListChildren( urn api.DSPathSpec) ( []api.DSPathSpec, error) { - TraceDirectory(config_obj, "ListChildren", urn) + TraceDirectory(self, config_obj, "ListChildren", urn) - all_children, err := listChildren(config_obj, urn) + all_children, err := self.listChildren(config_obj, urn) if err != nil { return nil, err } @@ -257,13 +257,18 @@ func (self *FileBaseDataStore) ListChildren( return children[i].ModTime().UnixNano() < children[j].ModTime().UnixNano() }) + db, err := GetDB(config_obj) + if err != nil { + return nil, err + } + // Slice the result according to the required offset and count. result := make([]api.DSPathSpec, 0, len(children)) for _, child := range children { var child_pathspec api.DSPathSpec if child.IsDir() { - name := utils.UnsanitizeComponent(child.Name()) + name := UncompressComponent(db, config_obj, child.Name()) result = append(result, urn.AddUnsafeChild(name).SetDir()) continue } @@ -275,7 +280,8 @@ func (self *FileBaseDataStore) ListChildren( continue } - name := utils.UnsanitizeComponent(child.Name()[:len(extension)]) + name := UncompressComponent(db, + config_obj, child.Name()[:len(extension)]) // Skip over files that do not belong in the data store. if spec_type == api.PATH_TYPE_DATASTORE_UNKNOWN { @@ -294,14 +300,15 @@ func (self *FileBaseDataStore) ListChildren( // Called to close all db handles etc. Not thread safe. func (self *FileBaseDataStore) Close() {} -func writeContentToFile(config_obj *config_proto.Config, +func writeContentToFile( + db DataStore, config_obj *config_proto.Config, urn api.DSPathSpec, data []byte) error { if config_obj.Datastore == nil { return datastoreNotConfiguredError } - filename := urn.AsDatastoreFilename(config_obj) + filename := AsDatastoreFilename(db, config_obj, urn) // Truncate the file immediately so we dont need to make a seocnd // syscall. Empirically on Linux, a truncate call always works, @@ -339,13 +346,14 @@ func writeContentToFile(config_obj *config_proto.Config, } func readContentFromFile( - config_obj *config_proto.Config, urn api.DSPathSpec) ([]byte, error) { + db DataStore, config_obj *config_proto.Config, + urn api.DSPathSpec) ([]byte, error) { if config_obj.Datastore == nil { return nil, datastoreNotConfiguredError } - file, err := os.Open(urn.AsDatastoreFilename(config_obj)) + file, err := os.Open(AsDatastoreFilename(db, config_obj, urn)) if err == nil { defer file.Close() @@ -363,9 +371,8 @@ func readContentFromFile( if os.IsNotExist(err) && urn.Type() == api.PATH_TYPE_DATASTORE_JSON { - file, err := os.Open(urn. - SetType(api.PATH_TYPE_DATASTORE_PROTO). - AsDatastoreFilename(config_obj)) + file, err := os.Open(AsDatastoreFilename( + db, config_obj, urn.SetType(api.PATH_TYPE_DATASTORE_PROTO))) if err == nil { defer file.Close() @@ -382,22 +389,25 @@ func readContentFromFile( return nil, errors.Wrap(err, 0) } -func Trace(config_obj *config_proto.Config, +func Trace( + db DataStore, + config_obj *config_proto.Config, name string, filename api.DSPathSpec) { return fmt.Printf("Trace FileBaseDataStore: %v: %v\n", name, - filename.AsDatastoreFilename(config_obj)) + AsDatastoreFilename(db, config_obj, filename)) } -func TraceDirectory(config_obj *config_proto.Config, +func TraceDirectory( + db DataStore, config_obj *config_proto.Config, name string, filename api.DSPathSpec) { return fmt.Printf("Trace FileBaseDataStore: %v: %v\n", name, - filename.AsDatastoreDirectory(config_obj)) + AsDatastoreDirectory(db, config_obj, filename)) } // Support RawDataStore interface @@ -405,7 +415,7 @@ func (self *FileBaseDataStore) GetBuffer( config_obj *config_proto.Config, urn api.DSPathSpec) ([]byte, error) { - return readContentFromFile(config_obj, urn) + return readContentFromFile(self, config_obj, urn) } func (self *FileBaseDataStore) Error() error { @@ -431,7 +441,7 @@ func (self *FileBaseDataStore) SetBuffer( return err } - err = writeContentToFile(config_obj, urn, data) + err = writeContentToFile(self, config_obj, urn, data) if completion != nil && !utils.CompareFuncs(completion, utils.SyncCompleter) { completion() diff --git a/datastore/filebased_generic.go b/datastore/filebased_generic.go index 9525e32a188..a9321e05108 100644 --- a/datastore/filebased_generic.go +++ b/datastore/filebased_generic.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package datastore diff --git a/datastore/filebased_test.go b/datastore/filebased_test.go index 6d91405f6e7..44edb460c01 100644 --- a/datastore/filebased_test.go +++ b/datastore/filebased_test.go @@ -78,8 +78,8 @@ func (self FilebasedTestSuite) TestFullDiskErrors() { assert.NoError(self.T(), err) // Fill the disk now - fd, err := os.OpenFile( - pad_path.AsDatastoreFilename(self.config_obj), + fd, err := os.OpenFile(datastore.AsDatastoreFilename( + self.datastore, self.config_obj, pad_path), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) assert.NoError(self.T(), err) fillUpDisk(fd) @@ -109,8 +109,9 @@ func (self FilebasedTestSuite) TestGetSubjectOfEmptyFileIsError() { path := path_specs.NewUnsafeDatastorePath("test") // Create an empty file - fd, err := os.OpenFile( - path.AsDatastoreFilename(self.config_obj), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) + fd, err := os.OpenFile(datastore.AsDatastoreFilename( + self.datastore, self.config_obj, path), + os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) assert.NoError(self.T(), err) fd.Close() @@ -131,6 +132,24 @@ func (self FilebasedTestSuite) TestSetGetJSON() { // self.DumpDirectory() } +// On linux maximum size of filename is 255 bytes. This means that +// with the addition of unicode escapes we might exceed this with even +// very short filenames. +func (self FilebasedTestSuite) TestVeryLongFilenameHashEncoding() { + very_long_filename := strings.Repeat("Very Long Filename", 100) + assert.Equal(self.T(), len(very_long_filename), 1800) + + path := path_specs.NewUnsafeDatastorePath("longfiles", very_long_filename) + filename := datastore.AsDatastoreFilename( + self.datastore, self.config_obj, path) + + // Filename should be smaller than the read filename because it is + // compressed into a hash. + assert.True(self.T(), len(filename) < 250) + assert.Equal(self.T(), filepath.Base(filename), + "#8ad0b37a7718f0403aa86f9c6bcfff35ef6ad39f.json.db") +} + func (self *FilebasedTestSuite) SetupTest() { var err error self.dirname, err = tempfile.TempDir("datastore_test") diff --git a/datastore/filebased_utils.go b/datastore/filebased_utils.go new file mode 100644 index 00000000000..1b436def466 --- /dev/null +++ b/datastore/filebased_utils.go @@ -0,0 +1,184 @@ +package datastore + +import ( + "crypto/sha1" + "fmt" + "path/filepath" + "runtime" + "strings" + + api_proto "www.velocidex.com/golang/velociraptor/api/proto" + config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/path_specs" + "www.velocidex.com/golang/velociraptor/utils" +) + +func AsFilestoreFilename( + db DataStore, config_obj *config_proto.Config, path api.FSPathSpec) string { + return AsFilestoreDirectory(db, config_obj, path) + + api.GetExtensionForFilestore(path) +} + +func AsFilestoreDirectory( + db DataStore, config_obj *config_proto.Config, + path api.FSPathSpec) string { + data_store_root := "" + if config_obj != nil && config_obj.Datastore != nil { + data_store_root = config_obj.Datastore.FilestoreDirectory + } + + if path.IsSafe() { + return asSafeDirWithRoot(config_obj, + path.AsDatastorePath(), data_store_root) + } + return asUnsafeDirWithRoot(db, config_obj, + path.AsDatastorePath(), data_store_root) +} + +func AsDatastoreDirectory( + db DataStore, config_obj *config_proto.Config, path api.DSPathSpec) string { + location := "" + if config_obj.Datastore != nil { + location = config_obj.Datastore.Location + } + + if path.IsSafe() { + return asSafeDirWithRoot(config_obj, path, location) + } + return asUnsafeDirWithRoot(db, config_obj, path, location) +} + +// When we are unsafe we need to Sanitize components hitting the +// filesystem. +func asUnsafeDirWithRoot( + db DataStore, + config_obj *config_proto.Config, + path api.DSPathSpec, root string) string { + sep := string(filepath.Separator) + new_components := make([]string, 0, len(path.Components())) + for _, i := range path.Components() { + if i != "" { + new_components = append(new_components, CompressComponent( + db, config_obj, i)) + } + } + result := sep + strings.Join(new_components, sep) + + // This relies on the filepath starting with a drive letter + // and having \ as path separators. Main's + // validateServerConfig() ensures this is the case. + if runtime.GOOS == "windows" { + return "\\\\?\\" + root + result + } + return root + result +} + +func AsDatastoreFilename( + db DataStore, config_obj *config_proto.Config, path api.DSPathSpec) string { + return AsDatastoreDirectory(db, config_obj, path) + + api.GetExtensionForDatastore(path) +} + +// If the path spec is already safe we can shortcut it and not +// sanitize. +func asSafeDirWithRoot( + config_obj *config_proto.Config, + path api.DSPathSpec, root string) string { + // No need to sanitize here because the DSPathSpec is already + // safe. + sep := string(filepath.Separator) + + // This relies on the filepath starting with a drive letter + // and having \ as path separators, and having a trailing + // \. Main's config.ValidateDatastoreConfig() ensures this is + // the case. + if runtime.GOOS == "windows" { + // Remove empty components which are broken on windows due to + // the long filename hack. + components := make([]string, 0, len(path.Components())) + for _, c := range path.Components() { + if c != "" { + components = append(components, c) + } + } + return WINDOWS_LFN_PREFIX + root + sep + + strings.Join(components, sep) + } + + return root + sep + strings.Join(path.Components(), sep) +} + +// This function is only really called when listing a directory - we +// find the hash compressed member and need to reconstruct its full +// name. +func UncompressComponent( + db DataStore, + config_obj *config_proto.Config, + component string) string { + if len(component) == 0 || component[0] != '#' { + return utils.UnsanitizeComponent(component) + } + + ds_pathspec := &api_proto.DSPathSpec{} + err := db.GetSubject(config_obj, + LFNCompressedHashPath(component), ds_pathspec) + if err != nil || len(ds_pathspec.Components) != 1 { + return component + } + + return ds_pathspec.Components[0] +} + +func CompressComponent( + db DataStore, + config_obj *config_proto.Config, + component string) string { + sanitized_component := utils.SanitizeString(component) + if len(sanitized_component) < 250 { + return sanitized_component + } + + // Hash compress the original component + hash := sha1.Sum([]byte(component)) + component_hash := fmt.Sprintf("#%x", hash) + db, err := GetDB(config_obj) + if err != nil { + return component_hash + } + + ds_pathspec := &api_proto.DSPathSpec{ + Components: []string{component}, + } + _ = db.SetSubject(config_obj, + LFNCompressedHashPath(component_hash), ds_pathspec) + return component_hash +} + +// Long file names are compressed into hashes and stored in the +// datastore. +func LFNCompressedHashPath(hash string) api.DSPathSpec { + res := path_specs.NewSafeDatastorePath("lfn_hashes"). + SetType(api.PATH_TYPE_DATASTORE_JSON) + + i := 0 + var part []byte + for _, c := range []byte(hash) { + if c == '#' { + continue + } + part = append(part, c) + i++ + + if i == 3 || i == 12 { + res = res.AddChild(string(part)) + part = nil + } + } + + if len(part) > 0 { + res = res.AddChild(string(part)) + } + + return res +} diff --git a/datastore/memcache.go b/datastore/memcache.go index 181e77c8b72..7068fdbaa19 100644 --- a/datastore/memcache.go +++ b/datastore/memcache.go @@ -11,7 +11,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/Velocidex/ttlcache/v2" "google.golang.org/protobuf/encoding/protojson" @@ -31,12 +30,19 @@ var ( errorNoDirectoryMetadata = errors.New("No Directory Metadata") ) +// Prometheus panics with multiple registrations which trigger on +// tests. +func registerGauge(g prometheus.Collector) { + prometheus.Unregister(g) + prometheus.Register(g) +} + func RegisterMemcacheDatastoreMetrics(db MemcacheStater) error { // These might return an error if they are called more than once, // but we assume under normal operation the config_obj does not // change, therefore the datastore does not really change. So it // is ok to ignore these errors. - _ = prometheus.Register(promauto.NewGaugeFunc( + registerGauge(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Name: "memcache_dir_lru_total", Help: "Total directories cached", @@ -45,7 +51,7 @@ func RegisterMemcacheDatastoreMetrics(db MemcacheStater) error { return float64(stats.DirItemCount) })) - _ = prometheus.Register(promauto.NewGaugeFunc( + registerGauge(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Name: "memcache_data_lru_total", Help: "Total files cached", @@ -54,7 +60,7 @@ func RegisterMemcacheDatastoreMetrics(db MemcacheStater) error { return float64(stats.DataItemCount) })) - _ = prometheus.Register(promauto.NewGaugeFunc( + registerGauge(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Name: "memcache_dir_lru_total_bytes", Help: "Total directories cached", @@ -63,7 +69,7 @@ func RegisterMemcacheDatastoreMetrics(db MemcacheStater) error { return float64(stats.DirItemSize) })) - _ = prometheus.Register(promauto.NewGaugeFunc( + registerGauge(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Name: "memcache_data_lru_total_bytes", Help: "Total bytes cached", @@ -300,7 +306,7 @@ type MemcacheDatastore struct { // errorNoDirectoryMetadata then we skip updating the metadata. get_dir_metadata func( dir_cache *DirectoryLRUCache, - config_obj *config_proto.Config, + db DataStore, config_obj *config_proto.Config, urn api.DSPathSpec) (*DirectoryMetadata, error) } @@ -308,11 +314,11 @@ type MemcacheDatastore struct { // return a DirectoryMetadata object for the urn. func get_dir_metadata( dir_cache *DirectoryLRUCache, - config_obj *config_proto.Config, urn api.DSPathSpec) ( + db DataStore, config_obj *config_proto.Config, urn api.DSPathSpec) ( *DirectoryMetadata, error) { // Check if the top level directory contains metadata. - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(db, config_obj, urn) md, pres := dir_cache.Get(path) if pres { return md, nil @@ -324,7 +330,7 @@ func get_dir_metadata( for len(urn.Components()) > 0 { parent := urn.Dir() - path := parent.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(db, config_obj, parent) intermediate_md, ok := dir_cache.Get(path) if !ok { @@ -358,7 +364,7 @@ func (self *MemcacheDatastore) GetSubject( defer Instrument("read", "MemcacheDatastore", urn)() - path := urn.AsDatastoreFilename(config_obj) + path := AsDatastoreFilename(self, config_obj, urn) bulk_data_any, err := self.data_cache.Get(path) if err != nil { // Second try the old DB without json. This supports @@ -367,7 +373,8 @@ func (self *MemcacheDatastore) GetSubject( // read old files. if urn.Type() == api.PATH_TYPE_DATASTORE_JSON { bulk_data_any, err = self.data_cache.Get( - urn.SetType(api.PATH_TYPE_DATASTORE_PROTO).AsDatastoreFilename(config_obj)) + AsDatastoreFilename(self, config_obj, + urn.SetType(api.PATH_TYPE_DATASTORE_PROTO))) } if err != nil { @@ -476,16 +483,17 @@ func (self *MemcacheDatastore) SetData( config_obj *config_proto.Config, urn api.DSPathSpec, data []byte) (err error) { - err = self.data_cache.Set(urn.AsDatastoreFilename(config_obj), &BulkData{ - data: data, - }) + err = self.data_cache.Set( + AsDatastoreFilename(self, config_obj, urn), &BulkData{ + data: data, + }) if err != nil { return err } // Try to update the DirectoryMetadata cache if possible. parent := urn.Dir() - md, err := self.get_dir_metadata(self.dir_cache, config_obj, parent) + md, err := self.get_dir_metadata(self.dir_cache, self, config_obj, parent) if err == errorNoDirectoryMetadata { return nil } @@ -515,13 +523,14 @@ func (self *MemcacheDatastore) DeleteSubject( urn api.DSPathSpec) error { defer Instrument("delete", "MemcacheDatastore", urn)() - err := self.data_cache.Remove(urn.AsDatastoreFilename(config_obj)) + err := self.data_cache.Remove( + AsDatastoreFilename(self, config_obj, urn)) if err != nil { return utils.Wrap(utils.NotFoundError, "DeleteSubject") } // Try to remove it from the DirectoryMetadata if it exists. - md, err := self.get_dir_metadata(self.dir_cache, config_obj, urn.Dir()) + md, err := self.get_dir_metadata(self.dir_cache, self, config_obj, urn.Dir()) // No DirectoryMetadata, nothing to do. if err == errorNoDirectoryMetadata { @@ -540,7 +549,7 @@ func (self *MemcacheDatastore) SetChildren( config_obj *config_proto.Config, urn api.DSPathSpec, children []api.DSPathSpec) { - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(self, config_obj, urn) md, pres := self.dir_cache.Get(path) if !pres { md = self.dir_cache.NewDirectoryMetadata(path) @@ -565,7 +574,7 @@ func (self *MemcacheDatastore) ListChildren( defer Instrument("list", "MemcacheDatastore", urn)() - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(self, config_obj, urn) md, pres := self.dir_cache.Get(path) if !pres { return nil, nil @@ -615,7 +624,7 @@ func (self *MemcacheDatastore) GetForTests( func (self *MemcacheDatastore) GetBuffer( config_obj *config_proto.Config, urn api.DSPathSpec) ([]byte, error) { - path := urn.AsDatastoreFilename(config_obj) + path := AsDatastoreFilename(self, config_obj, urn) bulk_data_any, err := self.data_cache.Get(path) bulk_data, ok := bulk_data_any.(*BulkData) if !ok { @@ -661,7 +670,7 @@ func (self *MemcacheDatastore) Dump() []api.DSPathSpec { func (self *MemcacheDatastore) SetDirLoader(cb func( dir_cache *DirectoryLRUCache, - config_obj *config_proto.Config, + db DataStore, config_obj *config_proto.Config, urn api.DSPathSpec) (*DirectoryMetadata, error)) { self.get_dir_metadata = cb } diff --git a/datastore/memcache_file.go b/datastore/memcache_file.go index 350d694884d..b64b2c382e2 100644 --- a/datastore/memcache_file.go +++ b/datastore/memcache_file.go @@ -133,10 +133,10 @@ func (self *MemcacheFileDataStore) invalidateDirCache( config_obj *config_proto.Config, urn api.DSPathSpec) { for len(urn.Components()) > 0 { - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(self, config_obj, urn) md, pres := self.cache.dir_cache.Get(path) if pres && !md.IsFull() { - key_path := urn.AsDatastoreDirectory(config_obj) + key_path := AsDatastoreDirectory(self, config_obj, urn) self.cache.dir_cache.Remove(key_path) } urn = urn.Dir() @@ -232,7 +232,7 @@ func (self *MemcacheFileDataStore) processMutation(mutation *Mutation) { metricIdleWriters.Dec() switch mutation.op { case MUTATION_OP_SET_SUBJECT: - writeContentToFile(mutation.org_config_obj, mutation.urn, mutation.data) + writeContentToFile(self, mutation.org_config_obj, mutation.urn, mutation.data) self.invalidateDirCache(mutation.org_config_obj, mutation.urn) // Call the completion function once we hit @@ -272,7 +272,7 @@ func (self *MemcacheFileDataStore) GetSubject( if errors.Is(err, os.ErrNotExist) { // The file is not in the cache, read it from the file system // instead. - serialized_content, err := readContentFromFile(config_obj, urn) + serialized_content, err := readContentFromFile(self, config_obj, urn) if err != nil { return err } @@ -404,7 +404,7 @@ func (self *MemcacheFileDataStore) SetSubject( return err } - err = writeContentToFile(config_obj, urn, serialized_content) + err = writeContentToFile(self, config_obj, urn, serialized_content) if err != nil { return err } @@ -548,7 +548,7 @@ func (self *MemcacheFileDataStore) GetBuffer( return bulk_data, err } - bulk_data, err = readContentFromFile(config_obj, urn) + bulk_data, err = readContentFromFile(self, config_obj, urn) if err != nil { return nil, err } @@ -598,11 +598,11 @@ func (self *MemcacheFileDataStore) SetBuffer( // Recursively makes sure the directories are added to the cache. func get_file_dir_metadata( dir_cache *DirectoryLRUCache, - config_obj *config_proto.Config, urn api.DSPathSpec) ( + db DataStore, config_obj *config_proto.Config, urn api.DSPathSpec) ( *DirectoryMetadata, error) { // Check if the top level directory contains metadata. - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(db, config_obj, urn) // Fast path - the directory exists in the cache. NOTE: We dont // need to maintain the directories on the filesystem as the @@ -626,10 +626,10 @@ func get_file_dir_metadata( // perform a filesystem op and fill in the cache if needed. urn = urn.Dir() for len(urn.Components()) > 0 { - path := urn.AsDatastoreDirectory(config_obj) + path := AsDatastoreDirectory(db, config_obj, urn) md, pres := dir_cache.Get(path) if pres && !md.IsFull() { - key_path := urn.AsDatastoreDirectory(config_obj) + key_path := AsDatastoreDirectory(db, config_obj, urn) dir_cache.Remove(key_path) } urn = urn.Dir() diff --git a/datastore/utils.go b/datastore/utils.go index 0385ead3ba8..f6dc2393afe 100644 --- a/datastore/utils.go +++ b/datastore/utils.go @@ -76,7 +76,7 @@ func Walk(config_obj *config_proto.Config, with_directories bool, walkFn WalkFunc) error { - TraceDirectory(config_obj, "Walk", root) + TraceDirectory(datastore, config_obj, "Walk", root) all_children, err := datastore.ListChildren(config_obj, root) if err != nil { return err diff --git a/file_store/api/paths.go b/file_store/api/paths.go index 5c35e74e062..c1fe937d956 100644 --- a/file_store/api/paths.go +++ b/file_store/api/paths.go @@ -1,9 +1,5 @@ package api -import ( - config_proto "www.velocidex.com/golang/velociraptor/config/proto" -) - /* # How paths are handled in Velociraptor. @@ -117,12 +113,13 @@ type _PathSpec interface { Type() PathType String() string + + // Does any of the components need escaping? + IsSafe() bool } type DSPathSpec interface { _PathSpec - AsDatastoreDirectory(config_obj *config_proto.Config) string - AsDatastoreFilename(config_obj *config_proto.Config) string Dir() DSPathSpec @@ -147,8 +144,6 @@ type DSPathSpec interface { type FSPathSpec interface { _PathSpec - AsFilestoreFilename(config_obj *config_proto.Config) string - AsFilestoreDirectory(config_obj *config_proto.Config) string Dir() FSPathSpec diff --git a/file_store/directory/directory.go b/file_store/directory/directory.go index e5769e5b97b..08d71a73bff 100644 --- a/file_store/directory/directory.go +++ b/file_store/directory/directory.go @@ -35,6 +35,7 @@ import ( "github.com/go-errors/errors" "www.velocidex.com/golang/velociraptor/accessors/file_store_file_info" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store/api" logging "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/utils" @@ -96,15 +97,23 @@ func (self *DirectoryFileWriter) Close() error { type DirectoryFileStore struct { config_obj *config_proto.Config + db datastore.DataStore } func NewDirectoryFileStore(config_obj *config_proto.Config) *DirectoryFileStore { - return &DirectoryFileStore{config_obj} + db, err := datastore.GetDB(config_obj) + if err != nil { + return nil + } + return &DirectoryFileStore{ + config_obj: config_obj, + db: db, + } } func (self *DirectoryFileStore) Move(src, dest api.FSPathSpec) error { - src_path := src.AsFilestoreFilename(self.config_obj) - dest_path := dest.AsFilestoreFilename(self.config_obj) + src_path := datastore.AsFilestoreFilename(self.db, self.config_obj, src) + dest_path := datastore.AsFilestoreFilename(self.db, self.config_obj, dest) return os.Rename(src_path, dest_path) } @@ -118,7 +127,8 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) ( defer api.InstrumentWithDelay("list", "DirectoryFileStore", dirname)() - file_path := dirname.AsFilestoreDirectory(self.config_obj) + file_path := datastore.AsFilestoreDirectory( + self.db, self.config_obj, dirname) files, err := utils.ReadDir(file_path) if err != nil { return nil, err @@ -139,7 +149,8 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) ( result = append(result, file_store_file_info.NewFileStoreFileInfo( self.config_obj, dirname.AddUnsafeChild( - utils.UnsanitizeComponent(name)). + datastore.UncompressComponent( + self.db, self.config_obj, name)). SetType(name_type), fileinfo)) } @@ -149,7 +160,8 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) ( func (self *DirectoryFileStore) ReadFile( filename api.FSPathSpec) (api.FileReader, error) { - file_path := filename.AsFilestoreFilename(self.config_obj) + file_path := datastore.AsFilestoreFilename( + self.db, self.config_obj, filename) defer api.InstrumentWithDelay("open_read", "DirectoryFileStore", filename)() @@ -168,7 +180,8 @@ func (self *DirectoryFileStore) StatFile( defer api.Instrument("stat", "DirectoryFileStore", filename)() - file_path := filename.AsFilestoreFilename(self.config_obj) + file_path := datastore.AsFilestoreFilename( + self.db, self.config_obj, filename) file, err := os.Stat(file_path) if err != nil { return nil, err @@ -188,7 +201,8 @@ func (self *DirectoryFileStore) WriteFileWithCompletion( defer api.InstrumentWithDelay("open_write", "DirectoryFileStore", filename)() - file_path := filename.AsFilestoreFilename(self.config_obj) + file_path := datastore.AsFilestoreFilename( + self.db, self.config_obj, filename) err := os.MkdirAll(filepath.Dir(file_path), 0700) if err != nil { logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent) @@ -214,7 +228,8 @@ func (self *DirectoryFileStore) Delete(filename api.FSPathSpec) error { defer api.InstrumentWithDelay("delete", "DirectoryFileStore", filename)() - file_path := filename.AsFilestoreFilename(self.config_obj) + file_path := datastore.AsFilestoreFilename( + self.db, self.config_obj, filename) err := os.Remove(file_path) if err != nil { return err diff --git a/file_store/memory/memory.go b/file_store/memory/memory.go index 55a46672a1c..d830cdd0a67 100644 --- a/file_store/memory/memory.go +++ b/file_store/memory/memory.go @@ -12,6 +12,7 @@ import ( "github.com/Velocidex/ordereddict" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/path_specs" "www.velocidex.com/golang/velociraptor/utils" @@ -34,10 +35,16 @@ func NewMemoryFileStore(config_obj *config_proto.Config) *MemoryFileStore { mu.Lock() defer mu.Unlock() + db, err := datastore.GetDB(config_obj) + if err != nil { + return nil + } + if Test_memory_file_store == nil { Test_memory_file_store = &MemoryFileStore{ Data: ordereddict.NewDict(), Paths: ordereddict.NewDict(), + db: db, config_obj: config_obj, } } @@ -234,6 +241,7 @@ type MemoryFileStore struct { config_obj *config_proto.Config Data *ordereddict.Dict Paths *ordereddict.Dict + db datastore.DataStore } func (self *MemoryFileStore) Debug() { @@ -268,7 +276,7 @@ func (self *MemoryFileStore) ReadFile(path api.FSPathSpec) (api.FileReader, erro self.mu.Lock() defer self.mu.Unlock() - filename := pathSpecToPath(path, self.config_obj) + filename := pathSpecToPath(self.db, self.config_obj, path) self.Trace("ReadFile", filename) _, pres := self.Data.Get(filename) if pres { @@ -294,7 +302,7 @@ func (self *MemoryFileStore) WriteFileWithCompletion( self.mu.Lock() defer self.mu.Unlock() - filename := pathSpecToPath(path, self.config_obj) + filename := pathSpecToPath(self.db, self.config_obj, path) self.Trace("WriteFile", filename) buf, pres := self.Data.Get(filename) if !pres { @@ -317,7 +325,7 @@ func (self *MemoryFileStore) StatFile(path api.FSPathSpec) (api.FileInfo, error) self.mu.Lock() defer self.mu.Unlock() - filename := pathSpecToPath(path, self.config_obj) + filename := pathSpecToPath(self.db, self.config_obj, path) self.Trace("StatFile", filename) buff, pres := self.Data.Get(filename) if !pres { @@ -337,8 +345,8 @@ func (self *MemoryFileStore) Move(src, dest api.FSPathSpec) error { self.mu.Lock() defer self.mu.Unlock() - src_filename := pathSpecToPath(src, self.config_obj) - dest_filename := pathSpecToPath(dest, self.config_obj) + src_filename := pathSpecToPath(self.db, self.config_obj, src) + dest_filename := pathSpecToPath(self.db, self.config_obj, dest) buff, pres := self.Data.Get(src_filename) if !pres { return os.ErrNotExist @@ -355,7 +363,7 @@ func (self *MemoryFileStore) ListDirectory(root_path api.FSPathSpec) ([]api.File self.mu.Lock() defer self.mu.Unlock() - dirname := pathDirSpecToPath(root_path, self.config_obj) + dirname := pathDirSpecToPath(self.db, self.config_obj, root_path) self.Trace("ListDirectory", dirname) root_components := root_path.Components() @@ -440,7 +448,7 @@ func (self *MemoryFileStore) Delete(path api.FSPathSpec) error { self.mu.Lock() defer self.mu.Unlock() - filename := pathSpecToPath(path, self.config_obj) + filename := pathSpecToPath(self.db, self.config_obj, path) self.Trace("Delete", filename) self.Data.Delete(filename) self.Paths.Delete(filename) @@ -481,8 +489,9 @@ func (self *MemoryFileStore) Close() error { } func pathSpecToPath( - p api.FSPathSpec, config_obj *config_proto.Config) string { - return cleanPathForWindows(p.AsFilestoreFilename(config_obj)) + db datastore.DataStore, + config_obj *config_proto.Config, p api.FSPathSpec) string { + return cleanPathForWindows(datastore.AsFilestoreFilename(db, config_obj, p)) } func cleanPathForWindows(result string) string { @@ -496,7 +505,9 @@ func cleanPathForWindows(result string) string { return result } -func pathDirSpecToPath(p api.FSPathSpec, - config_obj *config_proto.Config) string { - return cleanPathForWindows(p.AsFilestoreDirectory(config_obj)) +func pathDirSpecToPath( + db datastore.DataStore, + config_obj *config_proto.Config, p api.FSPathSpec) string { + return cleanPathForWindows( + datastore.AsFilestoreDirectory(db, config_obj, p)) } diff --git a/file_store/path_specs/fs_path_spec.go b/file_store/path_specs/fs_path_spec.go index 7c1be67ad9a..bb821cd7ea4 100644 --- a/file_store/path_specs/fs_path_spec.go +++ b/file_store/path_specs/fs_path_spec.go @@ -3,7 +3,6 @@ package path_specs import ( "strconv" - config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/utils" ) @@ -72,25 +71,6 @@ func (self FSPathSpec) SetType(ext api.PathType) api.FSPathSpec { }} } -func (self FSPathSpec) AsFilestoreFilename( - config_obj *config_proto.Config) string { - return self.AsFilestoreDirectory(config_obj) + - api.GetExtensionForFilestore(self) -} - -func (self FSPathSpec) AsFilestoreDirectory( - config_obj *config_proto.Config) string { - data_store_root := "" - if config_obj != nil && config_obj.Datastore != nil { - data_store_root = config_obj.Datastore.FilestoreDirectory - } - - if self.is_safe { - return self.asSafeDirWithRoot(data_store_root) - } - return self.asUnsafeDirWithRoot(data_store_root) -} - func (self FSPathSpec) AsClientPath() string { return utils.JoinComponents(self.components, "/") + api.GetExtensionForFilestore(self) diff --git a/file_store/path_specs/path_specs.go b/file_store/path_specs/path_specs.go index 6a23630eab3..7bc5af0fb44 100644 --- a/file_store/path_specs/path_specs.go +++ b/file_store/path_specs/path_specs.go @@ -1,12 +1,8 @@ package path_specs import ( - "path/filepath" - "runtime" "strconv" - "strings" - config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/utils" ) @@ -35,6 +31,10 @@ func (self DSPathSpec) IsDir() bool { return self.is_dir } +func (self DSPathSpec) IsSafe() bool { + return self.is_safe +} + func (self DSPathSpec) SetDir() api.DSPathSpec { self.is_dir = true self.path_type = api.PATH_TYPE_DATASTORE_DIRECTORY @@ -112,46 +112,6 @@ func (self DSPathSpec) AsClientPath() string { api.GetExtensionForDatastore(self) } -func (self DSPathSpec) AsDatastoreDirectory( - config_obj *config_proto.Config) string { - location := "" - if config_obj.Datastore != nil { - location = config_obj.Datastore.Location - } - - if self.is_safe { - return self.asSafeDirWithRoot(location) - } - return self.asUnsafeDirWithRoot(location) -} - -// When we are unsafe we need to Sanitize components hitting the -// filesystem. -func (self DSPathSpec) asUnsafeDirWithRoot(root string) string { - sep := string(filepath.Separator) - new_components := make([]string, 0, len(self.components)) - for _, i := range self.components { - if i != "" { - new_components = append(new_components, utils.SanitizeString(i)) - } - } - result := sep + strings.Join(new_components, sep) - - // This relies on the filepath starting with a drive letter - // and having \ as path separators. Main's - // validateServerConfig() ensures this is the case. - if runtime.GOOS == "windows" { - return "\\\\?\\" + root + result - } - return root + result -} - -func (self DSPathSpec) AsDatastoreFilename( - config_obj *config_proto.Config) string { - return self.AsDatastoreDirectory(config_obj) + - api.GetExtensionForDatastore(self) -} - func (self DSPathSpec) AsFilestorePath() api.FSPathSpec { return &FSPathSpec{DSPathSpec: DSPathSpec{ components: self.components, @@ -180,29 +140,3 @@ func NewSafeDatastorePath(path_components ...string) DSPathSpec { return result } - -// If the path spec is already safe we can shortcut it and not -// sanitize. -func (self DSPathSpec) asSafeDirWithRoot(root string) string { - // No need to sanitize here because the DSPathSpec is already - // safe. - sep := string(filepath.Separator) - - // This relies on the filepath starting with a drive letter - // and having \ as path separators, and having a trailing - // \. Main's config.ValidateDatastoreConfig() ensures this is - // the case. - if runtime.GOOS == "windows" { - // Remove empty components which are broken on windows due to - // the long filename hack. - components := make([]string, 0, len(self.components)) - for _, c := range self.components { - if c != "" { - components = append(components, c) - } - } - return WINDOWS_LFN_PREFIX + root + sep + strings.Join(components, sep) - } - - return root + sep + strings.Join(self.components, sep) -} diff --git a/file_store/tests/testsuite.go b/file_store/tests/testsuite.go index 7ef602b0f42..f626758e4d3 100644 --- a/file_store/tests/testsuite.go +++ b/file_store/tests/testsuite.go @@ -202,8 +202,7 @@ func (self *FileStoreTestSuite) TestListDirectory() { names = nil err = api.Walk(self.filestore, filename.AddChild("nonexistant"), func(path api.FSPathSpec, info os.FileInfo) error { - names = append(names, path.AsFilestoreFilename( - self.config_obj)) + names = append(names, path.String()) return nil }) assert.NoError(self.T(), err) diff --git a/paths/artifacts/paths_test.go b/paths/artifacts/paths_test.go index b2ced335921..eb700cbf086 100644 --- a/paths/artifacts/paths_test.go +++ b/paths/artifacts/paths_test.go @@ -11,6 +11,7 @@ import ( "github.com/Velocidex/ordereddict" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store/memory" "www.velocidex.com/golang/velociraptor/file_store/path_specs" "www.velocidex.com/golang/velociraptor/file_store/test_utils" @@ -97,6 +98,9 @@ func (self *PathManageTestSuite) TestPathManager() { closer := utils.MockTime(utils.NewMockClock(time.Unix(ts, 0))) defer closer() + db, err := datastore.GetDB(self.ConfigObj) + assert.NoError(self.T(), err) + for _, testcase := range path_tests { path_manager, err := artifacts.NewArtifactPathManager( self.Ctx, self.ConfigObj, @@ -108,7 +112,8 @@ func (self *PathManageTestSuite) TestPathManager() { path, err := path_manager.GetPathForWriting() assert.NoError(self.T(), err) assert.Equal(self.T(), - cleanPath(path.AsFilestoreFilename(self.ConfigObj)), + cleanPath(datastore.AsFilestoreFilename( + db, self.ConfigObj, path)), cleanPath(self.dirname+"/"+testcase.expected)) file_store := memory.NewMemoryFileStore(self.ConfigObj) diff --git a/paths/paths_test.go b/paths/paths_test.go index 1147cb1dbee..d38d6360afc 100644 --- a/paths/paths_test.go +++ b/paths/paths_test.go @@ -59,7 +59,7 @@ func (self *PathManagerTestSuite) getDatastorePath(path_spec api.DSPathSpec) str continue } results = append(results, normalize_path( - k.AsDatastoreFilename(self.config_obj))) + datastore.AsDatastoreFilename(ds, self.config_obj, k))) } assert.Equal(self.T(), 1, len(results)) diff --git a/server/server_test.go b/server/server_test.go index e6ee1cc6cf0..75bc2841c7b 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -478,8 +478,11 @@ func (self *ServerTestSuite) RequiredFilestoreContains( file_store_factory := test_utils.GetMemoryFileStore(self.T(), self.ConfigObj) - value, pres := file_store_factory.Get(filename.AsFilestoreFilename( - self.ConfigObj)) + db, err := datastore.GetDB(self.ConfigObj) + assert.NoError(self.T(), err) + + value, pres := file_store_factory.Get(datastore.AsFilestoreFilename( + db, self.ConfigObj, filename)) if !pres { self.T().FailNow() } diff --git a/services/repository/manager_test.go b/services/repository/manager_test.go index fa37ef22406..48893588c24 100644 --- a/services/repository/manager_test.go +++ b/services/repository/manager_test.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/proto" artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store/test_utils" "www.velocidex.com/golang/velociraptor/paths/artifacts" "www.velocidex.com/golang/velociraptor/services" @@ -62,8 +63,11 @@ name: TestArtifact self.ConfigObj, "", "", "Server.Internal.ArtifactModification") assert.NoError(self.T(), err) + db, err := datastore.GetDB(self.ConfigObj) + assert.NoError(self.T(), err) + data, pres = file_store.Get( - path_manager.Path().AsFilestoreFilename(self.ConfigObj)) + datastore.AsFilestoreFilename(db, self.ConfigObj, path_manager.Path())) assert.True(self.T(), pres) assert.Contains(self.T(), string(data), `"op":"set"`) diff --git a/services/repository/plugin_test.go b/services/repository/plugin_test.go index ecf4b7ae48f..1ae6800bd6b 100644 --- a/services/repository/plugin_test.go +++ b/services/repository/plugin_test.go @@ -1,23 +1,24 @@ /* - Velociraptor - Dig Deeper - Copyright (C) 2019-2024 Rapid7 Inc. +Velociraptor - Dig Deeper +Copyright (C) 2019-2024 Rapid7 Inc. - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package repository_test import ( + "strings" "testing" "time" @@ -53,10 +54,16 @@ func (self *PluginTestSuite) TestArtifactsSyntax() { manager, err := services.GetRepositoryManager(self.ConfigObj) assert.NoError(self.T(), err) + logging.ClearMemoryLogs() + err = repository.LoadBuiltInArtifacts( self.Ctx, self.ConfigObj, manager.(*repository.RepositoryManager)) assert.NoError(self.T(), err) + assert.NotContains(self.T(), + strings.Join(logging.GetMemoryLogs(), "\n"), + "Cant parse asset") + ConfigObj := self.ConfigObj repository, err := manager.GetGlobalRepository(ConfigObj) assert.NoError(self.T(), err) diff --git a/utils/sanitize.go b/utils/sanitize.go index f839914b47d..98322b4a0a4 100644 --- a/utils/sanitize.go +++ b/utils/sanitize.go @@ -50,6 +50,12 @@ func SanitizeString(component string) string { return "%2E" + SanitizeString(component[1:]) } + // Escape components that start with # as the data store + // represents those as hashes. + if component[0] == '#' { + return "%23" + SanitizeString(component[1:]) + } + // Windows can not have a trailing "." instead swallowing it // completely. if component[length-1] == '.' { diff --git a/vql/server/file_store.go b/vql/server/file_store.go index 646f25af17e..e6d19f1a40c 100644 --- a/vql/server/file_store.go +++ b/vql/server/file_store.go @@ -172,36 +172,43 @@ func (self *FileStore) Call(ctx context.Context, return vfilter.Null{} } + db, err := datastore.GetDB(config_obj) + if err != nil { + scope.Log("file_store: %v", err) + return vfilter.Null{} + } + vfs_path := arg.VFSPath.Reduce(ctx) switch t := vfs_path.(type) { case *path_specs.FSPathSpec: - return t.AsFilestoreFilename(config_obj) + return datastore.AsFilestoreFilename(db, config_obj, t) case path_specs.FSPathSpec: - return t.AsFilestoreFilename(config_obj) + return datastore.AsFilestoreFilename(db, config_obj, t) case *path_specs.DSPathSpec: - return t.AsDatastoreFilename(config_obj) + return datastore.AsDatastoreFilename(db, config_obj, t) case path_specs.DSPathSpec: - return t.AsDatastoreFilename(config_obj) + return datastore.AsDatastoreFilename(db, config_obj, t) case *accessors.OSPath: - return path_specs.NewUnsafeFilestorePath(t.Components...).AsFilestoreFilename(config_obj) + return datastore.AsFilestoreFilename(db, config_obj, + path_specs.NewUnsafeFilestorePath(t.Components...)) case string: // Things that produce strings normally encode the path spec // with a prefix to let us know if this is a data store path // or a filestore path.. if strings.HasPrefix(t, "ds:") { - return paths.DSPathSpecFromClientPath( - strings.TrimPrefix(t, "ds:")). - AsDatastoreFilename(config_obj) + return datastore.AsDatastoreFilename(db, config_obj, + paths.DSPathSpecFromClientPath( + strings.TrimPrefix(t, "ds:"))) } - return paths.FSPathSpecFromClientPath( - strings.TrimPrefix(t, "fs:")). - AsFilestoreFilename(config_obj) + return datastore.AsFilestoreFilename(db, config_obj, + paths.FSPathSpecFromClientPath( + strings.TrimPrefix(t, "fs:"))) } return vfilter.Null{}