Entry cache component

CHANGELOG.md

@@ -1,4 +1,7 @@
 ## 2.4.0 (Unreleased)
+**Features**
+- Entry cache to hold directory listing results in cache for a given timeout. This will reduce REST calls going to storage while listing the blobs in parallel.
+
 **Bug Fixes**
 - [#1426](https://github.com/Azure/azure-storage-fuse/issues/1426) Read panic in block-cache due to boundary conditions.
 

MIGRATION.md

@@ -99,7 +99,7 @@ Note: Blobfuse2 accepts all CLI parameters that Blobfuse does, but may ignore pa
 | --log-level=LOG_WARNING                 | --log-level=LOG_WARNING                       | logging.level                         |                                                           |
 | --use-attr-cache=true                   | --use-attr-cache=true                         | attr_cache                            | Add attr_cache to the components list                     |
 | --use-adls=false                        | --use-adls=false                              | azstorage.type                        | Specify either 'block' or 'adls'                          |
-| --no-symlinks=false                     | --no-symlinks=false                           | attr_cache.no-symlinks                |                                                           |
+| --no-symlinks=false                     | --no-symlinks=true                            | attr_cache.no-symlinks                |                                                           |
 | --cache-on-list=true                    | --cache-on-list=true                          | attr_cache.no-cache-on-list           | This parameter has the opposite boolean semantics         |
 | --upload-modified-only=true             | --upload-modified-only=true                   |                                       | Always on in blobfuse2                                    |
 | --max-concurrency=12                    | --max-concurrency=12                          | azstorage.max-concurrency             |                                                           |

blobfuse2-release.yaml

@@ -1697,6 +1697,13 @@ stages:
                   echo "##vso[task.setvariable variable=is_preview]$is_preview"
                 fi
 
+                is_preview="false"
+                echo "##vso[task.setvariable variable=is_preview]$is_preview"
+                if [[ $marinerFuse3AmdRpm == *"preview"* ]]; then
+                  is_preview="true"
+                  echo "##vso[task.setvariable variable=is_preview]$is_preview"
+                fi
+
                 while IFS=, read -r distro fuseArchType repoName releaseName; do
 
                   # If the package is preview, publish to mariner preview package

cmd/imports.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/Azure/azure-storage-fuse/v2/component/azstorage"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/block_cache"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/custom"
+	_ "github.com/Azure/azure-storage-fuse/v2/component/entry_cache"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/file_cache"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/libfuse"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/loopback"

cmd/mount.go

@@ -94,6 +94,7 @@ type mountOptions struct {
 	AttrCache      bool     `config:"use-attr-cache"`
 	LibfuseOptions []string `config:"libfuse-options"`
 	BlockCache     bool     `config:"block-cache"`
+	EntryCache     int      `config:"list-cache-timeout"`
 }
 
 var options mountOptions
@@ -313,6 +314,10 @@ var mountCmd = &cobra.Command{
 			options.Components = pipeline
 		}
 
+		if config.IsSet("entry_cache.timeout-sec") || options.EntryCache > 0 {
+			options.Components = append(options.Components[:1], append([]string{"entry_cache"}, options.Components[1:]...)...)
+		}
+
 		if config.IsSet("libfuse-options") {
 			for _, v := range options.LibfuseOptions {
 				parameter := strings.Split(v, "=")

common/types.go

@@ -47,7 +47,7 @@ import (
 
 // Standard config default values
 const (
-	blobfuse2Version_ = "2.3.3"
+	blobfuse2Version_ = "2.4.0"
 
 	DefaultMaxLogFileSize = 512
 	DefaultLogFileCount   = 10

component/attr_cache/attr_cache.go

@@ -55,7 +55,6 @@ const defaultAttrCacheTimeout uint32 = (120)
 type AttrCache struct {
 	internal.BaseComponent
 	cacheTimeout uint32
-	cacheOnList  bool
 	noSymlinks   bool
 	maxFiles     int
 	cacheMap     map[string]*attrCacheItem
@@ -139,22 +138,18 @@ func (ac *AttrCache) Configure(_ bool) error {
 		ac.cacheTimeout = defaultAttrCacheTimeout
 	}
 
-	if config.IsSet(compName + ".cache-on-list") {
-		ac.cacheOnList = conf.CacheOnList
-	} else {
-		ac.cacheOnList = !conf.NoCacheOnList
-	}
-
 	if config.IsSet(compName + ".max-files") {
 		ac.maxFiles = conf.MaxFiles
 	} else {
 		ac.maxFiles = defaultMaxFiles
 	}
 
-	ac.noSymlinks = conf.NoSymlinks
+	if config.IsSet(compName + ".no-symlinks") {
+		ac.noSymlinks = conf.NoSymlinks
+	}
 
-	log.Info("AttrCache::Configure : cache-timeout %d, symlink %t, cache-on-list %t, max-files %d",
-		ac.cacheTimeout, ac.noSymlinks, ac.cacheOnList, ac.maxFiles)
+	log.Info("AttrCache::Configure : cache-timeout %d, symlink %t, max-files %d",
+		ac.cacheTimeout, ac.noSymlinks, ac.maxFiles)
 
 	return nil
 }
@@ -283,7 +278,7 @@ func (ac *AttrCache) StreamDir(options internal.StreamDirOptions) ([]*internal.O
 // cacheAttributes : On dir listing cache the attributes for all files
 func (ac *AttrCache) cacheAttributes(pathList []*internal.ObjAttr) {
 	// Check whether or not we are supposed to cache on list
-	if ac.cacheOnList && len(pathList) > 0 {
+	if len(pathList) > 0 {
 		// Putting this inside loop is heavy as for each item we will do a kernel call to get current time
 		// If there are millions of blobs then cost of this is very high.
 		currTime := time.Now()
@@ -478,14 +473,8 @@ func (ac *AttrCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr
 			// no entry if path does not exist
 			return &internal.ObjAttr{}, syscall.ENOENT
 		} else {
-			// IsMetadataRetrieved is false in the case of ADLS List since the API does not support metadata.
-			// Once migration of ADLS list to blob endpoint is done (in future service versions), we can remove this.
-			// options.RetrieveMetadata is set by CopyFromFile and WriteFile which need metadata to ensure it is preserved.
-			if value.getAttr().IsMetadataRetrieved() || (ac.noSymlinks && !options.RetrieveMetadata) {
-				// path exists and we have all the metadata required or we do not care about metadata
-				log.Debug("AttrCache::GetAttr : %s served from cache", options.Name)
-				return value.getAttr(), nil
-			}
+			log.Debug("AttrCache::GetAttr : %s served from cache", options.Name)
+			return value.getAttr(), nil
 		}
 	}
 

component/attr_cache/attr_cache_test.go

@@ -80,9 +80,6 @@ func newTestAttrCache(next internal.Component, configuration string) *AttrCache
 
 func getPathAttr(path string, size int64, mode os.FileMode, metadata bool) *internal.ObjAttr {
 	flags := internal.NewFileBitMap()
-	if metadata {
-		flags.Set(internal.PropFlagMetadataRetrieved)
-	}
 	return &internal.ObjAttr{
 		Path:     path,
 		Name:     filepath.Base(path),
@@ -210,8 +207,7 @@ func (suite *attrCacheTestSuite) TestDefault() {
 	defer suite.cleanupTest()
 	suite.assert.Equal(suite.attrCache.Name(), "attr_cache")
 	suite.assert.EqualValues(suite.attrCache.cacheTimeout, 120)
-	suite.assert.Equal(suite.attrCache.cacheOnList, true)
-	suite.assert.Equal(suite.attrCache.noSymlinks, false)
+	// suite.assert.Equal(suite.attrCache.noSymlinks, false)
 }
 
 // Tests configuration
@@ -223,7 +219,6 @@ func (suite *attrCacheTestSuite) TestConfig() {
 
 	suite.assert.Equal(suite.attrCache.Name(), "attr_cache")
 	suite.assert.EqualValues(suite.attrCache.cacheTimeout, 60)
-	suite.assert.Equal(suite.attrCache.cacheOnList, false)
 	suite.assert.Equal(suite.attrCache.noSymlinks, true)
 }
 
@@ -246,7 +241,6 @@ func (suite *attrCacheTestSuite) TestConfigZero() {
 
 	suite.assert.Equal(suite.attrCache.Name(), "attr_cache")
 	suite.assert.EqualValues(suite.attrCache.cacheTimeout, 0)
-	suite.assert.Equal(suite.attrCache.cacheOnList, false)
 	suite.assert.Equal(suite.attrCache.noSymlinks, true)
 }
 
@@ -426,29 +420,6 @@ func (suite *attrCacheTestSuite) TestReadDirExists() {
 	}
 }
 
-func (suite *attrCacheTestSuite) TestReadDirNoCacheOnList() {
-	defer suite.cleanupTest()
-	suite.cleanupTest() // clean up the default attr cache generated
-	cacheOnList := false
-	config := fmt.Sprintf("attr_cache:\n  no-cache-on-list: %t", !cacheOnList)
-	suite.setupTestHelper(config) // setup a new attr cache with a custom config (clean up will occur after the test as usual)
-	suite.assert.EqualValues(suite.attrCache.cacheOnList, cacheOnList)
-	path := "a"
-	size := int64(1024)
-	mode := os.FileMode(0)
-	aAttr := generateNestedPathAttr(path, size, mode)
-
-	options := internal.ReadDirOptions{Name: path}
-	suite.mock.EXPECT().ReadDir(options).Return(aAttr, nil)
-
-	suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty before call
-	returnedAttr, err := suite.attrCache.ReadDir(options)
-	suite.assert.Nil(err)
-	suite.assert.Equal(aAttr, returnedAttr)
-
-	suite.assert.Empty(suite.attrCache.cacheMap) // cacheMap should be empty after call
-}
-
 func (suite *attrCacheTestSuite) TestReadDirError() {
 	defer suite.cleanupTest()
 	var paths = []string{"a", "a/", "ab", "ab/"}
@@ -912,7 +883,6 @@ func (suite *attrCacheTestSuite) TestGetAttrExistsWithoutMetadataNoSymlinks() {
 		// This is a little janky but required since testify suite does not support running setup or clean up for subtests.
 		suite.cleanupTest()
 		suite.setupTestHelper(config) // setup a new attr cache with a custom config (clean up will occur after the test as usual)
-		suite.assert.EqualValues(suite.attrCache.cacheOnList, noSymlinks)
 		suite.Run(path, func() {
 			truncatedPath := internal.TruncateDirName(path)
 			addDirectoryToCache(suite.assert, suite.attrCache, "a", true) // add the paths to the cache with IsMetadataRetrived=true
@@ -941,7 +911,7 @@ func (suite *attrCacheTestSuite) TestGetAttrExistsWithoutMetadata() {
 
 			options := internal.GetAttrOptions{Name: path}
 			// attributes should not be accessible so call the mock
-			suite.mock.EXPECT().GetAttr(options).Return(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), false), nil)
+			//suite.mock.EXPECT().GetAttr(options).Return(getPathAttr(path, defaultSize, fs.FileMode(defaultMode), false), nil)
 
 			_, err := suite.attrCache.GetAttr(options)
 			suite.assert.Nil(err)

component/azstorage/azstorage.go

@@ -508,7 +508,7 @@ func (az *AzStorage) CreateLink(options internal.CreateLinkOptions) error {
 
 func (az *AzStorage) ReadLink(options internal.ReadLinkOptions) (string, error) {
 	log.Trace("AzStorage::ReadLink : Read symlink %s", options.Name)
-	data, err := az.storage.ReadBuffer(options.Name, 0, 0)
+	data, err := az.storage.ReadBuffer(options.Name, 0, options.Size)
 
 	if err != nil {
 		azStatsCollector.PushEvents(readLink, options.Name, nil)

component/azstorage/block_blob.go

@@ -450,7 +450,6 @@ func (bb *BlockBlob) getAttrUsingRest(name string) (attr *internal.ObjAttr, err
 
 	parseMetadata(attr, prop.Metadata)
 
-	attr.Flags.Set(internal.PropFlagMetadataRetrieved)
 	attr.Flags.Set(internal.PropFlagModeDefault)
 
 	return attr, nil
@@ -595,7 +594,6 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern
 				MD5:    blobInfo.Properties.ContentMD5,
 			}
 			parseMetadata(attr, blobInfo.Metadata)
-			attr.Flags.Set(internal.PropFlagMetadataRetrieved)
 			attr.Flags.Set(internal.PropFlagModeDefault)
 		}
 		blobList = append(blobList, attr)
@@ -634,7 +632,6 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern
 				attr.Atime = attr.Mtime
 				attr.Crtime = attr.Mtime
 				attr.Ctime = attr.Mtime
-				attr.Flags.Set(internal.PropFlagMetadataRetrieved)
 				attr.Flags.Set(internal.PropFlagModeDefault)
 				blobList = append(blobList, attr)
 			}

component/azstorage/block_blob_test.go

@@ -644,7 +644,6 @@ func (s *blockBlobTestSuite) TestReadDirNoVirtualDirectory() {
 			s.assert.EqualValues(name, entries[0].Path)
 			s.assert.EqualValues(name, entries[0].Name)
 			s.assert.True(entries[0].IsDir())
-			s.assert.True(entries[0].IsMetadataRetrieved())
 			s.assert.True(entries[0].IsModeDefault())
 		})
 	}
@@ -664,13 +663,11 @@ func (s *blockBlobTestSuite) TestReadDirHierarchy() {
 	s.assert.EqualValues(base+"/c1", entries[0].Path)
 	s.assert.EqualValues("c1", entries[0].Name)
 	s.assert.True(entries[0].IsDir())
-	s.assert.True(entries[0].IsMetadataRetrieved())
 	s.assert.True(entries[0].IsModeDefault())
 	// Check the file
 	s.assert.EqualValues(base+"/c2", entries[1].Path)
 	s.assert.EqualValues("c2", entries[1].Name)
 	s.assert.False(entries[1].IsDir())
-	s.assert.True(entries[1].IsMetadataRetrieved())
 	s.assert.True(entries[1].IsModeDefault())
 }
 
@@ -693,19 +690,16 @@ func (s *blockBlobTestSuite) TestReadDirRoot() {
 			s.assert.EqualValues(base, entries[0].Path)
 			s.assert.EqualValues(base, entries[0].Name)
 			s.assert.True(entries[0].IsDir())
-			s.assert.True(entries[0].IsMetadataRetrieved())
 			s.assert.True(entries[0].IsModeDefault())
 			// Check the baseb dir
 			s.assert.EqualValues(base+"b", entries[1].Path)
 			s.assert.EqualValues(base+"b", entries[1].Name)
 			s.assert.True(entries[1].IsDir())
-			s.assert.True(entries[1].IsMetadataRetrieved())
 			s.assert.True(entries[1].IsModeDefault())
 			// Check the basec file
 			s.assert.EqualValues(base+"c", entries[2].Path)
 			s.assert.EqualValues(base+"c", entries[2].Name)
 			s.assert.False(entries[2].IsDir())
-			s.assert.True(entries[2].IsMetadataRetrieved())
 			s.assert.True(entries[2].IsModeDefault())
 		})
 	}
@@ -725,7 +719,6 @@ func (s *blockBlobTestSuite) TestReadDirSubDir() {
 	s.assert.EqualValues(base+"/c1"+"/gc1", entries[0].Path)
 	s.assert.EqualValues("gc1", entries[0].Name)
 	s.assert.False(entries[0].IsDir())
-	s.assert.True(entries[0].IsMetadataRetrieved())
 	s.assert.True(entries[0].IsModeDefault())
 }
 
@@ -745,7 +738,6 @@ func (s *blockBlobTestSuite) TestReadDirSubDirPrefixPath() {
 	s.assert.EqualValues("c1"+"/gc1", entries[0].Path)
 	s.assert.EqualValues("gc1", entries[0].Name)
 	s.assert.False(entries[0].IsDir())
-	s.assert.True(entries[0].IsMetadataRetrieved())
 	s.assert.True(entries[0].IsModeDefault())
 }
 

component/azstorage/config.go

@@ -557,15 +557,14 @@ func ParseAndReadDynamicConfig(az *AzStorage, opt AzStorageOptions, reload bool)
 		az.stConfig.honourACL = false
 	}
 
+	// by default symlink will be disabled
+	az.stConfig.disableSymlink = true
+
 	if config.IsSet("attr_cache.no-symlinks") {
 		err := config.UnmarshalKey("attr_cache.no-symlinks", &az.stConfig.disableSymlink)
 		if err != nil {
-			az.stConfig.disableSymlink = true
 			log.Err("ParseAndReadDynamicConfig : Failed to unmarshal attr_cache.no-symlinks")
 		}
-	} else {
-		// by default symlink will be disabled
-		az.stConfig.disableSymlink = true
 	}
 
 	// Auth related reconfig

component/azstorage/datalake.go

@@ -402,8 +402,6 @@ func (dl *Datalake) GetAttr(name string) (attr *internal.ObjAttr, err error) {
 		attr.Mode = attr.Mode | os.ModeDir
 	}
 
-	attr.Flags.Set(internal.PropFlagMetadataRetrieved)
-
 	if dl.Config.honourACL && dl.Config.authConfig.ObjectID != "" {
 		acl, err := fileClient.GetAccessControl(context.Background(), nil)
 		if err != nil {

component/azstorage/datalake_test.go

@@ -498,13 +498,11 @@ func (s *datalakeTestSuite) TestReadDirHierarchy() {
 	s.assert.EqualValues(base+"/c1", entries[0].Path)
 	s.assert.EqualValues("c1", entries[0].Name)
 	s.assert.True(entries[0].IsDir())
-	s.assert.False(entries[0].IsMetadataRetrieved())
 	s.assert.False(entries[0].IsModeDefault())
 	// Check the file
 	s.assert.EqualValues(base+"/c2", entries[1].Path)
 	s.assert.EqualValues("c2", entries[1].Name)
 	s.assert.False(entries[1].IsDir())
-	s.assert.False(entries[1].IsMetadataRetrieved())
 	s.assert.False(entries[1].IsModeDefault())
 }
 
@@ -527,19 +525,16 @@ func (s *datalakeTestSuite) TestReadDirRoot() {
 			s.assert.EqualValues(base, entries[0].Path)
 			s.assert.EqualValues(base, entries[0].Name)
 			s.assert.True(entries[0].IsDir())
-			s.assert.False(entries[0].IsMetadataRetrieved())
 			s.assert.False(entries[0].IsModeDefault())
 			// Check the baseb dir
 			s.assert.EqualValues(base+"b", entries[1].Path)
 			s.assert.EqualValues(base+"b", entries[1].Name)
 			s.assert.True(entries[1].IsDir())
-			s.assert.False(entries[1].IsMetadataRetrieved())
 			s.assert.False(entries[1].IsModeDefault())
 			// Check the basec file
 			s.assert.EqualValues(base+"c", entries[2].Path)
 			s.assert.EqualValues(base+"c", entries[2].Name)
 			s.assert.False(entries[2].IsDir())
-			s.assert.False(entries[2].IsMetadataRetrieved())
 			s.assert.False(entries[2].IsModeDefault())
 		})
 	}
@@ -559,7 +554,6 @@ func (s *datalakeTestSuite) TestReadDirSubDir() {
 	s.assert.EqualValues(base+"/c1"+"/gc1", entries[0].Path)
 	s.assert.EqualValues("gc1", entries[0].Name)
 	s.assert.False(entries[0].IsDir())
-	s.assert.False(entries[0].IsMetadataRetrieved())
 	s.assert.False(entries[0].IsModeDefault())
 }
 
@@ -579,7 +573,6 @@ func (s *datalakeTestSuite) TestReadDirSubDirPrefixPath() {
 	s.assert.EqualValues(base+"/c1"+"/gc1", entries[0].Path)
 	s.assert.EqualValues("gc1", entries[0].Name)
 	s.assert.False(entries[0].IsDir())
-	s.assert.False(entries[0].IsMetadataRetrieved())
 	s.assert.False(entries[0].IsModeDefault())
 }