Skip to content

Commit

Permalink
Rework metadata cache setting
Browse files Browse the repository at this point in the history
  • Loading branch information
iychoi committed Jan 24, 2022
1 parent 9898968 commit a2e3559
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 25 deletions.
47 changes: 33 additions & 14 deletions fs/cache.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,26 @@
package fs

import (
"strings"
"time"

"github.com/cyverse/go-irodsclient/irods/types"
"github.com/cyverse/go-irodsclient/irods/util"
gocache "github.com/patrickmn/go-cache"
)

// MetadataCacheTimeoutSetting defines cache timeout for path
type MetadataCacheTimeoutSetting struct {
Path string
Timeout time.Duration
Inherit bool
}

// FileSystemCache manages filesystem caches
type FileSystemCache struct {
cacheTimeout time.Duration
cleanupTimeout time.Duration
cacheTimeoutPathMap map[string]time.Duration
cacheTimeoutPaths []MetadataCacheTimeoutSetting
cacheTimeoutPathMap map[string]MetadataCacheTimeoutSetting
entryCache *gocache.Cache
dirCache *gocache.Cache
metadataCache *gocache.Cache
Expand All @@ -25,7 +33,7 @@ type FileSystemCache struct {
}

// NewFileSystemCache creates a new FileSystemCache
func NewFileSystemCache(cacheTimeout time.Duration, cleanup time.Duration, cacheTimeoutPathMap map[string]time.Duration) *FileSystemCache {
func NewFileSystemCache(cacheTimeout time.Duration, cleanup time.Duration, cacheTimeoutSettings []MetadataCacheTimeoutSetting) *FileSystemCache {
entryCache := gocache.New(cacheTimeout, cleanup)
dirCache := gocache.New(cacheTimeout, cleanup)
metadataCache := gocache.New(cacheTimeout, cleanup)
Expand All @@ -36,14 +44,21 @@ func NewFileSystemCache(cacheTimeout time.Duration, cleanup time.Duration, cache
dirACLsCache := gocache.New(cacheTimeout, cleanup)
fileACLsCache := gocache.New(cacheTimeout, cleanup)

if cacheTimeoutPathMap == nil {
cacheTimeoutPathMap = map[string]time.Duration{}
if cacheTimeoutSettings == nil {
cacheTimeoutSettings = []MetadataCacheTimeoutSetting{}
}

// build a map for quick search
cacheTimeoutSettingMap := map[string]MetadataCacheTimeoutSetting{}
for _, timeoutSetting := range cacheTimeoutSettings {
cacheTimeoutSettingMap[timeoutSetting.Path] = timeoutSetting
}

return &FileSystemCache{
cacheTimeout: cacheTimeout,
cleanupTimeout: cleanup,
cacheTimeoutPathMap: cacheTimeoutPathMap,
cacheTimeoutPaths: cacheTimeoutSettings,
cacheTimeoutPathMap: cacheTimeoutSettingMap,
entryCache: entryCache,
dirCache: dirCache,
metadataCache: metadataCache,
Expand All @@ -58,17 +73,21 @@ func NewFileSystemCache(cacheTimeout time.Duration, cleanup time.Duration, cache

func (cache *FileSystemCache) getCacheTTLForPath(path string) time.Duration {
// check map first
if ttl, ok := cache.cacheTimeoutPathMap[path]; ok {
if timeoutSetting, ok := cache.cacheTimeoutPathMap[path]; ok {
// exact match
return ttl
return timeoutSetting.Timeout
}

for pathMatch, ttl := range cache.cacheTimeoutPathMap {
if strings.HasSuffix(pathMatch, "/*") {
// wildcard key
pathPrefix := pathMatch[0 : len(pathMatch)-1]
if strings.HasPrefix(path, pathPrefix) {
return ttl
// check inherit
parentPaths := util.GetParentDirs(path)
for i := len(parentPaths) - 1; i >= 0; i-- {
parentPath := parentPaths[i]

if timeoutSetting, ok := cache.cacheTimeoutPathMap[parentPath]; ok {
// parent match
if timeoutSetting.Inherit {
// inherit
return timeoutSetting.Timeout
}
}
}
Expand Down
12 changes: 4 additions & 8 deletions fs/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,19 @@ type FileSystemConfig struct {
ConnectionMax int
CacheTimeout time.Duration
CacheCleanupTime time.Duration
CacheTimeoutPathMap map[string]time.Duration
CacheTimeoutSettings []MetadataCacheTimeoutSetting
// for mysql iCAT backend, this should be true.
// for postgresql iCAT backend, this can be false.
StartNewTransaction bool
}

// NewFileSystemConfig create a FileSystemConfig
func NewFileSystemConfig(applicationName string, connectionLifespan time.Duration, operationTimeout time.Duration, connectionIdleTimeout time.Duration, connectionMax int, cacheTimeout time.Duration, cacheCleanupTime time.Duration, cacheTimeoutPathMap map[string]time.Duration, startNewTransaction bool) *FileSystemConfig {
func NewFileSystemConfig(applicationName string, connectionLifespan time.Duration, operationTimeout time.Duration, connectionIdleTimeout time.Duration, connectionMax int, cacheTimeout time.Duration, cacheCleanupTime time.Duration, cacheTimeoutSettings []MetadataCacheTimeoutSetting, startNewTransaction bool) *FileSystemConfig {
connMax := connectionMax
if connMax < FileSystemConnectionMaxMin {
connMax = FileSystemConnectionMaxMin
}

if cacheTimeoutPathMap == nil {
cacheTimeoutPathMap = map[string]time.Duration{}
}

return &FileSystemConfig{
ApplicationName: applicationName,
ConnectionLifespan: connectionLifespan,
Expand All @@ -47,7 +43,7 @@ func NewFileSystemConfig(applicationName string, connectionLifespan time.Duratio
ConnectionMax: connMax,
CacheTimeout: cacheTimeout,
CacheCleanupTime: cacheCleanupTime,
CacheTimeoutPathMap: cacheTimeoutPathMap,
CacheTimeoutSettings: cacheTimeoutSettings,
StartNewTransaction: startNewTransaction,
}
}
Expand All @@ -61,7 +57,7 @@ func NewFileSystemConfigWithDefault(applicationName string) *FileSystemConfig {
ConnectionIdleTimeout: FileSystemTimeoutDefault,
ConnectionMax: FileSystemConnectionMaxDefault,
CacheTimeout: FileSystemTimeoutDefault,
CacheTimeoutPathMap: map[string]time.Duration{},
CacheTimeoutSettings: []MetadataCacheTimeoutSetting{},
CacheCleanupTime: FileSystemTimeoutDefault,
StartNewTransaction: true,
}
Expand Down
6 changes: 3 additions & 3 deletions fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func NewFileSystem(account *types.IRODSAccount, config *FileSystemConfig) (*File
return nil, err
}

cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutPathMap)
cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutSettings)

return &FileSystem{
account: account,
Expand All @@ -51,7 +51,7 @@ func NewFileSystemWithDefault(account *types.IRODSAccount, applicationName strin
return nil, err
}

cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutPathMap)
cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutSettings)

return &FileSystem{
account: account,
Expand All @@ -70,7 +70,7 @@ func NewFileSystemWithSessionConfig(account *types.IRODSAccount, sessConfig *ses
return nil, err
}

cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutPathMap)
cache := NewFileSystemCache(config.CacheTimeout, config.CacheCleanupTime, config.CacheTimeoutSettings)

return &FileSystem{
account: account,
Expand Down
89 changes: 89 additions & 0 deletions irods/util/path.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package util

import (
"fmt"
"path"
"path/filepath"
"sort"
"strings"
)

// JoinPath makes the path from dir and file paths
func JoinPath(dirPath string, filePath string) string {
if strings.HasSuffix(dirPath, "/") {
return fmt.Sprintf("%s/%s", dirPath[0:len(dirPath)-1], filePath)
}
return fmt.Sprintf("%s/%s", dirPath, filePath)
}

// SplitPath splits the path into dir and file
func SplitPath(p string) (string, string) {
return filepath.Split(p)
}

// GetDirname returns the dir of the path
func GetDirname(p string) string {
return filepath.Dir(p)
}

// GetFileName returns the filename of the path
func GetFileName(p string) string {
return filepath.Base(p)
}

// IsAbsolutePath returns true if the path is absolute
func IsAbsolutePath(p string) bool {
return strings.HasPrefix(p, "/")
}

// GetPathDepth returns depth of the path
// "/" returns 0
// "abc" returns -1
// "/abc" returns 0
// "/a/b" returns 1
// "/a/b/c" returns 2
func GetPathDepth(p string) int {
if !strings.HasPrefix(p, "/") {
return -1
}

cleanPath := path.Clean(p)

if cleanPath == "/" {
return 0
}

pArr := strings.Split(p[1:], "/")
return len(pArr) - 1
}

// GetParentDirs returns all parent dirs
func GetParentDirs(p string) []string {
parents := []string{}

if p == "/" {
return parents
}

curPath := p
for len(curPath) > 0 && curPath != "/" {
curDir := GetDirname(curPath)
if len(curDir) > 0 {
parents = append(parents, curDir)
}

curPath = curDir
}

// sort
sort.Slice(parents, func(i int, j int) bool {
return len(parents[i]) < len(parents[j])
})

return parents
}

// GetRelativePath returns relative path
func GetRelativePath(p1 string, p2 string) (string, error) {
return filepath.Rel(p1, p2)
}

0 comments on commit a2e3559

Please sign in to comment.