From 9e7464c04a88c1734e79b56839ad2eff773b532f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 24 Jun 2022 11:57:11 -0700 Subject: [PATCH] UNC path may not contain forward slashes (#214) * UNC path may not contain forward slashes * update changelog --- CHANGELOG.md | 3 +++ commands/cmd_clonestore.go | 7 ++++--- commands/cmd_clonestore_test.go | 3 ++- commands/cmd_cp.go | 3 ++- commands/cmd_downsync.go | 13 +++++++------ commands/cmd_pack.go | 5 +++-- commands/cmd_printVersionUsage.go | 3 ++- commands/cmd_unpack.go | 11 ++++++----- commands/cmd_upsync.go | 3 ++- longtailstorelib/blobStore.go | 4 ++++ longtailstorelib/fsstore.go | 20 +++++++++++++------- longtailutils/folderscanner.go | 5 +++-- longtailutils/pathfilter.go | 13 ------------- remotestore/remotestore.go | 2 +- 14 files changed, 52 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9456cd5..bbe8a515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ ## +- **FIXED** Full support for windows extended length paths (UNC paths) + +## v0.3.5 - **UPDATED** Updated longtail to 0.3.4 ## v0.3.4 diff --git a/commands/cmd_clonestore.go b/commands/cmd_clonestore.go index b9424bb0..767e32fe 100644 --- a/commands/cmd_clonestore.go +++ b/commands/cmd_clonestore.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/DanEngelbrecht/golongtail/remotestore" "github.com/pkg/errors" @@ -291,7 +292,7 @@ func updateCurrentVersionFromLongtail( localVersionIndex, sourceVersionIndex, versionDiff, - longtailutils.NormalizePath(targetPath), + longtailstorelib.NormalizeFileSystemPath(targetPath), retainPermissions) localVersionIndex.Dispose() @@ -432,7 +433,7 @@ func cloneOneVersion( &writeContentProgress, versionMissingStoreIndex, targetVersionIndex, - longtailutils.NormalizePath(targetPath)) + longtailstorelib.NormalizeFileSystemPath(targetPath)) writeContentProgress.Dispose() if err != nil { err = errors.Wrap(err, fmt.Sprintf("failed writing content from `%s`", targetPath)) @@ -558,7 +559,7 @@ func cloneStore( var sourceCompressBlockStore longtaillib.Longtail_BlockStoreAPI if len(localCachePath) > 0 { - localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping) + localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping) cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, sourceRemoteIndexStore) diff --git a/commands/cmd_clonestore_test.go b/commands/cmd_clonestore_test.go index af7c1c21..2769dd06 100644 --- a/commands/cmd_clonestore_test.go +++ b/commands/cmd_clonestore_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "testing" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/pkg/errors" ) @@ -27,7 +28,7 @@ func createZipForFolder(searchPath string) (bytes.Buffer, error) { return nil } subPath := path[len(searchPath)+1:] - fw, err := zw.Create(longtailutils.NormalizePath(subPath)) + fw, err := zw.Create(longtailstorelib.NormalizeFileSystemPath(subPath)) if err != nil { return err diff --git a/commands/cmd_cp.go b/commands/cmd_cp.go index c1ba93bd..39a3542d 100644 --- a/commands/cmd_cp.go +++ b/commands/cmd_cp.go @@ -5,6 +5,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/DanEngelbrecht/golongtail/remotestore" "github.com/pkg/errors" @@ -63,7 +64,7 @@ func cpVersionIndex( if localCachePath == "" { compressBlockStore = longtaillib.CreateCompressBlockStore(remoteIndexStore, creg) } else { - localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping) + localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping) cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore) diff --git a/commands/cmd_downsync.go b/commands/cmd_downsync.go index d1a17b70..6efe97ec 100644 --- a/commands/cmd_downsync.go +++ b/commands/cmd_downsync.go @@ -6,6 +6,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/DanEngelbrecht/golongtail/remotestore" "github.com/pkg/errors" @@ -64,7 +65,7 @@ func downsync( resolvedTargetFolderPath := "" if targetFolderPath == "" { - normalizedSourceFilePath := longtailutils.NormalizePath(sourceFilePath) + normalizedSourceFilePath := longtailstorelib.NormalizeFileSystemPath(sourceFilePath) normalizedSourceFilePath = strings.ReplaceAll(normalizedSourceFilePath, "\\", "/") urlSplit := strings.Split(normalizedSourceFilePath, "/") sourceName := urlSplit[len(urlSplit)-1] @@ -85,7 +86,7 @@ func downsync( cacheTargetIndex = false } - cacheTargetIndexPath := resolvedTargetFolderPath + "/.longtail.index.cache.lvi" + cacheTargetIndexPath := longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath + "/.longtail.index.cache.lvi") if cacheTargetIndex { if longtaillib.FileExists(fs, cacheTargetIndexPath) { @@ -153,7 +154,7 @@ func downsync( if localCachePath == "" { compressBlockStore = longtaillib.CreateCompressBlockStore(remoteIndexStore, creg) } else { - localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping) + localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping) cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore) @@ -232,7 +233,7 @@ func downsync( targetVersionIndex, sourceVersionIndex, versionDiff, - longtailutils.NormalizePath(resolvedTargetFolderPath), + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath), retainPermissions) if err != nil { err = errors.Wrapf(err, "Failed writing version `%s` to `%s`", sourceFilePath, targetFolderPath) @@ -290,7 +291,7 @@ func downsync( validateFileInfos, err := longtaillib.GetFilesRecursively( fs, pathFilter, - longtailutils.NormalizePath(resolvedTargetFolderPath)) + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath)) if err != nil { err = errors.Wrapf(err, "Failed to scan `%s`", resolvedTargetFolderPath) return storeStats, timeStats, errors.Wrap(err, fname) @@ -308,7 +309,7 @@ func downsync( chunker, jobs, &createVersionIndexProgress, - longtailutils.NormalizePath(resolvedTargetFolderPath), + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath), validateFileInfos, nil, targetChunkSize, diff --git a/commands/cmd_pack.go b/commands/cmd_pack.go index b70e2cb6..59861037 100644 --- a/commands/cmd_pack.go +++ b/commands/cmd_pack.go @@ -7,6 +7,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/pkg/errors" @@ -55,7 +56,7 @@ func pack( resolvedTargetPath := "" if targetFilePath == "" { - urlSplit := strings.Split(longtailutils.NormalizePath(sourceFolderPath), "/") + urlSplit := strings.Split(longtailstorelib.NormalizeFileSystemPath(sourceFolderPath), "/") sourceName := urlSplit[len(urlSplit)-1] sourceNameSplit := strings.Split(sourceName, ".") resolvedTargetPath = sourceNameSplit[0] @@ -159,7 +160,7 @@ func pack( &writeContentProgress, storeIndex, vindex, - longtailutils.NormalizePath(sourceFolderPath)) + longtailstorelib.NormalizeFileSystemPath(sourceFolderPath)) if err != nil { err = errors.Wrapf(err, "Failed writing content from `%s`", sourceFolderPath) return storeStats, timeStats, errors.Wrapf(err, fname) diff --git a/commands/cmd_printVersionUsage.go b/commands/cmd_printVersionUsage.go index dedfe07b..9ee5c390 100644 --- a/commands/cmd_printVersionUsage.go +++ b/commands/cmd_printVersionUsage.go @@ -5,6 +5,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/DanEngelbrecht/golongtail/remotestore" "github.com/pkg/errors" @@ -55,7 +56,7 @@ func printVersionUsage( indexStore = remoteIndexStore } else { localFS = longtaillib.CreateFSStorageAPI() - localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", false) + localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", false) cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore) diff --git a/commands/cmd_unpack.go b/commands/cmd_unpack.go index 4d6f5073..a83f468c 100644 --- a/commands/cmd_unpack.go +++ b/commands/cmd_unpack.go @@ -7,6 +7,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -56,7 +57,7 @@ func unpack( resolvedTargetFolderPath := "" if targetFolderPath == "" { - urlSplit := strings.Split(longtailutils.NormalizePath(sourceFilePath), "/") + urlSplit := strings.Split(longtailstorelib.NormalizeFileSystemPath(sourceFilePath), "/") sourceName := urlSplit[len(urlSplit)-1] sourceNameSplit := strings.Split(sourceName, ".") resolvedTargetFolderPath = sourceNameSplit[0] @@ -75,7 +76,7 @@ func unpack( cacheTargetIndex = false } - cacheTargetIndexPath := resolvedTargetFolderPath + "/.longtail.index.cache.lvi" + cacheTargetIndexPath := longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath + "/.longtail.index.cache.lvi") if cacheTargetIndex { if longtaillib.FileExists(fs, cacheTargetIndexPath) { @@ -204,7 +205,7 @@ func unpack( targetVersionIndex, sourceVersionIndex, versionDiff, - longtailutils.NormalizePath(resolvedTargetFolderPath), + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath), retainPermissions) if err != nil { err = errors.Wrapf(err, "Failed writing version `%s` to `%s`", sourceFilePath, targetFolderPath) @@ -252,7 +253,7 @@ func unpack( validateFileInfos, err := longtaillib.GetFilesRecursively( fs, pathFilter, - longtailutils.NormalizePath(resolvedTargetFolderPath)) + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath)) if err != nil { err = errors.Wrapf(err, "Failed to scan `%s`", resolvedTargetFolderPath) return storeStats, timeStats, errors.Wrap(err, fname) @@ -270,7 +271,7 @@ func unpack( chunker, jobs, &createVersionIndexProgress, - longtailutils.NormalizePath(resolvedTargetFolderPath), + longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath), validateFileInfos, nil, targetChunkSize, diff --git a/commands/cmd_upsync.go b/commands/cmd_upsync.go index b40a8a85..3d51ed3b 100644 --- a/commands/cmd_upsync.go +++ b/commands/cmd_upsync.go @@ -5,6 +5,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/DanEngelbrecht/golongtail/longtailutils" "github.com/DanEngelbrecht/golongtail/remotestore" "github.com/pkg/errors" @@ -151,7 +152,7 @@ func upsync( &writeContentProgress, versionMissingStoreIndex, vindex, - longtailutils.NormalizePath(sourceFolderPath)) + longtailstorelib.NormalizeFileSystemPath(sourceFolderPath)) if err != nil { err = errors.Wrapf(err, "Failed writing content from `%s`", sourceFolderPath) return storeStats, timeStats, errors.Wrapf(err, fname) diff --git a/longtailstorelib/blobStore.go b/longtailstorelib/blobStore.go index 5ccd4443..7b1343fe 100644 --- a/longtailstorelib/blobStore.go +++ b/longtailstorelib/blobStore.go @@ -65,6 +65,10 @@ func CreateBlobStoreForURI(uri string, opts ...BlobStoreOption) (BlobStore, erro if strings.HasPrefix(uri, "fsblob://") { return NewFSBlobStore(uri[len("fsblob://"):], false) } + // Special case for unc paths + if strings.HasPrefix(uri, UNCPrefix) { + return NewFSBlobStore(uri, false) + } blobStoreURL, err := url.Parse(uri) if err == nil { switch blobStoreURL.Scheme { diff --git a/longtailstorelib/fsstore.go b/longtailstorelib/fsstore.go index e4c04c0b..f50b9e17 100644 --- a/longtailstorelib/fsstore.go +++ b/longtailstorelib/fsstore.go @@ -30,11 +30,17 @@ type fsBlobObject struct { metageneration int64 } -func normalizePath(path string) string { - doubleForwardRemoved := strings.Replace(path, "//", "/", -1) - doubleBackwardRemoved := strings.Replace(doubleForwardRemoved, "\\\\", "/", -1) - backwardRemoved := strings.Replace(doubleBackwardRemoved, "\\", "/", -1) - return backwardRemoved +const UNCPrefix = "\\\\?\\" + +func NormalizeFileSystemPath(path string) string { + if strings.HasPrefix(path, UNCPrefix) { + forwardSlashReplaced := strings.Replace(path, "/", "\\", -1) + doubleBackwardRemoved := UNCPrefix + strings.Replace(forwardSlashReplaced[len(UNCPrefix):], "\\\\", "\\", -1) + return doubleBackwardRemoved + } + backwardRemoved := strings.Replace(path, "\\", "/", -1) + doubleForwardRemoved := strings.Replace(backwardRemoved, "//", "/", -1) + return doubleForwardRemoved } // NewFSBlobStore ... @@ -52,7 +58,7 @@ func (blobStore *fsBlobStore) String() string { } func (blobClient *fsBlobClient) NewObject(filepath string) (BlobObject, error) { - fsPath := path.Join(blobClient.store.prefix, filepath) + fsPath := NormalizeFileSystemPath(path.Join(blobClient.store.prefix, filepath)) return &fsBlobObject{client: blobClient, path: fsPath, metageneration: -1}, nil } @@ -71,7 +77,7 @@ func (blobClient *fsBlobClient) GetObjects(pathPrefix string) ([]BlobProperties, if strings.HasSuffix(path, "._lck") { return nil } - leafPath := normalizePath(path[len(searchPath)+1:]) + leafPath := NormalizeFileSystemPath(path[len(searchPath)+1:]) if len(leafPath) < len(pathPrefix) { return nil } diff --git a/longtailutils/folderscanner.go b/longtailutils/folderscanner.go index 61a99de1..510ea14c 100644 --- a/longtailutils/folderscanner.go +++ b/longtailutils/folderscanner.go @@ -7,6 +7,7 @@ import ( "time" "github.com/DanEngelbrecht/golongtail/longtaillib" + "github.com/DanEngelbrecht/golongtail/longtailstorelib" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -29,7 +30,7 @@ func (scanner *AsyncFolderScanner) Scan( fileInfos, err := longtaillib.GetFilesRecursively( fs, pathFilter, - NormalizePath(sourceFolderPath)) + longtailstorelib.NormalizeFileSystemPath(sourceFolderPath)) if err != nil { err := errors.Wrap(err, fmt.Sprintf("Failed getting folder structure for `%s`", sourceFolderPath)) scanner.err = errors.Wrap(err, fname) @@ -95,7 +96,7 @@ func GetFolderIndex( chunker, jobs, &createVersionIndexProgress, - NormalizePath(sourceFolderPath), + longtailstorelib.NormalizeFileSystemPath(sourceFolderPath), fileInfos, compressionTypes, targetChunkSize, diff --git a/longtailutils/pathfilter.go b/longtailutils/pathfilter.go index 99dbf47c..b53d22ac 100644 --- a/longtailutils/pathfilter.go +++ b/longtailutils/pathfilter.go @@ -3,7 +3,6 @@ package longtailutils import ( "context" "regexp" - "strings" "github.com/DanEngelbrecht/golongtail/longtaillib" "github.com/pkg/errors" @@ -11,18 +10,6 @@ import ( log "github.com/sirupsen/logrus" ) -const uncprefix = "\\\\?\\" - -func NormalizePath(path string) string { - doubleForwardRemoved := strings.Replace(path, "//", "/", -1) - if strings.HasPrefix(doubleForwardRemoved, uncprefix) { - doubleBackwardRemoved := uncprefix + strings.Replace(doubleForwardRemoved[4:], "\\\\", "\\", -1) - return doubleBackwardRemoved - } - doubleBackwardRemoved := strings.Replace(doubleForwardRemoved, "\\\\", "/", -1) - return doubleBackwardRemoved -} - type regexPathFilter struct { compiledIncludeRegexes []*regexp.Regexp compiledExcludeRegexes []*regexp.Regexp diff --git a/remotestore/remotestore.go b/remotestore/remotestore.go index 73f06922..e0c1207a 100644 --- a/remotestore/remotestore.go +++ b/remotestore/remotestore.go @@ -1516,7 +1516,7 @@ func getStoreIndexFromBlocks( } blockPath := getBlockPath("chunks", blockIndex.GetBlockHash()) - if blockPath == longtailutils.NormalizePath(blockKey) { + if blockPath == longtailstorelib.NormalizeFileSystemPath(blockKey) { batchBlockIndexes[batchPos] = blockIndex } else { log.Warnf("Block %s name does not match content hash, expected name %s", blockKey, blockPath)