UNC path may not contain forward slashes (#214)

* UNC path may not contain forward slashes * update changelog
DanEngelbrecht · Jun 24, 2022 · 9e7464c · 9e7464c
1 parent 8a70395
commit 9e7464c
Show file tree

Hide file tree

Showing 14 changed files with 52 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,7 @@
 ##
+- **FIXED** Full support for windows extended length paths (UNC paths)
+
+## v0.3.5
 - **UPDATED** Updated longtail to 0.3.4
 
 ## v0.3.4

diff --git a/commands/cmd_clonestore.go b/commands/cmd_clonestore.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/DanEngelbrecht/golongtail/remotestore"
 	"github.com/pkg/errors"
@@ -291,7 +292,7 @@ func updateCurrentVersionFromLongtail(
 		localVersionIndex,
 		sourceVersionIndex,
 		versionDiff,
-		longtailutils.NormalizePath(targetPath),
+		longtailstorelib.NormalizeFileSystemPath(targetPath),
 		retainPermissions)
 
 	localVersionIndex.Dispose()
@@ -432,7 +433,7 @@ func cloneOneVersion(
 			&writeContentProgress,
 			versionMissingStoreIndex,
 			targetVersionIndex,
-			longtailutils.NormalizePath(targetPath))
+			longtailstorelib.NormalizeFileSystemPath(targetPath))
 		writeContentProgress.Dispose()
 		if err != nil {
 			err = errors.Wrap(err, fmt.Sprintf("failed writing content from `%s`", targetPath))
@@ -558,7 +559,7 @@ func cloneStore(
 	var sourceCompressBlockStore longtaillib.Longtail_BlockStoreAPI
 
 	if len(localCachePath) > 0 {
-		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping)
+		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping)
 
 		cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, sourceRemoteIndexStore)
 

diff --git a/commands/cmd_clonestore_test.go b/commands/cmd_clonestore_test.go
@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/pkg/errors"
 )
@@ -27,7 +28,7 @@ func createZipForFolder(searchPath string) (bytes.Buffer, error) {
 			return nil
 		}
 		subPath := path[len(searchPath)+1:]
-		fw, err := zw.Create(longtailutils.NormalizePath(subPath))
+		fw, err := zw.Create(longtailstorelib.NormalizeFileSystemPath(subPath))
 
 		if err != nil {
 			return err

diff --git a/commands/cmd_cp.go b/commands/cmd_cp.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/DanEngelbrecht/golongtail/remotestore"
 	"github.com/pkg/errors"
@@ -63,7 +64,7 @@ func cpVersionIndex(
 	if localCachePath == "" {
 		compressBlockStore = longtaillib.CreateCompressBlockStore(remoteIndexStore, creg)
 	} else {
-		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping)
+		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping)
 
 		cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore)
 

diff --git a/commands/cmd_downsync.go b/commands/cmd_downsync.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/DanEngelbrecht/golongtail/remotestore"
 	"github.com/pkg/errors"
@@ -64,7 +65,7 @@ func downsync(
 
 	resolvedTargetFolderPath := ""
 	if targetFolderPath == "" {
-		normalizedSourceFilePath := longtailutils.NormalizePath(sourceFilePath)
+		normalizedSourceFilePath := longtailstorelib.NormalizeFileSystemPath(sourceFilePath)
 		normalizedSourceFilePath = strings.ReplaceAll(normalizedSourceFilePath, "\\", "/")
 		urlSplit := strings.Split(normalizedSourceFilePath, "/")
 		sourceName := urlSplit[len(urlSplit)-1]
@@ -85,7 +86,7 @@ func downsync(
 		cacheTargetIndex = false
 	}
 
-	cacheTargetIndexPath := resolvedTargetFolderPath + "/.longtail.index.cache.lvi"
+	cacheTargetIndexPath := longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath + "/.longtail.index.cache.lvi")
 
 	if cacheTargetIndex {
 		if longtaillib.FileExists(fs, cacheTargetIndexPath) {
@@ -153,7 +154,7 @@ func downsync(
 	if localCachePath == "" {
 		compressBlockStore = longtaillib.CreateCompressBlockStore(remoteIndexStore, creg)
 	} else {
-		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", enableFileMapping)
+		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", enableFileMapping)
 
 		cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore)
 
@@ -232,7 +233,7 @@ func downsync(
 		targetVersionIndex,
 		sourceVersionIndex,
 		versionDiff,
-		longtailutils.NormalizePath(resolvedTargetFolderPath),
+		longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath),
 		retainPermissions)
 	if err != nil {
 		err = errors.Wrapf(err, "Failed writing version `%s` to `%s`", sourceFilePath, targetFolderPath)
@@ -290,7 +291,7 @@ func downsync(
 		validateFileInfos, err := longtaillib.GetFilesRecursively(
 			fs,
 			pathFilter,
-			longtailutils.NormalizePath(resolvedTargetFolderPath))
+			longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
 		if err != nil {
 			err = errors.Wrapf(err, "Failed to scan `%s`", resolvedTargetFolderPath)
 			return storeStats, timeStats, errors.Wrap(err, fname)
@@ -308,7 +309,7 @@ func downsync(
 			chunker,
 			jobs,
 			&createVersionIndexProgress,
-			longtailutils.NormalizePath(resolvedTargetFolderPath),
+			longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath),
 			validateFileInfos,
 			nil,
 			targetChunkSize,

diff --git a/commands/cmd_pack.go b/commands/cmd_pack.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/pkg/errors"
 
@@ -55,7 +56,7 @@ func pack(
 
 	resolvedTargetPath := ""
 	if targetFilePath == "" {
-		urlSplit := strings.Split(longtailutils.NormalizePath(sourceFolderPath), "/")
+		urlSplit := strings.Split(longtailstorelib.NormalizeFileSystemPath(sourceFolderPath), "/")
 		sourceName := urlSplit[len(urlSplit)-1]
 		sourceNameSplit := strings.Split(sourceName, ".")
 		resolvedTargetPath = sourceNameSplit[0]
@@ -159,7 +160,7 @@ func pack(
 		&writeContentProgress,
 		storeIndex,
 		vindex,
-		longtailutils.NormalizePath(sourceFolderPath))
+		longtailstorelib.NormalizeFileSystemPath(sourceFolderPath))
 	if err != nil {
 		err = errors.Wrapf(err, "Failed writing content from `%s`", sourceFolderPath)
 		return storeStats, timeStats, errors.Wrapf(err, fname)

diff --git a/commands/cmd_printVersionUsage.go b/commands/cmd_printVersionUsage.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/DanEngelbrecht/golongtail/remotestore"
 	"github.com/pkg/errors"
@@ -55,7 +56,7 @@ func printVersionUsage(
 		indexStore = remoteIndexStore
 	} else {
 		localFS = longtaillib.CreateFSStorageAPI()
-		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailutils.NormalizePath(localCachePath), "", false)
+		localIndexStore = longtaillib.CreateFSBlockStore(jobs, localFS, longtailstorelib.NormalizeFileSystemPath(localCachePath), "", false)
 
 		cacheBlockStore = longtaillib.CreateCacheBlockStore(jobs, localIndexStore, remoteIndexStore)
 

diff --git a/commands/cmd_unpack.go b/commands/cmd_unpack.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -56,7 +57,7 @@ func unpack(
 
 	resolvedTargetFolderPath := ""
 	if targetFolderPath == "" {
-		urlSplit := strings.Split(longtailutils.NormalizePath(sourceFilePath), "/")
+		urlSplit := strings.Split(longtailstorelib.NormalizeFileSystemPath(sourceFilePath), "/")
 		sourceName := urlSplit[len(urlSplit)-1]
 		sourceNameSplit := strings.Split(sourceName, ".")
 		resolvedTargetFolderPath = sourceNameSplit[0]
@@ -75,7 +76,7 @@ func unpack(
 		cacheTargetIndex = false
 	}
 
-	cacheTargetIndexPath := resolvedTargetFolderPath + "/.longtail.index.cache.lvi"
+	cacheTargetIndexPath := longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath + "/.longtail.index.cache.lvi")
 
 	if cacheTargetIndex {
 		if longtaillib.FileExists(fs, cacheTargetIndexPath) {
@@ -204,7 +205,7 @@ func unpack(
 		targetVersionIndex,
 		sourceVersionIndex,
 		versionDiff,
-		longtailutils.NormalizePath(resolvedTargetFolderPath),
+		longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath),
 		retainPermissions)
 	if err != nil {
 		err = errors.Wrapf(err, "Failed writing version `%s` to `%s`", sourceFilePath, targetFolderPath)
@@ -252,7 +253,7 @@ func unpack(
 		validateFileInfos, err := longtaillib.GetFilesRecursively(
 			fs,
 			pathFilter,
-			longtailutils.NormalizePath(resolvedTargetFolderPath))
+			longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
 		if err != nil {
 			err = errors.Wrapf(err, "Failed to scan `%s`", resolvedTargetFolderPath)
 			return storeStats, timeStats, errors.Wrap(err, fname)
@@ -270,7 +271,7 @@ func unpack(
 			chunker,
 			jobs,
 			&createVersionIndexProgress,
-			longtailutils.NormalizePath(resolvedTargetFolderPath),
+			longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath),
 			validateFileInfos,
 			nil,
 			targetChunkSize,

diff --git a/commands/cmd_upsync.go b/commands/cmd_upsync.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/DanEngelbrecht/golongtail/longtailutils"
 	"github.com/DanEngelbrecht/golongtail/remotestore"
 	"github.com/pkg/errors"
@@ -151,7 +152,7 @@ func upsync(
 			&writeContentProgress,
 			versionMissingStoreIndex,
 			vindex,
-			longtailutils.NormalizePath(sourceFolderPath))
+			longtailstorelib.NormalizeFileSystemPath(sourceFolderPath))
 		if err != nil {
 			err = errors.Wrapf(err, "Failed writing content from `%s`", sourceFolderPath)
 			return storeStats, timeStats, errors.Wrapf(err, fname)

diff --git a/longtailstorelib/blobStore.go b/longtailstorelib/blobStore.go
@@ -65,6 +65,10 @@ func CreateBlobStoreForURI(uri string, opts ...BlobStoreOption) (BlobStore, erro
 	if strings.HasPrefix(uri, "fsblob://") {
 		return NewFSBlobStore(uri[len("fsblob://"):], false)
 	}
+	// Special case for unc paths
+	if strings.HasPrefix(uri, UNCPrefix) {
+		return NewFSBlobStore(uri, false)
+	}
 	blobStoreURL, err := url.Parse(uri)
 	if err == nil {
 		switch blobStoreURL.Scheme {

diff --git a/longtailstorelib/fsstore.go b/longtailstorelib/fsstore.go
@@ -30,11 +30,17 @@ type fsBlobObject struct {
 	metageneration int64
 }
 
-func normalizePath(path string) string {
-	doubleForwardRemoved := strings.Replace(path, "//", "/", -1)
-	doubleBackwardRemoved := strings.Replace(doubleForwardRemoved, "\\\\", "/", -1)
-	backwardRemoved := strings.Replace(doubleBackwardRemoved, "\\", "/", -1)
-	return backwardRemoved
+const UNCPrefix = "\\\\?\\"
+
+func NormalizeFileSystemPath(path string) string {
+	if strings.HasPrefix(path, UNCPrefix) {
+		forwardSlashReplaced := strings.Replace(path, "/", "\\", -1)
+		doubleBackwardRemoved := UNCPrefix + strings.Replace(forwardSlashReplaced[len(UNCPrefix):], "\\\\", "\\", -1)
+		return doubleBackwardRemoved
+	}
+	backwardRemoved := strings.Replace(path, "\\", "/", -1)
+	doubleForwardRemoved := strings.Replace(backwardRemoved, "//", "/", -1)
+	return doubleForwardRemoved
 }
 
 // NewFSBlobStore ...
@@ -52,7 +58,7 @@ func (blobStore *fsBlobStore) String() string {
 }
 
 func (blobClient *fsBlobClient) NewObject(filepath string) (BlobObject, error) {
-	fsPath := path.Join(blobClient.store.prefix, filepath)
+	fsPath := NormalizeFileSystemPath(path.Join(blobClient.store.prefix, filepath))
 	return &fsBlobObject{client: blobClient, path: fsPath, metageneration: -1}, nil
 }
 
@@ -71,7 +77,7 @@ func (blobClient *fsBlobClient) GetObjects(pathPrefix string) ([]BlobProperties,
 		if strings.HasSuffix(path, "._lck") {
 			return nil
 		}
-		leafPath := normalizePath(path[len(searchPath)+1:])
+		leafPath := NormalizeFileSystemPath(path[len(searchPath)+1:])
 		if len(leafPath) < len(pathPrefix) {
 			return nil
 		}

diff --git a/longtailutils/folderscanner.go b/longtailutils/folderscanner.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
+	"github.com/DanEngelbrecht/golongtail/longtailstorelib"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -29,7 +30,7 @@ func (scanner *AsyncFolderScanner) Scan(
 		fileInfos, err := longtaillib.GetFilesRecursively(
 			fs,
 			pathFilter,
-			NormalizePath(sourceFolderPath))
+			longtailstorelib.NormalizeFileSystemPath(sourceFolderPath))
 		if err != nil {
 			err := errors.Wrap(err, fmt.Sprintf("Failed getting folder structure for `%s`", sourceFolderPath))
 			scanner.err = errors.Wrap(err, fname)
@@ -95,7 +96,7 @@ func GetFolderIndex(
 			chunker,
 			jobs,
 			&createVersionIndexProgress,
-			NormalizePath(sourceFolderPath),
+			longtailstorelib.NormalizeFileSystemPath(sourceFolderPath),
 			fileInfos,
 			compressionTypes,
 			targetChunkSize,

diff --git a/longtailutils/pathfilter.go b/longtailutils/pathfilter.go
@@ -3,26 +3,13 @@ package longtailutils
 import (
 	"context"
 	"regexp"
-	"strings"
 
 	"github.com/DanEngelbrecht/golongtail/longtaillib"
 	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 )
 
-const uncprefix = "\\\\?\\"
-
-func NormalizePath(path string) string {
-	doubleForwardRemoved := strings.Replace(path, "//", "/", -1)
-	if strings.HasPrefix(doubleForwardRemoved, uncprefix) {
-		doubleBackwardRemoved := uncprefix + strings.Replace(doubleForwardRemoved[4:], "\\\\", "\\", -1)
-		return doubleBackwardRemoved
-	}
-	doubleBackwardRemoved := strings.Replace(doubleForwardRemoved, "\\\\", "/", -1)
-	return doubleBackwardRemoved
-}
-
 type regexPathFilter struct {
 	compiledIncludeRegexes []*regexp.Regexp
 	compiledExcludeRegexes []*regexp.Regexp

diff --git a/remotestore/remotestore.go b/remotestore/remotestore.go
@@ -1516,7 +1516,7 @@ func getStoreIndexFromBlocks(
 				}
 
 				blockPath := getBlockPath("chunks", blockIndex.GetBlockHash())
-				if blockPath == longtailutils.NormalizePath(blockKey) {
+				if blockPath == longtailstorelib.NormalizeFileSystemPath(blockKey) {
 					batchBlockIndexes[batchPos] = blockIndex
 				} else {
 					log.Warnf("Block %s name does not match content hash, expected name %s", blockKey, blockPath)