update to longtail 0.4.2-pre5 (#251)

- **FIXED** Networks share paths that starts with \\ no longer need to be manually escaped (fixes #249) - **UPDATED** Update longtaillib to v0.4.2
DanEngelbrecht · Mar 10, 2024 · f8a9c55 · f8a9c55
1 parent 3b418bc
commit f8a9c55
Show file tree

Hide file tree

Showing 21 changed files with 187 additions and 58 deletions.
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
@@ -42,7 +42,7 @@ jobs:
           go-version: 1.21
 
       - name: Check out source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build
         run: |
@@ -61,7 +61,7 @@ jobs:
           cp ${{matrix.target}} dist/
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dist-${{matrix.platform}}-${{matrix.arch}}
           path: dist
@@ -74,28 +74,28 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Download Linux artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v4
       with:
         name: dist-linux-amd64
         path: dist-linux-x64
 
     - name: Download MacOs x64 artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v4
       with:
         name: dist-macos-amd64
         path: dist-macos-x64
 
     - name: Download MacOs arm64 artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v4
       with:
         name: dist-macos-arm64
         path: dist-macos-arm64
 
     - name: Download Win32 artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v4
       with:
         name: dist-win32-amd64
         path: dist-win32-x64

diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.yml
@@ -25,7 +25,7 @@ jobs:
           go-version: 1.21
 
       - name: Check out source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: test
         if: ${{ matrix.arch == 'amd64' }}

diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.12.1
       with:
         access_token: ${{ github.token }}
 
@@ -34,7 +34,7 @@ jobs:
           go-version: 1.21
 
       - name: Check out source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: test
         if: ${{ matrix.arch == 'amd64' }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 ##
 - **FIXED** Networks share paths that starts with \\ no longer need to be manually escaped (fixes https://github.com/DanEngelbrecht/golongtail/issues/249)
+- **UPDATED** Update longtaillib to v0.4.2
+
+## v0.4.1
 - **UPDATED** Update to golang 1.21
 - **UPDATED** Update longtaillib to v0.4.1
 - **UPDATED** Updated all golang dependencies

diff --git a/commands/cmd_clonestore.go b/commands/cmd_clonestore.go
@@ -232,7 +232,7 @@ func updateCurrentVersionFromLongtail(
 		}
 	} else {
 		targetFolderScanner := longtailutils.AsyncFolderScanner{}
-		targetFolderScanner.Scan(targetPath, pathFilter, fs)
+		targetFolderScanner.Scan(targetPath, pathFilter, fs, jobs)
 
 		targetIndexReader := longtailutils.AsyncVersionIndexReader{}
 		targetIndexReader.Read(targetPath,
@@ -299,7 +299,9 @@ func updateCurrentVersionFromLongtail(
 			longtailstorelib.NormalizeFileSystemPath(targetPath),
 			retainPermissions)
 	} else {
-		concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, longtailstorelib.NormalizeFileSystemPath(targetPath))
+		concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, sourceVersionIndex,
+			versionDiff,
+			longtailstorelib.NormalizeFileSystemPath(targetPath))
 		defer concurrentChunkWriteAPI.Dispose()
 		err = longtaillib.ChangeVersion2(
 			sourceStore,
@@ -326,7 +328,7 @@ func updateCurrentVersionFromLongtail(
 		return longtaillib.Longtail_VersionIndex{}, hash, errors.Wrap(err, fname)
 	}
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
-	targetFolderScanner.Scan(targetPath, pathFilter, fs)
+	targetFolderScanner.Scan(targetPath, pathFilter, fs, jobs)
 
 	targetIndexReader := longtailutils.AsyncVersionIndexReader{}
 	targetIndexReader.Read(targetPath,

diff --git a/commands/cmd_downsync.go b/commands/cmd_downsync.go
@@ -121,7 +121,7 @@ func downsync(
 
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
 	if scanTarget && targetIndexPath == "" {
-		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs)
+		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs, jobs)
 	}
 
 	hashRegistry := longtaillib.CreateFullHashRegistry()
@@ -267,7 +267,7 @@ func downsync(
 	changeVersionStartTime := time.Now()
 	changeVersionProgress := longtailutils.CreateProgress("Updating version          ", 1)
 	defer changeVersionProgress.Dispose()
-	concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
+	concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, sourceVersionIndex, versionDiff, longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
 	defer concurrentChunkWriteAPI.Dispose()
 
 	if useLegacyWrite {

diff --git a/commands/cmd_pack.go b/commands/cmd_pack.go
@@ -72,16 +72,16 @@ func pack(
 	fs := longtaillib.CreateFSStorageAPI()
 	defer fs.Dispose()
 
-	sourceFolderScanner := longtailutils.AsyncFolderScanner{}
-	if sourceIndexPath == "" {
-		sourceFolderScanner.Scan(sourceFolderPath, pathFilter, fs)
-	}
-
 	jobs := longtaillib.CreateBikeshedJobAPI(uint32(numWorkerCount), 0)
 	defer jobs.Dispose()
 	hashRegistry := longtaillib.CreateFullHashRegistry()
 	defer hashRegistry.Dispose()
 
+	sourceFolderScanner := longtailutils.AsyncFolderScanner{}
+	if sourceIndexPath == "" {
+		sourceFolderScanner.Scan(sourceFolderPath, pathFilter, fs, jobs)
+	}
+
 	compressionType, err := longtailutils.GetCompressionType(compressionAlgorithm)
 	if err != nil {
 		return storeStats, timeStats, errors.Wrapf(err, fname)

diff --git a/commands/cmd_unpack.go b/commands/cmd_unpack.go
@@ -88,7 +88,7 @@ func unpack(
 
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
 	if scanTarget && targetIndexPath == "" {
-		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs)
+		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs, jobs)
 	}
 
 	hashRegistry := longtaillib.CreateFullHashRegistry()
@@ -217,7 +217,7 @@ func unpack(
 			longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath),
 			retainPermissions)
 	} else {
-		concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
+		concurrentChunkWriteAPI := longtaillib.CreateConcurrentChunkWriteAPI(fs, sourceVersionIndex, versionDiff, longtailstorelib.NormalizeFileSystemPath(resolvedTargetFolderPath))
 		defer concurrentChunkWriteAPI.Dispose()
 
 		err = longtaillib.ChangeVersion2(

diff --git a/commands/cmd_upsync.go b/commands/cmd_upsync.go
@@ -64,16 +64,16 @@ func upsync(
 	fs := longtaillib.CreateFSStorageAPI()
 	defer fs.Dispose()
 
-	sourceFolderScanner := longtailutils.AsyncFolderScanner{}
-	if sourceIndexPath == "" {
-		sourceFolderScanner.Scan(sourceFolderPath, pathFilter, fs)
-	}
-
 	jobs := longtaillib.CreateBikeshedJobAPI(uint32(numWorkerCount), 0)
 	defer jobs.Dispose()
 	hashRegistry := longtaillib.CreateFullHashRegistry()
 	defer hashRegistry.Dispose()
 
+	sourceFolderScanner := longtailutils.AsyncFolderScanner{}
+	if sourceIndexPath == "" {
+		sourceFolderScanner.Scan(sourceFolderPath, pathFilter, fs, jobs)
+	}
+
 	compressionType, err := longtailutils.GetCompressionType(compressionAlgorithm)
 	if err != nil {
 		return storeStats, timeStats, errors.Wrapf(err, fname)

diff --git a/go.work.sum b/go.work.sum
@@ -0,0 +1,8 @@
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
diff --git a/longtaillib/golongtail.h b/longtaillib/golongtail.h
@@ -258,11 +258,11 @@ static struct Longtail_VersionIndex* GetArchiveVersionIndex(struct Longtail_Arch
 
 static void EnableMemtrace() {
     Longtail_MemTracer_Init();
-    Longtail_SetAllocAndFree(Longtail_MemTracer_Alloc, Longtail_MemTracer_Free);
+    Longtail_SetReAllocAndFree(Longtail_MemTracer_ReAlloc, Longtail_MemTracer_Free);
 }
 
 static void DisableMemtrace() {
-    Longtail_SetAllocAndFree(0,  0);
+    Longtail_SetReAllocAndFree(0,  0);
     Longtail_MemTracer_Dispose();
 }
 

diff --git a/longtaillib/longtail/include/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h b/longtaillib/longtail/include/lib/concurrentchunkwrite/longtail_concurrentchunkwrite.h
@@ -6,7 +6,11 @@
 extern "C" {
 #endif
 
-LONGTAIL_EXPORT extern struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI(struct Longtail_StorageAPI* storageAPI, const char* base_path);
+LONGTAIL_EXPORT extern struct Longtail_ConcurrentChunkWriteAPI* Longtail_CreateConcurrentChunkWriteAPI(
+	struct Longtail_StorageAPI* storageAPI,
+	struct Longtail_VersionIndex* version_index,
+	struct Longtail_VersionDiff* version_diff,
+	const char* base_path);
 
 #ifdef __cplusplus
 }

diff --git a/longtaillib/longtail/include/lib/memtracer/longtail_memtracer.h b/longtaillib/longtail/include/lib/memtracer/longtail_memtracer.h
@@ -12,7 +12,7 @@ LONGTAIL_EXPORT extern uint32_t Longtail_GetMemTracerDetailed();
 LONGTAIL_EXPORT void Longtail_MemTracer_Init();
 LONGTAIL_EXPORT char* Longtail_MemTracer_GetStats(uint32_t log_level);
 LONGTAIL_EXPORT void Longtail_MemTracer_Dispose();
-LONGTAIL_EXPORT void* Longtail_MemTracer_Alloc(const char* context, size_t s);
+LONGTAIL_EXPORT void* Longtail_MemTracer_ReAlloc(const char* context, void* old, size_t s);
 LONGTAIL_EXPORT void Longtail_MemTracer_Free(void* p);
 
 LONGTAIL_EXPORT int Longtail_MemTracer_DumpStats(const char* name);