Set version in ZIP local header to ZIP64 when file offset is >4GB (#102053) (#103006)

* ZipArchiveEntry didn't set ZIP64 in local headers for small files if their offset are > 4GB.
* Added System.IO.Compression and System.IO.Packaging tests.

---------

Co-authored-by: Gan Keyu <gankeyu@hotmail.com>

Author: carlossanlop

src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs

@@ -469,7 +469,7 @@ internal void WriteCentralDirectoryFileHeader()
 
             bool zip64Needed = false;
 
-            if (SizesTooLarge()
+            if (AreSizesTooLarge
 #if DEBUG_FORCE_ZIP64
                 || _archive._forceZip64
 #endif
@@ -490,7 +490,7 @@ internal void WriteCentralDirectoryFileHeader()
             }
 
 
-            if (_offsetOfLocalHeader > uint.MaxValue
+            if (IsOffsetTooLarge
 #if DEBUG_FORCE_ZIP64
                 || _archive._forceZip64
 #endif
@@ -797,7 +797,11 @@ private bool IsOpenable(bool needToUncompress, bool needToLoadIntoMemory, out st
             return true;
         }
 
-        private bool SizesTooLarge() => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue;
+        private bool AreSizesTooLarge => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue;
+
+        private bool IsOffsetTooLarge => _offsetOfLocalHeader > uint.MaxValue;
+
+        private bool ShouldUseZIP64 => AreSizesTooLarge || IsOffsetTooLarge;
 
         // return value is true if we allocated an extra field for 64 bit headers, un/compressed size
         private bool WriteLocalFileHeader(bool isEmptyFile)
@@ -813,6 +817,9 @@ private bool WriteLocalFileHeader(bool isEmptyFile)
             bool zip64Used = false;
             uint compressedSizeTruncated, uncompressedSizeTruncated;
 
+            // save offset
+            _offsetOfLocalHeader = writer.BaseStream.Position;
+
             // if we already know that we have an empty file don't worry about anything, just do a straight shot of the header
             if (isEmptyFile)
             {
@@ -840,7 +847,7 @@ private bool WriteLocalFileHeader(bool isEmptyFile)
                 {
                     // We are in seekable mode so we will not need to write a data descriptor
                     _generalPurposeBitFlag &= ~BitFlagValues.DataDescriptor;
-                    if (SizesTooLarge()
+                    if (ShouldUseZIP64
 #if DEBUG_FORCE_ZIP64
                         || (_archive._forceZip64 && _archive.Mode == ZipArchiveMode.Update)
 #endif
@@ -865,9 +872,6 @@ private bool WriteLocalFileHeader(bool isEmptyFile)
                 }
             }
 
-            // save offset
-            _offsetOfLocalHeader = writer.BaseStream.Position;
-
             // calculate extra field. if zip64 stuff + original extraField aren't going to fit, dump the original extraField, because this is more important
             int bigExtraFieldLength = (zip64Used ? zip64ExtraField.TotalSize : 0)
                                       + (_lhUnknownExtraFields != null ? ZipGenericExtraField.TotalSize(_lhUnknownExtraFields) : 0);
@@ -964,7 +968,7 @@ private void WriteCrcAndSizesInLocalHeader(bool zip64HeaderUsed)
             long finalPosition = _archive.ArchiveStream.Position;
             BinaryWriter writer = new BinaryWriter(_archive.ArchiveStream);
 
-            bool zip64Needed = SizesTooLarge()
+            bool zip64Needed = ShouldUseZIP64
 #if DEBUG_FORCE_ZIP64
                 || _archive._forceZip64
 #endif
@@ -1048,7 +1052,7 @@ private void WriteDataDescriptor()
 
             writer.Write(ZipLocalFileHeader.DataDescriptorSignature);
             writer.Write(_crc32);
-            if (SizesTooLarge())
+            if (AreSizesTooLarge)
             {
                 writer.Write(_compressedSize);
                 writer.Write(_uncompressedSize);

src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs

@@ -1,48 +1,129 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Collections.Generic;
-using System.Linq;
+using System.Reflection;
 using Xunit;
 
-namespace System.IO.Compression.Tests
+namespace System.IO.Compression.Tests;
+
+[Collection(nameof(DisableParallelization))]
+public class zip_LargeFiles : ZipFileTestBase
 {
-    [Collection(nameof(DisableParallelization))]
-    public class zip_LargeFiles : ZipFileTestBase
+    [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes
+    [OuterLoop("It requires almost 12 GB of free disk space")]
+    public static void UnzipOver4GBZipFile()
     {
-        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes
-        [OuterLoop("It requires almost 12 GB of free disk space")]
-        public static void UnzipOver4GBZipFile()
+        byte[] buffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // 1 GB
+
+        string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip");
+        DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB"));
+
+        try
+        {
+            for (byte i = 0; i < 6; i++)
+            {
+                File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer);
+            }
+
+            ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false);
+
+            using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath);
+            foreach (ZipArchiveEntry entry in zipArchive.Entries)
+            {
+                using Stream entryStream = entry.Open();
+
+                Assert.True(entryStream.CanRead);
+                Assert.Equal(buffer.Length, entryStream.Length);
+            }
+        }
+        finally
         {
-            byte[] buffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // 1 GB
+            File.Delete(zipArchivePath);
+
+            tempDir.Delete(recursive: true);
+        }
+    }
+
+    private static void FillWithHardToCompressData(byte[] buffer)
+    {
+        Random.Shared.NextBytes(buffer);
+    }
 
-            string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip");
-            DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB"));
+    [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes
+    [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")]
+    [InlineData(false)]
+    [InlineData(true)]
+    public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed)
+    {
+        // issue #94899
+
+        CompressionLevel compressLevel = isCompressed ? CompressionLevel.Optimal : CompressionLevel.NoCompression;
+        byte[] smallBuffer = GC.AllocateUninitializedArray<byte>(1000);
+        byte[] largeBuffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // ~1 GB
+        string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip");
+        string LargeFileName = "largefile";
+        string SmallFileName = "smallfile";
+        uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4;
+        ushort Zip64Version = 45;
 
-            try
+        try
+        {
+            using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite);
+
+            // Create
+            using (ZipArchive archive = new(fs, ZipArchiveMode.Create, true))
             {
-                for (byte i = 0; i < 6; i++)
+                ZipArchiveEntry file = archive.CreateEntry(LargeFileName, compressLevel);
+
+                using (Stream stream = file.Open())
                 {
-                    File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer);
+                    // Write 5GB of data
+                    for (var i = 0; i < 5; i++)
+                    {
+                        if (isCompressed)
+                        {
+                            FillWithHardToCompressData(largeBuffer);
+                        }
+
+                        stream.Write(largeBuffer);
+                    }
                 }
 
-                ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false);
+                file = archive.CreateEntry(SmallFileName, compressLevel);
 
-                using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath);
-                foreach (ZipArchiveEntry entry in zipArchive.Entries)
+                using (Stream stream = file.Open())
                 {
-                    using Stream entryStream = entry.Open();
-
-                    Assert.True(entryStream.CanRead);
-                    Assert.Equal(buffer.Length, entryStream.Length);
+                    stream.Write(smallBuffer);
                 }
             }
-            finally
+
+            fs.Position = 0;
+
+            // Validate
+            using (ZipArchive archive = new(fs, ZipArchiveMode.Read))
             {
-                File.Delete(zipArchivePath);
+                using var reader = new BinaryReader(fs);
 
-                tempDir.Delete(recursive: true);
+                FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance);
+
+                if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long))
+                {
+                    Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written.");
+                }
+
+                foreach (ZipArchiveEntry entry in archive.Entries)
+                {
+                    fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart;
+                    ushort versionNeeded = reader.ReadUInt16();
+
+                    // Version is not ZIP64 for files with Local Header at >4GB offset.
+                    Assert.Equal(Zip64Version, versionNeeded);
+                }
             }
         }
+        finally
+        {
+            File.Delete(zipArchivePath);
+        }
     }
 }

src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs

@@ -0,0 +1,97 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.IO.Compression;
+using System.Net.Mime;
+using System.Reflection;
+using Xunit;
+
+namespace System.IO.Packaging.Tests;
+
+public partial class LargeFilesTests
+{
+    private static void FillWithHardToCompressData(byte[] buffer)
+    {
+        Random.Shared.NextBytes(buffer);
+    }
+
+    [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes
+    [InlineData(false)]
+    [InlineData(true)]
+    [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")]
+    public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed)
+    {
+        // issue #94899
+
+        CompressionOption compressionOption = isCompressed ? CompressionOption.Normal : CompressionOption.NotCompressed;
+        byte[] smallBuffer = GC.AllocateUninitializedArray<byte>(1000);
+        byte[] largeBuffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // ~1 GB
+        string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip");
+        Uri largePartUri = PackUriHelper.CreatePartUri(new Uri("large.bin", UriKind.Relative));
+        Uri smallPartUri = PackUriHelper.CreatePartUri(new Uri("small.bin", UriKind.Relative));
+        uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4;
+        ushort Zip64Version = 45;
+
+        try
+        {
+            using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite);
+
+            // Create
+            using (Package package = Package.Open(fs, FileMode.Create, FileAccess.Write))
+            {
+                PackagePart partLarge = package.CreatePart(largePartUri, MediaTypeNames.Application.Octet, compressionOption);
+
+                using (Stream streamLarge = partLarge.GetStream())
+                {
+                    // Write 5GB of data
+
+                    for (var i = 0; i < 5; i++)
+                    {
+                        if (isCompressed)
+                        {
+                            FillWithHardToCompressData(largeBuffer);
+                        }
+
+                        streamLarge.Write(largeBuffer);
+                    }
+                }
+
+                PackagePart partSmall = package.CreatePart(smallPartUri, MediaTypeNames.Application.Octet, compressionOption);
+
+                using (Stream streamSmall = partSmall.GetStream())
+                {
+                    streamSmall.Write(smallBuffer);
+                }
+            }
+
+
+            fs.Position = 0;
+
+            // Validate
+            using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Read))
+            {
+                using var reader = new BinaryReader(fs);
+
+                FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance);
+
+                if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long))
+                {
+                    Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written.");
+                }
+
+                foreach (ZipArchiveEntry entry in archive.Entries)
+                {
+                    fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart;
+                    ushort versionNeeded = reader.ReadUInt16();
+
+                    // Version is not ZIP64 for files with Local Header at >4GB offset.
+                    Assert.Equal(Zip64Version, versionNeeded);
+                }
+            }
+        }
+        finally
+        {
+            File.Delete(zipArchivePath);
+        }
+    }
+}