Properly handle filesizes larger than 8 Gb (#76707)

* Properly handle filesizes larger than 8 Gb

* Add simulated data stream and address feedback

* Move tests for 8Gb files to outerloop and disable parallelization

* Change TarSizeFieldTooLargeForEntryType on .resx

Author: jozkee

src/libraries/System.Formats.Tar/src/Resources/Strings.resx

@@ -238,7 +238,7 @@
     <value>The size field is negative in a tar entry.</value>
   </data>
   <data name="TarSizeFieldTooLargeForEntryType" xml:space="preserve">
-    <value>The value of the size field for the current entry of type '{0}' is beyond the expected length.</value>
+    <value>The value of the size field for the current entry of type '{0}' is greater than the expected length.</value>
   </data>
   <data name="TarSymbolicLinkTargetNotExists" xml:space="preserve">
     <value>Cannot create the symbolic link '{0}' because the specified target '{1}' does not exist.</value>
@@ -264,4 +264,7 @@
   <data name="TarEntryFieldExceedsMaxLength" xml:space="preserve">
     <value>The field '{0}' exceeds the maximum allowed length for this format.</value>
   </data>
+  <data name="TarSizeFieldTooLargeForEntryFormat" xml:space="preserve">
+    <value>The value of the size field for the current entry of format '{0}' is greater than the format allows.</value>
+  </data>
 </root>

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs

@@ -376,7 +376,8 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca
                 return null;
             }
 
-            long size = (int)TarHelpers.ParseOctal<uint>(buffer.Slice(FieldLocations.Size, FieldLengths.Size));
+            long size = (long)TarHelpers.ParseOctal<ulong>(buffer.Slice(FieldLocations.Size, FieldLengths.Size));
+            Debug.Assert(size <= TarHelpers.MaxSizeLength, "size exceeded the max value possible with 11 octal digits. Actual size " + size);
             if (size < 0)
             {
                 throw new InvalidDataException(string.Format(SR.TarSizeFieldNegative));

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs

@@ -28,13 +28,13 @@ internal sealed partial class TarHeader
         // Writes the current header as a V7 entry into the archive stream.
         internal void WriteAsV7(Stream archiveStream, Span<byte> buffer)
         {
-            long actualLength = WriteV7FieldsToBuffer(buffer);
+            WriteV7FieldsToBuffer(buffer);
 
             archiveStream.Write(buffer);
 
             if (_dataStream != null)
             {
-                WriteData(archiveStream, _dataStream, actualLength);
+                WriteData(archiveStream, _dataStream, _size);
             }
         }
 
@@ -43,39 +43,37 @@ internal async Task WriteAsV7Async(Stream archiveStream, Memory<byte> buffer, Ca
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            long actualLength = WriteV7FieldsToBuffer(buffer.Span);
+            WriteV7FieldsToBuffer(buffer.Span);
 
             await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
             if (_dataStream != null)
             {
-                await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
+                await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
             }
         }
 
         // Writes the V7 header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
-        private long WriteV7FieldsToBuffer(Span<byte> buffer)
+        private void WriteV7FieldsToBuffer(Span<byte> buffer)
         {
-            long actualLength = GetTotalDataBytesToWrite();
+            _size = GetTotalDataBytesToWrite();
             TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.V7, _typeFlag);
 
             int tmpChecksum = WriteName(buffer);
-            tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType);
+            tmpChecksum += WriteCommonFields(buffer, actualEntryType);
             _checksum = WriteChecksum(tmpChecksum, buffer);
-
-            return actualLength;
         }
 
         // Writes the current header as a Ustar entry into the archive stream.
         internal void WriteAsUstar(Stream archiveStream, Span<byte> buffer)
         {
-            long actualLength = WriteUstarFieldsToBuffer(buffer);
+            WriteUstarFieldsToBuffer(buffer);
 
             archiveStream.Write(buffer);
 
             if (_dataStream != null)
             {
-                WriteData(archiveStream, _dataStream, actualLength);
+                WriteData(archiveStream, _dataStream, _size);
             }
         }
 
@@ -84,29 +82,27 @@ internal async Task WriteAsUstarAsync(Stream archiveStream, Memory<byte> buffer,
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            long actualLength = WriteUstarFieldsToBuffer(buffer.Span);
+            WriteUstarFieldsToBuffer(buffer.Span);
 
             await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
             if (_dataStream != null)
             {
-                await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
+                await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
             }
         }
 
         // Writes the Ustar header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
-        private long WriteUstarFieldsToBuffer(Span<byte> buffer)
+        private void WriteUstarFieldsToBuffer(Span<byte> buffer)
         {
-            long actualLength = GetTotalDataBytesToWrite();
+            _size = GetTotalDataBytesToWrite();
             TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Ustar, _typeFlag);
 
             int tmpChecksum = WriteUstarName(buffer);
-            tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType);
+            tmpChecksum += WriteCommonFields(buffer, actualEntryType);
             tmpChecksum += WritePosixMagicAndVersion(buffer);
             tmpChecksum += WritePosixAndGnuSharedFields(buffer);
             _checksum = WriteChecksum(tmpChecksum, buffer);
-
-            return actualLength;
         }
 
         // Writes the current header as a PAX Global Extended Attributes entry into the archive stream.
@@ -144,6 +140,7 @@ internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
             // First, we write the preceding extended attributes header
             TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
             // Fill the current header's dict
+            _size = GetTotalDataBytesToWrite();
             CollectExtendedAttributesFromStandardFieldsIfNeeded();
             // And pass the attributes to the preceding extended attributes header for writing
             extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1);
@@ -157,12 +154,12 @@ internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
         internal async Task WriteAsPaxAsync(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
         {
             Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);
-
             cancellationToken.ThrowIfCancellationRequested();
 
             // First, we write the preceding extended attributes header
             TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
             // Fill the current header's dict
+            _size = GetTotalDataBytesToWrite();
             CollectExtendedAttributesFromStandardFieldsIfNeeded();
             // And pass the attributes to the preceding extended attributes header for writing
             await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
@@ -243,13 +240,13 @@ private static TarHeader GetGnuLongMetadataHeader(TarEntryType entryType, string
         // Writes the current header as a GNU entry into the archive stream.
         internal void WriteAsGnuInternal(Stream archiveStream, Span<byte> buffer)
         {
-            WriteAsGnuSharedInternal(buffer, out long actualLength);
+            WriteAsGnuSharedInternal(buffer);
 
             archiveStream.Write(buffer);
 
             if (_dataStream != null)
             {
-                WriteData(archiveStream, _dataStream, actualLength);
+                WriteData(archiveStream, _dataStream, _size);
             }
         }
 
@@ -258,23 +255,23 @@ internal async Task WriteAsGnuInternalAsync(Stream archiveStream, Memory<byte> b
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            WriteAsGnuSharedInternal(buffer.Span, out long actualLength);
+            WriteAsGnuSharedInternal(buffer.Span);
 
             await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
             if (_dataStream != null)
             {
-                await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
+                await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
             }
         }
 
         // Shared checksum and data length calculations for GNU entry writing.
-        private void WriteAsGnuSharedInternal(Span<byte> buffer, out long actualLength)
+        private void WriteAsGnuSharedInternal(Span<byte> buffer)
         {
-            actualLength = GetTotalDataBytesToWrite();
+            _size = GetTotalDataBytesToWrite();
 
             int tmpChecksum = WriteName(buffer);
-            tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag));
+            tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag));
             tmpChecksum += WriteGnuMagicAndVersion(buffer);
             tmpChecksum += WritePosixAndGnuSharedFields(buffer);
             tmpChecksum += WriteGnuFields(buffer);
@@ -285,45 +282,44 @@ private void WriteAsGnuSharedInternal(Span<byte> buffer, out long actualLength)
         // Writes the current header as a PAX Extended Attributes entry into the archive stream.
         private void WriteAsPaxExtendedAttributes(Stream archiveStream, Span<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber)
         {
-            WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber);
-            _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
+            WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
             WriteAsPaxInternal(archiveStream, buffer);
         }
 
         // Asynchronously writes the current header as a PAX Extended Attributes entry into the archive stream and returns the value of the final checksum.
         private Task WriteAsPaxExtendedAttributesAsync(Stream archiveStream, Memory<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
-
-            WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber);
-            _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
+            WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
             return WriteAsPaxInternalAsync(archiveStream, buffer, cancellationToken);
         }
 
         // Initializes the name, mode and type flag of a PAX extended attributes entry.
-        private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber)
+        private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber, Dictionary<string, string> extendedAttributes)
         {
             Debug.Assert(isGea && globalExtendedAttributesEntryNumber >= 0 || !isGea && globalExtendedAttributesEntryNumber < 0);
 
+            _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
             _name = isGea ?
                 GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber) :
                 GenerateExtendedAttributeName();
 
             _mode = TarHelpers.GetDefaultMode(_typeFlag);
+            _size = GetTotalDataBytesToWrite();
             _typeFlag = isGea ? TarEntryType.GlobalExtendedAttributes : TarEntryType.ExtendedAttributes;
         }
 
         // Both the Extended Attributes and Global Extended Attributes entry headers are written in a similar way, just the data changes
         // This method writes an entry as both entries require, using the data from the current header instance.
         private void WriteAsPaxInternal(Stream archiveStream, Span<byte> buffer)
         {
-            WriteAsPaxSharedInternal(buffer, out long actualLength);
+            WriteAsPaxSharedInternal(buffer);
 
             archiveStream.Write(buffer);
 
             if (_dataStream != null)
             {
-                WriteData(archiveStream, _dataStream, actualLength);
+                WriteData(archiveStream, _dataStream, _size);
             }
         }
 
@@ -333,23 +329,21 @@ private async Task WriteAsPaxInternalAsync(Stream archiveStream, Memory<byte> bu
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            WriteAsPaxSharedInternal(buffer.Span, out long actualLength);
+            WriteAsPaxSharedInternal(buffer.Span);
 
             await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
             if (_dataStream != null)
             {
-                await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
+                await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
             }
         }
 
         // Shared checksum and data length calculations for PAX entry writing.
-        private void WriteAsPaxSharedInternal(Span<byte> buffer, out long actualLength)
+        private void WriteAsPaxSharedInternal(Span<byte> buffer)
         {
-            actualLength = GetTotalDataBytesToWrite();
-
             int tmpChecksum = WriteName(buffer);
-            tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag));
+            tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag));
             tmpChecksum += WritePosixMagicAndVersion(buffer);
             tmpChecksum += WritePosixAndGnuSharedFields(buffer);
 
@@ -446,7 +440,7 @@ private int WriteUstarName(Span<byte> buffer)
         }
 
         // Writes all the common fields shared by all formats into the specified spans.
-        private int WriteCommonFields(Span<byte> buffer, long actualLength, TarEntryType actualEntryType)
+        private int WriteCommonFields(Span<byte> buffer, TarEntryType actualEntryType)
         {
             // Don't write an empty LinkName if the entry is a hardlink or symlink
             Debug.Assert(!string.IsNullOrEmpty(_linkName) ^ (_typeFlag is not TarEntryType.SymbolicLink and not TarEntryType.HardLink));
@@ -468,11 +462,21 @@ private int WriteCommonFields(Span<byte> buffer, long actualLength, TarEntryType
                 checksum += FormatOctal(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid));
             }
 
-            _size = actualLength;
-
             if (_size > 0)
             {
-                checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size));
+                if (_size <= TarHelpers.MaxSizeLength)
+                {
+                    checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size));
+                }
+                else if (_format is not TarEntryFormat.Pax)
+                {
+                    throw new ArgumentException(SR.Format(SR.TarSizeFieldTooLargeForEntryFormat, _format));
+                }
+                else
+                {
+                    Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes and not TarEntryType.GlobalExtendedAttributes);
+                    Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength);
+                }
             }
 
             checksum += WriteAsTimestamp(_mTime, buffer.Slice(FieldLocations.MTime, FieldLengths.MTime));
@@ -732,10 +736,14 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded()
                 ExtendedAttributes[PaxEaLinkName] = _linkName;
             }
 
-            if (_size > 99_999_999)
+            if (_size > TarHelpers.MaxSizeLength)
             {
                 ExtendedAttributes[PaxEaSize] = _size.ToString();
             }
+            else
+            {
+                ExtendedAttributes.Remove(PaxEaSize);
+            }
 
             // Sets the specified string to the dictionary if it's longer than the specified max byte length; otherwise, remove it.
             static void TryAddStringField(Dictionary<string, string> extendedAttributes, string key, string? value, int maxLength)

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs

@@ -19,6 +19,7 @@ internal static partial class TarHelpers
     {
         internal const short RecordSize = 512;
         internal const int MaxBufferLength = 4096;
+        internal const long MaxSizeLength = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb.
 
         // Default mode for TarEntry created for a file-type.
         private const UnixFileMode DefaultFileMode =