-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Tar: Fix PAX regression when handling the size of really long unseeka…
…ble data streams (#88280) * Fix regression introduced by #84279 preventing PAX entries with really long data streams to get its size correctly stored in the extended attributes when the data stream is unseekable. * Move tests for large files to a new manual tests project.
- Loading branch information
1 parent
bf78b40
commit 2268fb3
Showing
9 changed files
with
450 additions
and
303 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
366 changes: 243 additions & 123 deletions
366
src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using Xunit; | ||
|
|
||
| namespace System.Formats.Tar.Tests; | ||
|
|
||
| [OuterLoop] | ||
| [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time | ||
| public class ManualTests : TarTestsBase | ||
| { | ||
| public static bool ManualTestsEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MANUAL_TESTS")); | ||
|
|
||
| public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryData() | ||
| { | ||
| foreach (bool unseekableStream in new[] { false, true }) | ||
| { | ||
| foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) | ||
| { | ||
| yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream }; | ||
| } | ||
|
|
||
| // Pax supports unlimited size files. | ||
| yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream }; | ||
| } | ||
| } | ||
|
|
||
| [ConditionalTheory(nameof(ManualTestsEnabled))] | ||
| [MemberData(nameof(WriteEntry_LongFileSize_TheoryData))] | ||
| [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")] | ||
| public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream) | ||
| { | ||
| // Write archive with a 8 Gb long entry. | ||
| using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose }); | ||
| Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile; | ||
|
|
||
| using (TarWriter writer = new(s, leaveOpen: true)) | ||
| { | ||
| TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); | ||
| writeEntry.DataStream = new SimulatedDataStream(size); | ||
| writer.WriteEntry(writeEntry); | ||
| } | ||
|
|
||
| tarFile.Position = 0; | ||
|
|
||
| // Read archive back. | ||
| using TarReader reader = new TarReader(s); | ||
| TarEntry entry = reader.GetNextEntry(); | ||
| Assert.Equal(size, entry.Length); | ||
|
|
||
| Stream dataStream = entry.DataStream; | ||
| Assert.Equal(size, dataStream.Length); | ||
| Assert.Equal(0, dataStream.Position); | ||
|
|
||
| ReadOnlySpan<byte> dummyData = SimulatedDataStream.DummyData.Span; | ||
|
|
||
| // Read the first bytes. | ||
| Span<byte> buffer = new byte[dummyData.Length]; | ||
| Assert.Equal(buffer.Length, dataStream.Read(buffer)); | ||
| AssertExtensions.SequenceEqual(dummyData, buffer); | ||
| Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct. | ||
| buffer.Clear(); | ||
|
|
||
| // Read the last bytes. | ||
| long dummyDataOffset = size - dummyData.Length - 1; | ||
| if (dataStream.CanSeek) | ||
| { | ||
| Assert.False(unseekableStream); | ||
| dataStream.Seek(dummyDataOffset, SeekOrigin.Begin); | ||
| } | ||
| else | ||
| { | ||
| Assert.True(unseekableStream); | ||
| Span<byte> seekBuffer = new byte[4_096]; | ||
|
|
||
| while (dataStream.Position < dummyDataOffset) | ||
| { | ||
| int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position); | ||
| int res = dataStream.Read(seekBuffer.Slice(0, bufSize)); | ||
| Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong"); | ||
| } | ||
| } | ||
|
|
||
| Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct. | ||
| Assert.Equal(buffer.Length, dataStream.Read(buffer)); | ||
| AssertExtensions.SequenceEqual(dummyData, buffer); | ||
| Assert.Equal(size, dataStream.Position); | ||
|
|
||
| Assert.Null(reader.GetNextEntry()); | ||
| } | ||
| } |
82 changes: 82 additions & 0 deletions
82
src/libraries/System.Formats.Tar/tests/Manual/ManualTestsAsync.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Threading.Tasks; | ||
| using Xunit; | ||
|
|
||
| namespace System.Formats.Tar.Tests; | ||
|
|
||
| [OuterLoop] | ||
| [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time | ||
| public class ManualTestsAsync : TarTestsBase | ||
| { | ||
| public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryDataAsync() | ||
| // Fixes error xUnit1015: MemberData needs to be in the same class | ||
| => ManualTests.WriteEntry_LongFileSize_TheoryData(); | ||
|
|
||
| [ConditionalTheory(nameof(ManualTests.ManualTestsEnabled))] | ||
| [MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))] | ||
| [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")] | ||
| public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream) | ||
| { | ||
| // Write archive with a 8 Gb long entry. | ||
| await using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose }); | ||
| Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile; | ||
|
|
||
| await using (TarWriter writer = new(s, leaveOpen: true)) | ||
| { | ||
| TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); | ||
| writeEntry.DataStream = new SimulatedDataStream(size); | ||
| await writer.WriteEntryAsync(writeEntry); | ||
| } | ||
|
|
||
| tarFile.Position = 0; | ||
|
|
||
| // Read the archive back. | ||
| await using TarReader reader = new TarReader(s); | ||
| TarEntry entry = await reader.GetNextEntryAsync(); | ||
| Assert.Equal(size, entry.Length); | ||
|
|
||
| Stream dataStream = entry.DataStream; | ||
| Assert.Equal(size, dataStream.Length); | ||
| Assert.Equal(0, dataStream.Position); | ||
|
|
||
| ReadOnlyMemory<byte> dummyData = SimulatedDataStream.DummyData; | ||
|
|
||
| // Read the first bytes. | ||
| byte[] buffer = new byte[dummyData.Length]; | ||
| Assert.Equal(buffer.Length, dataStream.Read(buffer)); | ||
| AssertExtensions.SequenceEqual(dummyData.Span, buffer); | ||
| Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct. | ||
| buffer.AsSpan().Clear(); | ||
|
|
||
| // Read the last bytes. | ||
| long dummyDataOffset = size - dummyData.Length - 1; | ||
| if (dataStream.CanSeek) | ||
| { | ||
| Assert.False(unseekableStream); | ||
| dataStream.Seek(dummyDataOffset, SeekOrigin.Begin); | ||
| } | ||
| else | ||
| { | ||
| Assert.True(unseekableStream); | ||
| Memory<byte> seekBuffer = new byte[4_096]; | ||
|
|
||
| while (dataStream.Position < dummyDataOffset) | ||
| { | ||
| int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position); | ||
| int res = await dataStream.ReadAsync(seekBuffer.Slice(0, bufSize)); | ||
| Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong"); | ||
| } | ||
| } | ||
|
|
||
| Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct. | ||
| Assert.Equal(buffer.Length, dataStream.Read(buffer)); | ||
| AssertExtensions.SequenceEqual(dummyData.Span, buffer); | ||
| Assert.Equal(size, dataStream.Position); | ||
|
|
||
| Assert.Null(await reader.GetNextEntryAsync()); | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
src/libraries/System.Formats.Tar/tests/Manual/System.Formats.Tar.Manual.Tests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <TargetFramework>$(NetCoreAppCurrent)</TargetFramework> | ||
| <IncludeRemoteExecutor>true</IncludeRemoteExecutor> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <Compile Include="ManualTests.cs" /> | ||
| <Compile Include="ManualTestsAsync.cs" /> | ||
| <Compile Include="..\TarTestsBase.cs" /> | ||
| <Compile Include="..\SimulatedDataStream.cs" /> | ||
| <Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" /> | ||
| <Compile Include="$(CommonTestPath)System\IO\TempDirectory.cs" Link="Common\System\IO\TempDirectory.cs" /> | ||
| <Compile Include="$(CommonTestPath)System\IO\WrappedStream.cs" Link="Common\System\IO\WrappedStream.cs" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 0 additions & 92 deletions
92
src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.