Skip to content

feat(zip): make it possible to skip header tests #689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 28, 2022
Merged

feat(zip): make it possible to skip header tests #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b939aa
feat(zip): make it possible to skip header tests
piksel Nov 19, 2021
64a8cb7
refactor for readability and more specific errors
piksel Nov 19, 2021
a7af855
refactor tests to provide details on failure
piksel Nov 19, 2021
fc92777
Merge branch 'master' into feat/opt-ziptest
piksel Apr 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/ICSharpCode.SharpZipLib/Zip/ZipConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,21 @@ public enum GeneralBitFlags
/// </summary>
ReservedPkware15 = 0x8000
}

/// <summary>
/// Helpers for <see cref="GeneralBitFlags"/>
/// </summary>
public static class GeneralBitFlagsExtensions
{
/// <summary>
/// This is equivalent of <see cref="Enum.HasFlag"/> in .NET Core, but since the .NET FW
/// version is really slow (due to un-/boxing and reflection) we use this wrapper.
/// </summary>
/// <param name="flagData"></param>
/// <param name="flag"></param>
/// <returns></returns>
public static bool Includes(this GeneralBitFlags flagData, GeneralBitFlags flag) => (flag & flagData) != 0;
}

#endregion Enumerations

Expand Down
153 changes: 77 additions & 76 deletions src/ICSharpCode.SharpZipLib/Zip/ZipFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,7 @@ public bool TestArchive(bool testData, TestStrategy strategy, ZipTestResultHandl
[Flags]
private enum HeaderTest
{
None = 0x0,
Extract = 0x01, // Check that this header represents an entry whose data can be extracted
Header = 0x02, // Check that this header contents are valid
}
Expand All @@ -1110,13 +1111,12 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)

if (signature != ZipConstants.LocalHeaderSignature)
{
throw new ZipException(string.Format("Wrong local header signature at 0x{0:x}, expected 0x{1:x8}, actual 0x{2:x8}",
entryAbsOffset, ZipConstants.LocalHeaderSignature, signature));
throw new ZipException($"Wrong local header signature at 0x{entryAbsOffset:x}, expected 0x{ZipConstants.LocalHeaderSignature:x8}, actual 0x{signature:x8}");
}

var extractVersion = (short)(ReadLEUshort() & 0x00ff);
var localFlags = (GeneralBitFlags)ReadLEUshort();
var compressionMethod = (short)ReadLEUshort();
var compressionMethod = (CompressionMethod)ReadLEUshort();
var fileTime = (short)ReadLEUshort();
var fileDate = (short)ReadLEUshort();
uint crcValue = ReadLEUint();
Expand All @@ -1134,7 +1134,7 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
var localExtraData = new ZipExtraData(extraData);

// Extra data / zip64 checks
if (localExtraData.Find(1))
if (localExtraData.Find(headerID: 1))
{
// 2010-03-04 Forum 10512: removed checks for version >= ZipConstants.VersionZip64
// and size or compressedSize = MaxValue, due to rogue creators.
Expand Down Expand Up @@ -1175,15 +1175,19 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
throw new ZipException("Compression method not supported");
}

if ((extractVersion > ZipConstants.VersionMadeBy)
|| ((extractVersion > 20) && (extractVersion < ZipConstants.VersionZip64)))
if (extractVersion > ZipConstants.VersionMadeBy
|| (extractVersion > 20 && extractVersion < ZipConstants.VersionZip64))
{
throw new ZipException(string.Format("Version required to extract this entry not supported ({0})", extractVersion));
throw new ZipException($"Version required to extract this entry not supported ({extractVersion})");
}

if (localFlags.HasAny(GeneralBitFlags.Patched | GeneralBitFlags.StrongEncryption | GeneralBitFlags.EnhancedCompress | GeneralBitFlags.HeaderMasked))
const GeneralBitFlags notSupportedFlags = GeneralBitFlags.Patched
| GeneralBitFlags.StrongEncryption
| GeneralBitFlags.EnhancedCompress
| GeneralBitFlags.HeaderMasked;
if (localFlags.HasAny(notSupportedFlags))
{
throw new ZipException("The library does not support the zip version required to extract this entry");
throw new ZipException($"The library does not support the zip features required to extract this entry ({localFlags & notSupportedFlags:F})");
}
}
}
Expand All @@ -1207,7 +1211,7 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
(extractVersion != 63)
)
{
throw new ZipException(string.Format("Version required to extract this entry is invalid ({0})", extractVersion));
throw new ZipException($"Version required to extract this entry is invalid ({extractVersion})");
}

var localEncoding = _stringCodec.ZipInputEncoding(localFlags);
Expand All @@ -1221,7 +1225,7 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
// Encryption requires extract version >= 20
if (localFlags.HasAny(GeneralBitFlags.Encrypted) && extractVersion < 20)
{
throw new ZipException(string.Format("Version required to extract this entry is too low for encryption ({0})", extractVersion));
throw new ZipException($"Version required to extract this entry is too low for encryption ({extractVersion})");
}

// Strong encryption requires encryption flag to be set and extract version >= 50.
Expand All @@ -1234,26 +1238,26 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)

if (extractVersion < 50)
{
throw new ZipException(string.Format("Version required to extract this entry is too low for encryption ({0})", extractVersion));
throw new ZipException($"Version required to extract this entry is too low for encryption ({extractVersion})");
}
}

// Patched entries require extract version >= 27
if (localFlags.HasAny(GeneralBitFlags.Patched) && extractVersion < 27)
{
throw new ZipException(string.Format("Patched data requires higher version than ({0})", extractVersion));
throw new ZipException($"Patched data requires higher version than ({extractVersion})");
}

// Central header flags match local entry flags.
if ((int)localFlags != entry.Flags)
{
throw new ZipException("Central header/local header flags mismatch");
throw new ZipException($"Central header/local header flags mismatch ({(GeneralBitFlags)entry.Flags:F} vs {localFlags:F})");
}

// Central header compression method matches local entry
if (entry.CompressionMethodForHeader != (CompressionMethod)compressionMethod)
if (entry.CompressionMethodForHeader != compressionMethod)
{
throw new ZipException("Central header/local header compression method mismatch");
throw new ZipException($"Central header/local header compression method mismatch ({entry.CompressionMethodForHeader:G} vs {compressionMethod:G})");
}

if (entry.Version != extractVersion)
Expand All @@ -1272,7 +1276,7 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)

if (localFlags.HasAny(GeneralBitFlags.HeaderMasked))
{
if ((fileTime != 0) || (fileDate != 0))
if (fileTime != 0 || fileDate != 0)
{
throw new ZipException("Header masked set but date/time values non-zero");
}
Expand All @@ -1287,8 +1291,8 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
}

// Crc valid for empty entry.
// This will also apply to streamed entries where size isnt known and the header cant be patched
if ((size == 0) && (compressedSize == 0))
// This will also apply to streamed entries where size isn't known and the header cant be patched
if (size == 0 && compressedSize == 0)
{
if (crcValue != 0)
{
Expand Down Expand Up @@ -1351,20 +1355,15 @@ private long TestLocalHeader(ZipEntry entry, HeaderTest tests)
if (!localFlags.HasAny(GeneralBitFlags.Descriptor) ||
((size > 0 || compressedSize > 0) && entry.Size > 0))
{
if ((size != 0)
&& (size != entry.Size))
if (size != 0 && size != entry.Size)
{
throw new ZipException(
string.Format("Size mismatch between central header({0}) and local header({1})",
entry.Size, size));
throw new ZipException($"Size mismatch between central header ({entry.Size}) and local header ({size})");
}

if ((compressedSize != 0)
if (compressedSize != 0
&& (compressedSize != entry.CompressedSize && compressedSize != 0xFFFFFFFF && compressedSize != -1))
{
throw new ZipException(
string.Format("Compressed size mismatch between central header({0}) and local header({1})",
entry.CompressedSize, compressedSize));
throw new ZipException($"Compressed size mismatch between central header({entry.CompressedSize}) and local header({compressedSize})");
}
}

Expand Down Expand Up @@ -3502,20 +3501,16 @@ private void ReadEntries()
}

bool isZip64 = false;
bool requireZip64 = false;


// Check if zip64 header information is required.
if ((thisDiskNumber == 0xffff) ||
(startCentralDirDisk == 0xffff) ||
(entriesForThisDisk == 0xffff) ||
(entriesForWholeCentralDir == 0xffff) ||
(centralDirSize == 0xffffffff) ||
(offsetOfCentralDir == 0xffffffff))
{
requireZip64 = true;
}

// #357 - always check for the existance of the Zip64 central directory.
bool requireZip64 = thisDiskNumber == 0xffff ||
startCentralDirDisk == 0xffff ||
entriesForThisDisk == 0xffff ||
entriesForWholeCentralDir == 0xffff ||
centralDirSize == 0xffffffff ||
offsetOfCentralDir == 0xffffffff;

// #357 - always check for the existence of the Zip64 central directory.
// #403 - Take account of the fixed size of the locator when searching.
// Subtract from locatedEndOfCentralDir so that the endLocation is the location of EndOfCentralDirectorySignature,
// rather than the data following the signature.
Expand Down Expand Up @@ -3549,7 +3544,7 @@ private void ReadEntries()

if (sig64 != ZipConstants.Zip64CentralFileHeaderSignature)
{
throw new ZipException(string.Format("Invalid Zip64 Central directory signature at {0:X}", offset64));
throw new ZipException($"Invalid Zip64 Central directory signature at {offset64:X}");
}

// NOTE: Record size = SizeOfFixedFields + SizeOfVariableData - 12.
Expand Down Expand Up @@ -3604,8 +3599,11 @@ private void ReadEntries()
int extraLen = ReadLEUshort();
int commentLen = ReadLEUshort();

int diskStartNo = ReadLEUshort(); // Not currently used
int internalAttributes = ReadLEUshort(); // Not currently used

// ReSharper disable once UnusedVariable, Currently unused but needs to be read to offset the stream
int diskStartNo = ReadLEUshort();
// ReSharper disable once UnusedVariable, Currently unused but needs to be read to offset the stream
int internalAttributes = ReadLEUshort();

uint externalAttributes = ReadLEUint();
long offset = ReadLEUint();
Expand All @@ -3629,7 +3627,7 @@ private void ReadEntries()
ExternalFileAttributes = (int)externalAttributes
};

if ((bitFlags & 8) == 0)
if (!entry.HasFlag(GeneralBitFlags.Descriptor))
{
entry.CryptoCheckValue = (byte)(crc >> 24);
}
Expand Down Expand Up @@ -3672,9 +3670,15 @@ private void ReadEntries()
/// </exception>
private long LocateEntry(ZipEntry entry)
{
return TestLocalHeader(entry, HeaderTest.Extract);
return TestLocalHeader(entry, SkipLocalEntryTestsOnLocate ? HeaderTest.None : HeaderTest.Extract);
}

/// <summary>
/// Skip the verification of the local header when reading an archive entry. Set this to attempt to read the
/// entries even if the headers should indicate that doing so would fail or produce an unexpected output.
/// </summary>
public bool SkipLocalEntryTestsOnLocate { get; set; } = false;

private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry)
{
CryptoStream result = null;
Expand All @@ -3691,15 +3695,15 @@ private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry)
}
int saltLen = entry.AESSaltLen;
byte[] saltBytes = new byte[saltLen];
int saltIn = StreamUtils.ReadRequestedBytes(baseStream, saltBytes, 0, saltLen);
if (saltIn != saltLen)
throw new ZipException("AES Salt expected " + saltLen + " got " + saltIn);
//
int saltIn = StreamUtils.ReadRequestedBytes(baseStream, saltBytes, offset: 0, saltLen);

if (saltIn != saltLen) throw new ZipException($"AES Salt expected {saltLen} git {saltIn}");

byte[] pwdVerifyRead = new byte[2];
StreamUtils.ReadFully(baseStream, pwdVerifyRead);
int blockSize = entry.AESKeySize / 8; // bits to bytes

var decryptor = new ZipAESTransform(rawPassword_, saltBytes, blockSize, false);
var decryptor = new ZipAESTransform(rawPassword_, saltBytes, blockSize, writeMode: false);
byte[] pwdVerifyCalc = decryptor.PwdVerifier;
if (pwdVerifyCalc[0] != pwdVerifyRead[0] || pwdVerifyCalc[1] != pwdVerifyRead[1])
throw new ZipException("Invalid password for AES");
Expand All @@ -3712,8 +3716,7 @@ private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry)
}
else
{
if ((entry.Version < ZipConstants.VersionStrongEncryption)
|| (entry.Flags & (int)GeneralBitFlags.StrongEncryption) == 0)
if (entry.Version < ZipConstants.VersionStrongEncryption || !entry.HasFlag(GeneralBitFlags.StrongEncryption))
{
var classicManaged = new PkzipClassicManaged();

Expand All @@ -3738,31 +3741,29 @@ private Stream CreateAndInitDecryptionStream(Stream baseStream, ZipEntry entry)

private Stream CreateAndInitEncryptionStream(Stream baseStream, ZipEntry entry)
{
CryptoStream result = null;
if ((entry.Version < ZipConstants.VersionStrongEncryption)
|| (entry.Flags & (int)GeneralBitFlags.StrongEncryption) == 0)
{
var classicManaged = new PkzipClassicManaged();
if (entry.Version >= ZipConstants.VersionStrongEncryption &&
entry.HasFlag(GeneralBitFlags.StrongEncryption)) return null;

OnKeysRequired(entry.Name);
if (HaveKeys == false)
{
throw new ZipException("No password available for encrypted stream");
}
var classicManaged = new PkzipClassicManaged();

// Closing a CryptoStream will close the base stream as well so wrap it in an UncompressedStream
// which doesnt do this.
result = new CryptoStream(new UncompressedStream(baseStream),
classicManaged.CreateEncryptor(key, null), CryptoStreamMode.Write);
OnKeysRequired(entry.Name);
if (HaveKeys == false)
{
throw new ZipException("No password available for encrypted stream");
}

if ((entry.Crc < 0) || (entry.Flags & 8) != 0)
{
WriteEncryptionHeader(result, entry.DosTime << 16);
}
else
{
WriteEncryptionHeader(result, entry.Crc);
}
// Closing a CryptoStream will close the base stream as well so wrap it in an UncompressedStream
// which doesnt do this.
var result = new CryptoStream(new UncompressedStream(baseStream),
classicManaged.CreateEncryptor(key, null), CryptoStreamMode.Write);

if (entry.Crc < 0 || entry.HasFlag(GeneralBitFlags.Descriptor))
{
WriteEncryptionHeader(result, entry.DosTime << 16);
}
else
{
WriteEncryptionHeader(result, entry.Crc);
}
return result;
}
Expand All @@ -3785,7 +3786,7 @@ private static void WriteEncryptionHeader(Stream stream, long crcValue)
rng.GetBytes(cryptBuffer);
}
cryptBuffer[11] = (byte)(crcValue >> 24);
stream.Write(cryptBuffer, 0, cryptBuffer.Length);
stream.Write(cryptBuffer, offset: 0, cryptBuffer.Length);
}

#endregion Internal routines
Expand Down
1 change: 0 additions & 1 deletion test/ICSharpCode.SharpZipLib.Tests/TestSupport/Utils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using System;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace ICSharpCode.SharpZipLib.Tests.TestSupport
Expand Down
Loading