feat(zip): enable ZipOuputStream to write precompressed files (#683)

Co-authored-by: nils måsén <nils@piksel.se>

Author: Temtaime

src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs

@@ -270,7 +270,76 @@ public void PutNextEntry(ZipEntry entry)
 				WriteOutput(GetEntryEncryptionHeader(entry));
 			}
 		}
-		
+
+		/// <summary>
+		/// Starts a new passthrough Zip entry. It automatically closes the previous
+		/// entry if present.
+		/// Passthrough entry is an entry that is created from compressed data. 
+		/// It is useful to avoid recompression to save CPU resources if compressed data is already disposable.
+		/// All entry elements bar name, crc, size and compressed size are optional, but must be correct if present.
+		/// Compression should be set to Deflated.
+		/// </summary>
+		/// <param name="entry">
+		/// the entry.
+		/// </param>
+		/// <exception cref="System.ArgumentNullException">
+		/// if entry passed is null.
+		/// </exception>
+		/// <exception cref="System.IO.IOException">
+		/// if an I/O error occurred.
+		/// </exception>
+		/// <exception cref="System.InvalidOperationException">
+		/// if stream was finished.
+		/// </exception>
+		/// <exception cref="ZipException">
+		/// Crc is not set<br/>
+		/// Size is not set<br/>
+		/// CompressedSize is not set<br/>
+		/// CompressionMethod is not Deflate<br/>
+		/// Too many entries in the Zip file<br/>
+		/// Entry name is too long<br/>
+		/// Finish has already been called<br/>
+		/// </exception>
+		/// <exception cref="System.NotImplementedException">
+		/// The Compression method specified for the entry is unsupported<br/>
+		/// Entry is encrypted<br/>
+		/// </exception>
+		public void PutNextPassthroughEntry(ZipEntry entry) 
+		{
+			if(curEntry != null) 
+			{
+				CloseEntry();
+			}
+
+			if(entry.Crc < 0) 
+			{
+				throw new ZipException("Crc must be set for passthrough entry");
+			}
+
+			if(entry.Size < 0) 
+			{
+				throw new ZipException("Size must be set for passthrough entry");
+			}
+
+			if(entry.CompressedSize < 0) 
+			{
+				throw new ZipException("CompressedSize must be set for passthrough entry");
+			}
+
+			if(entry.CompressionMethod != CompressionMethod.Deflated)
+			{
+				throw new NotImplementedException("Only Deflated entries are supported for passthrough");
+			}
+
+			if(!string.IsNullOrEmpty(Password)) 
+			{
+				throw new NotImplementedException("Encrypted passthrough entries are not supported");
+			}
+
+			PutNextEntry(baseOutputStream_, entry, 0, true);
+		}
+
+
 		private void WriteOutput(byte[] bytes) 
 			=> baseOutputStream_.Write(bytes, 0, bytes.Length);
 
@@ -282,7 +351,7 @@ private byte[] GetEntryEncryptionHeader(ZipEntry entry) =>
 				? InitializeAESPassword(entry, Password)
 				: CreateZipCryptoHeader(entry.Crc < 0 ? entry.DosTime << 16 : entry.Crc);
 
-		internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0)
+		internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0, bool passthroughEntry = false)
 		{
 			if (entry == null)
 			{
@@ -313,6 +382,8 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0)
 				throw new InvalidOperationException("The Password property must be set before AES encrypted entries can be added");
 			}
 
+			entryIsPassthrough = passthroughEntry;
+
 			int compressionLevel = defaultCompressionLevel;
 
 			// Clear flags that the library manages internally
@@ -322,7 +393,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0)
 			bool headerInfoAvailable;
 
 			// No need to compress - definitely no data.
-			if (entry.Size == 0)
+			if (entry.Size == 0 && !entryIsPassthrough)
 			{
 				entry.CompressedSize = entry.Size;
 				entry.Crc = 0;
@@ -406,14 +477,17 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0)
 
 			// Activate the entry.
 			curEntry = entry;
+			size = 0;
+
+			if(entryIsPassthrough)
+				return;
+
 			crc.Reset();
 			if (method == CompressionMethod.Deflated)
 			{
 				deflater_.Reset();
 				deflater_.SetLevel(compressionLevel);
 			}
-			size = 0;
-			
 		}
 
 		/// <summary>
@@ -506,6 +580,17 @@ internal void WriteEntryFooter(Stream stream)
 				throw new InvalidOperationException("No open entry");
 			}
 
+			if(entryIsPassthrough) 
+			{
+				if(curEntry.CompressedSize != size) 
+				{
+					throw new ZipException($"compressed size was {size}, but {curEntry.CompressedSize} expected");
+				}
+
+				offset += size;
+				return;
+			}
+
 			long csize = size;
 
 			// First finish the deflater, if appropriate
@@ -695,30 +780,28 @@ public override void Write(byte[] buffer, int offset, int count)
 				throw new ArgumentException("Invalid offset/count combination");
 			}
 
-			if (curEntry.AESKeySize == 0)
+			if (curEntry.AESKeySize == 0 && !entryIsPassthrough)
 			{
-				// Only update CRC if AES is not enabled
+				// Only update CRC if AES is not enabled and entry is not a passthrough one
 				crc.Update(new ArraySegment<byte>(buffer, offset, count));
 			}
 
 			size += count;
 
-			switch (curMethod)
+			if(curMethod == CompressionMethod.Stored || entryIsPassthrough)
 			{
-				case CompressionMethod.Deflated:
-					base.Write(buffer, offset, count);
-					break;
-
-				case CompressionMethod.Stored:
-					if (Password != null)
-					{
-						CopyAndEncrypt(buffer, offset, count);
-					}
-					else
-					{
-						baseOutputStream_.Write(buffer, offset, count);
-					}
-					break;
+				if (Password != null)
+				{
+					CopyAndEncrypt(buffer, offset, count);
+				}
+				else
+				{
+					baseOutputStream_.Write(buffer, offset, count);
+				}
+			}
+			else
+			{
+				base.Write(buffer, offset, count);
 			}
 		}
 
@@ -844,6 +927,8 @@ public override void Flush()
 		/// </summary>
 		private ZipEntry curEntry;
 
+		private bool entryIsPassthrough;
+
 		private int defaultCompressionLevel = Deflater.DEFAULT_COMPRESSION;
 
 		private CompressionMethod curMethod = CompressionMethod.Deflated;

test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs

@@ -1,5 +1,7 @@
-using ICSharpCode.SharpZipLib.Tests.TestSupport;
+using ICSharpCode.SharpZipLib.Checksum;
+using ICSharpCode.SharpZipLib.Tests.TestSupport;
 using ICSharpCode.SharpZipLib.Zip;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using NUnit.Framework;
 using System;
 using System.IO;

test/ICSharpCode.SharpZipLib.Tests/Zip/PassthroughTests.cs

@@ -0,0 +1,140 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
+using ICSharpCode.SharpZipLib.Checksum;
+using ICSharpCode.SharpZipLib.Tests.TestSupport;
+using ICSharpCode.SharpZipLib.Zip;
+using NUnit.Framework;
+
+namespace ICSharpCode.SharpZipLib.Tests.Zip
+{
+	[TestFixture]
+	public class PassthroughTests
+	{
+		[Test]
+		[Category("Zip")]
+		public void AddingValidPrecompressedEntryToZipOutputStream()
+		{
+			using var ms = new MemoryStream();
+
+			using (var outStream = new ZipOutputStream(ms){IsStreamOwner = false})
+			{
+				var (compressedData, crc, size) = CreateDeflatedData();
+				var entry = new ZipEntry("dummyfile.tst")
+				{
+					CompressionMethod = CompressionMethod.Deflated,
+					Size = size,
+					Crc = (uint)crc.Value,
+					CompressedSize = compressedData.Length,
+				};
+
+				outStream.PutNextPassthroughEntry(entry);
+
+				compressedData.CopyTo(outStream);
+			}
+
+			Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray()));
+		}
+
+		private static (MemoryStream, Crc32, int) CreateDeflatedData()
+		{
+			var data = Encoding.UTF8.GetBytes("Hello, world");
+
+			var crc = new Crc32();
+			crc.Update(data);
+
+			var compressedData = new MemoryStream();
+			using(var gz = new DeflateStream(compressedData, CompressionMode.Compress, leaveOpen: true)) 
+			{
+				gz.Write(data, 0, data.Length);
+			}
+			compressedData.Position = 0;
+
+			return (compressedData, crc, data.Length);
+		}
+
+		[Test]
+		[Category("Zip")]
+		public void AddingPrecompressedEntryToZipOutputStreamWithInvalidSize()
+		{
+			using var outStream = new ZipOutputStream(new MemoryStream());
+			var (compressedData, crc, size) = CreateDeflatedData();
+			outStream.Password = "mockpassword";
+			var entry = new ZipEntry("dummyfile.tst")
+			{
+				CompressionMethod = CompressionMethod.Stored,
+				Crc = (uint)crc.Value,
+				CompressedSize = compressedData.Length,
+			};
+			
+			Assert.Throws<ZipException>(() =>
+			{
+				outStream.PutNextPassthroughEntry(entry);
+			});
+		}
+		
+		
+		[Test]
+		[Category("Zip")]
+		public void AddingPrecompressedEntryToZipOutputStreamWithInvalidCompressedSize()
+		{
+			using var outStream = new ZipOutputStream(new MemoryStream());
+			var (compressedData, crc, size) = CreateDeflatedData();
+			outStream.Password = "mockpassword";
+			var entry = new ZipEntry("dummyfile.tst")
+			{
+				CompressionMethod = CompressionMethod.Stored,
+				Size = size,
+				Crc = (uint)crc.Value,
+			};
+			
+			Assert.Throws<ZipException>(() =>
+			{
+				outStream.PutNextPassthroughEntry(entry);
+			});
+		}
+
+		[Test]
+		[Category("Zip")]
+		public void AddingPrecompressedEntryToZipOutputStreamWithNonSupportedMethod()
+		{
+			using var outStream = new ZipOutputStream(new MemoryStream());
+			var (compressedData, crc, size) = CreateDeflatedData();
+			outStream.Password = "mockpassword";
+			var entry = new ZipEntry("dummyfile.tst")
+			{
+				CompressionMethod = CompressionMethod.LZMA,
+				Size = size,
+				Crc = (uint)crc.Value,
+				CompressedSize = compressedData.Length,
+			};
+			
+			Assert.Throws<NotImplementedException>(() =>
+			{
+				outStream.PutNextPassthroughEntry(entry);
+			});
+		}
+
+		[Test]
+		[Category("Zip")]
+		public void AddingPrecompressedEntryToZipOutputStreamWithEncryption()
+		{
+			using var outStream = new ZipOutputStream(new MemoryStream());
+			var (compressedData, crc, size) = CreateDeflatedData();
+			outStream.Password = "mockpassword";
+			var entry = new ZipEntry("dummyfile.tst")
+			{
+				CompressionMethod = CompressionMethod.Deflated,
+				Size = size,
+				Crc = (uint)crc.Value,
+				CompressedSize = compressedData.Length,
+			};
+			
+			Assert.Throws<NotImplementedException>(() =>
+			{
+				outStream.PutNextPassthroughEntry(entry);
+			});
+		}
+	}
+}