-
Notifications
You must be signed in to change notification settings - Fork 51
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implementation of streaming compression
- Loading branch information
1 parent
f9b360b
commit 963af94
Showing
12 changed files
with
524 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| BenchmarkDotNet.Artifacts |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| using System; | ||
| using BenchmarkDotNet.Attributes; | ||
|
|
||
| namespace ZstdNet.Benchmarks | ||
| { | ||
| [MemoryDiagnoser] | ||
| public class CompressionOverheadBenchmarks | ||
| { | ||
| private const int TestSize = 1024; | ||
|
|
||
| private readonly byte[] UncompressedData = new byte[TestSize]; | ||
| private readonly byte[] CompressedData; | ||
|
|
||
| private readonly byte[] Buffer = new byte[Compressor.GetCompressBound(TestSize)]; | ||
|
|
||
| private readonly Compressor Compressor = new Compressor(new CompressionOptions(1)); | ||
| private readonly Decompressor Decompressor = new Decompressor(); | ||
|
|
||
| public CompressionOverheadBenchmarks() | ||
| { | ||
| var r = new Random(0); | ||
| r.NextBytes(UncompressedData); | ||
|
|
||
| CompressedData = Compressor.Wrap(UncompressedData); | ||
| } | ||
|
|
||
| [Benchmark] public void Compress1KBRandom() => Compressor.Wrap(UncompressedData, Buffer, 0); | ||
| [Benchmark] public void Decompress1KBRandom() => Decompressor.Unwrap(CompressedData, Buffer, 0); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| using BenchmarkDotNet.Running; | ||
|
|
||
| namespace ZstdNet.Benchmarks | ||
| { | ||
| class Program | ||
| { | ||
| static void Main() | ||
| { | ||
| BenchmarkRunner.Run<CompressionOverheadBenchmarks>(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>netcoreapp2.0</TargetFramework> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="BenchmarkDotNet" Version="0.10.11" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\ZstdNet\ZstdNet.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| using System; | ||
| using System.IO; | ||
| using NUnit.Framework; | ||
|
|
||
| namespace ZstdNet.Tests | ||
| { | ||
| public enum DataFill | ||
| { | ||
| Random, | ||
| Sequential | ||
| } | ||
|
|
||
| internal static class DataGenerator | ||
| { | ||
| private static readonly Random Random = new Random(1234); | ||
|
|
||
| public const int LargeBufferSize = 1 * 1024 * 1024; | ||
| public const int SmallBufferSize = 1 * 1024; | ||
|
|
||
| public static MemoryStream GetSmallStream(DataFill dataFill) => new MemoryStream(GetBuffer(SmallBufferSize, dataFill)); | ||
| public static MemoryStream GetLargeStream(DataFill dataFill) => new MemoryStream(GetBuffer(LargeBufferSize, dataFill)); | ||
| public static MemoryStream GetStream(int length, DataFill dataFill) => new MemoryStream(GetBuffer(length, dataFill)); | ||
|
|
||
| public static byte[] GetSmallBuffer(DataFill dataFill) => GetBuffer(SmallBufferSize, dataFill); | ||
| public static byte[] GetLargeBuffer(DataFill dataFill) => GetBuffer(LargeBufferSize, dataFill); | ||
|
|
||
| public static byte[] GetBuffer(int length, DataFill dataFill) | ||
| { | ||
| var buffer = new byte[length]; | ||
| if(dataFill == DataFill.Random) | ||
| Random.NextBytes(buffer); | ||
| else | ||
| { | ||
| for(int i = 0; i < buffer.Length; i++) | ||
| buffer[i] = (byte)(i % 256); | ||
| } | ||
|
|
||
| return buffer; | ||
| } | ||
| } | ||
|
|
||
| [TestFixture] | ||
| public class SteamingTests | ||
| { | ||
| [Test] | ||
| public void CompressionImprovesWithDictionary() | ||
| { | ||
| var trainingData = new byte[100][]; | ||
| for(int i = 0; i < trainingData.Length; i++) | ||
| trainingData[i] = DataGenerator.GetSmallBuffer(DataFill.Random); | ||
|
|
||
| var dict = DictBuilder.TrainFromBuffer(trainingData); | ||
| var compressionOptions = new CompressionOptions(dict); | ||
|
|
||
| var testStream = DataGenerator.GetSmallStream(DataFill.Random); | ||
|
|
||
| var normalResultStream = new MemoryStream(); | ||
| using(var compressionStream = new CompressorStream(normalResultStream)) | ||
| testStream.CopyTo(compressionStream); | ||
|
|
||
| var dictResultStream = new MemoryStream(); | ||
| using(var compressionStream = new CompressorStream(dictResultStream, compressionOptions)) | ||
| testStream.CopyTo(compressionStream); | ||
|
|
||
| Assert.Greater(normalResultStream.Length, dictResultStream.Length); | ||
| } | ||
|
|
||
| [Test] | ||
| public void CompressionShrinksData() | ||
| { | ||
| var inStream = DataGenerator.GetLargeStream(DataFill.Sequential); | ||
|
|
||
| var outStream = new MemoryStream(); | ||
| using(var compressionStream = new CompressorStream(outStream)) | ||
| inStream.CopyTo(compressionStream); | ||
|
|
||
| Assert.Greater(inStream.Length, outStream.Length); | ||
| } | ||
|
|
||
| [Test] | ||
| public void RoundTrip_BatchToStreaming() | ||
| { | ||
| var testBuffer = DataGenerator.GetLargeBuffer(DataFill.Sequential); | ||
|
|
||
| byte[] compressedBuffer; | ||
| using(var compressor = new Compressor()) | ||
| compressedBuffer = compressor.Wrap(testBuffer); | ||
|
|
||
| var resultStream = new MemoryStream(); | ||
| using(var decompressionStream = new DecompressorStream(new MemoryStream(compressedBuffer))) | ||
| decompressionStream.CopyTo(resultStream); | ||
|
|
||
| Validate(testBuffer, resultStream.ToArray()); | ||
| } | ||
|
|
||
| [Test] | ||
| public void RoundTrip_StreamingToBatch() | ||
| { | ||
| var testStream = DataGenerator.GetLargeStream(DataFill.Sequential); | ||
|
|
||
| var tempStream = new MemoryStream(); | ||
| using(var compressorStream = new CompressorStream(testStream)) | ||
| tempStream.CopyTo(compressorStream); | ||
|
|
||
| byte[] resultBuffer; | ||
| using(var decompressor = new Decompressor()) | ||
| resultBuffer = decompressor.Unwrap(tempStream.ToArray()); | ||
|
|
||
| Validate(tempStream.ToArray(), resultBuffer); | ||
| } | ||
|
|
||
| [Test] | ||
| public void RoundTrip_StreamingStreaming() | ||
| { | ||
| var testStream = DataGenerator.GetLargeStream(DataFill.Sequential); | ||
|
|
||
| var tempStream = new MemoryStream(); | ||
| using(var compressionStream = new CompressorStream(tempStream)) | ||
| testStream.CopyTo(compressionStream); | ||
|
|
||
| tempStream.Position = 0; | ||
|
|
||
| var resultStream = new MemoryStream(); | ||
| using(var decompressionStream = new DecompressorStream(tempStream)) | ||
| decompressionStream.CopyTo(resultStream); | ||
|
|
||
| Validate(testStream.ToArray(), resultStream.ToArray()); | ||
| } | ||
|
|
||
| private static void Validate(byte[] expected, byte[] actual) | ||
| { | ||
| Assert.AreEqual(expected.Length, actual.Length, "Decompressed Stream length is different than input stream"); | ||
|
|
||
| for(int i = 0; i < expected.Length; i++) | ||
| Assert.AreEqual(expected[i], actual[i], $"Decompressed byte index {i} is different than input stream"); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.