Skip to content

Commit

Permalink
make StreamProcessor automatically aware of maxInputBufferSize
Browse files Browse the repository at this point in the history
  • Loading branch information
@nitram509
nitram509 committed Jan 19, 2016
1 parent 1412d35 commit 5dd70e3
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 39 deletions.
2 changes: 1 addition & 1 deletion ...erformance/src/main/java/com/meteogroup/jbrotli/performance/Brotli_vs_Gzip_Benchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public void brotli_compression_with_BrotliStreamCompressor_using_byte_array() {
@Benchmark
public void brotli_compression_with_BrotliStreamCompressor_using_ByteBuffer() {
cpHtmlDataByteBuffer.position(0);
outByteBuffer = brotliStreamCompressor.compress(cpHtmlDataByteBuffer, true);
outByteBuffer = brotliStreamCompressor.compressNextBuffer(cpHtmlDataByteBuffer, true);
if (outByteBuffer.capacity() == 0) throw new AssertionError("epic fail");
}

Expand Down
15 changes: 10 additions & 5 deletions jbrotli/src/main/java/com/meteogroup/jbrotli/BrotliStreamCompressor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.nio.ByteBuffer;

import static com.meteogroup.jbrotli.BrotliErrorChecker.assertBrotliOk;
import static java.lang.Math.min;

public final class BrotliStreamCompressor implements Closeable {

Expand All @@ -39,7 +40,7 @@ public BrotliStreamCompressor() {

/**
* @param parameter parameter to use for this compressor
* @throws BrotliException in case of something in native code went wrong
* @throws BrotliException in case of something in native code went wrong
*/
public BrotliStreamCompressor(Brotli.Parameter parameter) throws BrotliException {
assertBrotliOk(initBrotliCompressor(parameter.getMode().mode, parameter.getQuality(), parameter.getLgwin(), parameter.getLgblock()));
Expand Down Expand Up @@ -69,16 +70,20 @@ public final byte[] compress(byte[] in, int inPosition, int inLength, boolean do
}

/**
* One may use {@link ByteBuffer#position(int)} and {@link ByteBuffer#limit(int)} to adjust
* how the buffers are used for reading and writing.
* Compressing larger {@link ByteBuffer}s is more easy, because this method
* automatically compresses the maximum partial buffer. For example, if you have a 10mb buffer and the
* {@link #getInputBlockSize()} is 2mb, you can call {@link #compressNextBuffer(ByteBuffer, boolean)}
* 5 times in total. The input {@link ByteBuffer#position(int)} will be set accordingly.
* You may use {@link ByteBuffer#position(int)} and {@link ByteBuffer#limit(int)} to adjust
* how the buffers are used for reading.
*
* @param in input buffer
* @param doFlush do flush
* @return a direct baked {@link ByteBuffer} containing the compressed output
*/
public final ByteBuffer compress(ByteBuffer in, boolean doFlush) {
public final ByteBuffer compressNextBuffer(ByteBuffer in, boolean doFlush) {
int inPosition = in.position();
int inLimit = in.limit();
int inLimit = min(in.limit(), inPosition + getMaxInputBufferSize());
int inRemain = inLimit - inPosition;
if (inRemain < 0)
throw new IllegalArgumentException("The source (in) position must me smaller then the source ByteBuffer's limit.");
Expand Down
73 changes: 40 additions & 33 deletions jbrotli/src/test/java/com/meteogroup/jbrotli/BrotliStreamCompressorByteBufferTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,37 @@ public void tearDown() throws Exception {
compressor.close();
}

@Test
public void compress_sets_position_to_the_next_compressed_block__until_hits_limit() throws Exception {
// setup
int maxInputBufferSize = compressor.getMaxInputBufferSize();
int expectedLargeBufferSize = (int) (maxInputBufferSize * 2.5);

// given
ByteBuffer in = wrapDirect(createFilledByteArray(expectedLargeBufferSize, 'x'));

// when #1
compressor.compressNextBuffer(in, false);
// then
assertThat(in.position()).isEqualTo(maxInputBufferSize);
assertThat(in.limit()).isEqualTo(expectedLargeBufferSize);
assertThat(in.capacity()).isEqualTo(expectedLargeBufferSize);

// when #2
compressor.compressNextBuffer(in, false);
// then
assertThat(in.position()).isEqualTo(maxInputBufferSize * 2);
assertThat(in.limit()).isEqualTo(expectedLargeBufferSize);
assertThat(in.capacity()).isEqualTo(expectedLargeBufferSize);

// when #2.5
compressor.compressNextBuffer(in, false);
// then
assertThat(in.position()).isEqualTo(expectedLargeBufferSize);
assertThat(in.limit()).isEqualTo(expectedLargeBufferSize);
assertThat(in.capacity()).isEqualTo(expectedLargeBufferSize);
}


//
// *** direct ByteBuffer **********
Expand All @@ -60,7 +91,7 @@ public void compress_with_direct_ByteBuffer_and_flushing() throws Exception {
inBuffer.position(0);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, true);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, true);

assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);
assertThat(getByteArray(outBuffer)).startsWith(A_BYTES_COMPRESSED);
Expand All @@ -73,12 +104,12 @@ public void compress_with_direct_ByteBuffer_without_flushing() throws Exception
inBuffer.position(0);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, false);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, false);
// then
assertThat(outBuffer.capacity()).isEqualTo(0);

// when
outBuffer = compressor.compress(ByteBuffer.allocateDirect(0), true);
outBuffer = compressor.compressNextBuffer(ByteBuffer.allocateDirect(0), true);
// then
assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);

Expand All @@ -100,7 +131,7 @@ public void compress_with_direct_ByteBuffer_using_position_and_length() throws E
inBuffer.limit(testPosition + A_BYTES.length);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, true);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, true);

// then
assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);
Expand All @@ -110,18 +141,6 @@ public void compress_with_direct_ByteBuffer_using_position_and_length() throws E
assertThat(getByteArray(outBuffer)).startsWith(A_BYTES_COMPRESSED);
}

@Test(expectedExceptions = IllegalArgumentException.class,
expectedExceptionsMessageRegExp = "BrotliStreamCompressor, input ByteBuffer size is larger than allowed input block size. Slice the input into smaller chunks.")
public void compress_with_array_direct_ByteBuffer_using_larger_input_buffer_throws_exception() throws Exception {
// given
ByteBuffer tmpBuffer = wrapDirect(new byte[compressor.getMaxInputBufferSize() + 1]);

// when
compressor.compress(tmpBuffer, true);

// expected exception
}

//
// *** array wrapped ByteBuffer **********

Expand All @@ -130,7 +149,7 @@ public void compress_with_array_wrapped_ByteBuffer_and_flushing() throws Excepti
ByteBuffer inBuffer = wrap(A_BYTES);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, true);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, true);

assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);
assertThat(getByteArray(outBuffer)).isEqualTo(A_BYTES_COMPRESSED);
Expand All @@ -141,12 +160,12 @@ public void compress_with_array_wrapped_ByteBuffer_without_flushing() throws Exc
ByteBuffer inBuffer = wrap(A_BYTES);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, false);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, false);
// then
assertThat(outBuffer.capacity()).isEqualTo(0);

// when
outBuffer = compressor.compress(wrap(new byte[0]), true);
outBuffer = compressor.compressNextBuffer(wrap(new byte[0]), true);
// then
assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);

Expand All @@ -167,7 +186,7 @@ public void compress_with_array_wrapped_ByteBuffer_using_position_and_length() t
inBuffer.limit(testPosition + A_BYTES.length);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, true);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, true);

// then
assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);
Expand All @@ -191,7 +210,7 @@ public void compress_with_array_wrapped_ByteBuffer_using_arrayOffset_and_length(
inBuffer.limit(A_BYTES.length);

// when
ByteBuffer outBuffer = compressor.compress(inBuffer, true);
ByteBuffer outBuffer = compressor.compressNextBuffer(inBuffer, true);

// then
assertThat(outBuffer.capacity()).isEqualTo(A_BYTES_COMPRESSED.length);
Expand All @@ -201,16 +220,4 @@ public void compress_with_array_wrapped_ByteBuffer_using_arrayOffset_and_length(
assertThat(getByteArray(outBuffer)).startsWith(A_BYTES_COMPRESSED);
}

@Test(expectedExceptions = IllegalArgumentException.class,
expectedExceptionsMessageRegExp = "BrotliStreamCompressor, input byte array length is larger than allowed input block size. Slice the input into smaller chunks.")
public void compress_with_array_wrapped_ByteBuffer_using_larger_input_buffer_throws_exception() throws Exception {
// given
ByteBuffer tmpBuffer = wrap(new byte[compressor.getMaxInputBufferSize() + 1]);

// when
compressor.compress(tmpBuffer, true);

// expected exception
}

}

0 comments on commit 5dd70e3

Please sign in to comment.