From 0b3ec2b55d4ecbaf3cbfecee79599a71ac1481d2 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 29 Jan 2014 10:23:02 -0800 Subject: [PATCH 01/14] HPACK: Write headers using indexed names where possible. --- .../okhttp/internal/spdy/HpackDraft05.java | 36 +++++-- .../internal/spdy/HpackDraft05Test.java | 102 +++++++++++------- .../internal/spdy/Http20Draft09Test.java | 9 +- 3 files changed, 96 insertions(+), 51 deletions(-) diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/HpackDraft05.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/HpackDraft05.java index 36620689d126..0bd8b0677934 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/HpackDraft05.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/HpackDraft05.java @@ -1,14 +1,17 @@ package com.squareup.okhttp.internal.spdy; import com.squareup.okhttp.internal.BitArray; -import com.squareup.okhttp.internal.bytes.ByteString; import com.squareup.okhttp.internal.Util; +import com.squareup.okhttp.internal.bytes.ByteString; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static com.squareup.okhttp.internal.Util.asciiLowerCase; @@ -188,7 +191,7 @@ void readHeaders() throws IOException { } else { // 0NNNNNNN if (b == 0x40) { // 01000000 readLiteralHeaderWithoutIndexingNewName(); - } else if ((b & 0xe0) == 0x40) { // 01NNNNNN + } else if ((b & 0x40) == 0x40) { // 01NNNNNN int index = readInt(b, PREFIX_6_BITS); readLiteralHeaderWithoutIndexingIndexedName(index - 1); } else if (b == 0) { // 00000000 @@ -375,6 +378,19 @@ ByteString readByteString(boolean asciiLowercase) throws IOException { } } + private static final Map NAME_TO_FIRST_INDEX = nameToFirstIndex(); + + private static Map nameToFirstIndex() { + Map result = + new LinkedHashMap(STATIC_HEADER_TABLE.length); + for (int i = 0; i < STATIC_HEADER_TABLE.length; i++) { + if (!result.containsKey(STATIC_HEADER_TABLE[i].name)) { + result.put(STATIC_HEADER_TABLE[i].name, i); + } + } + return Collections.unmodifiableMap(result); + } + static final class Writer { private final OutputStream out; @@ -383,11 +399,19 @@ static final class Writer { } void writeHeaders(List
headerBlock) throws IOException { - // TODO: implement a compression strategy. + // TODO: implement index tracking for (int i = 0, size = headerBlock.size(); i < size; i++) { - out.write(0x40); // Literal Header without Indexing - New Name. - writeByteString(headerBlock.get(i).name); - writeByteString(headerBlock.get(i).value); + ByteString name = headerBlock.get(i).name; + Integer staticIndex = NAME_TO_FIRST_INDEX.get(name); + if (staticIndex != null) { + // Literal Header Field without Indexing - Indexed Name. + writeInt(staticIndex + 1, PREFIX_6_BITS, 0x40); + writeByteString(headerBlock.get(i).value); + } else { + out.write(0x40); // Literal Header without Indexing - New Name. + writeByteString(name); + writeByteString(headerBlock.get(i).value); + } } } diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/HpackDraft05Test.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/HpackDraft05Test.java index 9e3bb1ed80e8..5f011835fa15 100644 --- a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/HpackDraft05Test.java +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/HpackDraft05Test.java @@ -27,6 +27,7 @@ import org.junit.Test; import static com.squareup.okhttp.internal.Util.headerEntries; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -36,9 +37,12 @@ public class HpackDraft05Test { private final MutableByteArrayInputStream bytesIn = new MutableByteArrayInputStream(); private HpackDraft05.Reader hpackReader; + private ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + private HpackDraft05.Writer hpackWriter; - @Before public void resetReader() { + @Before public void reset() { hpackReader = newReader(bytesIn); + hpackWriter = new HpackDraft05.Writer(new DataOutputStream(bytesOut)); } /** @@ -167,7 +171,7 @@ public class HpackDraft05Test { /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.1.1 */ - @Test public void decodeLiteralHeaderFieldWithIndexing() throws IOException { + @Test public void readLiteralHeaderFieldWithIndexing() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(0x00); // Literal indexed @@ -191,30 +195,61 @@ public class HpackDraft05Test { assertEquals(headerEntries("custom-key", "custom-header"), hpackReader.getAndReset()); } + /** + * Literal Header Field without Indexing - New Name + */ + @Test public void literalHeaderFieldWithoutIndexingNewName() throws IOException { + List
headerBlock = headerEntries("custom-key", "custom-header"); + + ByteArrayOutputStream expectedBytes = new ByteArrayOutputStream(); + + expectedBytes.write(0x40); // Not indexed + expectedBytes.write(0x0a); // Literal name (len = 10) + expectedBytes.write("custom-key".getBytes(), 0, 10); + + expectedBytes.write(0x0d); // Literal value (len = 13) + expectedBytes.write("custom-header".getBytes(), 0, 13); + + hpackWriter.writeHeaders(headerBlock); + assertArrayEquals(expectedBytes.toByteArray(), bytesOut.toByteArray()); + + bytesIn.set(bytesOut.toByteArray()); + hpackReader.readHeaders(); + hpackReader.emitReferenceSet(); + + assertEquals(0, hpackReader.headerCount); + + assertEquals(headerBlock, hpackReader.getAndReset()); + } + /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.1.2 */ - @Test public void decodeLiteralHeaderFieldWithoutIndexingIndexedName() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); + @Test public void literalHeaderFieldWithoutIndexingIndexedName() throws IOException { + List
headerBlock = headerEntries(":path", "/sample/path"); - out.write(0x44); // == Literal not indexed == - // Indexed name (idx = 4) -> :path - out.write(0x0c); // Literal value (len = 12) - out.write("/sample/path".getBytes(), 0, 12); + ByteArrayOutputStream expectedBytes = new ByteArrayOutputStream(); + expectedBytes.write(0x44); // == Literal not indexed == + // Indexed name (idx = 4) -> :path + expectedBytes.write(0x0c); // Literal value (len = 12) + expectedBytes.write("/sample/path".getBytes(), 0, 12); - bytesIn.set(out.toByteArray()); + hpackWriter.writeHeaders(headerBlock); + assertArrayEquals(expectedBytes.toByteArray(), bytesOut.toByteArray()); + + bytesIn.set(bytesOut.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); assertEquals(0, hpackReader.headerCount); - assertEquals(headerEntries(":path", "/sample/path"), hpackReader.getAndReset()); + assertEquals(headerBlock, hpackReader.getAndReset()); } /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.1.3 */ - @Test public void decodeIndexedHeaderField() throws IOException { + @Test public void readIndexedHeaderField() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(0x82); // == Indexed - Add == @@ -264,7 +299,7 @@ public class HpackDraft05Test { /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.1.4 */ - @Test public void decodeIndexedHeaderFieldFromStaticTableWithoutBuffering() throws IOException { + @Test public void readIndexedHeaderFieldFromStaticTableWithoutBuffering() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(0x82); // == Indexed - Add == @@ -284,24 +319,24 @@ public class HpackDraft05Test { /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.2 */ - @Test public void decodeRequestExamplesWithoutHuffman() throws IOException { + @Test public void readRequestExamplesWithoutHuffman() throws IOException { ByteArrayOutputStream out = firstRequestWithoutHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkFirstRequestWithoutHuffman(); + checkReadFirstRequestWithoutHuffman(); out = secondRequestWithoutHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkSecondRequestWithoutHuffman(); + checkReadSecondRequestWithoutHuffman(); out = thirdRequestWithoutHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkThirdRequestWithoutHuffman(); + checkReadThirdRequestWithoutHuffman(); } private ByteArrayOutputStream firstRequestWithoutHuffman() { @@ -321,7 +356,7 @@ private ByteArrayOutputStream firstRequestWithoutHuffman() { return out; } - private void checkFirstRequestWithoutHuffman() { + private void checkReadFirstRequestWithoutHuffman() { assertEquals(4, hpackReader.headerCount); // [ 1] (s = 57) :authority: www.example.com @@ -366,7 +401,7 @@ private ByteArrayOutputStream secondRequestWithoutHuffman() { return out; } - private void checkSecondRequestWithoutHuffman() { + private void checkReadSecondRequestWithoutHuffman() { assertEquals(5, hpackReader.headerCount); // [ 1] (s = 53) cache-control: no-cache @@ -427,7 +462,7 @@ private ByteArrayOutputStream thirdRequestWithoutHuffman() { return out; } - private void checkThirdRequestWithoutHuffman() { + private void checkReadThirdRequestWithoutHuffman() { assertEquals(8, hpackReader.headerCount); // [ 1] (s = 54) custom-key: custom-value @@ -486,24 +521,24 @@ private void checkThirdRequestWithoutHuffman() { /** * http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-E.3 */ - @Test public void decodeRequestExamplesWithHuffman() throws IOException { + @Test public void readRequestExamplesWithHuffman() throws IOException { ByteArrayOutputStream out = firstRequestWithHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkFirstRequestWithHuffman(); + checkReadFirstRequestWithHuffman(); out = secondRequestWithHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkSecondRequestWithHuffman(); + checkReadSecondRequestWithHuffman(); out = thirdRequestWithHuffman(); bytesIn.set(out.toByteArray()); hpackReader.readHeaders(); hpackReader.emitReferenceSet(); - checkThirdRequestWithHuffman(); + checkReadThirdRequestWithHuffman(); } private ByteArrayOutputStream firstRequestWithHuffman() { @@ -528,7 +563,7 @@ private ByteArrayOutputStream firstRequestWithHuffman() { return out; } - private void checkFirstRequestWithHuffman() { + private void checkReadFirstRequestWithHuffman() { assertEquals(4, hpackReader.headerCount); // [ 1] (s = 57) :authority: www.example.com @@ -577,7 +612,7 @@ private ByteArrayOutputStream secondRequestWithHuffman() { return out; } - private void checkSecondRequestWithHuffman() { + private void checkReadSecondRequestWithHuffman() { assertEquals(5, hpackReader.headerCount); // [ 1] (s = 53) cache-control: no-cache @@ -647,7 +682,7 @@ private ByteArrayOutputStream thirdRequestWithHuffman() { return out; } - private void checkThirdRequestWithHuffman() { + private void checkReadThirdRequestWithHuffman() { assertEquals(8, hpackReader.headerCount); // [ 1] (s = 54) custom-key: custom-value @@ -703,10 +738,6 @@ private void checkThirdRequestWithHuffman() { "custom-key", "custom-value"), hpackReader.getAndReset()); } - private ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - private final HpackDraft05.Writer hpackWriter = - new HpackDraft05.Writer(new DataOutputStream(bytesOut)); - @Test public void readSingleByteInt() throws IOException { assertEquals(10, newReader(byteStream()).readInt(10, 31)); assertEquals(10, newReader(byteStream()).readInt(0xe0 | 10, 31)); @@ -768,17 +799,6 @@ private void checkThirdRequestWithHuffman() { assertSame(ByteString.EMPTY, newReader(byteStream(0)).readByteString(false)); } - @Test public void headersRoundTrip() throws IOException { - List
sentHeaders = headerEntries("name", "value"); - hpackWriter.writeHeaders(sentHeaders); - ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray()); - HpackDraft05.Reader reader = newReader(bytesIn); - reader.readHeaders(); - reader.emitReferenceSet(); - List
receivedHeaders = reader.getAndReset(); - assertEquals(sentHeaders, receivedHeaders); - } - private HpackDraft05.Reader newReader(InputStream input) { return new HpackDraft05.Reader(false, 4096, input); } diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java index 8a8aaedfc741..4f94d27c4941 100644 --- a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java @@ -219,21 +219,22 @@ public void pushPromise(int streamId, int promisedStreamId, List
headerB // Decoding the first header will cross frame boundaries. byte[] headerBlock = literalHeaders(pushPromise); + int firstFrameLength = headerBlock.length - 1; { // Write the first headers frame. - dataOut.writeShort((headerBlock.length / 2) + 4); + dataOut.writeShort(firstFrameLength + 4); dataOut.write(Http20Draft09.TYPE_PUSH_PROMISE); dataOut.write(0); // no flags dataOut.writeInt(expectedStreamId & 0x7fffffff); dataOut.writeInt(expectedPromisedStreamId & 0x7fffffff); - dataOut.write(headerBlock, 0, headerBlock.length / 2); + dataOut.write(headerBlock, 0, firstFrameLength); } { // Write the continuation frame, specifying no more frames are expected. - dataOut.writeShort(headerBlock.length / 2); + dataOut.writeShort(1); dataOut.write(Http20Draft09.TYPE_CONTINUATION); dataOut.write(Http20Draft09.FLAG_END_HEADERS); dataOut.writeInt(expectedStreamId & 0x7fffffff); - dataOut.write(headerBlock, headerBlock.length / 2, headerBlock.length / 2); + dataOut.write(headerBlock, firstFrameLength, 1); } FrameReader fr = newReader(out); From 2d01579a18cc50c33147e3e51924515dbdbf950b Mon Sep 17 00:00:00 2001 From: jwilson Date: Thu, 30 Jan 2014 22:20:55 -0500 Subject: [PATCH 02/14] Adapters that go the other way, to java.io. These ones are slightly more awkward because they need to do their own internal buffering. --- .../com/squareup/okhttp/internal/Util.java | 2 +- .../okhttp/internal/bytes/OkBuffer.java | 18 +-- .../okhttp/internal/bytes/OkBuffers.java | 103 +++++++++++++++++- .../okhttp/internal/bytes/OkBufferTest.java | 80 +++++++++++++- 4 files changed, 186 insertions(+), 17 deletions(-) diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java index 0cd85e19172c..fbbf46fd0d3d 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java @@ -82,7 +82,7 @@ public static int getDefaultPort(String protocol) { return -1; } - public static void checkOffsetAndCount(int arrayLength, int offset, int count) { + public static void checkOffsetAndCount(long arrayLength, long offset, long count) { if ((offset | count) < 0 || offset > arrayLength || arrayLength - offset < count) { throw new ArrayIndexOutOfBoundsException(); } diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java index fc6c25f7df1a..e534dc526bbb 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java @@ -21,6 +21,8 @@ import java.util.Collections; import java.util.List; +import static com.squareup.okhttp.internal.Util.checkOffsetAndCount; + /** * A collection of bytes in memory. * @@ -147,7 +149,7 @@ public String readUtf8(int byteCount) { } private byte[] readBytes(int byteCount) { - checkByteCount(byteCount); + checkOffsetAndCount(this.byteCount, 0, byteCount); int offset = 0; byte[] result = new byte[byteCount]; @@ -301,7 +303,7 @@ Segment writableSegment(int minimumCapacity) { // yielding sink [51%, 91%, 30%] and source [62%, 82%]. if (source == this) throw new IllegalArgumentException("source == this"); - source.checkByteCount(byteCount); + checkOffsetAndCount(source.byteCount, 0, byteCount); while (byteCount > 0) { // Is a prefix of the source's head segment all that we need to move? @@ -365,7 +367,6 @@ public long indexOf(byte b) throws IOException { } @Override public void flush(Deadline deadline) { - throw new UnsupportedOperationException("Cannot flush() an OkBuffer"); } @Override public void close(Deadline deadline) { @@ -400,15 +401,4 @@ List segmentSizes() { } return new String(result); } - - /** Throws if this has fewer bytes than {@code requested}. */ - void checkByteCount(long requested) { - if (requested < 0) { - throw new IllegalArgumentException("requested < 0: " + requested); - } - if (requested > this.byteCount) { - throw new IllegalArgumentException( - String.format("requested %s > available %s", requested, this.byteCount)); - } - } } diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java index ec7c60a3ed01..c4a17ac9d862 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java @@ -19,6 +19,8 @@ import java.io.InputStream; import java.io.OutputStream; +import static com.squareup.okhttp.internal.Util.checkOffsetAndCount; + public final class OkBuffers { private OkBuffers() { } @@ -28,7 +30,7 @@ public static Sink sink(final OutputStream out) { return new Sink() { @Override public void write(OkBuffer source, long byteCount, Deadline deadline) throws IOException { - source.checkByteCount(byteCount); + checkOffsetAndCount(source.byteCount, 0, byteCount); while (byteCount > 0) { deadline.throwIfReached(); Segment head = source.head; @@ -60,6 +62,52 @@ public static Sink sink(final OutputStream out) { }; } + /** + * Returns an output stream that writes to {@code sink}. This may buffer data + * by deferring writes. + */ + public static OutputStream outputStream(final Sink sink) { + return new OutputStream() { + final OkBuffer buffer = new OkBuffer(); // Buffer at most one segment of data. + + @Override public void write(int b) throws IOException { + buffer.writeByte((byte) b); + if (buffer.byteCount == Segment.SIZE) { + sink.write(buffer, buffer.byteCount, Deadline.NONE); + } + } + + @Override public void write(byte[] data, int offset, int byteCount) throws IOException { + checkOffsetAndCount(data.length, offset, byteCount); + int limit = offset + byteCount; + while (offset < limit) { + Segment onlySegment = buffer.writableSegment(1); + int toCopy = Math.min(limit - offset, Segment.SIZE - onlySegment.limit); + System.arraycopy(data, offset, onlySegment.data, onlySegment.limit, toCopy); + offset += toCopy; + onlySegment.limit += toCopy; + buffer.byteCount += toCopy; + if (buffer.byteCount == Segment.SIZE) { + sink.write(buffer, buffer.byteCount, Deadline.NONE); + } + } + } + + @Override public void flush() throws IOException { + sink.write(buffer, buffer.byteCount, Deadline.NONE); + sink.flush(Deadline.NONE); + } + + @Override public void close() throws IOException { + sink.close(Deadline.NONE); + } + + @Override public String toString() { + return "outputStream(" + sink + ")"; + } + }; + } + /** Returns a source that reads from {@code in}. */ public static Source source(final InputStream in) { return new Source() { @@ -85,4 +133,57 @@ public static Source source(final InputStream in) { } }; } + + /** + * Returns an input stream that reads from {@code source}. This may buffer + * data by reading extra data eagerly. + */ + public static InputStream inputStream(final Source source) { + return new InputStream() { + final OkBuffer buffer = new OkBuffer(); + + @Override public int read() throws IOException { + if (buffer.byteCount == 0) { + long count = source.read(buffer, Segment.SIZE, Deadline.NONE); + if (count == -1) return -1; + } + return buffer.readByte(); + } + + @Override public int read(byte[] data, int offset, int byteCount) throws IOException { + checkOffsetAndCount(data.length, offset, byteCount); + + if (buffer.byteCount == 0) { + long count = source.read(buffer, Segment.SIZE, Deadline.NONE); + if (count == -1) return -1; + } + + Segment head = buffer.head; + int toCopy = Math.min(byteCount, head.limit - head.pos); + System.arraycopy(head.data, head.pos, data, offset, toCopy); + + head.pos += toCopy; + buffer.byteCount -= toCopy; + + if (head.pos == head.limit) { + buffer.head = head.pop(); + SegmentPool.INSTANCE.recycle(head); + } + + return toCopy; + } + + @Override public int available() throws IOException { + return (int) Math.min(buffer.byteCount, Integer.MAX_VALUE); + } + + @Override public void close() throws IOException { + super.close(); + } + + @Override public String toString() { + return "inputStream(" + source + ")"; + } + }; + } } diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/OkBufferTest.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/OkBufferTest.java index ed52cb68910b..eedb9def753d 100644 --- a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/OkBufferTest.java +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/OkBufferTest.java @@ -17,7 +17,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.Arrays; import java.util.List; import org.junit.Test; @@ -41,7 +43,7 @@ public final class OkBufferTest { try { buffer.readUtf8(1); fail(); - } catch (IllegalArgumentException expected) { + } catch (ArrayIndexOutOfBoundsException expected) { } } @@ -292,6 +294,26 @@ private List moveBytesBetweenBuffers(String... contents) { assertEquals("a" + repeat('b', 9998) + "c", out.toString("UTF-8")); } + @Test public void outputStreamFromSink() throws Exception { + OkBuffer sink = new OkBuffer(); + OutputStream out = OkBuffers.outputStream(sink); + out.write('a'); + out.write(repeat('b', 9998).getBytes(UTF_8)); + out.write('c'); + out.flush(); + assertEquals("a" + repeat('b', 9998) + "c", sink.readUtf8(10000)); + } + + @Test public void outputStreamFromSinkBounds() throws Exception { + OkBuffer sink = new OkBuffer(); + OutputStream out = OkBuffers.outputStream(sink); + try { + out.write(new byte[100], 50, 51); + fail(); + } catch (ArrayIndexOutOfBoundsException expected) { + } + } + @Test public void sourceFromInputStream() throws Exception { InputStream in = new ByteArrayInputStream( ("a" + repeat('b', Segment.SIZE * 2) + "c").getBytes(UTF_8)); @@ -316,6 +338,62 @@ private List moveBytesBetweenBuffers(String... contents) { assertEquals(-1, source.read(sink, 1, Deadline.NONE)); } + @Test public void sourceFromInputStreamBounds() throws Exception { + Source source = OkBuffers.source(new ByteArrayInputStream(new byte[100])); + try { + source.read(new OkBuffer(), -1, Deadline.NONE); + fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test public void inputStreamFromSource() throws Exception { + OkBuffer source = new OkBuffer(); + source.writeUtf8("a"); + source.writeUtf8(repeat('b', Segment.SIZE)); + source.writeUtf8("c"); + + InputStream in = OkBuffers.inputStream(source); + assertEquals(0, in.available()); + assertEquals(Segment.SIZE + 2, source.byteCount()); + + // Reading one byte buffers a full segment. + assertEquals('a', in.read()); + assertEquals(Segment.SIZE - 1, in.available()); + assertEquals(2, source.byteCount()); + + // Reading as much as possible reads the rest of that buffered segment. + byte[] data = new byte[Segment.SIZE * 2]; + assertEquals(Segment.SIZE - 1, in.read(data, 0, data.length)); + assertEquals(repeat('b', Segment.SIZE - 1), new String(data, 0, Segment.SIZE - 1, UTF_8)); + assertEquals(2, source.byteCount()); + + // Continuing to read buffers the next segment. + assertEquals('b', in.read()); + assertEquals(1, in.available()); + assertEquals(0, source.byteCount()); + + // Continuing to read reads from the buffer. + assertEquals('c', in.read()); + assertEquals(0, in.available()); + assertEquals(0, source.byteCount()); + + // Once we've exhausted the source, we're done. + assertEquals(-1, in.read()); + assertEquals(0, source.byteCount()); + } + + @Test public void inputStreamFromSourceBounds() throws IOException { + OkBuffer source = new OkBuffer(); + source.writeUtf8(repeat('a', 100)); + InputStream in = OkBuffers.inputStream(source); + try { + in.read(new byte[100], 50, 51); + fail(); + } catch (ArrayIndexOutOfBoundsException expected) { + } + } + @Test public void writeBytes() throws Exception { OkBuffer data = new OkBuffer(); data.writeByte(0xab); From 9bbdf3a9245e83dfb2a2f20afce7fd1dc41f95fb Mon Sep 17 00:00:00 2001 From: jwilson Date: Fri, 31 Jan 2014 22:24:05 -0500 Subject: [PATCH 03/14] Quick and dirty benchmark. Sample output: OkHttp [gzip, chunked, HTTP_11] bodyByteCount=1048576 headerCount=20 threadCount=10 Requests per second: 121.0 Requests per second: 304.6 Requests per second: 379.1 Requests per second: 386.9 Requests per second: 369.2 Requests per second: 390.8 Requests per second: 368.8 Requests per second: 325.2 Requests per second: 409.4 Requests per second: 389.1 --- benchmarks/pom.xml | 36 +++ .../squareup/okhttp/benchmarks/Benchmark.java | 238 ++++++++++++++++++ .../benchmarks/HttpURLConnectionRequest.java | 60 +++++ .../okhttp/mockwebserver/MockWebServer.java | 13 +- pom.xml | 1 + 5 files changed, 343 insertions(+), 5 deletions(-) create mode 100644 benchmarks/pom.xml create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml new file mode 100644 index 000000000000..e606d7f84d12 --- /dev/null +++ b/benchmarks/pom.xml @@ -0,0 +1,36 @@ + + + + 4.0.0 + + + com.squareup.okhttp + parent + 2.0.0-SNAPSHOT + + + benchmarks + Benchmarks + + + + com.squareup.okhttp + okhttp + ${project.version} + + + com.squareup.okhttp + mockwebserver + ${project.version} + + + org.bouncycastle + bcprov-jdk15on + + + org.mortbay.jetty.npn + npn-boot + provided + + + diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java new file mode 100644 index 000000000000..7637a3ba2482 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Protocol; +import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.mockwebserver.Dispatcher; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.RecordedRequest; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; + +/** + * This benchmark is fake, but may be useful for certain relative comparisons. + * It uses a local connection to a MockWebServer to measure how many identical + * requests per second can be carried over a fixed number of threads. + */ +public class Benchmark { + private static final int NUM_REPORTS = 10; + private final Random random = new Random(0); + + /** Which client to run.*/ + // TODO: implement additional candidates for other HTTP client libraries. + Candidate candidate = new OkHttp(); + + /** How many concurrent threads to execute. */ + int threadCount = 10; + + /** True to use TLS. */ + // TODO: compare different ciphers? + boolean tls = false; + + /** True to use gzip content-encoding for the response body. */ + boolean gzip = true; + + /** Don't combine chunked with SPDY_3 or HTTP_2; that's not allowed. */ + boolean chunked = true; + + /** The size of the HTTP response body, in uncompressed bytes. */ + int bodyByteCount = 1024 * 1024; + + /** How many additional headers were included, beyond the built-in ones. */ + int headerCount = 20; + + /** Which ALPN/NPN protocols are in use. Only useful with TLS. */ + List protocols = Arrays.asList(Protocol.HTTP_11); + + public static void main(String[] args) throws IOException { + new Benchmark().run(); + } + + public void run() throws IOException { + ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, + 1, TimeUnit.SECONDS, new LinkedBlockingQueue()); + + System.out.println(toString()); + + // Prepare the client & server + candidate.prepare(); + MockWebServer server = startServer(); + String url = server.getUrl("/").toString(); + + int targetBacklog = 10; + int requestCount = 0; + long reportStart = System.nanoTime(); + long reportPeriod = TimeUnit.SECONDS.toNanos(1); + int reports = 0; + + // Run until we've printed enough reports. + while (reports < NUM_REPORTS) { + // Print a report if we haven't recently. + long now = System.nanoTime(); + double reportDuration = now - reportStart; + if (reportDuration > reportPeriod) { + double requestsPerSecond = requestCount / reportDuration * TimeUnit.SECONDS.toNanos(1); + System.out.println(String.format("Requests per second: %.1f", requestsPerSecond)); + requestCount = 0; + reportStart = now; + reports++; + } + + // Fill the job queue with work. + while (executor.getQueue().size() < targetBacklog) { + executor.execute(candidate.request(url)); + requestCount++; + } + + // The job queue is full. Take a break. + sleep(10); + } + } + + @Override public String toString() { + List modifiers = new ArrayList(); + if (tls) modifiers.add("tls"); + if (gzip) modifiers.add("gzip"); + if (chunked) modifiers.add("chunked"); + modifiers.addAll(protocols); + + return String.format("%s %s\n" + + "bodyByteCount=%s headerCount=%s threadCount=%s", + candidate.getClass().getSimpleName(), modifiers, + bodyByteCount, headerCount, threadCount); + } + + private void sleep(int millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException ignored) { + } + } + + private MockWebServer startServer() throws IOException { + Logger.getLogger(MockWebServer.class.getName()).setLevel(Level.WARNING); + MockWebServer server = new MockWebServer(); + + if (tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + server.useHttps(sslContext.getSocketFactory(), false); + server.setNpnEnabled(true); + } + + final MockResponse response = newResponse(); + server.setDispatcher(new Dispatcher() { + @Override public MockResponse dispatch(RecordedRequest request) { + return response; + } + }); + + server.play(); + return server; + } + + private MockResponse newResponse() throws IOException { + byte[] body = new byte[bodyByteCount]; + random.nextBytes(body); + + MockResponse result = new MockResponse(); + + if (gzip) { + body = gzip(body); + result.addHeader("Content-Encoding: gzip"); + } + + if (chunked) { + result.setChunkedBody(body, 1024); + } else { + result.setBody(body); + } + + for (int i = 0; i < headerCount; i++) { + result.addHeader(randomString(12), randomString(20)); + } + + return result; + } + + private String randomString(int length) { + String alphabet = "-abcdefghijklmnopqrstuvwxyz"; + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = alphabet.charAt(random.nextInt(alphabet.length())); + } + return new String(result); + } + + /** Returns a gzipped copy of {@code bytes}. */ + private byte[] gzip(byte[] bytes) throws IOException { + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + OutputStream gzippedOut = new GZIPOutputStream(bytesOut); + gzippedOut.write(bytes); + gzippedOut.close(); + return bytesOut.toByteArray(); + } + + interface Candidate { + void prepare(); + Runnable request(String url); + } + + class OkHttp implements Candidate { + private OkHttpClient client; + + @Override public void prepare() { + client = new OkHttpClient(); + client.setProtocols(protocols); + + URL.setURLStreamHandlerFactory(client); + + if (tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override public boolean verify(String s, SSLSession session) { + return true; + } + }; + client.setSslSocketFactory(socketFactory); + client.setHostnameVerifier(hostnameVerifier); + } + } + + @Override public Runnable request(String url) { + return new HttpURLConnectionRequest(url); + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java new file mode 100644 index 000000000000..b1eb99eceda1 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.concurrent.TimeUnit; + +public class HttpURLConnectionRequest implements Runnable { + private static final boolean VERBOSE = false; + private final URL url; + + public HttpURLConnectionRequest(String url) { + try { + this.url = new URL(url); + } catch (MalformedURLException e) { + throw new AssertionError(); + } + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + InputStream in = urlConnection.getInputStream(); + + // Discard the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } +} diff --git a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java index 4ca28bab4cc8..90e2925e625a 100644 --- a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java +++ b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java @@ -18,10 +18,10 @@ package com.squareup.okhttp.mockwebserver; import com.squareup.okhttp.Protocol; -import com.squareup.okhttp.internal.bytes.ByteString; import com.squareup.okhttp.internal.NamedRunnable; import com.squareup.okhttp.internal.Platform; import com.squareup.okhttp.internal.Util; +import com.squareup.okhttp.internal.bytes.ByteString; import com.squareup.okhttp.internal.spdy.Header; import com.squareup.okhttp.internal.spdy.IncomingStreamHandler; import com.squareup.okhttp.internal.spdy.SpdyConnection; @@ -72,7 +72,6 @@ * replays them upon request in sequence. */ public final class MockWebServer { - private static final X509TrustManager UNTRUSTED_TRUST_MANAGER = new X509TrustManager() { @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { @@ -381,7 +380,9 @@ private boolean processOneRequest(Socket socket, InputStream in, OutputStream ou } else if (response.getSocketPolicy() == SocketPolicy.SHUTDOWN_OUTPUT_AT_END) { socket.shutdownOutput(); } - logger.info("Received request: " + request + " and responded: " + response); + if (logger.isLoggable(Level.INFO)) { + logger.info("Received request: " + request + " and responded: " + response); + } sequenceNumber++; return true; } @@ -611,8 +612,10 @@ private SpdySocketHandler(Socket socket, Protocol protocol) { throw new AssertionError(e); } writeResponse(stream, response); - logger.info("Received request: " + request + " and responded: " + response - + " protocol is " + protocol.name.utf8()); + if (logger.isLoggable(Level.INFO)) { + logger.info("Received request: " + request + " and responded: " + response + + " protocol is " + protocol.name.utf8()); + } } private RecordedRequest readRequest(SpdyStream stream) throws IOException { diff --git a/pom.xml b/pom.xml index a4ab7ea59988..aea71da07add 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,7 @@ okhttp-protocols mockwebserver samples + benchmarks From 7c21992db56623b9d77f682d7e4edea078c3985b Mon Sep 17 00:00:00 2001 From: jwilson Date: Sat, 1 Feb 2014 00:52:44 -0500 Subject: [PATCH 04/14] New benchmark targets for Apache HTTP Client and URLConnection. OkHttp [HTTP_11] bodyByteCount=1048576 headerCount=20 threadCount=10 Requests per second: 690.9 UrlConnection [HTTP_11] bodyByteCount=1048576 headerCount=20 threadCount=10 Requests per second: 671.3 ApacheHttpClient [HTTP_11] bodyByteCount=1048576 headerCount=20 threadCount=10 Requests per second: 317.4 --- benchmarks/pom.xml | 4 ++ .../benchmarks/ApacheHttpClientRequest.java | 64 ++++++++++++++++++ .../squareup/okhttp/benchmarks/Benchmark.java | 56 +++++++++++++--- ...nectionRequest.java => OkHttpRequest.java} | 12 ++-- .../benchmarks/UrlConnectionRequest.java | 66 +++++++++++++++++++ 5 files changed, 190 insertions(+), 12 deletions(-) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java rename benchmarks/src/main/java/com/squareup/okhttp/benchmarks/{HttpURLConnectionRequest.java => OkHttpRequest.java} (84%) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index e606d7f84d12..7b8ce5278531 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -32,5 +32,9 @@ npn-boot provided + + org.apache.httpcomponents + httpclient + diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java new file mode 100644 index 000000000000..a507738dfe2e --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; + +class ApacheHttpClientRequest implements Runnable { + private static final boolean VERBOSE = false; + private final HttpClient client; + private final String url; + + public ApacheHttpClientRequest(String url, HttpClient client) { + this.client = client; + this.url = url; + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpResponse response = client.execute(new HttpGet(url)); + InputStream in = response.getEntity().getContent(); + Header contentEncoding = response.getFirstHeader("Content-Encoding"); + if (contentEncoding != null && contentEncoding.getValue().equals("gzip")) { + in = new GZIPInputStream(in); + } + + // Consume the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index 7637a3ba2482..3b0887af8a8d 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -18,6 +18,7 @@ import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Protocol; import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.internal.http.HttpsURLConnectionImpl; import com.squareup.okhttp.mockwebserver.Dispatcher; import com.squareup.okhttp.mockwebserver.MockResponse; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -25,7 +26,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -40,6 +40,11 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; +import org.apache.http.client.HttpClient; +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingClientConnectionManager; /** * This benchmark is fake, but may be useful for certain relative comparisons. @@ -51,8 +56,7 @@ public class Benchmark { private final Random random = new Random(0); /** Which client to run.*/ - // TODO: implement additional candidates for other HTTP client libraries. - Candidate candidate = new OkHttp(); + Candidate candidate = new UrlConnection(); // new OkHttp(); // new ApacheHttpClient(); /** How many concurrent threads to execute. */ int threadCount = 10; @@ -62,10 +66,10 @@ public class Benchmark { boolean tls = false; /** True to use gzip content-encoding for the response body. */ - boolean gzip = true; + boolean gzip = false; /** Don't combine chunked with SPDY_3 or HTTP_2; that's not allowed. */ - boolean chunked = true; + boolean chunked = false; /** The size of the HTTP response body, in uncompressed bytes. */ int bodyByteCount = 1024 * 1024; @@ -216,8 +220,6 @@ class OkHttp implements Candidate { client = new OkHttpClient(); client.setProtocols(protocols); - URL.setURLStreamHandlerFactory(client); - if (tls) { SSLContext sslContext = SslContextBuilder.localhost(); SSLSocketFactory socketFactory = sslContext.getSocketFactory(); @@ -232,7 +234,45 @@ class OkHttp implements Candidate { } @Override public Runnable request(String url) { - return new HttpURLConnectionRequest(url); + return new OkHttpRequest(client, url); + } + } + + class UrlConnection implements Candidate { + @Override public void prepare() { + if (tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override public boolean verify(String s, SSLSession session) { + return true; + } + }; + HttpsURLConnectionImpl.setDefaultHostnameVerifier(hostnameVerifier); + HttpsURLConnectionImpl.setDefaultSSLSocketFactory(socketFactory); + } + } + + @Override public Runnable request(String url) { + return new UrlConnectionRequest(url); + } + } + + class ApacheHttpClient implements Candidate { + private HttpClient client; + + @Override public void prepare() { + ClientConnectionManager connectionManager = new PoolingClientConnectionManager(); + if (tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + connectionManager.getSchemeRegistry().register( + new Scheme("https", 443, new org.apache.http.conn.ssl.SSLSocketFactory(sslContext))); + } + client = new DefaultHttpClient(connectionManager); + } + + @Override public Runnable request(String url) { + return new ApacheHttpClientRequest(url, client); } } } diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java similarity index 84% rename from benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java rename to benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java index b1eb99eceda1..e4ab485627ed 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpURLConnectionRequest.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java @@ -15,6 +15,7 @@ */ package com.squareup.okhttp.benchmarks; +import com.squareup.okhttp.OkHttpClient; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -22,12 +23,15 @@ import java.net.URL; import java.util.concurrent.TimeUnit; -public class HttpURLConnectionRequest implements Runnable { +class OkHttpRequest implements Runnable { private static final boolean VERBOSE = false; + + private final OkHttpClient client; private final URL url; - public HttpURLConnectionRequest(String url) { + public OkHttpRequest(OkHttpClient client, String url) { try { + this.client = client; this.url = new URL(url); } catch (MalformedURLException e) { throw new AssertionError(); @@ -38,10 +42,10 @@ public void run() { byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { - HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + HttpURLConnection urlConnection = client.open(url); InputStream in = urlConnection.getInputStream(); - // Discard the response body. + // Consume the response body. int total = 0; for (int count; (count = in.read(buffer)) != -1; ) { total += count; diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java new file mode 100644 index 000000000000..ab5c9b43e078 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +/** Uses the default java.net.HttpURLConnection implementation. */ +class UrlConnectionRequest implements Runnable { + private static final boolean VERBOSE = false; + + private final URL url; + + public UrlConnectionRequest(String url) { + try { + this.url = new URL(url); + } catch (MalformedURLException e) { + throw new AssertionError(); + } + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + InputStream in = urlConnection.getInputStream(); + if ("gzip".equals(urlConnection.getHeaderField("Content-Encoding"))) { + in = new GZIPInputStream(in); + } + + // Consume the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } +} From 13ce4a09bec4cb9c999f5e45d4cf3d3a90c1c95d Mon Sep 17 00:00:00 2001 From: jwilson Date: Sat, 1 Feb 2014 12:20:28 -0500 Subject: [PATCH 05/14] New benchmark target for Netty. This required some reworking to the harness since Netty doesn't want to be used with an ExecutorService. NettyHttpClient [HTTP_11] bodyByteCount=1048576 headerCount=20 concurrencyLevel=10 Requests per second: 597.1 OkHttp [HTTP_11] bodyByteCount=1048576 headerCount=20 concurrencyLevel=10 Requests per second: 705.2 --- benchmarks/pom.xml | 15 ++ .../okhttp/benchmarks/ApacheHttpClient.java | 91 +++++++++ .../benchmarks/ApacheHttpClientRequest.java | 64 ------- .../squareup/okhttp/benchmarks/Benchmark.java | 109 ++--------- .../okhttp/benchmarks/HttpClient.java | 25 +++ .../okhttp/benchmarks/NettyHttpClient.java | 174 ++++++++++++++++++ .../squareup/okhttp/benchmarks/OkHttp.java | 88 +++++++++ .../okhttp/benchmarks/OkHttpRequest.java | 64 ------- .../benchmarks/SynchronousHttpClient.java | 42 +++++ .../okhttp/benchmarks/UrlConnection.java | 87 +++++++++ .../benchmarks/UrlConnectionRequest.java | 66 ------- 11 files changed, 535 insertions(+), 290 deletions(-) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java delete mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpClient.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java delete mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java delete mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 7b8ce5278531..14d93eeefd8b 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -36,5 +36,20 @@ org.apache.httpcomponents httpclient + + io.netty + netty-transport + 4.0.15.Final + + + io.netty + netty-handler + 4.0.15.Final + + + io.netty + netty-codec-http + 4.0.15.Final + diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java new file mode 100644 index 000000000000..cd2cba31cff9 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.internal.SslContextBuilder; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; +import javax.net.ssl.SSLContext; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.ssl.SSLSocketFactory; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingClientConnectionManager; + +/** Benchmark Apache HTTP client. */ +class ApacheHttpClient extends SynchronousHttpClient { + private static final boolean VERBOSE = false; + + private HttpClient client; + + @Override public void prepare(Benchmark benchmark) { + super.prepare(benchmark); + ClientConnectionManager connectionManager = new PoolingClientConnectionManager(); + if (benchmark.tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + connectionManager.getSchemeRegistry().register( + new Scheme("https", 443, new SSLSocketFactory(sslContext))); + } + client = new DefaultHttpClient(connectionManager); + } + + @Override public Runnable request(URL url) { + return new ApacheHttpClientRequest(url); + } + + class ApacheHttpClientRequest implements Runnable { + private final URL url; + + public ApacheHttpClientRequest(URL url) { + this.url = url; + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpResponse response = client.execute(new HttpGet(url.toString())); + InputStream in = response.getEntity().getContent(); + Header contentEncoding = response.getFirstHeader("Content-Encoding"); + if (contentEncoding != null && contentEncoding.getValue().equals("gzip")) { + in = new GZIPInputStream(in); + } + + // Consume the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java deleted file mode 100644 index a507738dfe2e..000000000000 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClientRequest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2014 Square, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.squareup.okhttp.benchmarks; - -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; - -class ApacheHttpClientRequest implements Runnable { - private static final boolean VERBOSE = false; - private final HttpClient client; - private final String url; - - public ApacheHttpClientRequest(String url, HttpClient client) { - this.client = client; - this.url = url; - } - - public void run() { - byte[] buffer = new byte[1024]; - long start = System.nanoTime(); - try { - HttpResponse response = client.execute(new HttpGet(url)); - InputStream in = response.getEntity().getContent(); - Header contentEncoding = response.getFirstHeader("Content-Encoding"); - if (contentEncoding != null && contentEncoding.getValue().equals("gzip")) { - in = new GZIPInputStream(in); - } - - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); - long finish = System.nanoTime(); - - if (VERBOSE) { - System.out.println(String.format("Transferred % 8d bytes in %4d ms", - total, TimeUnit.NANOSECONDS.toMillis(finish - start))); - } - } catch (IOException e) { - System.out.println("Failed: " + e); - } - } -} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index 3b0887af8a8d..16bd063508a5 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -15,10 +15,8 @@ */ package com.squareup.okhttp.benchmarks; -import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Protocol; import com.squareup.okhttp.internal.SslContextBuilder; -import com.squareup.okhttp.internal.http.HttpsURLConnectionImpl; import com.squareup.okhttp.mockwebserver.Dispatcher; import com.squareup.okhttp.mockwebserver.MockResponse; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -26,25 +24,16 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.net.ssl.SSLSocketFactory; -import org.apache.http.client.HttpClient; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.PoolingClientConnectionManager; /** * This benchmark is fake, but may be useful for certain relative comparisons. @@ -56,10 +45,10 @@ public class Benchmark { private final Random random = new Random(0); /** Which client to run.*/ - Candidate candidate = new UrlConnection(); // new OkHttp(); // new ApacheHttpClient(); + HttpClient httpClient = new NettyHttpClient(); - /** How many concurrent threads to execute. */ - int threadCount = 10; + /** How many concurrent requests to execute. */ + int concurrencyLevel = 10; /** True to use TLS. */ // TODO: compare different ciphers? @@ -80,22 +69,18 @@ public class Benchmark { /** Which ALPN/NPN protocols are in use. Only useful with TLS. */ List protocols = Arrays.asList(Protocol.HTTP_11); - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws Exception { new Benchmark().run(); } - public void run() throws IOException { - ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, - 1, TimeUnit.SECONDS, new LinkedBlockingQueue()); - + public void run() throws Exception { System.out.println(toString()); // Prepare the client & server - candidate.prepare(); + httpClient.prepare(this); MockWebServer server = startServer(); - String url = server.getUrl("/").toString(); + URL url = server.getUrl("/"); - int targetBacklog = 10; int requestCount = 0; long reportStart = System.nanoTime(); long reportPeriod = TimeUnit.SECONDS.toNanos(1); @@ -115,8 +100,8 @@ public void run() throws IOException { } // Fill the job queue with work. - while (executor.getQueue().size() < targetBacklog) { - executor.execute(candidate.request(url)); + while (httpClient.acceptingJobs()) { + httpClient.enqueue(url); requestCount++; } @@ -133,9 +118,9 @@ public void run() throws IOException { modifiers.addAll(protocols); return String.format("%s %s\n" - + "bodyByteCount=%s headerCount=%s threadCount=%s", - candidate.getClass().getSimpleName(), modifiers, - bodyByteCount, headerCount, threadCount); + + "bodyByteCount=%s headerCount=%s concurrencyLevel=%s", + httpClient.getClass().getSimpleName(), modifiers, + bodyByteCount, headerCount, concurrencyLevel); } private void sleep(int millis) { @@ -207,72 +192,4 @@ private byte[] gzip(byte[] bytes) throws IOException { gzippedOut.close(); return bytesOut.toByteArray(); } - - interface Candidate { - void prepare(); - Runnable request(String url); - } - - class OkHttp implements Candidate { - private OkHttpClient client; - - @Override public void prepare() { - client = new OkHttpClient(); - client.setProtocols(protocols); - - if (tls) { - SSLContext sslContext = SslContextBuilder.localhost(); - SSLSocketFactory socketFactory = sslContext.getSocketFactory(); - HostnameVerifier hostnameVerifier = new HostnameVerifier() { - @Override public boolean verify(String s, SSLSession session) { - return true; - } - }; - client.setSslSocketFactory(socketFactory); - client.setHostnameVerifier(hostnameVerifier); - } - } - - @Override public Runnable request(String url) { - return new OkHttpRequest(client, url); - } - } - - class UrlConnection implements Candidate { - @Override public void prepare() { - if (tls) { - SSLContext sslContext = SslContextBuilder.localhost(); - SSLSocketFactory socketFactory = sslContext.getSocketFactory(); - HostnameVerifier hostnameVerifier = new HostnameVerifier() { - @Override public boolean verify(String s, SSLSession session) { - return true; - } - }; - HttpsURLConnectionImpl.setDefaultHostnameVerifier(hostnameVerifier); - HttpsURLConnectionImpl.setDefaultSSLSocketFactory(socketFactory); - } - } - - @Override public Runnable request(String url) { - return new UrlConnectionRequest(url); - } - } - - class ApacheHttpClient implements Candidate { - private HttpClient client; - - @Override public void prepare() { - ClientConnectionManager connectionManager = new PoolingClientConnectionManager(); - if (tls) { - SSLContext sslContext = SslContextBuilder.localhost(); - connectionManager.getSchemeRegistry().register( - new Scheme("https", 443, new org.apache.http.conn.ssl.SSLSocketFactory(sslContext))); - } - client = new DefaultHttpClient(connectionManager); - } - - @Override public Runnable request(String url) { - return new ApacheHttpClientRequest(url, client); - } - } } diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpClient.java new file mode 100644 index 000000000000..136c5d86d300 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/HttpClient.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import java.net.URL; + +/** An HTTP client to benchmark. */ +interface HttpClient { + void prepare(Benchmark benchmark); + void enqueue(URL url) throws Exception; + boolean acceptingJobs(); +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java new file mode 100644 index 000000000000..d5adec5a053b --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.internal.Util; +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpContentDecompressor; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.ssl.SslHandler; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +/** Netty isn't an HTTP client, but it's almost one. */ +public class NettyHttpClient implements HttpClient { + private static final boolean VERBOSE = false; + + // Guarded by this. Real apps need more capable connection management. + private final List freeChannels = new ArrayList(); + private int totalChannels = 0; + + private int concurrencyLevel; + private Bootstrap bootstrap; + + @Override public void prepare(final Benchmark benchmark) { + this.concurrencyLevel = benchmark.concurrencyLevel; + + ChannelInitializer channelInitializer = new ChannelInitializer() { + @Override public void initChannel(SocketChannel channel) throws Exception { + ChannelPipeline pipeline = channel.pipeline(); + + if (benchmark.tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLEngine engine = sslContext.createSSLEngine(); + engine.setUseClientMode(true); + pipeline.addLast("ssl", new SslHandler(engine)); + } + + pipeline.addLast("codec", new HttpClientCodec()); + pipeline.addLast("inflater", new HttpContentDecompressor()); + pipeline.addLast("handler", new HttpChannel(channel)); + } + }; + + EventLoopGroup group = new NioEventLoopGroup(); + bootstrap = new Bootstrap(); + bootstrap.group(group) + .channel(NioSocketChannel.class) + .handler(channelInitializer); + } + + @Override public void enqueue(URL url) throws Exception { + acquireChannel(url).sendRequest(url); + } + + @Override public synchronized boolean acceptingJobs() { + int activeChannels = totalChannels - freeChannels.size(); + return activeChannels < concurrencyLevel; + } + + private HttpChannel acquireChannel(URL url) throws InterruptedException { + synchronized (this) { + if (!freeChannels.isEmpty()) { + return freeChannels.remove(freeChannels.size() - 1); + } else { + totalChannels++; + } + } + + Channel channel = bootstrap.connect(url.getHost(), Util.getEffectivePort(url)).sync().channel(); + return (HttpChannel) channel.pipeline().last(); + } + + private synchronized void release(HttpChannel httpChannel) { + freeChannels.add(httpChannel); + } + + class HttpChannel extends SimpleChannelInboundHandler { + private final SocketChannel channel; + byte[] buffer = new byte[1024]; + int total; + long start; + + public HttpChannel(SocketChannel channel) { + this.channel = channel; + } + + private void sendRequest(URL url) { + start = System.nanoTime(); + total = 0; + HttpRequest request = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, HttpMethod.GET, url.getPath()); + request.headers().set(HttpHeaders.Names.HOST, url.getHost()); + request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); + channel.writeAndFlush(request); + } + + @Override protected void channelRead0( + ChannelHandlerContext context, HttpObject message) throws Exception { + if (message instanceof HttpResponse) { + receive((HttpResponse) message); + } + if (message instanceof HttpContent) { + receive((HttpContent) message); + if (message instanceof LastHttpContent) { + release(this); + } + } + } + + @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + } + + void receive(HttpResponse response) { + // Don't do anything with headers. + } + + void receive(HttpContent content) { + // Consume the response body. + ByteBuf byteBuf = content.content(); + for (int toRead; (toRead = byteBuf.readableBytes()) > 0; ) { + byteBuf.readBytes(buffer, 0, Math.min(buffer.length, toRead)); + total += toRead; + } + + if (VERBOSE && content instanceof LastHttpContent) { + long finish = System.nanoTime(); + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } + + @Override public void exceptionCaught(ChannelHandlerContext context, Throwable cause) { + System.out.println("Failed: " + cause); + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java new file mode 100644 index 000000000000..6fdb40e9acbb --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.internal.SslContextBuilder; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; + +class OkHttp extends SynchronousHttpClient { + private static final boolean VERBOSE = false; + + private OkHttpClient client; + + @Override public void prepare(Benchmark benchmark) { + super.prepare(benchmark); + client = new OkHttpClient(); + client.setProtocols(benchmark.protocols); + + if (benchmark.tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override public boolean verify(String s, SSLSession session) { + return true; + } + }; + client.setSslSocketFactory(socketFactory); + client.setHostnameVerifier(hostnameVerifier); + } + } + + @Override public Runnable request(URL url) { + return new OkHttpRequest(url); + } + + class OkHttpRequest implements Runnable { + private final URL url; + + public OkHttpRequest(URL url) { + this.url = url; + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpURLConnection urlConnection = client.open(url); + InputStream in = urlConnection.getInputStream(); + + // Consume the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java deleted file mode 100644 index e4ab485627ed..000000000000 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpRequest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2014 Square, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.squareup.okhttp.benchmarks; - -import com.squareup.okhttp.OkHttpClient; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.concurrent.TimeUnit; - -class OkHttpRequest implements Runnable { - private static final boolean VERBOSE = false; - - private final OkHttpClient client; - private final URL url; - - public OkHttpRequest(OkHttpClient client, String url) { - try { - this.client = client; - this.url = new URL(url); - } catch (MalformedURLException e) { - throw new AssertionError(); - } - } - - public void run() { - byte[] buffer = new byte[1024]; - long start = System.nanoTime(); - try { - HttpURLConnection urlConnection = client.open(url); - InputStream in = urlConnection.getInputStream(); - - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); - long finish = System.nanoTime(); - - if (VERBOSE) { - System.out.println(String.format("Transferred % 8d bytes in %4d ms", - total, TimeUnit.NANOSECONDS.toMillis(finish - start))); - } - } catch (IOException e) { - System.out.println("Failed: " + e); - } - } -} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java new file mode 100644 index 000000000000..9a0851c7ff66 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import java.net.URL; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** Any HTTP client with a blocking API. */ +abstract class SynchronousHttpClient implements HttpClient { + int targetBacklog = 10; + ThreadPoolExecutor executor; + + @Override public void prepare(Benchmark benchmark) { + executor = new ThreadPoolExecutor(benchmark.concurrencyLevel, benchmark.concurrencyLevel, + 1, TimeUnit.SECONDS, new LinkedBlockingQueue()); + } + + @Override public void enqueue(URL url) { + executor.execute(request(url)); + } + + @Override public boolean acceptingJobs() { + return executor.getQueue().size() < targetBacklog; + } + + abstract Runnable request(URL url); +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java new file mode 100644 index 000000000000..a2c3f3af407e --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.internal.http.HttpsURLConnectionImpl; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; + +class UrlConnection extends SynchronousHttpClient { + private static final boolean VERBOSE = false; + + @Override public void prepare(Benchmark benchmark) { + super.prepare(benchmark); + if (benchmark.tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override public boolean verify(String s, SSLSession session) { + return true; + } + }; + HttpsURLConnectionImpl.setDefaultHostnameVerifier(hostnameVerifier); + HttpsURLConnectionImpl.setDefaultSSLSocketFactory(socketFactory); + } + } + + @Override public Runnable request(URL url) { + return new UrlConnectionRequest(url); + } + + class UrlConnectionRequest implements Runnable { + private final URL url; + + public UrlConnectionRequest(URL url) { + this.url = url; + } + + public void run() { + byte[] buffer = new byte[1024]; + long start = System.nanoTime(); + try { + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + InputStream in = urlConnection.getInputStream(); + if ("gzip".equals(urlConnection.getHeaderField("Content-Encoding"))) { + in = new GZIPInputStream(in); + } + + // Consume the response body. + int total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + long finish = System.nanoTime(); + + if (VERBOSE) { + System.out.println(String.format("Transferred % 8d bytes in %4d ms", + total, TimeUnit.NANOSECONDS.toMillis(finish - start))); + } + } catch (IOException e) { + System.out.println("Failed: " + e); + } + } + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java deleted file mode 100644 index ab5c9b43e078..000000000000 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnectionRequest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2014 Square, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.squareup.okhttp.benchmarks; - -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -/** Uses the default java.net.HttpURLConnection implementation. */ -class UrlConnectionRequest implements Runnable { - private static final boolean VERBOSE = false; - - private final URL url; - - public UrlConnectionRequest(String url) { - try { - this.url = new URL(url); - } catch (MalformedURLException e) { - throw new AssertionError(); - } - } - - public void run() { - byte[] buffer = new byte[1024]; - long start = System.nanoTime(); - try { - HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); - InputStream in = urlConnection.getInputStream(); - if ("gzip".equals(urlConnection.getHeaderField("Content-Encoding"))) { - in = new GZIPInputStream(in); - } - - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); - long finish = System.nanoTime(); - - if (VERBOSE) { - System.out.println(String.format("Transferred % 8d bytes in %4d ms", - total, TimeUnit.NANOSECONDS.toMillis(finish - start))); - } - } catch (IOException e) { - System.out.println("Failed: " + e); - } - } -} From 08bc3c8808ec4495a1284ed6eb2325d44b435d33 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 1 Feb 2014 11:00:49 -0800 Subject: [PATCH 06/14] Add MockWebServer.setNpnProtocols --- .../squareup/okhttp/benchmarks/Benchmark.java | 1 + .../okhttp/mockwebserver/MockWebServer.java | 25 +++++++++++++++++-- .../internal/http/URLConnectionTest.java | 5 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index 16bd063508a5..fe23182fc9a6 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -138,6 +138,7 @@ private MockWebServer startServer() throws IOException { SSLContext sslContext = SslContextBuilder.localhost(); server.useHttps(sslContext.getSocketFactory(), false); server.setNpnEnabled(true); + server.setNpnProtocols(protocols); } final MockResponse response = newResponse(); diff --git a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java index 90e2925e625a..71c7c860fa9a 100644 --- a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java +++ b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java @@ -106,6 +106,7 @@ public final class MockWebServer { private int port = -1; private boolean npnEnabled = true; + private List npnProtocols = Protocol.HTTP2_SPDY3_AND_HTTP; public int getPort() { if (port == -1) throw new IllegalStateException("Cannot retrieve port before calling play()"); @@ -165,6 +166,27 @@ public void setNpnEnabled(boolean npnEnabled) { this.npnEnabled = npnEnabled; } + /** + * Indicates the protocols supported by NPN on incoming HTTPS connections. + * This list is ignored when npn is disabled. + * + * @param protocols the protocols to use, in order of preference. The list + * must contain "http/1.1". It must not contain null. + */ + public void setNpnProtocols(List protocols) { + protocols = Util.immutableList(protocols); + if (!protocols.contains(Protocol.HTTP_11)) { + throw new IllegalArgumentException("protocols doesn't contain http/1.1: " + protocols); + } + if (protocols.contains(null)) { + throw new IllegalArgumentException("protocols must not contain null"); + } + if (protocols.contains(ByteString.EMPTY)) { + throw new IllegalArgumentException("protocols contains an empty string"); + } + this.npnProtocols = Util.immutableList(protocols); + } + /** * Serve requests with HTTPS rather than otherwise. * @param tunnelProxy true to expect the HTTP CONNECT method before @@ -304,8 +326,7 @@ public void processConnection() throws Exception { openClientSockets.put(socket, true); if (npnEnabled) { - // TODO: expose means to select which protocols to advertise. - Platform.get().setNpnProtocols(sslSocket, Protocol.HTTP2_SPDY3_AND_HTTP); + Platform.get().setNpnProtocols(sslSocket, npnProtocols); } sslSocket.startHandshake(); diff --git a/okhttp/src/test/java/com/squareup/okhttp/internal/http/URLConnectionTest.java b/okhttp/src/test/java/com/squareup/okhttp/internal/http/URLConnectionTest.java index d9d3d3d2f483..6ed031d54395 100644 --- a/okhttp/src/test/java/com/squareup/okhttp/internal/http/URLConnectionTest.java +++ b/okhttp/src/test/java/com/squareup/okhttp/internal/http/URLConnectionTest.java @@ -2834,10 +2834,11 @@ private static class FakeProxySelector extends ProxySelector { * -Xbootclasspath/p:/tmp/npn-boot-8.1.2.v20120308.jar} */ private void enableNpn(Protocol protocol) { - server.useHttps(sslContext.getSocketFactory(), false); - server.setNpnEnabled(true); client.setSslSocketFactory(sslContext.getSocketFactory()); client.setHostnameVerifier(new RecordingHostnameVerifier()); client.setProtocols(Arrays.asList(protocol, Protocol.HTTP_11)); + server.useHttps(sslContext.getSocketFactory(), false); + server.setNpnEnabled(true); + server.setNpnProtocols(client.getProtocols()); } } From 66b25823795f14f156576d633294eb8c62382c4a Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 1 Feb 2014 11:01:27 -0800 Subject: [PATCH 07/14] Add instructions to execute the benchmark with correct JVM parameters. --- benchmarks/README.md | 8 ++++++++ benchmarks/pom.xml | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 benchmarks/README.md diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..59f571fc5548 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,8 @@ +OkHttp Benchmarks +======================================= + +This module allows you to test the performance of HTTP clients. + +### Running + 1. If you made modifications to `com.squareup.okhttp.benchmarks.Benchmark` run `mvn compile`. + 2. Run `mvn exec:exec` to launch a new JVM, which will execute the benchmark. diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 14d93eeefd8b..cecf86480d72 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -52,4 +52,30 @@ 4.0.15.Final + + + + org.codehaus.mojo + exec-maven-plugin + + + + java + + + + + java + + -Xms512m + -Xmx512m + -Xbootclasspath/p:${settings.localRepository}/org/mortbay/jetty/npn/npn-boot/${npn.version}/npn-boot-${npn.version}.jar + -classpath + + com.squareup.okhttp.benchmarks.Benchmark + + + + + From 18ef05d70d3b5ac90e8df8b80cd83af82b82197b Mon Sep 17 00:00:00 2001 From: Norman Maurer Date: Sat, 1 Feb 2014 20:09:25 +0100 Subject: [PATCH 08/14] Use the pooled allocator for performance reasons --- .../java/com/squareup/okhttp/benchmarks/NettyHttpClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java index d5adec5a053b..cf3912384850 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java @@ -19,9 +19,11 @@ import com.squareup.okhttp.internal.Util; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; @@ -81,6 +83,7 @@ public class NettyHttpClient implements HttpClient { EventLoopGroup group = new NioEventLoopGroup(); bootstrap = new Bootstrap(); bootstrap.group(group) + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .channel(NioSocketChannel.class) .handler(channelInitializer); } From 99841aecb4273e0cbd22e7d67399bee20b806c52 Mon Sep 17 00:00:00 2001 From: jwilson Date: Sat, 1 Feb 2014 15:35:54 -0500 Subject: [PATCH 09/14] Use Caliper for benchmarking. --- benchmarks/pom.xml | 11 ++++ .../okhttp/benchmarks/ApacheHttpClient.java | 8 +-- .../squareup/okhttp/benchmarks/Benchmark.java | 57 +++++++++++++------ .../squareup/okhttp/benchmarks/Client.java | 44 ++++++++++++++ .../okhttp/benchmarks/NettyHttpClient.java | 2 +- .../squareup/okhttp/benchmarks/OkHttp.java | 11 +--- .../benchmarks/SynchronousHttpClient.java | 12 ++++ .../okhttp/benchmarks/UrlConnection.java | 8 +-- 8 files changed, 111 insertions(+), 42 deletions(-) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 14d93eeefd8b..960296b11810 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -13,6 +13,11 @@ Benchmarks + + com.google.caliper + caliper + 1.0-beta-1 + com.squareup.okhttp okhttp @@ -51,5 +56,11 @@ netty-codec-http 4.0.15.Final + + + com.jcraft + jzlib + 1.1.2 + diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java index cd2cba31cff9..cb8e719111c4 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java @@ -61,7 +61,6 @@ public ApacheHttpClientRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpResponse response = client.execute(new HttpGet(url.toString())); @@ -71,12 +70,7 @@ public void run() { in = new GZIPInputStream(in); } - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(in); long finish = System.nanoTime(); if (VERBOSE) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index 16bd063508a5..3b2b5e59dfb9 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -15,6 +15,9 @@ */ package com.squareup.okhttp.benchmarks; +import com.google.caliper.Param; +import com.google.caliper.model.ArbitraryMeasurement; +import com.google.caliper.runner.CaliperMain; import com.squareup.okhttp.Protocol; import com.squareup.okhttp.internal.SslContextBuilder; import com.squareup.okhttp.mockwebserver.Dispatcher; @@ -40,41 +43,57 @@ * It uses a local connection to a MockWebServer to measure how many identical * requests per second can be carried over a fixed number of threads. */ -public class Benchmark { +public class Benchmark extends com.google.caliper.Benchmark { private static final int NUM_REPORTS = 10; + private static final boolean VERBOSE = false; + private final Random random = new Random(0); /** Which client to run.*/ - HttpClient httpClient = new NettyHttpClient(); + @Param + Client client; /** How many concurrent requests to execute. */ - int concurrencyLevel = 10; + @Param({ "1", "10" }) + int concurrencyLevel; /** True to use TLS. */ // TODO: compare different ciphers? - boolean tls = false; + @Param + boolean tls; /** True to use gzip content-encoding for the response body. */ - boolean gzip = false; + @Param + boolean gzip; /** Don't combine chunked with SPDY_3 or HTTP_2; that's not allowed. */ - boolean chunked = false; + @Param + boolean chunked; /** The size of the HTTP response body, in uncompressed bytes. */ - int bodyByteCount = 1024 * 1024; + @Param({ "128", "1048576" }) + int bodyByteCount; /** How many additional headers were included, beyond the built-in ones. */ - int headerCount = 20; + @Param({ "0", "20" }) + int headerCount; /** Which ALPN/NPN protocols are in use. Only useful with TLS. */ List protocols = Arrays.asList(Protocol.HTTP_11); - public static void main(String[] args) throws Exception { - new Benchmark().run(); + public static void main(String[] args) { + List allArgs = new ArrayList(); + allArgs.add("--instrument"); + allArgs.add("arbitrary"); + allArgs.addAll(Arrays.asList(args)); + + CaliperMain.main(Benchmark.class, allArgs.toArray(new String[allArgs.size()])); } - public void run() throws Exception { - System.out.println(toString()); + @ArbitraryMeasurement(description = "requests per second") + public double run() throws Exception { + if (VERBOSE) System.out.println(toString()); + HttpClient httpClient = client.create(); // Prepare the client & server httpClient.prepare(this); @@ -85,6 +104,7 @@ public void run() throws Exception { long reportStart = System.nanoTime(); long reportPeriod = TimeUnit.SECONDS.toNanos(1); int reports = 0; + double best = 0.0; // Run until we've printed enough reports. while (reports < NUM_REPORTS) { @@ -93,7 +113,10 @@ public void run() throws Exception { double reportDuration = now - reportStart; if (reportDuration > reportPeriod) { double requestsPerSecond = requestCount / reportDuration * TimeUnit.SECONDS.toNanos(1); - System.out.println(String.format("Requests per second: %.1f", requestsPerSecond)); + if (VERBOSE) { + System.out.println(String.format("Requests per second: %.1f", requestsPerSecond)); + } + best = Math.max(best, requestsPerSecond); requestCount = 0; reportStart = now; reports++; @@ -108,6 +131,8 @@ public void run() throws Exception { // The job queue is full. Take a break. sleep(10); } + + return best; } @Override public String toString() { @@ -117,10 +142,8 @@ public void run() throws Exception { if (chunked) modifiers.add("chunked"); modifiers.addAll(protocols); - return String.format("%s %s\n" - + "bodyByteCount=%s headerCount=%s concurrencyLevel=%s", - httpClient.getClass().getSimpleName(), modifiers, - bodyByteCount, headerCount, concurrencyLevel); + return String.format("%s %s\nbodyByteCount=%s headerCount=%s concurrencyLevel=%s", + client, modifiers, bodyByteCount, headerCount, concurrencyLevel); } private void sleep(int millis) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java new file mode 100644 index 000000000000..0f076a610d14 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +enum Client { + OkHttp { + @Override HttpClient create() { + return new OkHttp(); + } + }, + + Apache { + @Override HttpClient create() { + return new ApacheHttpClient(); + } + }, + + UrlConnection { + @Override HttpClient create() { + return new UrlConnection(); + } + }, + + Netty { + @Override HttpClient create() { + return new NettyHttpClient(); + } + }; + + abstract HttpClient create(); +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java index d5adec5a053b..c3b06512eb5d 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java @@ -48,7 +48,7 @@ import javax.net.ssl.SSLEngine; /** Netty isn't an HTTP client, but it's almost one. */ -public class NettyHttpClient implements HttpClient { +class NettyHttpClient implements HttpClient { private static final boolean VERBOSE = false; // Guarded by this. Real apps need more capable connection management. diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java index 6fdb40e9acbb..03b9e3c6f10a 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java @@ -18,7 +18,6 @@ import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.internal.SslContextBuilder; import java.io.IOException; -import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.TimeUnit; @@ -62,18 +61,10 @@ public OkHttpRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpURLConnection urlConnection = client.open(url); - InputStream in = urlConnection.getInputStream(); - - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(urlConnection.getInputStream()); long finish = System.nanoTime(); if (VERBOSE) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java index 9a0851c7ff66..f4bb02c79b07 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java @@ -15,6 +15,8 @@ */ package com.squareup.okhttp.benchmarks; +import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -38,5 +40,15 @@ abstract class SynchronousHttpClient implements HttpClient { return executor.getQueue().size() < targetBacklog; } + protected long readAllAndClose(InputStream in) throws IOException { + byte[] buffer = new byte[1024]; + long total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + return total; + } + abstract Runnable request(URL url); } diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java index a2c3f3af407e..53d2a4b9e9f3 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java @@ -58,7 +58,6 @@ public UrlConnectionRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); @@ -67,12 +66,7 @@ public void run() { in = new GZIPInputStream(in); } - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(in); long finish = System.nanoTime(); if (VERBOSE) { From 15099af48064c22bc3357b01c9efcb1aa43c4819 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 1 Feb 2014 12:52:12 -0800 Subject: [PATCH 10/14] Flush control frames; don't flush user frames. --- .../com/squareup/okhttp/internal/spdy/Http20Draft09.java | 5 +++++ .../main/java/com/squareup/okhttp/internal/spdy/Spdy3.java | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java index fd465f9c8b1a..307a4a52b6f6 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java @@ -317,6 +317,7 @@ static final class Writer implements FrameWriter { @Override public synchronized void connectionHeader() throws IOException { if (!client) return; // Nothing to write; servers don't send connection headers! out.write(CONNECTION_HEADER); + out.flush(); } @Override @@ -409,6 +410,7 @@ void dataFrame(int streamId, byte flags, byte[] data, int offset, int length) out.writeInt(i & 0xffffff); out.writeInt(settings.get(i)); } + out.flush(); } @Override public synchronized void ping(boolean ack, int payload1, int payload2) @@ -420,6 +422,7 @@ void dataFrame(int streamId, byte flags, byte[] data, int offset, int length) frameHeader(length, type, flags, streamId); out.writeInt(payload1); out.writeInt(payload2); + out.flush(); } @Override @@ -436,6 +439,7 @@ public synchronized void goAway(int lastGoodStreamId, ErrorCode errorCode, byte[ if (debugData.length > 0) { out.write(debugData); } + out.flush(); } @Override public synchronized void windowUpdate(int streamId, long windowSizeIncrement) @@ -449,6 +453,7 @@ public synchronized void goAway(int lastGoodStreamId, ErrorCode errorCode, byte[ byte flags = FLAG_NONE; frameHeader(length, type, flags, streamId); out.writeInt((int) windowSizeIncrement); + out.flush(); } @Override public void close() throws IOException { diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java index 393b6ee00217..75060445aea0 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java @@ -369,7 +369,6 @@ public synchronized void synStream(boolean outFinished, boolean inFinished, int out.writeInt((flags & 0xff) << 24 | length & 0xffffff); out.writeInt(streamId & 0x7fffffff); headerBlockBuffer.writeTo(out); - out.flush(); } @Override public synchronized void rstStream(int streamId, ErrorCode errorCode) From 220c30ba20ab4ed80788aac18582b989ccf2c1b2 Mon Sep 17 00:00:00 2001 From: jwilson Date: Sun, 2 Feb 2014 17:53:43 -0500 Subject: [PATCH 11/14] InflaterSource. Like InflaterInputStream. --- .../okhttp/internal/bytes/InflaterSource.java | 100 ++++++++++++++++ .../okhttp/internal/bytes/OkBuffer.java | 1 - .../okhttp/internal/bytes/OkBuffers.java | 3 +- .../internal/bytes/InflaterSourceTest.java | 111 ++++++++++++++++++ 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/InflaterSource.java create mode 100644 okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/InflaterSourceTest.java diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/InflaterSource.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/InflaterSource.java new file mode 100644 index 000000000000..3a11d115dbfa --- /dev/null +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/InflaterSource.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.internal.bytes; + +import java.io.EOFException; +import java.io.IOException; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +/** A source that inflates another source. */ +public final class InflaterSource implements Source { + private final Source source; + private final Inflater inflater; + private final OkBuffer buffer = new OkBuffer(); + + /** + * When we call Inflater.setInput(), the inflater keeps our byte array until + * it needs input again. This tracks how many bytes the inflater is currently + * holding on to. + */ + private int bufferBytesHeldByInflater; + private boolean closed; + + public InflaterSource(Source source, Inflater inflater) { + if (source == null) throw new IllegalArgumentException("source == null"); + if (inflater == null) throw new IllegalArgumentException("inflater == null"); + this.source = source; + this.inflater = inflater; + } + + @Override public long read( + OkBuffer sink, long byteCount, Deadline deadline) throws IOException { + if (byteCount < 0) throw new IllegalArgumentException("byteCount < 0: " + byteCount); + if (closed) throw new IllegalStateException("closed"); + if (byteCount == 0) return 0; + + while (true) { + boolean sourceExhausted = false; + if (inflater.needsInput()) { + // Release buffer bytes from the inflater. + if (bufferBytesHeldByInflater > 0) { + Segment head = buffer.head; + head.pos += bufferBytesHeldByInflater; + buffer.byteCount -= bufferBytesHeldByInflater; + if (head.pos == head.limit) { + buffer.head = head.pop(); + SegmentPool.INSTANCE.recycle(head); + } + } + + // Refill the buffer with compressed data from the source. + if (buffer.byteCount == 0) { + sourceExhausted = source.read(buffer, Segment.SIZE, deadline) == -1; + } + + // Acquire buffer bytes for the inflater. + if (buffer.byteCount > 0) { + Segment head = buffer.head; + bufferBytesHeldByInflater = head.limit - head.pos; + inflater.setInput(head.data, head.pos, bufferBytesHeldByInflater); + } + } + + // Decompress the inflater's compressed data into the sink. + try { + Segment tail = sink.writableSegment(1); + int bytesInflated = inflater.inflate(tail.data, tail.limit, Segment.SIZE - tail.limit); + if (bytesInflated > 0) { + tail.limit += bytesInflated; + sink.byteCount += bytesInflated; + return bytesInflated; + } + if (inflater.finished() || inflater.needsDictionary()) return -1; + if (sourceExhausted) throw new EOFException("source exhausted prematurely"); + } catch (DataFormatException e) { + throw new IOException(e); + } + } + } + + @Override public void close(Deadline deadline) throws IOException { + if (closed) return; + inflater.end(); + closed = true; + source.close(deadline); + } +} diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java index e534dc526bbb..7fd538c92a55 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffer.java @@ -370,7 +370,6 @@ public long indexOf(byte b) throws IOException { } @Override public void close(Deadline deadline) { - throw new UnsupportedOperationException("Cannot close() an OkBuffer"); } /** For testing. This returns the sizes of the segments in this buffer. */ diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java index c4a17ac9d862..230ab4c76daf 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/OkBuffers.java @@ -94,11 +94,12 @@ public static OutputStream outputStream(final Sink sink) { } @Override public void flush() throws IOException { - sink.write(buffer, buffer.byteCount, Deadline.NONE); + sink.write(buffer, buffer.byteCount, Deadline.NONE); // Flush the buffer. sink.flush(Deadline.NONE); } @Override public void close() throws IOException { + sink.write(buffer, buffer.byteCount, Deadline.NONE); // Flush the buffer. sink.close(Deadline.NONE); } diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/InflaterSourceTest.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/InflaterSourceTest.java new file mode 100644 index 000000000000..df07f6564916 --- /dev/null +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/bytes/InflaterSourceTest.java @@ -0,0 +1,111 @@ +package com.squareup.okhttp.internal.bytes; + +import com.squareup.okhttp.internal.Base64; +import com.squareup.okhttp.internal.Util; +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class InflaterSourceTest { + @Test public void inflate() throws Exception { + OkBuffer deflated = decodeBase64("eJxzz09RyEjNKVAoLdZRKE9VL0pVyMxTKMlIVchIzEspVshPU0jNS8/MS00tK" + + "tYDAF6CD5s="); + OkBuffer inflated = inflate(deflated); + assertEquals("God help us, we're in the hands of engineers.", readUtf8(inflated)); + } + + @Test public void inflateTruncated() throws Exception { + OkBuffer deflated = decodeBase64("eJxzz09RyEjNKVAoLdZRKE9VL0pVyMxTKMlIVchIzEspVshPU0jNS8/MS00tK" + + "tYDAF6CDw=="); + try { + inflate(deflated); + fail(); + } catch (EOFException expected) { + } + } + + @Test public void inflateWellCompressed() throws Exception { + OkBuffer deflated = decodeBase64("eJztwTEBAAAAwqCs61/CEL5AAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB8B" + + "tFeWvE=\n"); + String original = repeat('a', 1024 * 1024); + OkBuffer inflated = inflate(deflated); + assertEquals(original, readUtf8(inflated)); + } + + @Test public void inflatePoorlyCompressed() throws Exception { + ByteString original = randomBytes(1024 * 1024); + OkBuffer deflated = deflate(toBuffer(original)); + OkBuffer inflated = inflate(deflated); + assertEquals(original, inflated.readByteString((int) inflated.byteCount())); + } + + private OkBuffer decodeBase64(String s) { + OkBuffer result = new OkBuffer(); + byte[] data = Base64.decode(s.getBytes(Util.UTF_8)); + result.write(data, 0, data.length); + return result; + } + + private String readUtf8(OkBuffer buffer) { + return buffer.readUtf8((int) buffer.byteCount()); + } + + /** Use DeflaterOutputStream to deflate source. */ + private OkBuffer deflate(OkBuffer buffer) throws IOException { + OkBuffer result = new OkBuffer(); + Sink sink = OkBuffers.sink(new DeflaterOutputStream(OkBuffers.outputStream(result))); + sink.write(buffer, buffer.byteCount(), Deadline.NONE); + sink.close(Deadline.NONE); + return result; + } + + private OkBuffer toBuffer(ByteString byteString) { + OkBuffer byteStringBuffer = new OkBuffer(); + byteStringBuffer.write(byteString); + return byteStringBuffer; + } + + /** Returns a new buffer containing the inflated contents of {@code deflated}. */ + private OkBuffer inflate(OkBuffer deflated) throws IOException { + OkBuffer result = new OkBuffer(); + InflaterSource source = new InflaterSource(deflated, new Inflater()); + while (source.read(result, Integer.MAX_VALUE, Deadline.NONE) != -1) { + } + return result; + } + + private ByteString randomBytes(int length) { + Random random = new Random(0); + byte[] randomBytes = new byte[length]; + random.nextBytes(randomBytes); + return ByteString.of(randomBytes); + } + + private String repeat(char c, int count) { + char[] array = new char[count]; + Arrays.fill(array, c); + return new String(array); + } +} From 4313b7af81f7023aa7210f08f2992eaf9fa15633 Mon Sep 17 00:00:00 2001 From: jwilson Date: Sun, 2 Feb 2014 10:43:43 -0500 Subject: [PATCH 12/14] Promote the target backlog to a parameter. --- .../squareup/okhttp/benchmarks/Benchmark.java | 6 +- .../squareup/okhttp/benchmarks/Client.java | 6 ++ .../okhttp/benchmarks/NettyHttpClient.java | 57 +++++++---- .../okhttp/benchmarks/OkHttpAsync.java | 94 +++++++++++++++++++ .../benchmarks/SynchronousHttpClient.java | 5 +- 5 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpAsync.java diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index fb4ecf14f1ef..151128d6c055 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -57,6 +57,10 @@ public class Benchmark extends com.google.caliper.Benchmark { @Param({ "1", "10" }) int concurrencyLevel; + /** How many requests to enqueue to await threads to execute them. */ + @Param({ "10" }) + int targetBacklog; + /** True to use TLS. */ // TODO: compare different ciphers? @Param @@ -129,7 +133,7 @@ public double run() throws Exception { } // The job queue is full. Take a break. - sleep(10); + sleep(1); } return best; diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java index 0f076a610d14..bd777aa3595e 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java @@ -22,6 +22,12 @@ enum Client { } }, + OkHttpAsync { + @Override HttpClient create() { + return new OkHttpAsync(); + } + }, + Apache { @Override HttpClient create() { return new ApacheHttpClient(); diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java index 980367c7135a..9044d0a33c9b 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java @@ -25,7 +25,6 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; @@ -43,8 +42,8 @@ import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.ssl.SslHandler; import java.net.URL; -import java.util.ArrayList; -import java.util.List; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; @@ -54,14 +53,17 @@ class NettyHttpClient implements HttpClient { private static final boolean VERBOSE = false; // Guarded by this. Real apps need more capable connection management. - private final List freeChannels = new ArrayList(); - private int totalChannels = 0; + private final Deque freeChannels = new ArrayDeque(); + private final Deque backlog = new ArrayDeque(); + private int totalChannels = 0; private int concurrencyLevel; + private int targetBacklog; private Bootstrap bootstrap; @Override public void prepare(final Benchmark benchmark) { this.concurrencyLevel = benchmark.concurrencyLevel; + this.targetBacklog = benchmark.targetBacklog; ChannelInitializer channelInitializer = new ChannelInitializer() { @Override public void initChannel(SocketChannel channel) throws Exception { @@ -80,38 +82,55 @@ class NettyHttpClient implements HttpClient { } }; - EventLoopGroup group = new NioEventLoopGroup(); bootstrap = new Bootstrap(); - bootstrap.group(group) + bootstrap.group(new NioEventLoopGroup(concurrencyLevel)) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .channel(NioSocketChannel.class) .handler(channelInitializer); } @Override public void enqueue(URL url) throws Exception { - acquireChannel(url).sendRequest(url); + HttpChannel httpChannel = null; + synchronized (this) { + if (!freeChannels.isEmpty()) { + httpChannel = freeChannels.pop(); + } else if (totalChannels < concurrencyLevel) { + totalChannels++; // Create a new channel. (outside of the synchronized block). + } else { + backlog.add(url); // Enqueue this for later, to be picked up when another request completes. + return; + } + } + if (httpChannel == null) { + Channel channel = bootstrap.connect(url.getHost(), Util.getEffectivePort(url)) + .sync().channel(); + httpChannel = (HttpChannel) channel.pipeline().last(); + } + httpChannel.sendRequest(url); } @Override public synchronized boolean acceptingJobs() { + return backlog.size() < targetBacklog || hasFreeChannels(); + } + + private boolean hasFreeChannels() { int activeChannels = totalChannels - freeChannels.size(); return activeChannels < concurrencyLevel; } - private HttpChannel acquireChannel(URL url) throws InterruptedException { + private void release(HttpChannel httpChannel) { + URL url; synchronized (this) { - if (!freeChannels.isEmpty()) { - return freeChannels.remove(freeChannels.size() - 1); - } else { - totalChannels++; + url = backlog.pop(); + if (url == null) { + // There were no URLs in the backlog. Pool this channel for later. + freeChannels.push(httpChannel); + return; } } - Channel channel = bootstrap.connect(url.getHost(), Util.getEffectivePort(url)).sync().channel(); - return (HttpChannel) channel.pipeline().last(); - } - - private synchronized void release(HttpChannel httpChannel) { - freeChannels.add(httpChannel); + // We removed a URL from the backlog. Schedule it right away. + httpChannel.sendRequest(url); } class HttpChannel extends SimpleChannelInboundHandler { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpAsync.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpAsync.java new file mode 100644 index 000000000000..b7633b76aa4c --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttpAsync.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +import com.squareup.okhttp.Dispatcher; +import com.squareup.okhttp.Failure; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import com.squareup.okhttp.internal.SslContextBuilder; +import java.io.IOException; +import java.net.URL; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; + +class OkHttpAsync implements HttpClient { + private static final boolean VERBOSE = false; + + private final AtomicInteger requestsInFlight = new AtomicInteger(); + + private OkHttpClient client; + private Response.Receiver receiver; + private int concurrencyLevel; + private int targetBacklog; + + @Override public void prepare(final Benchmark benchmark) { + concurrencyLevel = benchmark.concurrencyLevel; + targetBacklog = benchmark.targetBacklog; + + client = new OkHttpClient(); + client.setProtocols(benchmark.protocols); + client.setDispatcher(new Dispatcher(new ThreadPoolExecutor(benchmark.concurrencyLevel, + benchmark.concurrencyLevel, 60, TimeUnit.SECONDS, new LinkedBlockingQueue()))); + + if (benchmark.tls) { + SSLContext sslContext = SslContextBuilder.localhost(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override public boolean verify(String s, SSLSession session) { + return true; + } + }; + client.setSslSocketFactory(socketFactory); + client.setHostnameVerifier(hostnameVerifier); + } + + receiver = new Response.Receiver() { + @Override public void onFailure(Failure failure) { + System.out.println("Failed: " + failure.exception()); + } + + @Override public boolean onResponse(Response response) throws IOException { + Response.Body body = response.body(); + long total = SynchronousHttpClient.readAllAndClose(body.byteStream()); + long finish = System.nanoTime(); + if (VERBOSE) { + long start = (Long) response.request().tag(); + System.out.printf("Transferred % 8d bytes in %4d ms%n", + total, TimeUnit.NANOSECONDS.toMillis(finish - start)); + } + requestsInFlight.decrementAndGet(); + return true; + } + }; + } + + @Override public void enqueue(URL url) throws Exception { + requestsInFlight.incrementAndGet(); + client.enqueue(new Request.Builder().tag(System.nanoTime()).url(url).build(), receiver); + } + + @Override public synchronized boolean acceptingJobs() { + return requestsInFlight.get() < (concurrencyLevel + targetBacklog); + } +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java index f4bb02c79b07..b15eedcd8813 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java @@ -24,10 +24,11 @@ /** Any HTTP client with a blocking API. */ abstract class SynchronousHttpClient implements HttpClient { - int targetBacklog = 10; ThreadPoolExecutor executor; + int targetBacklog; @Override public void prepare(Benchmark benchmark) { + this.targetBacklog = benchmark.targetBacklog; executor = new ThreadPoolExecutor(benchmark.concurrencyLevel, benchmark.concurrencyLevel, 1, TimeUnit.SECONDS, new LinkedBlockingQueue()); } @@ -40,7 +41,7 @@ abstract class SynchronousHttpClient implements HttpClient { return executor.getQueue().size() < targetBacklog; } - protected long readAllAndClose(InputStream in) throws IOException { + static long readAllAndClose(InputStream in) throws IOException { byte[] buffer = new byte[1024]; long total = 0; for (int count; (count = in.read(buffer)) != -1; ) { From fdee6f13a4c5bf5bd24f2c237c2996aea01cc5ff Mon Sep 17 00:00:00 2001 From: Adrian Cole and Jesse Wilson Date: Wed, 5 Feb 2014 11:49:21 +0100 Subject: [PATCH 13/14] FindBugs sweep. --- .../okhttp/benchmarks/UrlConnection.java | 2 +- .../okhttp/internal/spdy/SpdyServer.java | 15 ++++++---- .../okhttp/mockwebserver/MockWebServer.java | 3 -- .../com/squareup/okhttp/internal/Util.java | 15 ---------- .../okhttp/internal/bytes/ByteString.java | 2 +- .../okhttp/internal/spdy/Http20Draft09.java | 6 ++-- .../squareup/okhttp/internal/spdy/Spdy3.java | 6 ++-- .../okhttp/internal/spdy/SpdyStream.java | 2 +- .../internal/spdy/Http20Draft09Test.java | 24 ++++++++++++++++ .../internal/spdy/SpdyConnectionTest.java | 2 +- .../com/squareup/okhttp/ConnectionPool.java | 12 ++++---- .../com/squareup/okhttp/OkHttpClient.java | 3 -- .../okhttp/internal/DiskLruCache.java | 28 ++++++++++--------- .../okhttp/sample/OkHttpContributors.java | 2 +- .../squareup/okhttp/sample/SampleServer.java | 9 ++++-- 15 files changed, 71 insertions(+), 60 deletions(-) diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java index 53d2a4b9e9f3..79abb69eeb85 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java @@ -50,7 +50,7 @@ class UrlConnection extends SynchronousHttpClient { return new UrlConnectionRequest(url); } - class UrlConnectionRequest implements Runnable { + static class UrlConnectionRequest implements Runnable { private final URL url; public UrlConnectionRequest(URL url) { diff --git a/mockwebserver/src/main/java/com/squareup/okhttp/internal/spdy/SpdyServer.java b/mockwebserver/src/main/java/com/squareup/okhttp/internal/spdy/SpdyServer.java index e135ef7ff6b5..79dc4bbb411f 100644 --- a/mockwebserver/src/main/java/com/squareup/okhttp/internal/spdy/SpdyServer.java +++ b/mockwebserver/src/main/java/com/squareup/okhttp/internal/spdy/SpdyServer.java @@ -18,6 +18,7 @@ import com.squareup.okhttp.Protocol; import com.squareup.okhttp.internal.SslContextBuilder; +import com.squareup.okhttp.internal.Util; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -130,17 +131,21 @@ private void serveDirectory(SpdyStream stream, String[] files) throws IOExceptio } private void serveFile(SpdyStream stream, File file) throws IOException { - InputStream in = new FileInputStream(file); byte[] buffer = new byte[8192]; stream.reply( headerEntries(":status", "200", ":version", "HTTP/1.1", "content-type", contentType(file)), true); + InputStream in = new FileInputStream(file); OutputStream out = stream.getOutputStream(); - int count; - while ((count = in.read(buffer)) != -1) { - out.write(buffer, 0, count); + try { + int count; + while ((count = in.read(buffer)) != -1) { + out.write(buffer, 0, count); + } + } finally { + Util.closeQuietly(in); + Util.closeQuietly(out); } - out.close(); } private String contentType(File file) { diff --git a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java index 71c7c860fa9a..3fdaf676e8b5 100644 --- a/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java +++ b/mockwebserver/src/main/java/com/squareup/okhttp/mockwebserver/MockWebServer.java @@ -181,9 +181,6 @@ public void setNpnProtocols(List protocols) { if (protocols.contains(null)) { throw new IllegalArgumentException("protocols must not contain null"); } - if (protocols.contains(ByteString.EMPTY)) { - throw new IllegalArgumentException("protocols contains an empty string"); - } this.npnProtocols = Util.immutableList(protocols); } diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java index fbbf46fd0d3d..e609db9528f9 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Util.java @@ -32,7 +32,6 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; -import java.nio.ByteOrder; import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -88,20 +87,6 @@ public static void checkOffsetAndCount(long arrayLength, long offset, long count } } - public static void pokeInt(byte[] dst, int offset, int value, ByteOrder order) { - if (order == ByteOrder.BIG_ENDIAN) { - dst[offset++] = (byte) ((value >> 24) & 0xff); - dst[offset++] = (byte) ((value >> 16) & 0xff); - dst[offset++] = (byte) ((value >> 8) & 0xff); - dst[offset] = (byte) ((value >> 0) & 0xff); - } else { - dst[offset++] = (byte) ((value >> 0) & 0xff); - dst[offset++] = (byte) ((value >> 8) & 0xff); - dst[offset++] = (byte) ((value >> 16) & 0xff); - dst[offset] = (byte) ((value >> 24) & 0xff); - } - } - /** Returns true if two possibly-null objects are equal. */ public static boolean equal(Object a, Object b) { return a == b || (a != null && a.equals(b)); diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/ByteString.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/ByteString.java index 64a1183a5566..9a6a799edcd8 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/ByteString.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/bytes/ByteString.java @@ -71,7 +71,7 @@ public boolean equalsAscii(String ascii) { if (ascii == null || data.length != ascii.length()) { return false; } - if (ascii == this.utf8) { + if (ascii == this.utf8) { // not using String.equals to avoid looping twice. return true; } for (int i = 0; i < data.length; i++) { diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java index 307a4a52b6f6..0bc07a87bfe1 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Http20Draft09.java @@ -460,11 +460,9 @@ public synchronized void goAway(int lastGoodStreamId, ErrorCode errorCode, byte[ out.close(); } - private void frameHeader(int length, byte type, byte flags, int streamId) - throws IOException { + void frameHeader(int length, byte type, byte flags, int streamId) throws IOException { if (length > 16383) throw illegalArgument("FRAME_SIZE_ERROR length > 16383: %s", length); - if ((streamId & 0x80000000) == 1) throw illegalArgument("(streamId & 0x80000000) == 1: %s", - streamId); + if ((streamId & 0x80000000) != 0) throw illegalArgument("reserved bit set: %s", streamId); out.writeInt((length & 0x3fff) << 16 | (type & 0xff) << 8 | (flags & 0xff)); out.writeInt(streamId & 0x7fffffff); } diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java index 75060445aea0..75afc37555b3 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/Spdy3.java @@ -200,7 +200,7 @@ private void readSynStream(Handler handler, int flags, int length) throws IOExce int streamId = w1 & 0x7fffffff; int associatedStreamId = w2 & 0x7fffffff; int priority = (s3 & 0xe000) >>> 13; - int slot = s3 & 0xff; + // int slot = s3 & 0xff; List
headerBlock = headerBlockReader.readNameValueBlock(length - 10); boolean inFinished = (flags & FLAG_FIN) != 0; @@ -248,7 +248,7 @@ private void readWindowUpdate(Handler handler, int flags, int length) throws IOE private void readPing(Handler handler, int flags, int length) throws IOException { if (length != 4) throw ioException("TYPE_PING length: %d != 4", length); int id = in.readInt(); - boolean ack = client == ((id % 2) == 1); + boolean ack = client == ((id & 1) == 1); handler.ping(ack, id, 0); } @@ -439,7 +439,7 @@ private void writeNameValueBlockToBuffer(List
headerBlock) throws IOExce @Override public synchronized void ping(boolean reply, int payload1, int payload2) throws IOException { - boolean payloadIsReply = client != ((payload1 % 2) == 1); + boolean payloadIsReply = client != ((payload1 & 1) == 1); if (reply != payloadIsReply) throw new IllegalArgumentException("payload != reply"); int type = TYPE_PING; int flags = 0; diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/SpdyStream.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/SpdyStream.java index 87ce18a0ba28..e36c8ebadf0a 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/SpdyStream.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/spdy/SpdyStream.java @@ -108,7 +108,7 @@ public synchronized boolean isOpen() { /** Returns true if this stream was created by this peer. */ public boolean isLocallyInitiated() { - boolean streamIsClient = (id % 2 == 1); + boolean streamIsClient = ((id & 1) == 1); return connection.client == streamIsClient; } diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java index 4f94d27c4941..80237341302f 100644 --- a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/Http20Draft09Test.java @@ -477,6 +477,30 @@ private Http20Draft09.Reader newReader(ByteArrayOutputStream out) { return new Http20Draft09.Reader(new ByteArrayInputStream(out.toByteArray()), 4096, false); } + @Test public void frameSizeError() throws IOException { + Http20Draft09.Writer writer = new Http20Draft09.Writer(new ByteArrayOutputStream(), true); + + try { + writer.frameHeader(16384, Http20Draft09.TYPE_DATA, Http20Draft09.FLAG_NONE, 0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("FRAME_SIZE_ERROR length > 16383: 16384", e.getMessage()); + } + } + + @Test public void streamIdHasReservedBit() throws IOException { + Http20Draft09.Writer writer = new Http20Draft09.Writer(new ByteArrayOutputStream(), true); + + try { + int streamId = 3; + streamId |= 1L << 31; // set reserved bit + writer.frameHeader(16383, Http20Draft09.TYPE_DATA, Http20Draft09.FLAG_NONE, streamId); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("reserved bit set: -2147483645", e.getMessage()); + } + } + private byte[] literalHeaders(List
sentHeaders) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); new HpackDraft05.Writer(new DataOutputStream(out)).writeHeaders(sentHeaders); diff --git a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/SpdyConnectionTest.java b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/SpdyConnectionTest.java index c5634915e089..acf2f5c44d3d 100644 --- a/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/SpdyConnectionTest.java +++ b/okhttp-protocols/src/test/java/com/squareup/okhttp/internal/spdy/SpdyConnectionTest.java @@ -1376,7 +1376,7 @@ private void clientSendsEmptyDataServerDoesntSendWindowUpdate(Variant variant) new Header(Header.TARGET_AUTHORITY, "squareup.com"), new Header(Header.TARGET_PATH, "/cached") )); - peer.sendFrame().synReply(true, 1, Arrays.asList( + peer.sendFrame().synReply(true, 2, Arrays.asList( new Header(Header.RESPONSE_STATUS, "200") )); peer.acceptFrame(); // RST_STREAM diff --git a/okhttp/src/main/java/com/squareup/okhttp/ConnectionPool.java b/okhttp/src/main/java/com/squareup/okhttp/ConnectionPool.java index 5eb4874aa41c..240cf83bebc1 100644 --- a/okhttp/src/main/java/com/squareup/okhttp/ConnectionPool.java +++ b/okhttp/src/main/java/com/squareup/okhttp/ConnectionPool.java @@ -23,7 +23,6 @@ import java.util.LinkedList; import java.util.List; import java.util.ListIterator; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -83,8 +82,8 @@ public class ConnectionPool { private final ExecutorService executorService = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue(), Util.threadFactory("OkHttp ConnectionPool", true)); - private final Callable connectionsCleanupCallable = new Callable() { - @Override public Void call() throws Exception { + private final Runnable connectionsCleanupRunnable = new Runnable() { + @Override public void run() { List expiredConnections = new ArrayList(MAX_CONNECTIONS_TO_CLEANUP); int idleConnectionCount = 0; synchronized (ConnectionPool.this) { @@ -113,7 +112,6 @@ public class ConnectionPool { for (Connection expiredConnection : expiredConnections) { Util.closeQuietly(expiredConnection); } - return null; } }; @@ -205,7 +203,7 @@ public synchronized Connection get(Address address) { connections.addFirst(foundConnection); // Add it back after iteration. } - executorService.submit(connectionsCleanupCallable); + executorService.execute(connectionsCleanupRunnable); return foundConnection; } @@ -239,7 +237,7 @@ public void recycle(Connection connection) { connection.resetIdleStartTime(); } - executorService.submit(connectionsCleanupCallable); + executorService.execute(connectionsCleanupRunnable); } /** @@ -247,7 +245,7 @@ public void recycle(Connection connection) { * continue to use {@code connection}. */ public void maybeShare(Connection connection) { - executorService.submit(connectionsCleanupCallable); + executorService.execute(connectionsCleanupRunnable); if (!connection.isSpdy()) { // Only SPDY connections are sharable. return; diff --git a/okhttp/src/main/java/com/squareup/okhttp/OkHttpClient.java b/okhttp/src/main/java/com/squareup/okhttp/OkHttpClient.java index 951f59a416eb..68e1cfadb9bc 100644 --- a/okhttp/src/main/java/com/squareup/okhttp/OkHttpClient.java +++ b/okhttp/src/main/java/com/squareup/okhttp/OkHttpClient.java @@ -349,9 +349,6 @@ public OkHttpClient setProtocols(List protocols) { if (protocols.contains(null)) { throw new IllegalArgumentException("protocols must not contain null"); } - if (protocols.contains(ByteString.EMPTY)) { - throw new IllegalArgumentException("protocols contains an empty string"); - } this.protocols = Util.immutableList(protocols); return this; } diff --git a/okhttp/src/main/java/com/squareup/okhttp/internal/DiskLruCache.java b/okhttp/src/main/java/com/squareup/okhttp/internal/DiskLruCache.java index 19e4cee61f46..1dbaa88c0e8c 100644 --- a/okhttp/src/main/java/com/squareup/okhttp/internal/DiskLruCache.java +++ b/okhttp/src/main/java/com/squareup/okhttp/internal/DiskLruCache.java @@ -33,7 +33,6 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -160,19 +159,22 @@ public final class DiskLruCache implements Closeable { /** This cache uses a single background thread to evict entries. */ final ThreadPoolExecutor executorService = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue(), Util.threadFactory("OkHttp DiskLruCache", true)); - private final Callable cleanupCallable = new Callable() { - public Void call() throws Exception { + private final Runnable cleanupRunnable = new Runnable() { + public void run() { synchronized (DiskLruCache.this) { if (journalWriter == null) { - return null; // Closed. + return; // Closed. } - trimToSize(); - if (journalRebuildRequired()) { - rebuildJournal(); - redundantOpCount = 0; + try { + trimToSize(); + if (journalRebuildRequired()) { + rebuildJournal(); + redundantOpCount = 0; + } + } catch (IOException e) { + throw new RuntimeException(e); } } - return null; } }; @@ -431,7 +433,7 @@ public synchronized Snapshot get(String key) throws IOException { redundantOpCount++; journalWriter.append(READ + ' ' + key + '\n'); if (journalRebuildRequired()) { - executorService.submit(cleanupCallable); + executorService.execute(cleanupRunnable); } return new Snapshot(key, entry.sequenceNumber, ins, entry.lengths); @@ -488,7 +490,7 @@ public long getMaxSize() { */ public synchronized void setMaxSize(long maxSize) { this.maxSize = maxSize; - executorService.submit(cleanupCallable); + executorService.execute(cleanupRunnable); } /** @@ -551,7 +553,7 @@ private synchronized void completeEdit(Editor editor, boolean success) throws IO journalWriter.flush(); if (size > maxSize || journalRebuildRequired()) { - executorService.submit(cleanupCallable); + executorService.execute(cleanupRunnable); } } @@ -593,7 +595,7 @@ public synchronized boolean remove(String key) throws IOException { lruEntries.remove(key); if (journalRebuildRequired()) { - executorService.submit(cleanupCallable); + executorService.execute(cleanupRunnable); } return true; diff --git a/samples/simple-client/src/main/java/com/squareup/okhttp/sample/OkHttpContributors.java b/samples/simple-client/src/main/java/com/squareup/okhttp/sample/OkHttpContributors.java index 8969f472364b..c6424e2dece4 100644 --- a/samples/simple-client/src/main/java/com/squareup/okhttp/sample/OkHttpContributors.java +++ b/samples/simple-client/src/main/java/com/squareup/okhttp/sample/OkHttpContributors.java @@ -18,7 +18,7 @@ public class OkHttpContributors { new TypeToken>() { }; - class Contributor { + static class Contributor { String login; int contributions; } diff --git a/samples/static-server/src/main/java/com/squareup/okhttp/sample/SampleServer.java b/samples/static-server/src/main/java/com/squareup/okhttp/sample/SampleServer.java index 274bf9dd4885..cb0e24e37aaa 100644 --- a/samples/static-server/src/main/java/com/squareup/okhttp/sample/SampleServer.java +++ b/samples/static-server/src/main/java/com/squareup/okhttp/sample/SampleServer.java @@ -9,6 +9,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.SecureRandom; @@ -116,8 +117,12 @@ public static void main(String[] args) throws Exception { private static SSLContext sslContext(String keystoreFile, String password) throws GeneralSecurityException, IOException { KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType()); - keystore.load(new FileInputStream(keystoreFile), password.toCharArray()); - + InputStream in = new FileInputStream(keystoreFile); + try { + keystore.load(in, password.toCharArray()); + } finally { + Util.closeQuietly(in); + } KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(keystore, password.toCharArray()); From 0ca4c82dd1032625831a5814ea2ddcf165029bdc Mon Sep 17 00:00:00 2001 From: Adam Speakman Date: Thu, 6 Feb 2014 12:06:17 +1300 Subject: [PATCH 14/14] Fix for NPE when the OpenSSLSocketImpl returns null from getNpnSelectedProtocol. --- .../src/main/java/com/squareup/okhttp/internal/Platform.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Platform.java b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Platform.java index eee229533e58..ed4ba10e1cb4 100644 --- a/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Platform.java +++ b/okhttp-protocols/src/main/java/com/squareup/okhttp/internal/Platform.java @@ -293,7 +293,9 @@ private Android( byte[] alpnResult = (byte[]) getAlpnSelectedProtocol.invoke(socket); if (alpnResult != null) return ByteString.of(alpnResult); } - return ByteString.of((byte[]) getNpnSelectedProtocol.invoke(socket)); + byte[] npnResult = (byte[]) getNpnSelectedProtocol.invoke(socket); + if (npnResult == null) return null; + return ByteString.of(npnResult); } catch (InvocationTargetException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) {