From 99841aecb4273e0cbd22e7d67399bee20b806c52 Mon Sep 17 00:00:00 2001 From: jwilson Date: Sat, 1 Feb 2014 15:35:54 -0500 Subject: [PATCH] Use Caliper for benchmarking. --- benchmarks/pom.xml | 11 ++++ .../okhttp/benchmarks/ApacheHttpClient.java | 8 +-- .../squareup/okhttp/benchmarks/Benchmark.java | 57 +++++++++++++------ .../squareup/okhttp/benchmarks/Client.java | 44 ++++++++++++++ .../okhttp/benchmarks/NettyHttpClient.java | 2 +- .../squareup/okhttp/benchmarks/OkHttp.java | 11 +--- .../benchmarks/SynchronousHttpClient.java | 12 ++++ .../okhttp/benchmarks/UrlConnection.java | 8 +-- 8 files changed, 111 insertions(+), 42 deletions(-) create mode 100644 benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 14d93eeefd8b..960296b11810 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -13,6 +13,11 @@ Benchmarks + + com.google.caliper + caliper + 1.0-beta-1 + com.squareup.okhttp okhttp @@ -51,5 +56,11 @@ netty-codec-http 4.0.15.Final + + + com.jcraft + jzlib + 1.1.2 + diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java index cd2cba31cff9..cb8e719111c4 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/ApacheHttpClient.java @@ -61,7 +61,6 @@ public ApacheHttpClientRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpResponse response = client.execute(new HttpGet(url.toString())); @@ -71,12 +70,7 @@ public void run() { in = new GZIPInputStream(in); } - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(in); long finish = System.nanoTime(); if (VERBOSE) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java index 16bd063508a5..3b2b5e59dfb9 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Benchmark.java @@ -15,6 +15,9 @@ */ package com.squareup.okhttp.benchmarks; +import com.google.caliper.Param; +import com.google.caliper.model.ArbitraryMeasurement; +import com.google.caliper.runner.CaliperMain; import com.squareup.okhttp.Protocol; import com.squareup.okhttp.internal.SslContextBuilder; import com.squareup.okhttp.mockwebserver.Dispatcher; @@ -40,41 +43,57 @@ * It uses a local connection to a MockWebServer to measure how many identical * requests per second can be carried over a fixed number of threads. */ -public class Benchmark { +public class Benchmark extends com.google.caliper.Benchmark { private static final int NUM_REPORTS = 10; + private static final boolean VERBOSE = false; + private final Random random = new Random(0); /** Which client to run.*/ - HttpClient httpClient = new NettyHttpClient(); + @Param + Client client; /** How many concurrent requests to execute. */ - int concurrencyLevel = 10; + @Param({ "1", "10" }) + int concurrencyLevel; /** True to use TLS. */ // TODO: compare different ciphers? - boolean tls = false; + @Param + boolean tls; /** True to use gzip content-encoding for the response body. */ - boolean gzip = false; + @Param + boolean gzip; /** Don't combine chunked with SPDY_3 or HTTP_2; that's not allowed. */ - boolean chunked = false; + @Param + boolean chunked; /** The size of the HTTP response body, in uncompressed bytes. */ - int bodyByteCount = 1024 * 1024; + @Param({ "128", "1048576" }) + int bodyByteCount; /** How many additional headers were included, beyond the built-in ones. */ - int headerCount = 20; + @Param({ "0", "20" }) + int headerCount; /** Which ALPN/NPN protocols are in use. Only useful with TLS. */ List protocols = Arrays.asList(Protocol.HTTP_11); - public static void main(String[] args) throws Exception { - new Benchmark().run(); + public static void main(String[] args) { + List allArgs = new ArrayList(); + allArgs.add("--instrument"); + allArgs.add("arbitrary"); + allArgs.addAll(Arrays.asList(args)); + + CaliperMain.main(Benchmark.class, allArgs.toArray(new String[allArgs.size()])); } - public void run() throws Exception { - System.out.println(toString()); + @ArbitraryMeasurement(description = "requests per second") + public double run() throws Exception { + if (VERBOSE) System.out.println(toString()); + HttpClient httpClient = client.create(); // Prepare the client & server httpClient.prepare(this); @@ -85,6 +104,7 @@ public void run() throws Exception { long reportStart = System.nanoTime(); long reportPeriod = TimeUnit.SECONDS.toNanos(1); int reports = 0; + double best = 0.0; // Run until we've printed enough reports. while (reports < NUM_REPORTS) { @@ -93,7 +113,10 @@ public void run() throws Exception { double reportDuration = now - reportStart; if (reportDuration > reportPeriod) { double requestsPerSecond = requestCount / reportDuration * TimeUnit.SECONDS.toNanos(1); - System.out.println(String.format("Requests per second: %.1f", requestsPerSecond)); + if (VERBOSE) { + System.out.println(String.format("Requests per second: %.1f", requestsPerSecond)); + } + best = Math.max(best, requestsPerSecond); requestCount = 0; reportStart = now; reports++; @@ -108,6 +131,8 @@ public void run() throws Exception { // The job queue is full. Take a break. sleep(10); } + + return best; } @Override public String toString() { @@ -117,10 +142,8 @@ public void run() throws Exception { if (chunked) modifiers.add("chunked"); modifiers.addAll(protocols); - return String.format("%s %s\n" - + "bodyByteCount=%s headerCount=%s concurrencyLevel=%s", - httpClient.getClass().getSimpleName(), modifiers, - bodyByteCount, headerCount, concurrencyLevel); + return String.format("%s %s\nbodyByteCount=%s headerCount=%s concurrencyLevel=%s", + client, modifiers, bodyByteCount, headerCount, concurrencyLevel); } private void sleep(int millis) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java new file mode 100644 index 000000000000..0f076a610d14 --- /dev/null +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/Client.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 Square, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.squareup.okhttp.benchmarks; + +enum Client { + OkHttp { + @Override HttpClient create() { + return new OkHttp(); + } + }, + + Apache { + @Override HttpClient create() { + return new ApacheHttpClient(); + } + }, + + UrlConnection { + @Override HttpClient create() { + return new UrlConnection(); + } + }, + + Netty { + @Override HttpClient create() { + return new NettyHttpClient(); + } + }; + + abstract HttpClient create(); +} diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java index d5adec5a053b..c3b06512eb5d 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/NettyHttpClient.java @@ -48,7 +48,7 @@ import javax.net.ssl.SSLEngine; /** Netty isn't an HTTP client, but it's almost one. */ -public class NettyHttpClient implements HttpClient { +class NettyHttpClient implements HttpClient { private static final boolean VERBOSE = false; // Guarded by this. Real apps need more capable connection management. diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java index 6fdb40e9acbb..03b9e3c6f10a 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/OkHttp.java @@ -18,7 +18,6 @@ import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.internal.SslContextBuilder; import java.io.IOException; -import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.TimeUnit; @@ -62,18 +61,10 @@ public OkHttpRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpURLConnection urlConnection = client.open(url); - InputStream in = urlConnection.getInputStream(); - - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(urlConnection.getInputStream()); long finish = System.nanoTime(); if (VERBOSE) { diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java index 9a0851c7ff66..f4bb02c79b07 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/SynchronousHttpClient.java @@ -15,6 +15,8 @@ */ package com.squareup.okhttp.benchmarks; +import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -38,5 +40,15 @@ abstract class SynchronousHttpClient implements HttpClient { return executor.getQueue().size() < targetBacklog; } + protected long readAllAndClose(InputStream in) throws IOException { + byte[] buffer = new byte[1024]; + long total = 0; + for (int count; (count = in.read(buffer)) != -1; ) { + total += count; + } + in.close(); + return total; + } + abstract Runnable request(URL url); } diff --git a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java index a2c3f3af407e..53d2a4b9e9f3 100644 --- a/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java +++ b/benchmarks/src/main/java/com/squareup/okhttp/benchmarks/UrlConnection.java @@ -58,7 +58,6 @@ public UrlConnectionRequest(URL url) { } public void run() { - byte[] buffer = new byte[1024]; long start = System.nanoTime(); try { HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); @@ -67,12 +66,7 @@ public void run() { in = new GZIPInputStream(in); } - // Consume the response body. - int total = 0; - for (int count; (count = in.read(buffer)) != -1; ) { - total += count; - } - in.close(); + long total = readAllAndClose(in); long finish = System.nanoTime(); if (VERBOSE) {