From acd45e124e05b3f1a8624a193480791acfb51419 Mon Sep 17 00:00:00 2001 From: Jake Wharton Date: Thu, 6 Feb 2014 14:26:30 -0500 Subject: [PATCH] Initial implementation of an OkHttp-backed curl clone. --- okcurl/pom.xml | 94 ++++++++ .../java/com/squareup/okhttp/curl/Main.java | 203 ++++++++++++++++++ .../main/resources/okcurl-version.properties | 1 + okhttp/pom.xml | 12 +- .../java/com/squareup/okhttp/Dispatcher.java | 22 +- .../com/squareup/okhttp/DispatcherTest.java | 26 ++- pom.xml | 13 ++ 7 files changed, 352 insertions(+), 19 deletions(-) create mode 100644 okcurl/pom.xml create mode 100644 okcurl/src/main/java/com/squareup/okhttp/curl/Main.java create mode 100644 okcurl/src/main/resources/okcurl-version.properties diff --git a/okcurl/pom.xml b/okcurl/pom.xml new file mode 100644 index 000000000000..af0ba2e21234 --- /dev/null +++ b/okcurl/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + + com.squareup.okhttp + parent + 2.0.0-SNAPSHOT + + + okcurl + OkCurl + + + + com.squareup.okhttp + okhttp + ${project.version} + + + org.bouncycastle + bcprov-jdk15on + + + org.mortbay.jetty.npn + npn-boot + + + io.airlift + airline + + + com.google.guava + guava + + + + junit + junit + test + + + + + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + com.squareup.okhttp.curl.Main + + + + + + package + + single + + + + + + org.skife.maven + really-executable-jar-maven-plugin + 1.1.0 + + + package + + really-executable-jar + + + + + -Xbootclasspath/p:$0 + + + + + diff --git a/okcurl/src/main/java/com/squareup/okhttp/curl/Main.java b/okcurl/src/main/java/com/squareup/okhttp/curl/Main.java new file mode 100644 index 000000000000..c5d2d12e1006 --- /dev/null +++ b/okcurl/src/main/java/com/squareup/okhttp/curl/Main.java @@ -0,0 +1,203 @@ +package com.squareup.okhttp.curl; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.squareup.okhttp.ConnectionPool; +import com.squareup.okhttp.Failure; +import com.squareup.okhttp.Headers; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Protocol; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import io.airlift.command.Arguments; +import io.airlift.command.Command; +import io.airlift.command.HelpOption; +import io.airlift.command.Option; +import io.airlift.command.SingleCommand; +import java.io.IOException; +import java.io.InputStream; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import static java.util.concurrent.TimeUnit.SECONDS; + +@Command(name = Main.NAME, description = "A curl for the next-generation web.") +public class Main extends HelpOption implements Runnable, Response.Receiver { + static final String NAME = "okcurl"; + static final int DEFAULT_TIMEOUT = -1; + + public static void main(String... args) { + SingleCommand.singleCommand(Main.class).parse(args).run(); + } + + private static String versionString() { + try { + Properties prop = new Properties(); + InputStream in = Main.class.getResourceAsStream("/okcurl-version.properties"); + prop.load(in); + in.close(); + return prop.getProperty("version"); + } catch (IOException e) { + throw new AssertionError("Could not load okcurl-version.properties."); + } + } + + private static String protocols() { + return Joiner.on(", ").join(Lists.transform(Arrays.asList(Protocol.values()), + new Function() { + @Override public String apply(Protocol protocol) { + return protocol.name.utf8(); + } + })); + } + + @Option(name = { "-H", "--header" }, description = "Custom header to pass to server") + public List headers; + + @Option(name = { "-A", "--user-agent" }, description = "User-Agent to send to server") + public String userAgent = NAME + "/" + versionString(); + + @Option(name = "--connect-timeout", description = "Maximum time allowed for connection (seconds)") + public int connectTimeout = DEFAULT_TIMEOUT; + + @Option(name = "--read-timeout", description = "Maximum time allowed for reading data (seconds)") + public int readTimeout = DEFAULT_TIMEOUT; + + @Option(name = { "-L", "--location" }, description = "Follow redirects") + public boolean followRedirects; + + @Option(name = { "-k", "--insecure" }, + description = "Allow connections to SSL sites without certs") + public boolean allowInsecure; + + @Option(name = { "-i", "--include" }, description = "Include protocol headers in the output") + public boolean showHeaders; + + @Option(name = { "-e", "--referer" }, description = "Referer URL") + public String referer; + + @Option(name = { "-V", "--version" }, description = "Show version number and quit") + public boolean version; + + @Arguments(title = "url", description = "Remote resource URL") + public String url; + + private OkHttpClient client; + + @Override public void run() { + if (showHelpIfRequested()) { + return; + } + if (version) { + System.out.println(NAME + " " + versionString()); + System.out.println("Protocols: " + protocols()); + return; + } + + client = getConfiguredClient(); + Request request = getConfiguredRequest(); + client.enqueue(request, this); + + // Immediately begin triggering an executor shutdown so that after execution of the above + // request the threads do not stick around until timeout. + client.getDispatcher().getExecutorService().shutdown(); + } + + private OkHttpClient getConfiguredClient() { + OkHttpClient client = new OkHttpClient(); + client.setFollowProtocolRedirects(followRedirects); + if (connectTimeout != DEFAULT_TIMEOUT) { + client.setConnectTimeout(connectTimeout, SECONDS); + } + if (readTimeout != DEFAULT_TIMEOUT) { + client.setReadTimeout(readTimeout, SECONDS); + } + if (allowInsecure) { + client.setSslSocketFactory(createInsecureSslSocketFactory()); + } + // If we don't set this reference, there's no way to clean shutdown persistent connections. + client.setConnectionPool(ConnectionPool.getDefault()); + return client; + } + + private Request getConfiguredRequest() { + Request.Builder request = new Request.Builder(); + request.url(url); + if (headers != null) { + for (String header : headers) { + String[] parts = header.split(":", -1); + request.header(parts[0], parts[1]); + } + } + if (referer != null) { + request.header("Referer", referer); + } + request.header("User-Agent", userAgent); + return request.build(); + } + + @Override public void onFailure(Failure failure) { + failure.exception().printStackTrace(); + close(); + } + + @Override public boolean onResponse(Response response) throws IOException { + if (showHeaders) { + System.out.println(response.statusLine()); + Headers headers = response.headers(); + for (int i = 0, count = headers.size(); i < count; i++) { + System.out.println(headers.name(i) + ": " + headers.value(i)); + } + System.out.println(); + } + + Response.Body body = response.body(); + byte[] buffer = new byte[1024]; + while (body.ready()) { + int c = body.byteStream().read(buffer); + if (c == -1) { + close(); + return true; + } + + System.out.write(buffer, 0, c); + } + close(); + return false; + } + + private void close() { + client.getConnectionPool().evictAll(); // Close any persistent connections. + } + + private static SSLSocketFactory createInsecureSslSocketFactory() { + try { + SSLContext context = SSLContext.getInstance("TLS"); + TrustManager permissive = new X509TrustManager() { + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) + throws CertificateException { + } + + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) + throws CertificateException { + } + + @Override public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + context.init(null, new TrustManager[] { permissive }, null); + return context.getSocketFactory(); + } catch (Exception e) { + throw new AssertionError(e); + } + } +} diff --git a/okcurl/src/main/resources/okcurl-version.properties b/okcurl/src/main/resources/okcurl-version.properties new file mode 100644 index 000000000000..defbd48204e4 --- /dev/null +++ b/okcurl/src/main/resources/okcurl-version.properties @@ -0,0 +1 @@ +version=${project.version} diff --git a/okhttp/pom.xml b/okhttp/pom.xml index 17f4cdac203c..a262e3f91a62 100644 --- a/okhttp/pom.xml +++ b/okhttp/pom.xml @@ -18,12 +18,6 @@ okhttp-protocols ${project.version} - - com.squareup.okhttp - mockwebserver - ${project.version} - test - org.mortbay.jetty.npn npn-boot @@ -35,6 +29,12 @@ junit test + + com.squareup.okhttp + mockwebserver + ${project.version} + test + diff --git a/okhttp/src/main/java/com/squareup/okhttp/Dispatcher.java b/okhttp/src/main/java/com/squareup/okhttp/Dispatcher.java index 198fd4ea80d9..58e06be40bce 100644 --- a/okhttp/src/main/java/com/squareup/okhttp/Dispatcher.java +++ b/okhttp/src/main/java/com/squareup/okhttp/Dispatcher.java @@ -19,7 +19,7 @@ import java.util.ArrayDeque; import java.util.Deque; import java.util.Iterator; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -27,7 +27,7 @@ /** * Policy on when async requests are executed. * - *

Each dispatcher uses an {@link Executor} to run jobs internally. If you + *

Each dispatcher uses an {@link ExecutorService} to run jobs internally. If you * supply your own executor, it should be able to run {@link #getMaxRequests the * configured maximum} number of jobs concurrently. */ @@ -36,7 +36,7 @@ public final class Dispatcher { private int maxRequestsPerHost = 5; /** Executes jobs. Created lazily. */ - private Executor executor; + private ExecutorService executorService; /** Ready jobs in the order they'll be run. */ private final Deque readyJobs = new ArrayDeque(); @@ -44,19 +44,19 @@ public final class Dispatcher { /** Running jobs. Includes canceled jobs that haven't finished yet. */ private final Deque runningJobs = new ArrayDeque(); - public Dispatcher(Executor executor) { - this.executor = executor; + public Dispatcher(ExecutorService executorService) { + this.executorService = executorService; } public Dispatcher() { } - public synchronized Executor getExecutor() { - if (executor == null) { - executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, + public synchronized ExecutorService getExecutorService() { + if (executorService == null) { + executorService = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new LinkedBlockingQueue(), Util.threadFactory("OkHttp Dispatcher", false)); } - return executor; + return executorService; } /** @@ -107,7 +107,7 @@ synchronized void enqueue(OkHttpClient client, Request request, Response.Receive if (runningJobs.size() < maxRequests && runningJobsForHost(job) < maxRequestsPerHost) { runningJobs.add(job); - getExecutor().execute(job); + getExecutorService().execute(job); } else { readyJobs.add(job); } @@ -143,7 +143,7 @@ private void promoteJobs() { if (runningJobsForHost(job) < maxRequestsPerHost) { i.remove(); runningJobs.add(job); - getExecutor().execute(job); + getExecutorService().execute(job); } if (runningJobs.size() >= maxRequests) return; // Reached max capacity. diff --git a/okhttp/src/test/java/com/squareup/okhttp/DispatcherTest.java b/okhttp/src/test/java/com/squareup/okhttp/DispatcherTest.java index 034ed84b6465..a42362fbff44 100644 --- a/okhttp/src/test/java/com/squareup/okhttp/DispatcherTest.java +++ b/okhttp/src/test/java/com/squareup/okhttp/DispatcherTest.java @@ -4,7 +4,8 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.concurrent.Executor; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.Test; @@ -129,7 +130,7 @@ public final class DispatcherTest { executor.assertJobs("http://a/2"); } - class RecordingExecutor implements Executor { + class RecordingExecutor extends AbstractExecutorService { private List jobs = new ArrayList(); @Override public void execute(Runnable command) { @@ -155,6 +156,27 @@ public void finishJob(String url) { } throw new AssertionError("No such job: " + url); } + + @Override public void shutdown() { + throw new UnsupportedOperationException(); + } + + @Override public List shutdownNow() { + throw new UnsupportedOperationException(); + } + + @Override public boolean isShutdown() { + throw new UnsupportedOperationException(); + } + + @Override public boolean isTerminated() { + throw new UnsupportedOperationException(); + } + + @Override public boolean awaitTermination(long timeout, TimeUnit unit) + throws InterruptedException { + throw new UnsupportedOperationException(); + } } private Request newRequest(String url) { diff --git a/pom.xml b/pom.xml index aea71da07add..82861f385904 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ okhttp okhttp-apache okhttp-protocols + okcurl mockwebserver samples benchmarks @@ -36,6 +37,8 @@ 1.48 2.2.3 4.2.2 + 0.6 + 16.0 4.11 @@ -87,6 +90,16 @@ httpclient ${apache.http.version} + + io.airlift + airline + ${airlift.version} + + + com.google.guava + guava + ${guava.version} +