Skip to content

Commit 665e1cd

Browse files
author
Amos Shi
committed
8269258: java/net/httpclient/ManyRequestsLegacy.java failed with connection timeout
Reviewed-by: rrich Backport-of: 37921e30803449c06b4d542fdfeed9928cce8a7d
1 parent 4c81a1c commit 665e1cd

File tree

3 files changed

+283
-116
lines changed

3 files changed

+283
-116
lines changed

‎test/jdk/java/net/httpclient/ManyRequests.java

Lines changed: 144 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@
3232
* @compile ../../../com/sun/net/httpserver/LogFilter.java
3333
* @compile ../../../com/sun/net/httpserver/EchoHandler.java
3434
* @compile ../../../com/sun/net/httpserver/FileServerHandler.java
35-
* @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=ssl ManyRequests
36-
* @run main/othervm/timeout=40 -Dtest.insertDelay=true ManyRequests
37-
* @run main/othervm/timeout=40 -Dtest.chunkSize=64 ManyRequests
38-
* @run main/othervm/timeout=40 -Dtest.insertDelay=true -Dtest.chunkSize=64 ManyRequests
35+
* @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=ssl,channel ManyRequests
36+
* @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=channel -Dtest.insertDelay=true ManyRequests
37+
* @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=channel -Dtest.chunkSize=64 ManyRequests
38+
* @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=channel -Dtest.insertDelay=true -Dtest.chunkSize=64 ManyRequests
3939
* @summary Send a large number of requests asynchronously
4040
*/
41-
// * @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=ssl ManyRequests
41+
// * @run main/othervm/timeout=40 -Djdk.httpclient.HttpClient.log=ssl,channel ManyRequests
4242

4343
import com.sun.net.httpserver.HttpsConfigurator;
4444
import com.sun.net.httpserver.HttpsParameters;
@@ -47,32 +47,49 @@
4747
import java.io.IOException;
4848
import java.io.InputStream;
4949
import java.io.OutputStream;
50+
import java.net.ConnectException;
5051
import java.net.InetAddress;
5152
import java.net.InetSocketAddress;
5253
import java.net.URI;
5354
import java.net.http.HttpClient;
5455
import java.net.http.HttpRequest;
5556
import java.net.http.HttpRequest.BodyPublishers;
57+
import java.net.http.HttpResponse;
5658
import java.net.http.HttpResponse.BodyHandlers;
5759
import java.util.Arrays;
5860
import java.util.Formatter;
5961
import java.util.HashMap;
6062
import java.util.LinkedList;
63+
import java.util.Map;
6164
import java.util.Random;
6265
import java.util.logging.Logger;
6366
import java.util.logging.Level;
6467
import java.util.concurrent.CompletableFuture;
68+
import java.util.concurrent.CompletionException;
69+
import java.util.concurrent.CompletionStage;
70+
import java.util.concurrent.ConcurrentHashMap;
71+
import java.util.concurrent.atomic.AtomicInteger;
72+
import java.util.stream.Stream;
6573
import javax.net.ssl.SSLContext;
74+
75+
import jdk.test.lib.Platform;
76+
import jdk.test.lib.RandomFactory;
6677
import jdk.test.lib.net.SimpleSSLContext;
78+
import jdk.test.lib.net.URIBuilder;
6779

6880
public class ManyRequests {
6981

70-
volatile static int counter = 0;
82+
static final int MAX_COUNT = 20;
83+
static final int MAX_LIMIT = 40;
84+
static final AtomicInteger COUNT = new AtomicInteger();
85+
static final AtomicInteger LIMIT = new AtomicInteger(MAX_LIMIT);
86+
static final Random RANDOM = RandomFactory.getRandom();
7187

7288
public static void main(String[] args) throws Exception {
7389
Logger logger = Logger.getLogger("com.sun.net.httpserver");
7490
logger.setLevel(Level.ALL);
7591
logger.info("TEST");
92+
Stream.of(Logger.getLogger("").getHandlers()).forEach((h) -> h.setLevel(Level.ALL));
7693
System.out.println("Sending " + REQUESTS
7794
+ " requests; delay=" + INSERT_DELAY
7895
+ ", chunks=" + CHUNK_SIZE
@@ -95,14 +112,14 @@ public static void main(String[] args) throws Exception {
95112
}
96113

97114
//static final int REQUESTS = 1000;
98-
static final int REQUESTS = 20;
115+
static final int REQUESTS = MAX_COUNT;
99116
static final boolean INSERT_DELAY = Boolean.getBoolean("test.insertDelay");
100117
static final int CHUNK_SIZE = Math.max(0,
101118
Integer.parseInt(System.getProperty("test.chunkSize", "0")));
102119
static final boolean XFIXED = Boolean.getBoolean("test.XFixed");
103120

104121
static class TestEchoHandler extends EchoHandler {
105-
final Random rand = new Random();
122+
final Random rand = RANDOM;
106123
@Override
107124
public void handle(HttpExchange e) throws IOException {
108125
System.out.println("Server: received " + e.getRequestURI());
@@ -128,60 +145,126 @@ protected void close(HttpExchange t, InputStream is) throws IOException {
128145
}
129146
}
130147

148+
static String now(long start) {
149+
long elapsed = System.nanoTime() - start;
150+
long ms = elapsed / 1000_000L;
151+
long s = ms / 1000L;
152+
if (s == 0) return ms + "ms: ";
153+
return s + "s, " + (ms - s * 1000L) + "ms: ";
154+
}
155+
156+
static String failure(Throwable t) {
157+
String s = "\n\t failed: " + t;
158+
for (t = t.getCause(); t != null ; t = t.getCause()) {
159+
s = s + "\n\t\t Caused by: " + t;
160+
}
161+
return s;
162+
}
163+
131164
static void test(HttpsServer server, HttpClient client) throws Exception {
132165
int port = server.getAddress().getPort();
133-
URI baseURI = new URI("https://localhost:" + port + "/foo/x");
166+
167+
URI baseURI = URIBuilder.newBuilder()
168+
.scheme("https")
169+
.host(InetAddress.getLoopbackAddress().getHostName())
170+
.port(port)
171+
.path("/foo/x").build();
134172
server.createContext("/foo", new TestEchoHandler());
135173
server.start();
136174

137-
RequestLimiter limiter = new RequestLimiter(40);
138-
Random rand = new Random();
139-
CompletableFuture<?>[] results = new CompletableFuture<?>[REQUESTS];
140-
HashMap<HttpRequest,byte[]> bodies = new HashMap<>();
141-
142-
for (int i=0; i<REQUESTS; i++) {
143-
byte[] buf = new byte[(i+1)*CHUNK_SIZE+i+1]; // different size bodies
144-
rand.nextBytes(buf);
145-
URI uri = new URI(baseURI.toString() + String.valueOf(i+1));
146-
HttpRequest r = HttpRequest.newBuilder(uri)
147-
.header("XFixed", "true")
148-
.POST(BodyPublishers.ofByteArray(buf))
149-
.build();
150-
bodies.put(r, buf);
151-
152-
results[i] =
153-
limiter.whenOkToSend()
154-
.thenCompose((v) -> {
155-
System.out.println("Client: sendAsync: " + r.uri());
156-
return client.sendAsync(r, BodyHandlers.ofByteArray());
157-
})
158-
.thenCompose((resp) -> {
159-
limiter.requestComplete();
160-
if (resp.statusCode() != 200) {
161-
String s = "Expected 200, got: " + resp.statusCode();
162-
System.out.println(s + " from "
163-
+ resp.request().uri().getPath());
164-
return completedWithIOException(s);
165-
} else {
166-
counter++;
167-
System.out.println("Result (" + counter + ") from "
168-
+ resp.request().uri().getPath());
169-
}
170-
return CompletableFuture.completedStage(resp.body())
171-
.thenApply((b) -> new Pair<>(resp, b));
172-
})
173-
.thenAccept((pair) -> {
174-
HttpRequest request = pair.t.request();
175-
byte[] requestBody = bodies.get(request);
176-
check(Arrays.equals(requestBody, pair.u),
177-
"bodies not equal:[" + bytesToHexString(requestBody)
178-
+ "] [" + bytesToHexString(pair.u) + "]");
179-
180-
});
181-
}
175+
// This loop implements a retry mechanism to work around an issue
176+
// on some systems (observed on Windows 10) that seem to be trying to
177+
// throttle the number of connections that can be made concurrently by
178+
// rejecting connection attempts.
179+
// On the first iteration of this loop, we will attempt 20 concurrent
180+
// requests. If this fails with ConnectException, we will retry the
181+
// 20 requests, but limiting the concurrency to 10 (LIMIT <- 10).
182+
// If this fails again, the test will fail.
183+
boolean done = false;
184+
LOOP: do {
185+
RequestLimiter limiter = new RequestLimiter(LIMIT.get());
186+
Random rand = RANDOM;
187+
CompletableFuture<?>[] results = new CompletableFuture<?>[REQUESTS];
188+
Map<HttpRequest,byte[]> bodies = new ConcurrentHashMap<>();
189+
190+
long start = System.nanoTime();
191+
192+
for (int i = 0; i < REQUESTS; i++) {
193+
byte[] buf = new byte[(i + 1) * CHUNK_SIZE + i + 1]; // different size bodies
194+
rand.nextBytes(buf);
195+
URI uri = new URI(baseURI.toString() + String.valueOf(i + 1));
196+
HttpRequest r = HttpRequest.newBuilder(uri)
197+
.header("XFixed", "true")
198+
.POST(BodyPublishers.ofByteArray(buf))
199+
.build();
200+
bodies.put(r, buf);
201+
202+
results[i] =
203+
limiter.whenOkToSend()
204+
.thenCompose((v) -> {
205+
System.out.println("Client: sendAsync: " + r.uri());
206+
return client.sendAsync(r, BodyHandlers.ofByteArray());
207+
})
208+
.handle((resp, t) -> {
209+
limiter.requestComplete();
210+
CompletionStage<Pair<HttpResponse<byte[]>, byte[]>> res;
211+
String now = now(start);
212+
if (t == null) {
213+
if (resp.statusCode() != 200) {
214+
String s = "Expected 200, got: " + resp.statusCode();
215+
System.out.println(now + s + " from "
216+
+ resp.request().uri().getPath());
217+
res = completedWithIOException(s);
218+
return res;
219+
} else {
220+
int counter = COUNT.incrementAndGet();
221+
System.out.println(now + "Result (" + counter + ") from "
222+
+ resp.request().uri().getPath());
223+
}
224+
res = CompletableFuture.completedStage(resp.body())
225+
.thenApply((b) -> new Pair<>(resp, b));
226+
return res;
227+
} else {
228+
int counter = COUNT.incrementAndGet();
229+
System.out.println(now + "Result (" + counter + ") from "
230+
+ r.uri().getPath()
231+
+ failure(t));
232+
res = CompletableFuture.failedFuture(t);
233+
return res;
234+
}
235+
})
236+
.thenCompose(c -> c)
237+
.thenAccept((pair) -> {
238+
HttpRequest request = pair.t.request();
239+
byte[] requestBody = bodies.get(request);
240+
check(Arrays.equals(requestBody, pair.u),
241+
"bodies not equal:[" + bytesToHexString(requestBody)
242+
+ "] [" + bytesToHexString(pair.u) + "]");
243+
244+
});
245+
}
246+
247+
// wait for them all to complete and throw exception in case of err
248+
try {
249+
CompletableFuture.allOf(results).join();
250+
done = true;
251+
} catch (CompletionException e) {
252+
if (!Platform.isWindows()) throw e;
253+
if (LIMIT.get() < REQUESTS) throw e;
254+
Throwable cause = e;
255+
while ((cause = cause.getCause()) != null) {
256+
if (cause instanceof ConnectException) {
257+
// try again, limit concurrency by half
258+
COUNT.set(0);
259+
LIMIT.set(REQUESTS/2);
260+
System.out.println("*** Retrying due to " + cause);
261+
continue LOOP;
262+
}
263+
}
264+
throw e;
265+
}
266+
} while (!done);
182267

183-
// wait for them all to complete and throw exception in case of error
184-
CompletableFuture.allOf(results).join();
185268
}
186269

187270
static <T> CompletableFuture<T> completedWithIOException(String message) {
@@ -203,11 +286,13 @@ static String bytesToHexString(byte[] bytes) {
203286
}
204287

205288
static final class Pair<T,U> {
289+
private final T t;
290+
private final U u;
291+
206292
Pair(T t, U u) {
207-
this.t = t; this.u = u;
293+
this.t = t;
294+
this.u = u;
208295
}
209-
T t;
210-
U u;
211296
}
212297

213298
/**

‎test/jdk/java/net/httpclient/ManyRequests2.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -33,10 +33,14 @@
3333
* @compile ../../../com/sun/net/httpserver/EchoHandler.java
3434
* @compile ../../../com/sun/net/httpserver/FileServerHandler.java
3535
* @build ManyRequests ManyRequests2
36-
* @run main/othervm/timeout=40 -Dtest.XFixed=true ManyRequests2
37-
* @run main/othervm/timeout=40 -Dtest.XFixed=true -Dtest.insertDelay=true ManyRequests2
38-
* @run main/othervm/timeout=40 -Dtest.XFixed=true -Dtest.chunkSize=64 ManyRequests2
36+
* @run main/othervm/timeout=40 -Dtest.XFixed=true
37+
* -Djdk.httpclient.HttpClient.log=channel ManyRequests2
38+
* @run main/othervm/timeout=40 -Dtest.XFixed=true -Dtest.insertDelay=true
39+
* -Djdk.httpclient.HttpClient.log=channel ManyRequests2
40+
* @run main/othervm/timeout=40 -Dtest.XFixed=true -Dtest.chunkSize=64
41+
* -Djdk.httpclient.HttpClient.log=channel ManyRequests2
3942
* @run main/othervm/timeout=40 -Djdk.internal.httpclient.debug=true
43+
* -Djdk.httpclient.HttpClient.log=channel
4044
* -Dtest.XFixed=true -Dtest.insertDelay=true
4145
* -Dtest.chunkSize=64 ManyRequests2
4246
* @summary Send a large number of requests asynchronously.

0 commit comments

Comments
 (0)