Skip to content

HDFS-16714: Replace okhttp by apache http client #4684

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

HDFS-16714: Replace okhttp by apache http client #4684

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d56fbd9
Repalce okhttp by apache http client
pan3793 Aug 3, 2022
7d67c60
nit
pan3793 Aug 3, 2022
1c3e894
nit
pan3793 Aug 3, 2022
b5e1ed5
nit
pan3793 Aug 3, 2022
0056f0d
nit
pan3793 Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion LICENSE-binary
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,6 @@ com.google.guava:guava:27.0-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.microsoft.azure:azure-storage:7.0.0
com.nimbusds:nimbus-jose-jwt:9.8.1
com.squareup.okhttp3:okhttp:4.9.3
com.squareup.okio:okio:1.6.0
com.zaxxer:HikariCP:4.0.3
commons-beanutils:commons-beanutils:1.9.3
Expand Down
12 changes: 0 additions & 12 deletions hadoop-client-modules/hadoop-client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,18 +114,6 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
<exclusion>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
Expand Down
13 changes: 0 additions & 13 deletions hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,17 +94,4 @@
<Bug pattern="EI_EXPOSE_REP" />
</Match>

<!--okhttp classes from Kotlin are not analysed for NP check. -->
<Match>
<Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
<Method name="refresh" />
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
</Match>

<Match>
<Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
<Method name="refresh" />
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
</Match>

</FindBugsFilter>
12 changes: 0 additions & 12 deletions hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</properties>

<dependencies>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
Expand Down
59 changes: 35 additions & 24 deletions ...main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,28 @@
package org.apache.hadoop.hdfs.web.oauth2;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;

import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;

import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
Expand All @@ -42,7 +49,6 @@
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.EXPIRES_IN;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.GRANT_TYPE;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.REFRESH_TOKEN;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.URLENCODED;
import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;

/**
Expand Down Expand Up @@ -103,30 +109,35 @@ public synchronized String getAccessToken() throws IOException {
}

void refresh() throws IOException {
OkHttpClient client =
new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
TimeUnit.MILLISECONDS)
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
HttpEntity reqEntity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_FORM_URLENCODED.withCharset(StandardCharsets.UTF_8))
.setParameters(
new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN),
new BasicNameValuePair(REFRESH_TOKEN, refreshToken),
new BasicNameValuePair(CLIENT_ID, clientId))
.build();

String bodyString =
Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
HttpUriRequest request = RequestBuilder
.post(refreshURL)
.setEntity(reqEntity)
.build();

RequestBody body = RequestBody.create(bodyString, URLENCODED);
RequestConfig reqConf = RequestConfig.custom()
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.build();

Request request = new Request.Builder().url(refreshURL).post(body).build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Unexpected code " + response);
}
if (response.code() != HttpStatus.SC_OK) {
HttpClientBuilder clientBuilder = HttpClientBuilder.create().setDefaultRequestConfig(reqConf);
try (CloseableHttpClient client = clientBuilder.build();
CloseableHttpResponse response = client.execute(request)) {
int statusCode = response.getStatusLine().getStatusCode();
String respText = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
if (statusCode != HttpStatus.SC_OK) {
throw new IllegalArgumentException(
"Received invalid http response: " + response.code() + ", text = "
+ response.toString());
"Received invalid http response: " + statusCode + ", text = " + respText);
}

Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());

Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(respText);
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
accessTokenTimer.setExpiresIn(newExpiresIn);

Expand Down
65 changes: 36 additions & 29 deletions ...t/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,28 @@
package org.apache.hadoop.hdfs.web.oauth2;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;

import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;

import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
Expand All @@ -43,7 +50,6 @@
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.CLIENT_SECRET;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.EXPIRES_IN;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.GRANT_TYPE;
import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.URLENCODED;
import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;

/**
Expand Down Expand Up @@ -97,34 +103,35 @@ public synchronized String getAccessToken() throws IOException {
}

void refresh() throws IOException {
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
HttpEntity reqEntity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_FORM_URLENCODED.withCharset(StandardCharsets.UTF_8))
.setParameters(
new BasicNameValuePair(CLIENT_SECRET, getCredential()),
new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS),
new BasicNameValuePair(CLIENT_ID, clientId))
.build();

String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
GRANT_TYPE, CLIENT_CREDENTIALS,
CLIENT_ID, clientId);

RequestBody body = RequestBody.create(bodyString, URLENCODED);
HttpUriRequest request = RequestBuilder
.post(refreshURL)
.setEntity(reqEntity)
.build();

Request request = new Request.Builder()
.url(refreshURL)
.post(body)
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Unexpected code " + response);
}
RequestConfig reqConf = RequestConfig.custom()
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.build();

if (response.code() != HttpStatus.SC_OK) {
throw new IllegalArgumentException("Received invalid http response: "
+ response.code() + ", text = " + response.toString());
HttpClientBuilder clientBuilder = HttpClientBuilder.create().setDefaultRequestConfig(reqConf);
try (CloseableHttpClient client = clientBuilder.build();
CloseableHttpResponse response = client.execute(request)) {
int statusCode = response.getStatusLine().getStatusCode();
String respText = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
if (statusCode != HttpStatus.SC_OK) {
throw new IllegalArgumentException(
"Received invalid http response: " + statusCode + ", text = " + respText);
}

Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
response.body().string());

Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(respText);
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
timer.setExpiresIn(newExpiresIn);

Expand Down
4 changes: 0 additions & 4 deletions ...t/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;

import okhttp3.MediaType;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand All @@ -30,9 +29,6 @@
public final class OAuth2Constants {
private OAuth2Constants() { /** Private constructor. **/ }

public static final MediaType URLENCODED
= MediaType.parse("application/x-www-form-urlencoded; charset=utf-8");

/* Constants for OAuth protocol */
public static final String ACCESS_TOKEN = "access_token";
public static final String BEARER = "bearer";
Expand Down
35 changes: 0 additions & 35 deletions hadoop-project/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,6 @@
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>4.0.3</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
<okhttp.version>2.7.5</okhttp.version>
<okhttp3.version>4.9.3</okhttp3.version>
<kotlin-stdlib.verion>1.4.10</kotlin-stdlib.verion>
<kotlin-stdlib-common.version>1.4.10</kotlin-stdlib-common.version>
<jdom2.version>2.0.6.1</jdom2.version>
<jna.version>5.2.0</jna.version>
<grizzly.version>2.2.21</grizzly.version>
Expand Down Expand Up @@ -221,37 +217,6 @@

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp3.version}</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin-stdlib.verion}</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
<version>${kotlin-stdlib-common.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId>
Expand Down