forked from apache/hadoop
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HADOOP-17705. S3A to add Config to set AWS region (apache#3020)
The option `fs.s3a.endpoint.region` can be used to explicitly set the AWS region of a bucket. This is needed when using AWS Private Link, as the region cannot be automatically determined. Contributed by Mehakmeet Singh
- Loading branch information
1 parent
a3b9c37
commit 452cd54
Showing
6 changed files
with
150 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.hadoop.fs.s3a; | ||
|
||
import com.amazonaws.ClientConfiguration; | ||
import com.amazonaws.client.builder.AwsClientBuilder; | ||
import com.amazonaws.util.AwsHostNameUtils; | ||
import org.assertj.core.api.Assertions; | ||
import org.junit.Test; | ||
|
||
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; | ||
|
||
/** | ||
* Test to check correctness of S3A endpoint regions in | ||
* {@link DefaultS3ClientFactory}. | ||
*/ | ||
public class ITestS3AEndpointRegion extends AbstractS3ATestBase { | ||
|
||
private static final String AWS_REGION_TEST = "test-region"; | ||
private static final String AWS_ENDPOINT_TEST = "test-endpoint"; | ||
private static final String AWS_ENDPOINT_TEST_WITH_REGION = | ||
"test-endpoint.some-region.amazonaws.com"; | ||
|
||
/** | ||
* Test to verify that setting a region with the config would bypass the | ||
* construction of region from endpoint. | ||
*/ | ||
@Test | ||
public void testWithRegionConfig() { | ||
getFileSystem().getConf().set(AWS_REGION, AWS_REGION_TEST); | ||
|
||
//Creating an endpoint config with a custom endpoint. | ||
AwsClientBuilder.EndpointConfiguration epr = createEpr(AWS_ENDPOINT_TEST, | ||
getFileSystem().getConf().getTrimmed(AWS_REGION)); | ||
//Checking if setting region config bypasses the endpoint region. | ||
Assertions.assertThat(epr.getSigningRegion()) | ||
.describedAs("There is a region mismatch") | ||
.isEqualTo(getFileSystem().getConf().get(AWS_REGION)); | ||
} | ||
|
||
/** | ||
* Test to verify that not setting the region config, would lead to using | ||
* endpoint to construct the region. | ||
*/ | ||
@Test | ||
public void testWithoutRegionConfig() { | ||
getFileSystem().getConf().unset(AWS_REGION); | ||
|
||
//Creating an endpoint config with a custom endpoint containing a region. | ||
AwsClientBuilder.EndpointConfiguration eprRandom = | ||
createEpr(AWS_ENDPOINT_TEST_WITH_REGION, | ||
getFileSystem().getConf().getTrimmed(AWS_REGION)); | ||
String regionFromEndpoint = | ||
AwsHostNameUtils | ||
.parseRegionFromAwsPartitionPattern(AWS_ENDPOINT_TEST_WITH_REGION); | ||
//Checking if not setting region config leads to constructing the region | ||
// from endpoint. | ||
Assertions.assertThat(eprRandom.getSigningRegion()) | ||
.describedAs("There is a region mismatch") | ||
.isNotEqualTo(getFileSystem().getConf().get(AWS_REGION)) | ||
.isEqualTo(regionFromEndpoint); | ||
} | ||
|
||
/** | ||
* Method to create EndpointConfiguration using an endpoint. | ||
* | ||
* @param endpoint the endpoint to be used for EndpointConfiguration creation. | ||
* @return an instance of EndpointConfiguration. | ||
*/ | ||
private AwsClientBuilder.EndpointConfiguration createEpr(String endpoint, | ||
String awsRegion) { | ||
return DefaultS3ClientFactory.createEndpointConfiguration(endpoint, | ||
new ClientConfiguration(), awsRegion); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters