Skip to content

Commit 419d572

Browse files
arunkumarchackoarunkumarchacko
committed
HADOOP-19343. Add additional authentication support
Add more tests. Add basic IOStatistics
1 parent 8fa7d83 commit 419d572

21 files changed

+936
-176
lines changed

‎hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,9 @@
2626
<Method name="createItemInfoForBlob" />
2727
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
2828
</Match>
29+
<Match>
30+
<Class name="org.apache.hadoop.fs.gs.GoogleCloudStorageExceptions" />
31+
<Method name="createCompositeException" />
32+
<Bug pattern="NP_NULL_ON_SOME_PATH" />
33+
</Match>
2934
</FindBugsFilter>

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GcsInstrumentation.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.gs;
20+
21+
import java.io.Closeable;
22+
import java.io.IOException;
23+
import java.util.EnumSet;
24+
25+
import org.apache.hadoop.fs.statistics.IOStatisticsSource;
26+
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
27+
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStoreBuilder;
28+
29+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
30+
31+
class GcsInstrumentation implements Closeable, IOStatisticsSource {
32+
private final IOStatisticsStore instanceIOStatistics;
33+
34+
GcsInstrumentation() {
35+
IOStatisticsStoreBuilder storeBuilder = iostatisticsStore();
36+
37+
// declare all counter statistics
38+
EnumSet.allOf(GcsStatistics.class).stream()
39+
.filter(statistic ->
40+
statistic.getType() == StatisticTypeEnum.TYPE_COUNTER)
41+
.forEach(stat -> {
42+
storeBuilder.withCounters(stat.getSymbol());
43+
});
44+
45+
EnumSet.allOf(GcsStatistics.class).stream()
46+
.filter(statistic ->
47+
statistic.getType() == StatisticTypeEnum.TYPE_DURATION)
48+
.forEach(stat -> {
49+
storeBuilder.withDurationTracking(stat.getSymbol());
50+
});
51+
52+
this.instanceIOStatistics = storeBuilder.build();
53+
}
54+
55+
@Override
56+
public void close() throws IOException {
57+
}
58+
59+
@Override
60+
public IOStatisticsStore getIOStatistics() {
61+
return instanceIOStatistics;
62+
}
63+
}

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GcsListOperation.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
import com.google.cloud.storage.Blob;
2525
import com.google.cloud.storage.Storage;
2626

27-
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
28-
2927
final class GcsListOperation {
3028
private static final int ALL = 0;
3129
private final Storage.BlobListOption[] listOptions;
@@ -72,7 +70,11 @@ Builder forRecursiveListing() {
7270
}
7371

7472
GcsListOperation build() {
75-
blobListOptions.add(Storage.BlobListOption.prefix(prefix));
73+
// Can be null while listing the root directory.
74+
if (prefix != null) {
75+
blobListOptions.add(Storage.BlobListOption.prefix(prefix));
76+
}
77+
7678
return new GcsListOperation(this);
7779
}
7880

@@ -82,13 +84,11 @@ Builder forCurrentDirectoryListing() {
8284
return this;
8385
}
8486

85-
Builder forCurrentDirectoryListingWithLimit(int theLimit) {
86-
checkArgument(
87-
theLimit > 0,
88-
"limit should be greater than 0. found %d; prefix=%s", theLimit, prefix);
89-
90-
this.limit = theLimit;
91-
prefix = StringPaths.toDirectoryPath(prefix);
87+
Builder forImplicitDirectoryCheck() {
88+
this.limit = 1;
89+
if (prefix != null) {
90+
prefix = StringPaths.toDirectoryPath(prefix);
91+
}
9292

9393
blobListOptions.add(Storage.BlobListOption.pageSize(1));
9494
forCurrentDirectoryListing();

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GcsStatistics.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.gs;
20+
21+
import org.apache.hadoop.fs.statistics.StoreStatisticNames;
22+
23+
import static org.apache.hadoop.fs.gs.StatisticTypeEnum.TYPE_DURATION;
24+
25+
enum GcsStatistics {
26+
INVOCATION_GET_FILE_STATUS(
27+
StoreStatisticNames.OP_GET_FILE_STATUS,
28+
"Calls of getFileStatus()",
29+
TYPE_DURATION),
30+
INVOCATION_CREATE(
31+
StoreStatisticNames.OP_CREATE,
32+
"Calls of create()",
33+
TYPE_DURATION),
34+
INVOCATION_DELETE(
35+
StoreStatisticNames.OP_DELETE,
36+
"Calls of delete()",
37+
TYPE_DURATION),
38+
INVOCATION_RENAME(
39+
StoreStatisticNames.OP_RENAME,
40+
"Calls of rename()",
41+
TYPE_DURATION),
42+
INVOCATION_OPEN(
43+
StoreStatisticNames.OP_OPEN,
44+
"Calls of open()",
45+
TYPE_DURATION),
46+
INVOCATION_MKDIRS(
47+
StoreStatisticNames.OP_MKDIRS,
48+
"Calls of mkdirs()",
49+
TYPE_DURATION),
50+
INVOCATION_LIST_STATUS(
51+
StoreStatisticNames.OP_LIST_STATUS,
52+
"Calls of listStatus()",
53+
TYPE_DURATION);
54+
55+
private final String description;
56+
private StatisticTypeEnum type;
57+
private String symbol;
58+
59+
StatisticTypeEnum getType() {
60+
return this.type;
61+
}
62+
63+
String getSymbol() {
64+
return this.symbol;
65+
}
66+
67+
GcsStatistics(String symbol, String description, StatisticTypeEnum type) {
68+
this.symbol = symbol;
69+
this.description = description;
70+
this.type = type;
71+
}
72+
}

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GcsStorageStatistics.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.gs;
20+
21+
import java.util.Iterator;
22+
23+
import org.apache.hadoop.classification.InterfaceAudience;
24+
import org.apache.hadoop.classification.InterfaceStability;
25+
import org.apache.hadoop.fs.statistics.IOStatistics;
26+
import org.apache.hadoop.fs.statistics.impl.StorageStatisticsFromIOStatistics;
27+
28+
@InterfaceAudience.Private
29+
@InterfaceStability.Evolving
30+
class GcsStorageStatistics
31+
extends StorageStatisticsFromIOStatistics {
32+
static final String NAME = "GhfsStorageStatistics";
33+
34+
GcsStorageStatistics(final IOStatistics ioStatistics) {
35+
super(NAME, Constants.SCHEME, ioStatistics);
36+
}
37+
38+
@Override
39+
public String toString() {
40+
StringBuilder sb = new StringBuilder();
41+
for (Iterator<LongStatistic> it = this.getLongStatistics(); it.hasNext();) {
42+
LongStatistic statistic = it.next();
43+
44+
if (sb.length() != 0) {
45+
sb.append(", ");
46+
}
47+
sb.append(String.format("%s=%s", statistic.getName(), statistic.getValue()));
48+
}
49+
50+
return String.format("[%s]", sb);
51+
}
52+
}

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.hadoop.fs.gs;
2020

21+
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
2122
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
2223
import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
2324
import static java.lang.Math.toIntExact;
@@ -27,7 +28,9 @@
2728
import com.google.api.client.util.ExponentialBackOff;
2829
import com.google.api.client.util.Sleeper;
2930
import com.google.api.gax.paging.Page;
31+
import com.google.auth.Credentials;
3032
import com.google.cloud.storage.*;
33+
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
3134
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
3235
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
3336
import org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding;
@@ -55,7 +58,7 @@
5558
* client</a>.
5659
*/
5760
class GoogleCloudStorage {
58-
static final Logger LOG = LoggerFactory.getLogger(GoogleHadoopFileSystem.class);
61+
static final Logger LOG = LoggerFactory.getLogger(GoogleCloudStorage.class);
5962
static final List<Storage.BlobField> BLOB_FIELDS =
6063
ImmutableList.of(
6164
Storage.BlobField.BUCKET, Storage.BlobField.CONTENT_ENCODING,
@@ -76,18 +79,19 @@ class GoogleCloudStorage {
7679
* Having an instance of gscImpl to redirect calls to Json client while new client implementation
7780
* is in WIP.
7881
*/
79-
GoogleCloudStorage(GoogleHadoopFileSystemConfiguration configuration) throws IOException {
80-
// TODO: Set credentials
81-
this.storage = createStorage(configuration.getProjectId());
82+
GoogleCloudStorage(GoogleHadoopFileSystemConfiguration configuration, Credentials credentials)
83+
throws IOException {
84+
this.storage = createStorage(configuration.getProjectId(), credentials);
8285
this.configuration = configuration;
8386
}
8487

85-
private static Storage createStorage(String projectId) {
88+
private static Storage createStorage(String projectId, Credentials credentials) {
89+
StorageOptions.Builder builder = StorageOptions.newBuilder();
8690
if (projectId != null) {
87-
return StorageOptions.newBuilder().setProjectId(projectId).build().getService();
91+
builder.setProjectId(projectId);
8892
}
8993

90-
return StorageOptions.newBuilder().build().getService();
94+
return builder.setCredentials(credentials).build().getService();
9195
}
9296

9397
WritableByteChannel create(final StorageResourceId resourceId, final CreateFileOptions options)
@@ -494,7 +498,9 @@ List<GoogleCloudStorageItemInfo> listDirectoryRecursive(String bucketName, Strin
494498
// TODO: Take delimiter from config
495499
// TODO: Set specific fields
496500

497-
checkArgument(objectName.endsWith("/"), String.format("%s should end with /", objectName));
501+
checkArgument(
502+
objectName == null || objectName.endsWith("/"),
503+
String.format("%s should end with /", objectName));
498504
try {
499505
List<Blob> blobs = new GcsListOperation.Builder(bucketName, objectName, storage)
500506
.forRecursiveListing().build()
@@ -887,7 +893,7 @@ List<GoogleCloudStorageItemInfo> getItemInfos(List<StorageResourceId> resourceId
887893
List<GoogleCloudStorageItemInfo> listDirectory(String bucketName, String objectNamePrefix)
888894
throws IOException {
889895
checkArgument(
890-
objectNamePrefix.endsWith("/"),
896+
objectNamePrefix == null || objectNamePrefix.endsWith("/"),
891897
String.format("%s should end with /", objectNamePrefix));
892898

893899
try {
@@ -971,7 +977,7 @@ GoogleCloudStorageItemInfo getFileOrDirectoryInfo(StorageResourceId resourceId)
971977
GoogleCloudStorageItemInfo getImplicitDirectory(StorageResourceId resourceId) {
972978
List<Blob> blobs = new GcsListOperation
973979
.Builder(resourceId.getBucketName(), resourceId.getObjectName(), storage)
974-
.forCurrentDirectoryListingWithLimit(1).build()
980+
.forImplicitDirectoryCheck().build()
975981
.execute();
976982

977983
if (blobs.isEmpty()) {
@@ -981,6 +987,34 @@ GoogleCloudStorageItemInfo getImplicitDirectory(StorageResourceId resourceId) {
981987
return GoogleCloudStorageItemInfo.createInferredDirectory(resourceId.toDirectoryId());
982988
}
983989

990+
public void deleteBuckets(List<String> bucketNames) throws IOException {
991+
LOG.trace("deleteBuckets({})", bucketNames);
992+
993+
// Validate all the inputs first.
994+
for (String bucketName : bucketNames) {
995+
checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty");
996+
}
997+
998+
// Gather exceptions to wrap in a composite exception at the end.
999+
List<IOException> innerExceptions = new ArrayList<>();
1000+
1001+
for (String bucketName : bucketNames) {
1002+
try {
1003+
boolean isDeleted = storage.delete(bucketName);
1004+
if (!isDeleted) {
1005+
innerExceptions.add(createFileNotFoundException(bucketName, null, null));
1006+
}
1007+
} catch (StorageException e) {
1008+
innerExceptions.add(
1009+
new IOException(String.format("Error deleting '%s' bucket", bucketName), e));
1010+
}
1011+
}
1012+
1013+
if (!innerExceptions.isEmpty()) {
1014+
throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions);
1015+
}
1016+
}
1017+
9841018
// Helper class to capture the results of list operation.
9851019
private class ListOperationResult {
9861020
private final Map<String, Blob> prefixes = new HashMap<>();

‎hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageExceptions.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,16 @@
1818

1919
package org.apache.hadoop.fs.gs;
2020

21+
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
22+
2123
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
2224
import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
2325
import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.nullToEmpty;
2426

2527
import java.io.FileNotFoundException;
2628
import java.io.IOException;
29+
import java.util.Collection;
30+
import java.util.Iterator;
2731
import javax.annotation.Nullable;
2832

2933
/**
@@ -55,4 +59,23 @@ static FileNotFoundException createFileNotFoundException(
5559
return createFileNotFoundException(
5660
resourceId.getBucketName(), resourceId.getObjectName(), cause);
5761
}
62+
63+
public static IOException createCompositeException(Collection<IOException> innerExceptions) {
64+
Preconditions.checkArgument(
65+
innerExceptions != null && !innerExceptions.isEmpty(),
66+
"innerExceptions (%s) must be not null and contain at least one element",
67+
innerExceptions);
68+
69+
Iterator<IOException> innerExceptionIterator = innerExceptions.iterator();
70+
71+
if (innerExceptions.size() == 1) {
72+
return innerExceptionIterator.next();
73+
}
74+
75+
IOException combined = new IOException("Multiple IOExceptions.");
76+
while (innerExceptionIterator.hasNext()) {
77+
combined.addSuppressed(innerExceptionIterator.next());
78+
}
79+
return combined;
80+
}
5881
}

0 commit comments

Comments
 (0)