typeProtos = new ArrayList<>();
for (OpenFilesType type : types) {
- OpenFilesTypeProto typeProto = OpenFilesTypeProto.valueOf(type.getMode());
+ OpenFilesTypeProto typeProto = OpenFilesTypeProto
+ .forNumber(type.getMode());
if (typeProto != null) {
typeProtos.add(typeProto);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
index baad11352f8a2..31ab9daed3006 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
@@ -22,7 +22,7 @@
import org.apache.commons.codec.binary.Base64;
-import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.GeneratedMessageV3;
import com.google.protobuf.Message;
import com.google.protobuf.Message.Builder;
import com.google.protobuf.MessageOrBuilder;
@@ -31,7 +31,7 @@
* Helper class for setting/getting data elements in an object backed by a
* protobuf implementation.
*/
-public class FederationProtocolPBTranslator {
/** Optional proto byte stream used to create this object. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index 77e40b4389d8c..1f17ee2201ffd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -276,12 +276,12 @@ public void add(Replica replica) {
try {
// zig-zag to reduce size of legacy blocks
cos.writeSInt64NoTag(replica.getBlockId());
- cos.writeRawVarint64(replica.getBytesOnDisk());
- cos.writeRawVarint64(replica.getGenerationStamp());
+ cos.writeUInt64NoTag(replica.getBytesOnDisk());
+ cos.writeUInt64NoTag(replica.getGenerationStamp());
ReplicaState state = replica.getState();
// although state is not a 64-bit value, using a long varint to
// allow for future use of the upper bits
- cos.writeRawVarint64(state.getValue());
+ cos.writeUInt64NoTag(state.getValue());
if (state == ReplicaState.FINALIZED) {
numFinalized++;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 51379b8657392..5950fa61d85b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -1015,8 +1015,9 @@ private SectionName(String name) {
}
}
- private static int getOndiskTrunkSize(com.google.protobuf.GeneratedMessage s) {
- return CodedOutputStream.computeRawVarint32Size(s.getSerializedSize())
+ private static int getOndiskTrunkSize(
+ com.google.protobuf.GeneratedMessageV3 s) {
+ return CodedOutputStream.computeUInt32SizeNoTag(s.getSerializedSize())
+ s.getSerializedSize();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index cec44f51fe046..e84dec598aa6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -433,7 +433,7 @@ private void dumpXattrs(INodeSection.XAttrFeatureProto xattrs) {
int ns = (XATTR_NAMESPACE_MASK & (encodedName >> XATTR_NAMESPACE_OFFSET)) |
((XATTR_NAMESPACE_EXT_MASK & (encodedName >> XATTR_NAMESPACE_EXT_OFFSET)) << 2);
o(INODE_SECTION_NS, XAttrProtos.XAttrProto.
- XAttrNamespaceProto.valueOf(ns).toString());
+ XAttrNamespaceProto.forNumber(ns).toString());
o(SECTION_NAME, SerialNumberManager.XATTR.getString(
XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET),
stringTable));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 22f84c5395946..7b635382574b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -910,7 +910,7 @@ public void testFSServerDefaultsHelper() {
b.setFileBufferSize(DFSConfigKeys.IO_FILE_BUFFER_SIZE_DEFAULT);
b.setEncryptDataTransfer(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT);
b.setTrashInterval(DFSConfigKeys.FS_TRASH_INTERVAL_DEFAULT);
- b.setChecksumType(HdfsProtos.ChecksumTypeProto.valueOf(
+ b.setChecksumType(HdfsProtos.ChecksumTypeProto.forNumber(
DataChecksum.Type.valueOf(DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT).id));
HdfsProtos.FsServerDefaultsProto proto = b.build();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestLargeBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestLargeBlockReport.java
index f5a68925c3a6a..99dc783c86a35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestLargeBlockReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestLargeBlockReport.java
@@ -75,6 +75,9 @@ public void tearDown() {
@Test
public void testBlockReportExceedsLengthLimit() throws Exception {
+ //protobuf's default limit increased to 2GB from protobuf 3.x onwards.
+ //So there will not be any exception thrown from protobuf.
+ conf.setInt(IPC_MAXIMUM_DATA_LENGTH, IPC_MAXIMUM_DATA_LENGTH_DEFAULT / 2);
initCluster();
// Create a large enough report that we expect it will go beyond the RPC
// server's length validation, and also protobuf length validation.
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0aa84dcdb179d..9f85148fbf91b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -84,7 +84,7 @@
- 2.5.0
+ 3.7.1
${env.HADOOP_PROTOC_PATH}
3.4.13
@@ -1918,6 +1918,9 @@
false
+
+ /opt/protobuf-3.7/bin/protoc
+
diff --git a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
index d4a632cf06dcf..a499c8ac000b0 100644
--- a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
+++ b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
@@ -266,8 +266,8 @@ synchronized void writeDirEntry(DirEntry e) throws IOException {
e.writeDelimitedTo(dirs);
}
- private static int getOndiskSize(com.google.protobuf.GeneratedMessage s) {
- return CodedOutputStream.computeRawVarint32Size(s.getSerializedSize())
+ private static int getOndiskSize(com.google.protobuf.GeneratedMessageV3 s) {
+ return CodedOutputStream.computeUInt32SizeNoTag(s.getSerializedSize())
+ s.getSerializedSize();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 21ace7d4638c7..91b5d8d22c390 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -55,6 +55,7 @@
org.apache.hadoop
hadoop-annotations
+
com.google.protobuf
protobuf-java
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
index 30f774136dcfe..caf254a2b3ef4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
@@ -41,7 +41,7 @@
import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
-import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.GeneratedMessageV3;
/**
* {@code PlacementConstraintToProtoConverter} generates a
@@ -50,7 +50,7 @@
*/
@Private
public class PlacementConstraintToProtoConverter
- implements PlacementConstraint.Visitor {
+ implements PlacementConstraint.Visitor {
private PlacementConstraint placementConstraint;
@@ -65,7 +65,7 @@ public PlacementConstraintProto convert() {
}
@Override
- public GeneratedMessage visit(SingleConstraint constraint) {
+ public GeneratedMessageV3 visit(SingleConstraint constraint) {
SimplePlacementConstraintProto.Builder sb =
SimplePlacementConstraintProto.newBuilder();
@@ -94,7 +94,7 @@ public GeneratedMessage visit(SingleConstraint constraint) {
}
@Override
- public GeneratedMessage visit(TargetExpression target) {
+ public GeneratedMessageV3 visit(TargetExpression target) {
PlacementConstraintTargetProto.Builder tb =
PlacementConstraintTargetProto.newBuilder();
@@ -109,16 +109,16 @@ public GeneratedMessage visit(TargetExpression target) {
}
@Override
- public GeneratedMessage visit(TargetConstraint constraint) {
+ public GeneratedMessageV3 visit(TargetConstraint constraint) {
throw new YarnRuntimeException("Unexpected TargetConstraint found.");
}
@Override
- public GeneratedMessage visit(CardinalityConstraint constraint) {
+ public GeneratedMessageV3 visit(CardinalityConstraint constraint) {
throw new YarnRuntimeException("Unexpected CardinalityConstraint found.");
}
- private GeneratedMessage visitAndOr(
+ private GeneratedMessageV3 visitAndOr(
CompositeConstraint composite, CompositeType type) {
CompositePlacementConstraintProto.Builder cb =
CompositePlacementConstraintProto.newBuilder();
@@ -137,17 +137,17 @@ private GeneratedMessage visitAndOr(
}
@Override
- public GeneratedMessage visit(And constraint) {
+ public GeneratedMessageV3 visit(And constraint) {
return visitAndOr(constraint, CompositeType.AND);
}
@Override
- public GeneratedMessage visit(Or constraint) {
+ public GeneratedMessageV3 visit(Or constraint) {
return visitAndOr(constraint, CompositeType.OR);
}
@Override
- public GeneratedMessage visit(DelayedOr constraint) {
+ public GeneratedMessageV3 visit(DelayedOr constraint) {
CompositePlacementConstraintProto.Builder cb =
CompositePlacementConstraintProto.newBuilder();
@@ -166,7 +166,7 @@ public GeneratedMessage visit(DelayedOr constraint) {
}
@Override
- public GeneratedMessage visit(TimedPlacementConstraint constraint) {
+ public GeneratedMessageV3 visit(TimedPlacementConstraint constraint) {
TimedPlacementConstraintProto.Builder tb =
TimedPlacementConstraintProto.newBuilder();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
index c3590230c5399..356385630091f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
@@ -154,10 +154,10 @@ public void setVolumeCapability(
CsiAdaptorProtos.VolumeCapability vc =
CsiAdaptorProtos.VolumeCapability.newBuilder()
.setAccessMode(CsiAdaptorProtos.VolumeCapability
- .AccessMode.valueOf(
+ .AccessMode.forNumber(
capability.getAccessMode().ordinal()))
.setVolumeType(CsiAdaptorProtos.VolumeCapability
- .VolumeType.valueOf(capability.getVolumeType().ordinal()))
+ .VolumeType.forNumber(capability.getVolumeType().ordinal()))
.addAllMountFlags(capability.getMountFlags())
.build();
builder.setVolumeCapability(vc);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
index 14bd89dc46cd9..bf3f4f55b24f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
@@ -68,9 +68,9 @@ public void addVolumeCapability(VolumeCapability volumeCapability) {
CsiAdaptorProtos.VolumeCapability vc =
CsiAdaptorProtos.VolumeCapability.newBuilder()
.setAccessMode(CsiAdaptorProtos.VolumeCapability.AccessMode
- .valueOf(volumeCapability.getAccessMode().ordinal()))
+ .forNumber(volumeCapability.getAccessMode().ordinal()))
.setVolumeType(CsiAdaptorProtos.VolumeCapability.VolumeType
- .valueOf(volumeCapability.getVolumeType().ordinal()))
+ .forNumber(volumeCapability.getVolumeType().ordinal()))
.addAllMountFlags(volumeCapability.getMountFlags())
.build();
builder.addVolumeCapabilities(vc);