Skip to content

HDFS-17641. Change badly distributed blocks metric protobuf requirement to optional #7274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 15, 2025
Merged

HDFS-17641. Change badly distributed blocks metric protobuf requirement to optional #7274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
417539c
change protobuf requirement to optional
Jan 7, 2025
7a278e9
modify badly distributed blocks to safely be optional
Apr 4, 2025
2ff5149
checkstyle
Apr 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions ...t/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,21 @@ public final class ECBlockGroupStats {
private final long missingBlockGroups;
private final long bytesInFutureBlockGroups;
private final long pendingDeletionBlocks;
private final long badlyDistributedBlocks;
private final Long badlyDistributedBlocks;
private final Long highestPriorityLowRedundancyBlocks;

public ECBlockGroupStats(long lowRedundancyBlockGroups,
long corruptBlockGroups, long missingBlockGroups,
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
long badlyDistributedBlocks) {
long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
bytesInFutureBlockGroups, pendingDeletionBlocks,
badlyDistributedBlocks, null);
null, null);
}

public ECBlockGroupStats(long lowRedundancyBlockGroups,
long corruptBlockGroups, long missingBlockGroups,
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
Long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
this.corruptBlockGroups = corruptBlockGroups;
this.missingBlockGroups = missingBlockGroups;
Expand Down Expand Up @@ -84,7 +83,11 @@ public long getPendingDeletionBlocks() {
return pendingDeletionBlocks;
}

public long getBadlyDistributedBlocks() {
public boolean hasBadlyDistributedBlocks() {
return getBadlyDistributedBlocks() != null;
}

public Long getBadlyDistributedBlocks() {
return badlyDistributedBlocks;
}

Expand All @@ -107,8 +110,11 @@ public String toString() {
.append(", BytesInFutureBlockGroups=").append(
getBytesInFutureBlockGroups())
.append(", PendingDeletionBlocks=").append(
getPendingDeletionBlocks())
.append(" , BadlyDistributedBlocks=").append(getBadlyDistributedBlocks());
getPendingDeletionBlocks());
if(hasBadlyDistributedBlocks()) {
statsBuilder.append(", BadlyDistributedBlocks=")
.append(getBadlyDistributedBlocks());
}
if (hasHighestPriorityLowRedundancyBlocks()) {
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
.append(getHighestPriorityLowRedundancyBlocks());
Expand Down Expand Up @@ -163,6 +169,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
long bytesInFutureBlockGroups = 0;
long pendingDeletionBlocks = 0;
long badlyDistributedBlocks = 0;
boolean hasBadlyDistributedBlocks = false;
long highestPriorityLowRedundancyBlocks = 0;
boolean hasHighestPriorityLowRedundancyBlocks = false;

Expand All @@ -172,20 +179,22 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
missingBlockGroups += stat.getMissingBlockGroups();
bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
if (stat.hasBadlyDistributedBlocks()) {
hasBadlyDistributedBlocks = true;
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
}
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
hasHighestPriorityLowRedundancyBlocks = true;
highestPriorityLowRedundancyBlocks +=
stat.getHighestPriorityLowRedundancyBlocks();
}
}
if (hasHighestPriorityLowRedundancyBlocks) {
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
}
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
badlyDistributedBlocks);
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
}
}
31 changes: 21 additions & 10 deletions ...adoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,22 @@ public final class ReplicatedBlockStats {
private final long missingReplicationOneBlocks;
private final long bytesInFutureBlocks;
private final long pendingDeletionBlocks;
private final long badlyDistributedBlocks;
private final Long badlyDistributedBlocks;
private final Long highestPriorityLowRedundancyBlocks;

public ReplicatedBlockStats(long lowRedundancyBlocks,
long corruptBlocks, long missingBlocks,
long missingReplicationOneBlocks, long bytesInFutureBlocks,
long pendingDeletionBlocks, long badlyDistributedBlocks) {
long pendingDeletionBlocks) {
this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
badlyDistributedBlocks, null);
null, null);
}

public ReplicatedBlockStats(long lowRedundancyBlocks,
long corruptBlocks, long missingBlocks,
long missingReplicationOneBlocks, long bytesInFutureBlocks,
long pendingDeletionBlocks, long badlyDistributedBlocks,
long pendingDeletionBlocks, Long badlyDistributedBlocks,
Long highestPriorityLowRedundancyBlocks) {
this.lowRedundancyBlocks = lowRedundancyBlocks;
this.corruptBlocks = corruptBlocks;
Expand Down Expand Up @@ -89,7 +89,11 @@ public long getPendingDeletionBlocks() {
return pendingDeletionBlocks;
}

public long getBadlyDistributedBlocks() {
public boolean hasBadlyDistributedBlocks() {
return getBadlyDistributedBlocks() != null;
}

public Long getBadlyDistributedBlocks() {
return badlyDistributedBlocks;
}

Expand All @@ -113,8 +117,10 @@ public String toString() {
getMissingReplicationOneBlocks())
.append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
.append(", PendingDeletionBlocks=").append(
getPendingDeletionBlocks())
.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
getPendingDeletionBlocks());
if (hasBadlyDistributedBlocks()) {
statsBuilder.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
}
if (hasHighestPriorityLowRedundancyBlocks()) {
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
getHighestPriorityLowRedundancyBlocks());
Expand All @@ -137,6 +143,7 @@ public static ReplicatedBlockStats merge(
long bytesInFutureBlocks = 0;
long pendingDeletionBlocks = 0;
long badlyDistributedBlocks = 0;
boolean hasBadlyDistributedBlocks = false;
long highestPriorityLowRedundancyBlocks = 0;
boolean hasHighestPriorityLowRedundancyBlocks = false;

Expand All @@ -148,20 +155,24 @@ public static ReplicatedBlockStats merge(
missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
bytesInFutureBlocks += stat.getBytesInFutureBlocks();
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();

if (stat.hasBadlyDistributedBlocks()) {
hasBadlyDistributedBlocks = true;
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
}
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
hasHighestPriorityLowRedundancyBlocks = true;
highestPriorityLowRedundancyBlocks +=
stat.getHighestPriorityLowRedundancyBlocks();
}
}
if (hasHighestPriorityLowRedundancyBlocks) {
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
}
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
pendingDeletionBlocks, badlyDistributedBlocks);
pendingDeletionBlocks);
}
}
21 changes: 12 additions & 9 deletions ...ct/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2033,7 +2033,7 @@ public static long[] convert(GetFsStatsResponseProto res) {

public static ReplicatedBlockStats convert(
GetFsReplicatedBlockStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
Expand All @@ -2043,21 +2043,20 @@ public static ReplicatedBlockStats convert(
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
res.getPendingDeletionBlocks());
}

public static ECBlockGroupStats convert(
GetFsECBlockGroupStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
res.getBadlyDistributedBlocks(), res.getHighestPrioLowRedundancyBlocks());
}
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
res.getBadlyDistributedBlocks());
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
}

public static DatanodeReportTypeProto convert(DatanodeReportType t) {
Expand Down Expand Up @@ -2526,8 +2525,10 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
replicatedBlockStats.getBytesInFutureBlocks());
result.setPendingDeletionBlocks(
replicatedBlockStats.getPendingDeletionBlocks());
result.setBadlyDistributedBlocks(
replicatedBlockStats.getBadlyDistributedBlocks());
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
result.setBadlyDistributedBlocks(
replicatedBlockStats.getBadlyDistributedBlocks());
}
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
Expand All @@ -2547,8 +2548,10 @@ public static GetFsECBlockGroupStatsResponseProto convert(
ecBlockGroupStats.getBytesInFutureBlockGroups());
result.setPendingDeletionBlocks(
ecBlockGroupStats.getPendingDeletionBlocks());
result.setBadlyDistributedBlocks(
ecBlockGroupStats.getBadlyDistributedBlocks());
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
result.setBadlyDistributedBlocks(
ecBlockGroupStats.getBadlyDistributedBlocks());
}
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
Expand Down
5 changes: 2 additions & 3 deletions hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,7 @@ message GetFsReplicatedBlockStatsResponseProto {
required uint64 blocks_in_future = 5;
required uint64 pending_deletion_blocks = 6;
optional uint64 highest_prio_low_redundancy_blocks = 7;
required uint64 badly_distributed_blocks = 8;

optional uint64 badly_distributed_blocks = 8;
}

message GetFsECBlockGroupStatsRequestProto { // no input paramters
Expand All @@ -386,7 +385,7 @@ message GetFsECBlockGroupStatsResponseProto {
required uint64 blocks_in_future = 4;
required uint64 pending_deletion_blocks = 5;
optional uint64 highest_prio_low_redundancy_blocks = 6;
required uint64 badly_distributed_blocks = 7;
optional uint64 badly_distributed_blocks = 7;
}

enum DatanodeReportTypeProto { // type of the datanode report
Expand Down
12 changes: 8 additions & 4 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,8 +554,10 @@ public void report(String[] argv, int i) throws IOException {
replicatedBlockStats.getMissingReplicaBlocks());
System.out.println("\tMissing blocks (with replication factor 1): " +
replicatedBlockStats.getMissingReplicationOneBlocks());
System.out.println("\tBadly Distributed Blocks: " +
replicatedBlockStats.getBadlyDistributedBlocks());
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
System.out.println("\tBadly Distributed Blocks: " +
replicatedBlockStats.getBadlyDistributedBlocks());
}
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
System.out.println("\tLow redundancy blocks with highest priority " +
"to recover: " +
Expand All @@ -573,8 +575,10 @@ public void report(String[] argv, int i) throws IOException {
ecBlockGroupStats.getCorruptBlockGroups());
System.out.println("\tMissing block groups: " +
ecBlockGroupStats.getMissingBlockGroups());
System.out.println("\tBadly Distributed Blocks: " +
ecBlockGroupStats.getBadlyDistributedBlocks());
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
System.out.println("\tBadly Distributed Blocks: " +
ecBlockGroupStats.getBadlyDistributedBlocks());
}
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
System.out.println("\tLow redundancy blocks with highest priority " +
"to recover: " +
Expand Down