Skip to content

HDFS-17063. Support to configure different capacity reserved for each disk of DataNode. #5793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 17, 2023
Merged

HDFS-17063. Support to configure different capacity reserved for each disk of DataNode. #5793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3f16169
HDFS-17063. Datanode configures different Capacity Reserved for each …
qijiale76 Jun 29, 2023
b5395d7
add doc
qijiale76 Jun 29, 2023
390e798
fix blanks checkstyle spotbugs
qijiale76 Jun 30, 2023
fb2f7ca
Add UT && Add space
qijiale76 Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...dfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ public class FsVolumeImpl implements FsVolumeSpi {
this.usage = usage;
if (this.usage != null) {
reserved = new ReservedSpaceCalculator.Builder(conf)
.setUsage(this.usage).setStorageType(storageType).build();
.setUsage(this.usage).setStorageType(storageType)
.setDir(currentDir != null ? currentDir.getParent() : "NULL").build();
boolean fixedSizeVolume = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_FIXED_VOLUME_SIZE_KEY,
DFSConfigKeys.DFS_DATANODE_FIXED_VOLUME_SIZE_DEFAULT);
Expand Down
43 changes: 30 additions & 13 deletions ...n/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public static class Builder {
private DF usage;
private StorageType storageType;

private String dir;

public Builder(Configuration conf) {
this.conf = conf;
}
Expand All @@ -61,6 +63,11 @@ public Builder setStorageType(
return this;
}

public Builder setDir(String newDir) {
this.dir = newDir;
return this;
}

ReservedSpaceCalculator build() {
try {
Class<? extends ReservedSpaceCalculator> clazz = conf.getClass(
Expand All @@ -69,10 +76,10 @@ ReservedSpaceCalculator build() {
ReservedSpaceCalculator.class);

Constructor constructor = clazz.getConstructor(
Configuration.class, DF.class, StorageType.class);
Configuration.class, DF.class, StorageType.class, String.class);

return (ReservedSpaceCalculator) constructor.newInstance(
conf, usage, storageType);
conf, usage, storageType, dir);
} catch (Exception e) {
throw new IllegalStateException(
"Error instantiating ReservedSpaceCalculator", e);
Expand All @@ -84,20 +91,30 @@ ReservedSpaceCalculator build() {
private final Configuration conf;
private final StorageType storageType;

private final String dir;

ReservedSpaceCalculator(Configuration conf, DF usage,
StorageType storageType) {
StorageType storageType, String dir) {
this.usage = usage;
this.conf = conf;
this.storageType = storageType;
this.dir = dir;
}

DF getUsage() {
return usage;
}

String getDir() {
return dir;
}

long getReservedFromConf(String key, long defaultValue) {
return conf.getLong(key + "." + StringUtils.toLowerCase(
storageType.toString()), conf.getLongBytes(key, defaultValue));
return conf.getLong(
key + "." + getDir() + "." + StringUtils.toLowerCase(storageType.toString()),
conf.getLong(key + "." + getDir(),
conf.getLong(key + "." + StringUtils.toLowerCase(storageType.toString()),
conf.getLongBytes(key, defaultValue))));
}

/**
Expand All @@ -117,8 +134,8 @@ public static class ReservedSpaceCalculatorAbsolute extends
private final long reservedBytes;

public ReservedSpaceCalculatorAbsolute(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
}
Expand All @@ -138,8 +155,8 @@ public static class ReservedSpaceCalculatorPercentage extends
private final long reservedPct;

public ReservedSpaceCalculatorPercentage(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedPct = getReservedFromConf(
DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY,
DFS_DATANODE_DU_RESERVED_PERCENTAGE_DEFAULT);
Expand All @@ -162,8 +179,8 @@ public static class ReservedSpaceCalculatorConservative extends
private final long reservedPct;

public ReservedSpaceCalculatorConservative(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
this.reservedPct = getReservedFromConf(
Expand Down Expand Up @@ -197,8 +214,8 @@ public static class ReservedSpaceCalculatorAggressive extends
private final long reservedPct;

public ReservedSpaceCalculatorAggressive(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
this.reservedPct = getReservedFromConf(
Expand Down
20 changes: 17 additions & 3 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,12 +397,19 @@
<name>dfs.datanode.du.reserved</name>
<value>0</value>
<description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
Specific directory based reservation is supported. The property can be followed with directory
name which is set at 'dfs.datanode.data.dir'. For example, reserved space for /data/hdfs1/data
can be configured using property 'dfs.datanode.du.reserved./data/hdfs1/data'. If specific directory
reservation is not configured then dfs.datanode.du.reserved will be used.
Specific storage type based reservation is also supported. The property can be followed with
corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage.
For example, reserved space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved will be used. Support multiple size unit suffix(case insensitive),
as described in dfs.blocksize.
as described in dfs.blocksize. Use directory name and storage type based reservation at the
same time is also allowed if both are configured.
Property priority example: dfs.datanode.du.reserved./data/hdfs1/data.ram_disk >
dfs.datanode.du.reserved./data/hdfs1/data > dfs.datanode.du.reserved.ram_disk > dfs.datanode.du.reserved
Note: In case of using tune2fs to set reserved-blocks-percentage, or other filesystem tools,
then you can possibly run into out of disk errors because hadoop will not check those
external tool configurations.
Expand All @@ -414,12 +421,19 @@
<value>0</value>
<description>Reserved space in percentage. Read dfs.datanode.du.reserved.calculator to see
when this takes effect. The actual number of bytes reserved will be calculated by using the
total capacity of the data directory in question. Specific storage type based reservation
total capacity of the data directory in question. Specific directory based reservation is
supported. The property can be followed with directory name which is set at 'dfs.datanode.data.dir'.
For example, reserved percentage space for /data/hdfs1/data can be configured using property
'dfs.datanode.du.reserved.pct./data/hdfs1/data'. If specific directory reservation is not
configured then dfs.datanode.du.reserved.pct will be used. Specific storage type based reservation
is also supported. The property can be followed with corresponding storage types
([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage.
For example, reserved percentage space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.pct.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved.pct will be used.
then dfs.datanode.du.reserved.pct will be used. Use directory and storage type based reservation
is also allowed if both are configured.
Priority example: dfs.datanode.du.reserved.pct./data/hdfs1/data.ram_disk > dfs.datanode.du.reserved.pct./data/hdfs1/data
> dfs.datanode.du.reserved.pct.ram_disk > dfs.datanode.du.reserved.pct
</description>
</property>

Expand Down
56 changes: 55 additions & 1 deletion ...va/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReservedSpaceCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,55 @@ public void testReservedSpaceAggresivePerStorageType() {
checkReserved(StorageType.ARCHIVE, 100000, 5000);
}

@Test
public void testReservedSpaceAbsolutePerDir() {
conf.setClass(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY, ReservedSpaceCalculatorAbsolute.class,
ReservedSpaceCalculator.class);

String dir1 = "/data/hdfs1/data";
String dir2 = "/data/hdfs2/data";
String dir3 = "/data/hdfs3/data";

conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir1 + ".ssd", 900);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir1, 1800);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir2, 2700);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + ".ssd", 3600);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY, 4500);

checkReserved(StorageType.SSD, 10000, 900, dir1);
checkReserved(StorageType.DISK, 10000, 1800, dir1);
checkReserved(StorageType.SSD, 10000, 2700, dir2);
checkReserved(StorageType.DISK, 10000, 2700, dir2);
checkReserved(StorageType.SSD, 10000, 3600, dir3);
checkReserved(StorageType.DISK, 10000, 4500, dir3);
}

@Test
public void testReservedSpacePercentagePerDir() {
conf.setClass(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY,
ReservedSpaceCalculatorPercentage.class,
ReservedSpaceCalculator.class);

String dir1 = "/data/hdfs1/data";
String dir2 = "/data/hdfs2/data";
String dir3 = "/data/hdfs3/data";

// Set percentage reserved values for different directories
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir1 + ".ssd", 20);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir1, 10);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir2, 25);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + ".ssd", 30);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY, 40);

// Verify reserved space calculations for different directories and storage types
checkReserved(StorageType.SSD, 10000, 2000, dir1);
checkReserved(StorageType.DISK, 10000, 1000, dir1);
checkReserved(StorageType.SSD, 10000, 2500, dir2);
checkReserved(StorageType.DISK, 10000, 2500, dir2);
checkReserved(StorageType.SSD, 10000, 3000, dir3);
checkReserved(StorageType.DISK, 10000, 4000, dir3);
}

@Test(expected = IllegalStateException.class)
public void testInvalidCalculator() {
conf.set(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY, "INVALIDTYPE");
Expand All @@ -179,10 +228,15 @@ public void testInvalidCalculator() {

private void checkReserved(StorageType storageType,
long totalCapacity, long reservedExpected) {
checkReserved(storageType, totalCapacity, reservedExpected, "NULL");
}

private void checkReserved(StorageType storageType,
long totalCapacity, long reservedExpected, String dir) {
when(usage.getCapacity()).thenReturn(totalCapacity);

reserved = new ReservedSpaceCalculator.Builder(conf).setUsage(usage)
.setStorageType(storageType).build();
.setStorageType(storageType).setDir(dir).build();
assertEquals(reservedExpected, reserved.getReserved());
}
}