Skip to content

Commit be404e1

Browse files
jian-hejian-he
committed
YARN-4405. Support node label store in non-appendable file system. Contributed by Wangda Tan
(cherry picked from commit 755dda8)
1 parent 33d4588 commit be404e1

File tree

11 files changed

+106
-61
lines changed

11 files changed

+106
-61
lines changed

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,22 @@
1818

1919
package org.apache.hadoop.conf;
2020

21-
import java.lang.Class;
21+
import org.junit.Before;
22+
import org.junit.Ignore;
23+
import org.junit.Test;
24+
2225
import java.lang.reflect.Field;
2326
import java.lang.reflect.Modifier;
24-
import java.util.ArrayList;
2527
import java.util.HashMap;
2628
import java.util.HashSet;
2729
import java.util.Iterator;
28-
import java.util.List;
2930
import java.util.Map;
30-
import java.util.Map.Entry;
3131
import java.util.Set;
3232
import java.util.regex.Matcher;
3333
import java.util.regex.Pattern;
3434

35-
import org.junit.Before;
36-
import org.junit.Ignore;
37-
import org.junit.Test;
3835
import static org.junit.Assert.assertTrue;
3936

40-
import org.apache.hadoop.conf.Configuration;
41-
4237
/**
4338
* Base class for comparing fields in one or more Configuration classes
4439
* against a corresponding .xml file. Usage is intended as follows:
@@ -331,6 +326,7 @@ public abstract class TestConfigurationFieldsBase {
331326
private static Set<String> compareConfigurationToXmlFields(Map<String,String> keyMap1, Map<String,String> keyMap2) {
332327
Set<String> retVal = new HashSet<String>(keyMap1.keySet());
333328
retVal.removeAll(keyMap2.keySet());
329+
334330
return retVal;
335331
}
336332

‎hadoop-yarn-project/CHANGES.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,9 @@ Release 2.8.0 - UNRELEASED
537537
YARN-4292. ResourceUtilization should be a part of NodeInfo REST API.
538538
(Sunil G via wangda)
539539

540+
YARN-4405. Support node label store in non-appendable file system. (Wangda
541+
Tan via jianhe)
542+
540543
OPTIMIZATIONS
541544

542545
YARN-3339. TestDockerContainerExecutor should pull a single image and not

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2072,6 +2072,12 @@ private static void addDeprecatedKeys() {
20722072
*/
20732073
public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
20742074

2075+
/** Node label store implementation class */
2076+
public static final String FS_NODE_LABELS_STORE_IMPL_CLASS = NODE_LABELS_PREFIX
2077+
+ "fs-store.impl.class";
2078+
public static final String DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS =
2079+
"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore";
2080+
20752081
/** URI for NodeLabelManager */
20762082
public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
20772083
+ "fs-store.root-dir";

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ public void initializeMemberVariables() {
4848
errorIfMissingXmlProps = true;
4949

5050
// Specific properties to skip
51+
configurationPropsToSkipCompare
52+
.add(YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS);
5153
configurationPropsToSkipCompare
5254
.add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
5355
configurationPropsToSkipCompare

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.hadoop.classification.InterfaceAudience.Private;
4343
import org.apache.hadoop.conf.Configuration;
4444
import org.apache.hadoop.service.AbstractService;
45+
import org.apache.hadoop.util.ReflectionUtils;
4546
import org.apache.hadoop.yarn.api.records.NodeId;
4647
import org.apache.hadoop.yarn.api.records.NodeLabel;
4748
import org.apache.hadoop.yarn.api.records.Resource;
@@ -224,10 +225,20 @@ protected void serviceInit(Configuration conf) throws Exception {
224225
labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
225226
}
226227

228+
boolean isCentralizedConfiguration() {
229+
return isCentralizedNodeLabelConfiguration;
230+
}
231+
227232
protected void initNodeLabelStore(Configuration conf) throws Exception {
228-
this.store = new FileSystemNodeLabelsStore(this);
233+
this.store =
234+
ReflectionUtils
235+
.newInstance(
236+
conf.getClass(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS,
237+
FileSystemNodeLabelsStore.class, NodeLabelsStore.class),
238+
conf);
239+
this.store.setNodeLabelsManager(this);
229240
this.store.init(conf);
230-
this.store.recover(!isCentralizedNodeLabelConfiguration);
241+
this.store.recover();
231242
}
232243

233244
// for UT purpose

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,6 @@
5252
import com.google.common.collect.Sets;
5353

5454
public class FileSystemNodeLabelsStore extends NodeLabelsStore {
55-
56-
public FileSystemNodeLabelsStore(CommonNodeLabelsManager mgr) {
57-
super(mgr);
58-
}
59-
6055
protected static final Log LOG = LogFactory.getLog(FileSystemNodeLabelsStore.class);
6156

6257
protected static final String DEFAULT_DIR_NAME = "node-labels";
@@ -69,8 +64,8 @@ protected enum SerializedLogType {
6964

7065
Path fsWorkingPath;
7166
FileSystem fs;
72-
FSDataOutputStream editlogOs;
73-
Path editLogPath;
67+
private FSDataOutputStream editlogOs;
68+
private Path editLogPath;
7469

7570
private String getDefaultFSNodeLabelsRootDir() throws IOException {
7671
// default is in local: /tmp/hadoop-yarn-${user}/node-labels/
@@ -160,12 +155,40 @@ public void removeClusterNodeLabels(Collection<String> labels)
160155
ensureCloseEditlogFile();
161156
}
162157
}
158+
159+
protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath)
160+
throws IOException {
161+
// If mirror.new exists, read from mirror.new,
162+
FSDataInputStream is = null;
163+
if (fs.exists(newMirrorPath)) {
164+
is = fs.open(newMirrorPath);
165+
} else if (fs.exists(oldMirrorPath)) {
166+
is = fs.open(oldMirrorPath);
167+
}
168+
169+
if (null != is) {
170+
List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl(
171+
AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is))
172+
.getNodeLabels();
173+
mgr.addToCluserNodeLabels(labels);
174+
175+
if (mgr.isCentralizedConfiguration()) {
176+
// Only load node to labels mapping while using centralized configuration
177+
Map<NodeId, Set<String>> nodeToLabels =
178+
new ReplaceLabelsOnNodeRequestPBImpl(
179+
ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
180+
.getNodeToLabels();
181+
mgr.replaceLabelsOnNode(nodeToLabels);
182+
}
183+
is.close();
184+
}
185+
}
163186

164187
/* (non-Javadoc)
165188
* @see org.apache.hadoop.yarn.nodelabels.NodeLabelsStore#recover(boolean)
166189
*/
167190
@Override
168-
public void recover(boolean ignoreNodeToLabelsMappings) throws YarnException,
191+
public void recover() throws YarnException,
169192
IOException {
170193
/*
171194
* Steps of recover
@@ -181,31 +204,13 @@ public void recover(boolean ignoreNodeToLabelsMappings) throws YarnException,
181204
// Open mirror from serialized file
182205
Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
183206
Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".old");
184-
185-
FSDataInputStream is = null;
186-
if (fs.exists(mirrorPath)) {
187-
is = fs.open(mirrorPath);
188-
} else if (fs.exists(oldMirrorPath)) {
189-
is = fs.open(oldMirrorPath);
190-
}
191-
192-
if (null != is) {
193-
List<NodeLabel> labels =
194-
new AddToClusterNodeLabelsRequestPBImpl(
195-
AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels();
196-
Map<NodeId, Set<String>> nodeToLabels =
197-
new ReplaceLabelsOnNodeRequestPBImpl(
198-
ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
199-
.getNodeToLabels();
200-
mgr.addToCluserNodeLabels(labels);
201-
mgr.replaceLabelsOnNode(nodeToLabels);
202-
is.close();
203-
}
207+
208+
loadFromMirror(mirrorPath, oldMirrorPath);
204209

205210
// Open and process editlog
206211
editLogPath = new Path(fsWorkingPath, EDITLOG_FILENAME);
207212
if (fs.exists(editLogPath)) {
208-
is = fs.open(editLogPath);
213+
FSDataInputStream is = fs.open(editLogPath);
209214

210215
while (true) {
211216
try {
@@ -233,7 +238,7 @@ public void recover(boolean ignoreNodeToLabelsMappings) throws YarnException,
233238
new ReplaceLabelsOnNodeRequestPBImpl(
234239
ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
235240
.getNodeToLabels();
236-
if (!ignoreNodeToLabelsMappings) {
241+
if (mgr.isCentralizedConfiguration()) {
237242
/*
238243
* In case of Distributed NodeLabels setup,
239244
* ignoreNodeToLabelsMappings will be set to true and recover will

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,7 @@
3131
import org.apache.hadoop.yarn.exceptions.YarnException;
3232

3333
public abstract class NodeLabelsStore implements Closeable {
34-
protected final CommonNodeLabelsManager mgr;
35-
36-
public NodeLabelsStore(CommonNodeLabelsManager mgr) {
37-
this.mgr = mgr;
38-
}
34+
protected CommonNodeLabelsManager mgr;
3935

4036
/**
4137
* Store node {@literal ->} label
@@ -62,16 +58,14 @@ public abstract void removeClusterNodeLabels(Collection<String> labels)
6258
* ignoreNodeToLabelsMappings will be set to true and recover will be invoked
6359
* as RM will collect the node labels from NM through registration/HB
6460
*
65-
* @param ignoreNodeToLabelsMappings
6661
* @throws IOException
6762
* @throws YarnException
6863
*/
69-
public abstract void recover(boolean ignoreNodeToLabelsMappings)
70-
throws IOException, YarnException;
64+
public abstract void recover() throws IOException, YarnException;
7165

7266
public void init(Configuration conf) throws Exception {}
73-
74-
public CommonNodeLabelsManager getNodeLabelsManager() {
75-
return mgr;
67+
68+
public void setNodeLabelsManager(CommonNodeLabelsManager mgr) {
69+
this.mgr = mgr;
7670
}
7771
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2451,4 +2451,12 @@
24512451
<name>yarn.am.blacklisting.disable-failure-threshold</name>
24522452
<value>0.8f</value>
24532453
</property>
2454+
2455+
<property>
2456+
<description>
2457+
Choose different implementation of node label's storage
2458+
</description>
2459+
<name>yarn.node-labels.fs-store.impl.class</name>
2460+
<value>org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore</value>
2461+
</property>
24542462
</configuration>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager {
3636

3737
@Override
3838
public void initNodeLabelStore(Configuration conf) {
39-
this.store = new NodeLabelsStore(this) {
39+
this.store = new NodeLabelsStore() {
4040

4141
@Override
42-
public void recover(boolean ignoreNodeToLabelsMappings)
42+
public void recover()
4343
throws IOException {
4444
}
4545

@@ -65,6 +65,8 @@ public void close() throws IOException {
6565
// do nothing
6666
}
6767
};
68+
69+
this.store.setNodeLabelsManager(this);
6870
}
6971

7072
@Override

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.io.File;
2222
import java.io.IOException;
2323
import java.util.Arrays;
24+
import java.util.Collection;
2425
import java.util.Map;
2526

2627
import org.apache.hadoop.conf.Configuration;
@@ -33,13 +34,17 @@
3334
import org.junit.Assert;
3435
import org.junit.Before;
3536
import org.junit.Test;
37+
import org.junit.runner.RunWith;
38+
import org.junit.runners.Parameterized;
39+
import org.mockito.Mockito;
3640

3741
import com.google.common.collect.ImmutableMap;
38-
import org.mockito.Mockito;
3942

43+
@RunWith(Parameterized.class)
4044
public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
4145
MockNodeLabelManager mgr = null;
4246
Configuration conf = null;
47+
String storeClassName = null;
4348

4449
private static class MockNodeLabelManager extends
4550
CommonNodeLabelsManager {
@@ -59,15 +64,23 @@ protected void stopDispatcher() {
5964
}
6065
}
6166

62-
private FileSystemNodeLabelsStore getStore() {
63-
return (FileSystemNodeLabelsStore) mgr.store;
67+
public TestFileSystemNodeLabelsStore(String className) {
68+
this.storeClassName = className;
69+
}
70+
71+
@Parameterized.Parameters
72+
public static Collection<String[]> getParameters() {
73+
return Arrays.asList(
74+
new String[][] { { FileSystemNodeLabelsStore.class.getCanonicalName() },
75+
{ NonAppendableFSNodeLabelStore.class.getCanonicalName() } });
6476
}
6577

6678
@Before
6779
public void before() throws IOException {
6880
mgr = new MockNodeLabelManager();
6981
conf = new Configuration();
7082
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
83+
conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS, storeClassName);
7184
File tempDir = File.createTempFile("nlb", ".tmp");
7285
tempDir.delete();
7386
tempDir.mkdirs();
@@ -80,7 +93,11 @@ public void before() throws IOException {
8093

8194
@After
8295
public void after() throws IOException {
83-
getStore().fs.delete(getStore().fsWorkingPath, true);
96+
if (mgr.store instanceof FileSystemNodeLabelsStore) {
97+
FileSystemNodeLabelsStore fsStore =
98+
((FileSystemNodeLabelsStore) mgr.store);
99+
fsStore.fs.delete(fsStore.fsWorkingPath, true);
100+
}
84101
mgr.stop();
85102
}
86103

@@ -324,11 +341,12 @@ public void testSerilizationAfterRecovery() throws Exception {
324341
@Test
325342
public void testRootMkdirOnInitStore() throws Exception {
326343
final FileSystem mockFs = Mockito.mock(FileSystem.class);
327-
FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) {
344+
FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() {
328345
void setFileSystem(Configuration conf) throws IOException {
329346
fs = mockFs;
330347
}
331348
};
349+
mockStore.setNodeLabelsManager(mgr);
332350
mockStore.fs = mockFs;
333351
verifyMkdirsCount(mockStore, true, 0);
334352
verifyMkdirsCount(mockStore, false, 1);

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
3737

3838
@Override
3939
public void initNodeLabelStore(Configuration conf) {
40-
this.store = new NodeLabelsStore(this) {
40+
this.store = new NodeLabelsStore() {
4141

4242
@Override
43-
public void recover(boolean ignoreNodeToLabelsMappings)
43+
public void recover()
4444
throws IOException {
4545
// do nothing
4646
}

0 commit comments

Comments
 (0)