Skip to content

Commit c001dcd

Browse files
zhengchenyuzhengchenyu
committed
MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0.
1 parent 512e39e commit c001dcd

File tree

5 files changed

+39
-37
lines changed

5 files changed

+39
-37
lines changed

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,20 @@ public void testCommandLine() throws Exception {
5858
app.verifyCompleted();
5959

6060
Assert.assertEquals(
61-
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
62-
" -Djava.net.preferIPv4Stack=true" +
63-
" -Dhadoop.metrics.log.level=WARN " +
64-
" -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
65-
" -Dlog4j.configuration=container-log4j.properties" +
66-
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
67-
" -Dyarn.app.container.log.filesize=0" +
68-
" -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
69-
" org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
70-
" 54321" +
71-
" attempt_0_0000_m_000000_0" +
72-
" 0" +
73-
" 1><LOG_DIR>/stdout" +
74-
" 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
61+
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
62+
" -Djava.net.preferIPv4Stack=true" +
63+
" -Dhadoop.metrics.log.level=WARN " +
64+
" -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
65+
" -Dlog4j.configuration=container-log4j.properties" +
66+
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
67+
" -Dyarn.app.container.log.filesize=10485760" +
68+
" -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
69+
" org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
70+
" 54321" +
71+
" attempt_0_0000_m_000000_0" +
72+
" 0" +
73+
" 1><LOG_DIR>/stdout" +
74+
" 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
7575

7676
Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
7777
app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
@@ -127,7 +127,7 @@ private void testReduceCommandLine(Configuration conf)
127127
" -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
128128
" -Dlog4j.configuration=container-log4j.properties" +
129129
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
130-
" -Dyarn.app.container.log.filesize=0" +
130+
" -Dyarn.app.container.log.filesize=10485760" +
131131
" -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
132132
" -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName +
133133
" -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" +
@@ -163,20 +163,20 @@ public void testCommandLineWithLog4JConifg() throws Exception {
163163
app.verifyCompleted();
164164

165165
Assert.assertEquals(
166-
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
167-
" -Djava.net.preferIPv4Stack=true" +
168-
" -Dhadoop.metrics.log.level=WARN " +
169-
" -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
170-
" -Dlog4j.configuration=" + testLogPropertieFile +
171-
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
172-
" -Dyarn.app.container.log.filesize=0" +
173-
" -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
174-
" org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
175-
" 54321" +
176-
" attempt_0_0000_m_000000_0" +
177-
" 0" +
178-
" 1><LOG_DIR>/stdout" +
179-
" 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
166+
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
167+
" -Djava.net.preferIPv4Stack=true" +
168+
" -Dhadoop.metrics.log.level=WARN " +
169+
" -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
170+
" -Dlog4j.configuration=" + testLogPropertieFile +
171+
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
172+
" -Dyarn.app.container.log.filesize=10485760" +
173+
" -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
174+
" org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
175+
" 54321" +
176+
" attempt_0_0000_m_000000_0" +
177+
" 0" +
178+
" 1><LOG_DIR>/stdout" +
179+
" 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
180180
}
181181

182182
@Test

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,8 @@ public static long getTaskLogLength(JobConf conf) {
466466
}
467467

468468
public static long getTaskLogLimitBytes(Configuration conf) {
469-
return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024;
469+
return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) *
470+
1024;
470471
}
471472

472473

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,8 @@ public interface MRJobConfig {
410410

411411
public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb";
412412

413+
public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240;
414+
413415
public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent";
414416

415417
public static final String MAP_INPUT_FILE = "mapreduce.map.input.file";
@@ -758,11 +760,11 @@ public interface MRJobConfig {
758760

759761
public static final String MR_AM_LOG_KB =
760762
MR_AM_PREFIX + "container.log.limit.kb";
761-
public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll
763+
public static final int DEFAULT_MR_AM_LOG_KB = 10240;
762764

763765
public static final String MR_AM_LOG_BACKUPS =
764766
MR_AM_PREFIX + "container.log.backups";
765-
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0;
767+
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
766768

767769
/**The number of splits when reporting progress in MR*/
768770
public static final String MR_AM_NUM_PROGRESS_SPLITS =

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -823,16 +823,15 @@
823823

824824
<property>
825825
<name>mapreduce.task.userlog.limit.kb</name>
826-
<value>0</value>
827-
<description>The maximum size of user-logs of each task in KB. 0 disables the cap.
826+
<value>10240</value>
827+
<description>The maximum size of user-logs of each task in KB.
828828
</description>
829829
</property>
830830

831831
<property>
832832
<name>yarn.app.mapreduce.am.container.log.limit.kb</name>
833-
<value>0</value>
833+
<value>10240</value>
834834
<description>The maximum size of the MRAppMaster attempt container logs in KB.
835-
0 disables the cap.
836835
</description>
837836
</property>
838837

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -945,7 +945,7 @@ private void verifyTailErrorLogOnContainerExit(Configuration conf,
945945
invalidCommand.add("-Djava.io.tmpdir=$PWD/tmp");
946946
invalidCommand.add("-Dlog4j.configuration=container-log4j.properties");
947947
invalidCommand.add("-Dyarn.app.container.log.dir=" + containerLogDir);
948-
invalidCommand.add("-Dyarn.app.container.log.filesize=0");
948+
invalidCommand.add("-Dyarn.app.container.log.filesize=10485760");
949949
invalidCommand.add("-Dhadoop.root.logger=INFO,CLA");
950950
invalidCommand.add("-Dhadoop.root.logfile=syslog");
951951
invalidCommand.add("-Xmx1024m");

0 commit comments

Comments
 (0)