Skip to content

Commit c1aa8d7

Browse files
virajjasanivirajjasani
committed
addendum
1 parent bace039 commit c1aa8d7

File tree

2 files changed

+169
-28
lines changed

2 files changed

+169
-28
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/AsyncRFAAppender.java

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.hdfs.util;
20+
21+
import java.io.IOException;
22+
23+
import org.apache.log4j.AsyncAppender;
24+
import org.apache.log4j.PatternLayout;
25+
import org.apache.log4j.RollingFileAppender;
26+
import org.apache.log4j.spi.LoggingEvent;
27+
28+
/**
29+
* Until we migrate to log4j2, use this appender for namenode audit logger as well as
30+
* datanode and namenode metric loggers with log4j1 properties.
31+
* This appender will take parameters necessary to supply RollingFileAppender to AsyncAppender.
32+
* While migrating to log4j2, we can directly specify appender ref to Async appender.
33+
*/
34+
public class AsyncRFAAppender extends AsyncAppender {
35+
36+
/**
37+
* The default maximum file size is 10MB.
38+
*/
39+
private String maxFileSize = String.valueOf(10*1024*1024);
40+
41+
/**
42+
* There is one backup file by default.
43+
*/
44+
private int maxBackupIndex = 1;
45+
46+
/**
47+
* The name of the log file.
48+
*/
49+
private String fileName = null;
50+
51+
private String conversionPattern = null;
52+
53+
/**
54+
* Does appender block when buffer is full.
55+
*/
56+
private boolean blocking = true;
57+
58+
/**
59+
* Buffer size.
60+
*/
61+
private int bufferSize = DEFAULT_BUFFER_SIZE;
62+
63+
private RollingFileAppender rollingFileAppender = null;
64+
65+
private final Object rollingFileAppenderLock = new Object();
66+
67+
@Override
68+
public void append(LoggingEvent event) {
69+
if (rollingFileAppender == null) {
70+
synchronized (rollingFileAppenderLock) {
71+
PatternLayout patternLayout;
72+
if (conversionPattern != null) {
73+
patternLayout = new PatternLayout(conversionPattern);
74+
} else {
75+
patternLayout = new PatternLayout();
76+
}
77+
try {
78+
rollingFileAppender = new RollingFileAppender(patternLayout, fileName, true);
79+
} catch (IOException e) {
80+
throw new RuntimeException(e);
81+
}
82+
rollingFileAppender.setMaxBackupIndex(maxBackupIndex);
83+
rollingFileAppender.setMaxFileSize(maxFileSize);
84+
this.addAppender(rollingFileAppender);
85+
super.setBlocking(blocking);
86+
super.setBufferSize(bufferSize);
87+
}
88+
}
89+
super.append(event);
90+
}
91+
92+
public String getMaxFileSize() {
93+
return maxFileSize;
94+
}
95+
96+
public void setMaxFileSize(String maxFileSize) {
97+
this.maxFileSize = maxFileSize;
98+
}
99+
100+
public int getMaxBackupIndex() {
101+
return maxBackupIndex;
102+
}
103+
104+
public void setMaxBackupIndex(int maxBackupIndex) {
105+
this.maxBackupIndex = maxBackupIndex;
106+
}
107+
108+
public String getFileName() {
109+
return fileName;
110+
}
111+
112+
public void setFileName(String fileName) {
113+
this.fileName = fileName;
114+
}
115+
116+
public String getConversionPattern() {
117+
return conversionPattern;
118+
}
119+
120+
public void setConversionPattern(String conversionPattern) {
121+
this.conversionPattern = conversionPattern;
122+
}
123+
124+
public boolean isBlocking() {
125+
return blocking;
126+
}
127+
128+
public void setBlocking(boolean blocking) {
129+
this.blocking = blocking;
130+
}
131+
132+
public int getBufferSize() {
133+
return bufferSize;
134+
}
135+
136+
public void setBufferSize(int bufferSize) {
137+
this.bufferSize = bufferSize;
138+
}
139+
}

‎hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,38 +22,41 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
2222
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
2323
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
2424

25+
# Only to be used for testing
26+
log4j.appender.PATTERNMATCHERAPPENDER=org.apache.hadoop.hdfs.server.namenode.PatternMatchingAppender
27+
2528
#
2629
# NameNode metrics logging.
2730
# The default is to retain two namenode-metrics.log files up to 64MB each.
2831
#
2932

30-
# TODO : log4j2 properties to provide example for using Async appender with other appenders
31-
namenode.metrics.logger=INFO,ASYNCAPPENDER,NNMETRICSRFA,PATTERNMATCHERAPPENDER
33+
# TODO : While migrating to log4j2, replace AsyncRFAAppender with AsyncAppender as
34+
# log4j2 properties support wrapping of other appenders to AsyncAppender using appender ref
35+
namenode.metrics.logger=INFO,ASYNCNNMETRICSRFA,PATTERNMATCHERAPPENDER
3236
log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}
3337
log4j.additivity.NameNodeMetricsLog=false
34-
log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender
35-
log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log
36-
log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout
37-
log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
38-
log4j.appender.NNMETRICSRFA.MaxBackupIndex=1
39-
log4j.appender.NNMETRICSRFA.MaxFileSize=64MB
40-
log4j.appender.PATTERNMATCHERAPPENDER=org.apache.hadoop.hdfs.server.namenode.PatternMatchingAppender
38+
log4j.appender.ASYNCNNMETRICSRFA=org.apache.hadoop.hdfs.util.AsyncRFAAppender
39+
log4j.appender.ASYNCNNMETRICSRFA.conversionPattern=%d{ISO8601} %m%n
40+
log4j.appender.ASYNCNNMETRICSRFA.maxFileSize=64MB
41+
log4j.appender.ASYNCNNMETRICSRFA.fileName=${hadoop.log.dir}/namenode-metrics.log
42+
log4j.appender.ASYNCNNMETRICSRFA.maxBackupIndex=1
4143

4244
#
4345
# DataNode metrics logging.
4446
# The default is to retain two datanode-metrics.log files up to 64MB each.
4547
#
4648

47-
# TODO : log4j2 properties to provide example for using Async appender with other appenders
48-
datanode.metrics.logger=INFO,ASYNCAPPENDER,DNMETRICSRFA,PATTERNMATCHERAPPENDER
49+
# TODO : While migrating to log4j2, replace AsyncRFAAppender with AsyncAppender as
50+
# log4j2 properties support wrapping of other appenders to AsyncAppender using appender ref
51+
datanode.metrics.logger=INFO,ASYNCDNMETRICSRFA,PATTERNMATCHERAPPENDER
4952
log4j.logger.DataNodeMetricsLog=${datanode.metrics.logger}
5053
log4j.additivity.DataNodeMetricsLog=false
51-
log4j.appender.DNMETRICSRFA=org.apache.log4j.RollingFileAppender
52-
log4j.appender.DNMETRICSRFA.File=${hadoop.log.dir}/datanode-metrics.log
53-
log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout
54-
log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
55-
log4j.appender.DNMETRICSRFA.MaxBackupIndex=1
56-
log4j.appender.DNMETRICSRFA.MaxFileSize=64MB
54+
log4j.appender.ASYNCDNMETRICSRFA=org.apache.hadoop.hdfs.util.AsyncRFAAppender
55+
log4j.appender.ASYNCDNMETRICSRFA.conversionPattern=%d{ISO8601} %m%n
56+
log4j.appender.ASYNCDNMETRICSRFA.maxFileSize=64MB
57+
log4j.appender.ASYNCDNMETRICSRFA.fileName=${hadoop.log.dir}/datanode-metrics.log
58+
log4j.appender.ASYNCDNMETRICSRFA.maxBackupIndex=1
59+
5760

5861
# Supress KMS error log
5962
log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF
@@ -62,18 +65,17 @@ log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGener
6265
# hdfs audit logging
6366
#
6467

65-
# TODO : log4j2 properties to provide example for using Async appender with other appenders
66-
hdfs.audit.logger=INFO,ASYNCAPPENDER,RFAAUDIT
68+
# TODO : While migrating to log4j2, replace AsyncRFAAppender with AsyncAppender as
69+
# log4j2 properties support wrapping of other appenders to AsyncAppender using appender ref
70+
hdfs.audit.logger=INFO,ASYNCAUDITAPPENDER
6771
hdfs.audit.log.maxfilesize=256MB
6872
hdfs.audit.log.maxbackupindex=20
6973
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
7074
log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
71-
log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
72-
log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
73-
log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
74-
log4j.appender.RFAAUDIT.layout.ConversionPattern=%m%n
75-
log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
76-
log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
77-
log4j.appender.ASYNCAPPENDER=org.apache.log4j.AsyncAppender
78-
log4j.appender.ASYNCAPPENDER.blocking=false
79-
log4j.appender.ASYNCAPPENDER.bufferSize=256
75+
log4j.appender.ASYNCAUDITAPPENDER=org.apache.hadoop.hdfs.util.AsyncRFAAppender
76+
log4j.appender.ASYNCAUDITAPPENDER.blocking=false
77+
log4j.appender.ASYNCAUDITAPPENDER.bufferSize=256
78+
log4j.appender.ASYNCAUDITAPPENDER.conversionPattern=%m%n
79+
log4j.appender.ASYNCAUDITAPPENDER.maxFileSize=${hdfs.audit.log.maxfilesize}
80+
log4j.appender.ASYNCAUDITAPPENDER.fileName=${hadoop.log.dir}/hdfs-audit.log
81+
log4j.appender.ASYNCAUDITAPPENDER.maxBackupIndex=${hdfs.audit.log.maxbackupindex}

0 commit comments

Comments
 (0)