Skip to content

Commit 13d932c

Browse files
Apache9Apache9
authored
HBASE-26947 Implement a special TestAppender to limit the size of test output (#4340)
Singed-off-by: Nick Dimiduk <ndimiduk@apache.org>
1 parent 8d8f956 commit 13d932c

File tree

5 files changed

+243
-1
lines changed

5 files changed

+243
-1
lines changed

‎hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestAppender.java

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.logging;
19+
20+
import java.io.Serializable;
21+
import java.nio.charset.StandardCharsets;
22+
import java.util.concurrent.atomic.AtomicBoolean;
23+
import java.util.concurrent.atomic.AtomicLong;
24+
import org.apache.logging.log4j.core.Appender;
25+
import org.apache.logging.log4j.core.Core;
26+
import org.apache.logging.log4j.core.Filter;
27+
import org.apache.logging.log4j.core.Layout;
28+
import org.apache.logging.log4j.core.LogEvent;
29+
import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
30+
import org.apache.logging.log4j.core.appender.ManagerFactory;
31+
import org.apache.logging.log4j.core.appender.OutputStreamManager;
32+
import org.apache.logging.log4j.core.appender.rolling.FileSize;
33+
import org.apache.logging.log4j.core.config.Property;
34+
import org.apache.logging.log4j.core.config.plugins.Plugin;
35+
import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute;
36+
import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
37+
import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required;
38+
39+
/**
40+
* Log4j2 appender to be used when running UTs.
41+
* <p/>
42+
* The main point here is to limit the total output size to prevent eating all the space of our ci
43+
* system when something wrong in our code.
44+
* <p/>
45+
* See HBASE-26947 for more details.
46+
*/
47+
@Plugin(name = HBaseTestAppender.PLUGIN_NAME, category = Core.CATEGORY_NAME,
48+
elementType = Appender.ELEMENT_TYPE, printObject = true)
49+
public final class HBaseTestAppender extends AbstractOutputStreamAppender<OutputStreamManager> {
50+
51+
public static final String PLUGIN_NAME = "HBaseTest";
52+
private static final HBaseTestManagerFactory FACTORY = new HBaseTestManagerFactory();
53+
54+
public static class Builder<B extends Builder<B>> extends AbstractOutputStreamAppender.Builder<B>
55+
implements org.apache.logging.log4j.core.util.Builder<HBaseTestAppender> {
56+
57+
@PluginBuilderAttribute
58+
@Required
59+
private Target target;
60+
61+
@PluginBuilderAttribute
62+
@Required
63+
private String maxSize;
64+
65+
public B setTarget(Target target) {
66+
this.target = target;
67+
return asBuilder();
68+
}
69+
70+
public B setMaxSize(String maxSize) {
71+
this.maxSize = maxSize;
72+
return asBuilder();
73+
}
74+
75+
@Override
76+
public HBaseTestAppender build() {
77+
long size = FileSize.parse(maxSize, -1);
78+
if (size <= 0) {
79+
LOGGER.error("Invalid maxSize {}", size);
80+
}
81+
Layout<? extends Serializable> layout = getOrCreateLayout(StandardCharsets.UTF_8);
82+
OutputStreamManager manager =
83+
OutputStreamManager.getManager(target.name(), FACTORY, new FactoryData(target, layout));
84+
return new HBaseTestAppender(getName(),
85+
layout,
86+
getFilter(),
87+
isIgnoreExceptions(),
88+
isImmediateFlush(),
89+
getPropertyArray(),
90+
manager,
91+
size);
92+
}
93+
}
94+
95+
/**
96+
* Data to pass to factory method.Unable to instantiate
97+
*/
98+
private static class FactoryData {
99+
private final Target target;
100+
private final Layout<? extends Serializable> layout;
101+
102+
public FactoryData(Target target, Layout<? extends Serializable> layout) {
103+
this.target = target;
104+
this.layout = layout;
105+
}
106+
}
107+
108+
/**
109+
* Factory to create the Appender.
110+
*/
111+
private static class HBaseTestManagerFactory
112+
implements ManagerFactory<HBaseTestOutputStreamManager, FactoryData> {
113+
114+
/**
115+
* Create an OutputStreamManager.
116+
* @param name The name of the entity to manage.
117+
* @param data The data required to create the entity.
118+
* @return The OutputStreamManager
119+
*/
120+
@Override
121+
public HBaseTestOutputStreamManager createManager(final String name, final FactoryData data) {
122+
return new HBaseTestOutputStreamManager(data.target, data.layout);
123+
}
124+
}
125+
126+
@PluginBuilderFactory
127+
public static <B extends Builder<B>> B newBuilder() {
128+
return new Builder<B>().asBuilder();
129+
}
130+
131+
private final long maxSize;
132+
133+
private final AtomicLong size = new AtomicLong(0);
134+
135+
private final AtomicBoolean stop = new AtomicBoolean(false);
136+
137+
private HBaseTestAppender(String name, Layout<? extends Serializable> layout, Filter filter,
138+
boolean ignoreExceptions, boolean immediateFlush, Property[] properties,
139+
OutputStreamManager manager, long maxSize) {
140+
super(name, layout, filter, ignoreExceptions, immediateFlush, properties, manager);
141+
this.maxSize = maxSize;
142+
}
143+
144+
@Override
145+
public void append(LogEvent event) {
146+
if (stop.get()) {
147+
return;
148+
}
149+
// for accounting, here we always convert the event to byte array first
150+
// this will effect performance a bit but it is OK since this is for UT only
151+
byte[] bytes = getLayout().toByteArray(event);
152+
if (bytes == null || bytes.length == 0) {
153+
return;
154+
}
155+
long sizeAfterAppend = size.addAndGet(bytes.length);
156+
if (sizeAfterAppend >= maxSize) {
157+
// stop logging if the log size exceeded the limit
158+
if (stop.compareAndSet(false, true)) {
159+
LOGGER.error("Log size exceeded the limit {}, will stop logging to prevent eating"
160+
+ " too much disk space", maxSize);
161+
}
162+
return;
163+
}
164+
super.append(event);
165+
}
166+
}

‎hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/HBaseTestOutputStreamManager.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.logging;
19+
20+
public class HBaseTestOutputStreamManager
21+
extends org.apache.logging.log4j.core.appender.OutputStreamManager {
22+
23+
public HBaseTestOutputStreamManager(Target target,
24+
org.apache.logging.log4j.core.Layout<?> layout) {
25+
super(target.output(), target.name(), layout, true);
26+
}
27+
}

‎hbase-logging/src/test/java/org/apache/hadoop/hbase/logging/Target.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.logging;
19+
20+
import java.io.PrintStream;
21+
22+
public enum Target {
23+
SYSTEM_OUT(System.out),
24+
SYSTEM_ERR(System.err);
25+
26+
private final PrintStream output;
27+
28+
private Target(PrintStream output) {
29+
this.output = output;
30+
}
31+
32+
public PrintStream output() {
33+
return output;
34+
}
35+
}

‎hbase-logging/src/test/resources/log4j2.properties

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@
1919
status = debug
2020
dest = err
2121
name = PropertiesConfig
22+
packages = org.apache.hadoop.hbase.logging
2223

23-
appender.console.type = Console
24+
appender.console.type = HBaseTest
2425
appender.console.target = SYSTEM_ERR
2526
appender.console.name = Console
27+
appender.console.maxSize = 1G
2628
appender.console.layout.type = PatternLayout
2729
appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
2830

‎pom.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1781,6 +1781,10 @@
17811781
<trimStackTrace>false</trimStackTrace>
17821782
<skip>${surefire.skipFirstPart}</skip>
17831783
<forkCount>${surefire.firstPartForkCount}</forkCount>
1784+
<!--
1785+
The counter in HBaseTestAppender will be broken if we set reuseForks to true, be
1786+
careful when you want to change this value. See HBASE-26947 for more details.
1787+
-->
17841788
<reuseForks>false</reuseForks>
17851789
<reportsDirectory>${surefire.reportsDirectory}</reportsDirectory>
17861790
<tempDir>${surefire.tempDir}</tempDir>
@@ -1825,6 +1829,10 @@
18251829
<configuration>
18261830
<skip>${surefire.skipSecondPart}</skip>
18271831
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
1832+
<!--
1833+
The counter in HBaseTestAppender will be broken if we set reuseForks to true, be
1834+
careful when you want to change this value. See HBASE-26947 for more details.
1835+
-->
18281836
<reuseForks>false</reuseForks>
18291837
<forkCount>${surefire.secondPartForkCount}</forkCount>
18301838
<groups>${surefire.secondPartGroups}</groups>
@@ -2331,6 +2339,10 @@
23312339
<bannedImport>org.apache.log4j.**</bannedImport>
23322340
<bannedImport>org.apache.logging.log4j.**</bannedImport>
23332341
</bannedImports>
2342+
<exclusions>
2343+
<!-- Exclude this one as it is a log4j2 appender implementation -->
2344+
<exclusion>org.apache.hadoop.hbase.logging.HBaseTestAppender</exclusion>
2345+
</exclusions>
23342346
</restrictImports>
23352347
<restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
23362348
<includeTestCode>false</includeTestCode>

0 commit comments

Comments
 (0)