Skip to content

Commit e198e59

Browse files
vineet4008vineet4008
committed
HBASE-28742 Fixes NPE for CompactionTool when mslab enabled
1 parent 90d323e commit e198e59

File tree

2 files changed

+145
-0
lines changed

2 files changed

+145
-0
lines changed

‎hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ public void setup(Context context) {
211211
} catch (IOException e) {
212212
throw new RuntimeException("Could not get the input FileSystem", e);
213213
}
214+
// Disable the MemStoreLAB as MemStore is not used by flow during compaction
215+
conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
214216
}
215217

216218
@Override
@@ -369,6 +371,8 @@ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs,
369371
*/
370372
private int doClient(final FileSystem fs, final Set<Path> toCompactDirs,
371373
final boolean compactOnce, final boolean major) throws IOException {
374+
// Disable the MemStoreLAB as MemStore is not used by flow during compaction
375+
getConf().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
372376
CompactionWorker worker = new CompactionWorker(fs, getConf());
373377
for (Path path : toCompactDirs) {
374378
worker.compact(path, compactOnce, major);

‎hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionToolNpeFix.java

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.regionserver;
19+
20+
import static org.junit.Assert.assertEquals;
21+
22+
import java.io.IOException;
23+
import org.apache.hadoop.conf.Configuration;
24+
import org.apache.hadoop.fs.FileStatus;
25+
import org.apache.hadoop.fs.FileSystem;
26+
import org.apache.hadoop.fs.Path;
27+
import org.apache.hadoop.hbase.HBaseClassTestRule;
28+
import org.apache.hadoop.hbase.HBaseConfiguration;
29+
import org.apache.hadoop.hbase.HBaseTestingUtility;
30+
import org.apache.hadoop.hbase.TableName;
31+
import org.apache.hadoop.hbase.client.Put;
32+
import org.apache.hadoop.hbase.testclassification.MediumTests;
33+
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
34+
import org.apache.hadoop.hbase.util.Bytes;
35+
import org.apache.hadoop.hbase.util.CommonFSUtils;
36+
import org.apache.hadoop.util.ToolRunner;
37+
import org.junit.After;
38+
import org.junit.AfterClass;
39+
import org.junit.Before;
40+
import org.junit.BeforeClass;
41+
import org.junit.ClassRule;
42+
import org.junit.Test;
43+
import org.junit.experimental.categories.Category;
44+
45+
@Category({ MediumTests.class, RegionServerTests.class })
46+
public class TestCompactionToolNpeFix {
47+
48+
@ClassRule
49+
public static final HBaseClassTestRule CLASS_RULE =
50+
HBaseClassTestRule.forClass(TestCompactionToolNpeFix.class);
51+
52+
private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
53+
54+
private HRegion region;
55+
private final static byte[] qualifier = Bytes.toBytes("qf");
56+
private static Path rootDir;
57+
private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
58+
59+
@BeforeClass
60+
public static void setUpAfterClass() throws Exception {
61+
TESTUTIL.getConfiguration().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
62+
TESTUTIL.startMiniCluster();
63+
rootDir = TESTUTIL.getDefaultRootDirPath();
64+
TESTUTIL.startMiniMapReduceCluster();
65+
66+
}
67+
68+
@AfterClass
69+
public static void tearDown() throws Exception {
70+
TESTUTIL.shutdownMiniMapReduceCluster();
71+
TESTUTIL.shutdownMiniCluster();
72+
TESTUTIL.cleanupTestDir();
73+
74+
}
75+
76+
@Before
77+
public void setUp() throws IOException {
78+
TESTUTIL.createTable(tableName, HBaseTestingUtility.fam1);
79+
this.region = TESTUTIL.getMiniHBaseCluster().getRegions(tableName).get(0);
80+
}
81+
82+
@After
83+
public void after() throws IOException {
84+
TESTUTIL.deleteTable(tableName);
85+
}
86+
87+
private void putAndFlush(int key) throws Exception {
88+
Put put = new Put(Bytes.toBytes(key));
89+
put.addColumn(HBaseTestingUtility.fam1, qualifier, Bytes.toBytes("val" + key));
90+
region.put(put);
91+
TESTUTIL.flush(tableName);
92+
}
93+
94+
private HStore prepareStoreWithMultiFiles() throws Exception {
95+
for (int i = 0; i < 5; i++) {
96+
this.putAndFlush(i);
97+
}
98+
HStore store = region.getStore(HBaseTestingUtility.fam1);
99+
assertEquals(5, store.getStorefilesCount());
100+
return store;
101+
}
102+
103+
@Test
104+
public void testCompactedFilesArchived() throws Exception {
105+
HStore store = prepareStoreWithMultiFiles();
106+
Path tableDir = CommonFSUtils.getTableDir(rootDir, region.getRegionInfo().getTable());
107+
FileSystem fs = store.getFileSystem();
108+
String storePath = tableDir + "/" + region.getRegionInfo().getEncodedName() + "/"
109+
+ Bytes.toString(HBaseTestingUtility.fam1);
110+
FileStatus[] regionDirFiles = fs.listStatus(new Path(storePath));
111+
assertEquals(5, regionDirFiles.length);
112+
String defaultFS = TESTUTIL.getMiniHBaseCluster().getConfiguration().get("fs.defaultFS");
113+
Configuration config = HBaseConfiguration.create();
114+
config.set("fs.defaultFS", defaultFS);
115+
int result = ToolRunner.run(config, new CompactionTool(),
116+
new String[] { "-compactOnce", "-major", storePath });
117+
assertEquals(0, result);
118+
regionDirFiles = fs.listStatus(new Path(storePath));
119+
assertEquals(1, regionDirFiles.length);
120+
}
121+
122+
@Test
123+
public void testCompactedFilesArchivedMapRed() throws Exception {
124+
HStore store = prepareStoreWithMultiFiles();
125+
Path tableDir = CommonFSUtils.getTableDir(rootDir, region.getRegionInfo().getTable());
126+
FileSystem fs = store.getFileSystem();
127+
String storePath = tableDir + "/" + region.getRegionInfo().getEncodedName() + "/"
128+
+ Bytes.toString(HBaseTestingUtility.fam1);
129+
FileStatus[] regionDirFiles = fs.listStatus(new Path(storePath));
130+
assertEquals(5, regionDirFiles.length);
131+
String defaultFS = TESTUTIL.getMiniHBaseCluster().getConfiguration().get("fs.defaultFS");
132+
Configuration config = HBaseConfiguration.create(TESTUTIL.getConfiguration());
133+
config.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
134+
config.set("fs.defaultFS", defaultFS);
135+
int result = ToolRunner.run(config, new CompactionTool(),
136+
new String[] { "-compactOnce", "-mapred", "-major", storePath });
137+
assertEquals(0, result);
138+
regionDirFiles = fs.listStatus(new Path(storePath));
139+
assertEquals(1, regionDirFiles.length);
140+
}
141+
}

0 commit comments

Comments
 (0)