Skip to content

Commit 693bd25

Browse files
Apache9Apache9
committed
HBASE-26280 Use store file tracker when snapshoting
1 parent 6581fef commit 693bd25

File tree

7 files changed

+53
-53
lines changed

7 files changed

+53
-53
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -612,9 +612,7 @@ private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg
612612
List<Path> mergedFiles = new ArrayList<>();
613613
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
614614
String family = hcd.getNameAsString();
615-
Configuration trackerConfig =
616-
StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd);
617-
StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs);
615+
StoreFileTracker tracker = StoreFileTrackerFactory.create(env.getMasterConfiguration(),htd, hcd, regionFs);
618616
final Collection<StoreFileInfo> storeFiles = tracker.load();
619617
if (storeFiles != null && storeFiles.size() > 0) {
620618
for (StoreFileInfo storeFileInfo : storeFiles) {

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -666,9 +666,8 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
666666
new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
667667
for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
668668
String family = cfd.getNameAsString();
669-
Configuration trackerConfig = StoreFileTrackerFactory.
670-
mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName()));
671-
StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs);
669+
StoreFileTracker tracker =
670+
StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs);
672671
Collection<StoreFileInfo> sfis = tracker.load();
673672
if (sfis == null) {
674673
continue;

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -622,12 +622,8 @@ private void insertRegionFilesIntoStoreTracker(List<Path> allFiles, MasterProced
622622
Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
623623
for(Path file : allFiles) {
624624
String familyName = file.getParent().getName();
625-
trackerMap.computeIfAbsent(familyName, t -> {
626-
Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc,
627-
tblDesc.getColumnFamily(Bytes.toBytes(familyName)));
628-
return StoreFileTrackerFactory.
629-
create(config, familyName, regionFs);
630-
});
625+
trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc,
626+
tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs));
631627
fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
632628
List<StoreFileInfo> infos = fileInfoMap.get(familyName);
633629
infos.add(new StoreFileInfo(conf, fs, file, true));

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -113,16 +113,15 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli
113113
* Used at master side when splitting/merging regions, as we do not have a Store, thus no
114114
* StoreContext at master side.
115115
*/
116-
public static StoreFileTracker create(Configuration conf, String family,
117-
HRegionFileSystem regionFs) {
118-
ColumnFamilyDescriptorBuilder fDescBuilder =
119-
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family));
120-
StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build())
121-
.withRegionFileSystem(regionFs).build();
116+
public static StoreFileTracker create(Configuration conf, TableDescriptor td,
117+
ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) {
118+
StoreContext ctx =
119+
StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs)
120+
.withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build();
122121
return StoreFileTrackerFactory.create(conf, true, ctx);
123122
}
124123

125-
public static Configuration mergeConfigurations(Configuration global, TableDescriptor table,
124+
private static Configuration mergeConfigurations(Configuration global, TableDescriptor table,
126125
ColumnFamilyDescriptor family) {
127126
return new CompoundConfiguration().add(global).addBytesMap(table.getValues())
128127
.addStringMap(family.getConfiguration()).addBytesMap(family.getValues());

‎hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java

Lines changed: 18 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
import org.apache.hadoop.hbase.regionserver.HStore;
4848
import org.apache.hadoop.hbase.regionserver.HStoreFile;
4949
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
50+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
51+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
5052
import org.apache.hadoop.hbase.util.Bytes;
5153
import org.apache.hadoop.hbase.util.CommonFSUtils;
5254
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -291,17 +293,17 @@ public void addRegion(final Path tableDir, final RegionInfo regionInfo) throws I
291293
addRegion(tableDir, regionInfo, visitor);
292294
}
293295

294-
protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor)
295-
throws IOException {
296+
protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor)
297+
throws IOException {
296298
boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
297299
try {
298300
Path baseDir = tableDir;
299301
// Open the RegionFS
300302
if (isMobRegion) {
301303
baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
302304
}
303-
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs,
304-
baseDir, regionInfo, true);
305+
HRegionFileSystem regionFs =
306+
HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true);
305307
monitor.rethrowException();
306308

307309
// 1. dump region meta info into the snapshot directory
@@ -317,26 +319,19 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio
317319
// in batches and may miss files being added/deleted. This could be more robust (iteratively
318320
// checking to see if we have all the files until we are sure), but the limit is currently
319321
// 1000 files/batch, far more than the number of store files under a single column family.
320-
Collection<String> familyNames = regionFs.getFamilies();
321-
if (familyNames != null) {
322-
for (String familyName: familyNames) {
323-
Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
324-
monitor.rethrowException();
325-
326-
Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
327-
if (storeFiles == null) {
328-
if (LOG.isDebugEnabled()) {
329-
LOG.debug("No files under family: " + familyName);
330-
}
331-
continue;
332-
}
333-
334-
// 2.1. build the snapshot reference for the store
335-
// iterate through all the store's files and create "references".
336-
addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
337-
338-
visitor.familyClose(regionData, familyData);
322+
for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
323+
Object familyData = visitor.familyOpen(regionData, cfd.getName());
324+
monitor.rethrowException();
325+
StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs);
326+
List<StoreFileInfo> storeFiles = tracker.load();
327+
if (storeFiles.isEmpty()) {
328+
LOG.debug("No files under family: {}", cfd.getNameAsString());
329+
continue;
339330
}
331+
// 2.1. build the snapshot reference for the store
332+
// iterate through all the store's files and create "references".
333+
addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
334+
visitor.familyClose(regionData, familyData);
340335
}
341336
visitor.regionClose(regionData);
342337
} catch (IOException e) {

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@
2626
import org.junit.BeforeClass;
2727
import org.junit.ClassRule;
2828
import org.junit.experimental.categories.Category;
29-
import org.slf4j.Logger;
30-
import org.slf4j.LoggerFactory;
3129

3230
/**
3331
* Test create/using/deleting snapshots from the client
@@ -41,8 +39,6 @@ public class TestMobSnapshotFromClient extends TestSnapshotFromClient {
4139
public static final HBaseClassTestRule CLASS_RULE =
4240
HBaseClassTestRule.forClass(TestMobSnapshotFromClient.class);
4341

44-
private static final Logger LOG = LoggerFactory.getLogger(TestMobSnapshotFromClient.class);
45-
4642
/**
4743
* Setup the config for the cluster
4844
* @throws Exception on failure
@@ -60,6 +56,7 @@ protected static void setupConf(Configuration conf) {
6056

6157
@Override
6258
protected void createTable() throws Exception {
59+
// TODO: create table with different store file tracker
6360
MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), TEST_FAM);
6461
}
6562
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static org.junit.Assert.fail;
2424

2525
import java.util.ArrayList;
26+
import java.util.Arrays;
2627
import java.util.List;
2728
import java.util.regex.Pattern;
2829
import org.apache.hadoop.conf.Configuration;
@@ -32,9 +33,11 @@
3233
import org.apache.hadoop.hbase.HBaseTestingUtil;
3334
import org.apache.hadoop.hbase.HConstants;
3435
import org.apache.hadoop.hbase.TableName;
36+
import org.apache.hadoop.hbase.TableNameTestRule;
3537
import org.apache.hadoop.hbase.TableNotFoundException;
3638
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
3739
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3841
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
3942
import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
4043
import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
@@ -51,7 +54,10 @@
5154
import org.junit.Rule;
5255
import org.junit.Test;
5356
import org.junit.experimental.categories.Category;
54-
import org.junit.rules.TestName;
57+
import org.junit.runner.RunWith;
58+
import org.junit.runners.Parameterized;
59+
import org.junit.runners.Parameterized.Parameter;
60+
import org.junit.runners.Parameterized.Parameters;
5561
import org.slf4j.Logger;
5662
import org.slf4j.LoggerFactory;
5763

@@ -64,7 +70,8 @@
6470
* <p>
6571
* This is an end-to-end test for the snapshot utility
6672
*/
67-
@Category({LargeTests.class, ClientTests.class})
73+
@RunWith(Parameterized.class)
74+
@Category({ LargeTests.class, ClientTests.class })
6875
public class TestSnapshotFromClient {
6976

7077
@ClassRule
@@ -82,7 +89,16 @@ public class TestSnapshotFromClient {
8289
private static final Pattern MATCH_ALL = Pattern.compile(".*");
8390

8491
@Rule
85-
public TestName name = new TestName();
92+
public TableNameTestRule name = new TableNameTestRule();
93+
94+
@Parameter
95+
public StoreFileTrackerFactory.Trackers trackerImpl;
96+
97+
@Parameters(name = "{index}: tracker={0}")
98+
public static List<Object[]> params() {
99+
return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
100+
new Object[] { StoreFileTrackerFactory.Trackers.FILE });
101+
}
86102

87103
/**
88104
* Setup the config for the cluster
@@ -109,7 +125,6 @@ protected static void setupConf(Configuration conf) {
109125
conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
110126
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
111127
ConstantSizeRegionSplitPolicy.class.getName());
112-
113128
}
114129

115130
@Before
@@ -119,7 +134,8 @@ public void setup() throws Exception {
119134

120135
protected void createTable() throws Exception {
121136
TableDescriptor htd =
122-
TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()).build();
137+
TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
138+
.setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
123139
UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
124140
}
125141

@@ -316,7 +332,7 @@ public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
316332
@Test
317333
public void testListTableSnapshots() throws Exception {
318334
Admin admin = null;
319-
final TableName tableName = TableName.valueOf(name.getMethodName());
335+
final TableName tableName = name.getTableName();
320336
try {
321337
admin = UTIL.getAdmin();
322338

@@ -401,7 +417,7 @@ public void testListTableSnapshotsWithRegex() throws Exception {
401417
@Test
402418
public void testDeleteTableSnapshots() throws Exception {
403419
Admin admin = null;
404-
final TableName tableName = TableName.valueOf(name.getMethodName());
420+
final TableName tableName = name.getTableName();
405421
try {
406422
admin = UTIL.getAdmin();
407423

0 commit comments

Comments
 (0)