Skip to content

Commit a7be992

Browse files
wchevreuilApache9
authored andcommitted
HBASE-26246 Persist the StoreFileTracker configurations to TableDescriptor when creating table (#3666)
Signed-off-by: Duo Zhang <zhangduo@apache.org>
1 parent a7e1229 commit a7be992

File tree

13 files changed

+113
-12
lines changed

13 files changed

+113
-12
lines changed

hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.hadoop.hbase.master.procedure;
2020

21+
2122
import java.io.IOException;
2223
import java.util.ArrayList;
2324
import java.util.List;
@@ -33,10 +34,12 @@
3334
import org.apache.hadoop.hbase.client.RegionInfo;
3435
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
3536
import org.apache.hadoop.hbase.client.TableDescriptor;
37+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3638
import org.apache.hadoop.hbase.client.TableState;
3739
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
3840
import org.apache.hadoop.hbase.master.MasterFileSystem;
3941
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
42+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
4043
import org.apache.hadoop.hbase.replication.ReplicationException;
4144
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
4245
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -285,6 +288,10 @@ private void preCreate(final MasterProcedureEnv env)
285288
getTableName(), (newRegions != null ? newRegions.size() : 0));
286289
}
287290

291+
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor);
292+
StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder);
293+
tableDescriptor = builder.build();
294+
288295
final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
289296
if (cpHost != null) {
290297
final RegionInfo[] regions = newRegions == null ? null :

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,7 +607,7 @@ public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path> allRegi
607607
writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
608608
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
609609
env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false);
610-
insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs);
610+
insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs);
611611
}
612612
return regionDir;
613613
}

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Collection;
2222
import java.util.List;
2323
import org.apache.hadoop.conf.Configuration;
24+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
2425
import org.apache.hadoop.hbase.regionserver.StoreContext;
2526
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
2627
import org.apache.yetus.audience.InterfaceAudience;
@@ -85,4 +86,11 @@ void set(List<StoreFileInfo> files) {
8586
throw new UnsupportedOperationException(
8687
"Should not call this method on " + getClass().getSimpleName());
8788
}
89+
90+
@Override
91+
public void persistConfiguration(TableDescriptorBuilder builder) {
92+
super.persistConfiguration(builder);
93+
builder.setValue(SRC_IMPL, src.getClass().getName());
94+
builder.setValue(DST_IMPL, dst.getClass().getName());
95+
}
8896
}

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import java.io.IOException;
2121
import java.util.Collection;
2222
import java.util.List;
23+
24+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
2325
import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
2426
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
2527
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
@@ -71,4 +73,10 @@ void replace(Collection<StoreFileInfo> compactedFiles, Collection<StoreFileInfo>
7173
* @return Writer for a new StoreFile
7274
*/
7375
StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException;
76+
77+
/**
78+
* Saves StoreFileTracker implementations specific configs into the table descriptors.
79+
* @param builder The table descriptor builder for the given table.
80+
*/
81+
void persistConfiguration(TableDescriptorBuilder builder);
7482
}

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,22 @@
1717
*/
1818
package org.apache.hadoop.hbase.regionserver.storefiletracker;
1919

20+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL;
21+
2022
import java.io.IOException;
2123
import java.util.Collection;
2224
import java.util.List;
2325
import org.apache.hadoop.conf.Configuration;
2426
import org.apache.hadoop.fs.Path;
2527
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
28+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
2629
import org.apache.hadoop.hbase.io.compress.Compression;
2730
import org.apache.hadoop.hbase.io.crypto.Encryption;
2831
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
2932
import org.apache.hadoop.hbase.io.hfile.HFile;
3033
import org.apache.hadoop.hbase.io.hfile.HFileContext;
3134
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
35+
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
3236
import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
3337
import org.apache.hadoop.hbase.regionserver.StoreContext;
3438
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -78,6 +82,15 @@ public final void replace(Collection<StoreFileInfo> compactedFiles,
7882
}
7983
}
8084

85+
@Override
86+
public void persistConfiguration(TableDescriptorBuilder builder) {
87+
if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) {
88+
String trackerImpl = StoreFileTrackerFactory.
89+
getStoreFileTrackerImpl(conf).getName();
90+
builder.setValue(TRACK_IMPL, trackerImpl).build();
91+
}
92+
}
93+
8194
private HFileContext createFileContext(Compression.Algorithm compression,
8295
boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) {
8396
if (compression == null) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@
66
* to you under the Apache License, Version 2.0 (the
77
* "License"); you may not use this file except in compliance
88
* with the License. You may obtain a copy of the License at
9-
*
10-
* http://www.apache.org/licenses/LICENSE-2.0
11-
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
1210
* Unless required by applicable law or agreed to in writing, software
1311
* distributed under the License is distributed on an "AS IS" BASIS,
1412
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +19,7 @@
2119
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
2220
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
2321
import org.apache.hadoop.hbase.client.TableDescriptor;
22+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
2423
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
2524
import org.apache.hadoop.hbase.regionserver.StoreContext;
2625
import org.apache.hadoop.hbase.regionserver.StoreUtils;
@@ -35,15 +34,17 @@
3534
/**
3635
* Factory method for creating store file tracker.
3736
*/
38-
@InterfaceAudience.Private
39-
public final class StoreFileTrackerFactory {
37+
@InterfaceAudience.Private public final class StoreFileTrackerFactory {
4038
public static final String TRACK_IMPL = "hbase.store.file-tracker.impl";
4139
private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class);
4240

41+
public static Class<? extends StoreFileTracker> getStoreFileTrackerImpl(Configuration conf) {
42+
return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class);
43+
}
44+
4345
public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica,
4446
StoreContext ctx) {
45-
Class<? extends StoreFileTracker> tracker =
46-
conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class);
47+
Class<? extends StoreFileTracker> tracker = getStoreFileTrackerImpl(conf);
4748
LOG.info("instantiating StoreFileTracker impl {}", tracker.getName());
4849
return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx);
4950
}
@@ -54,7 +55,7 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli
5455
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family));
5556
StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build())
5657
.withRegionFileSystem(regionFs).build();
57-
return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx);
58+
return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx);
5859
}
5960

6061
public static Configuration mergeConfigurations(Configuration global, TableDescriptor table,
@@ -75,4 +76,12 @@ static StoreFileTrackerBase create(Configuration conf, String configName,
7576
LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName);
7677
return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx);
7778
}
79+
80+
public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) {
81+
TableDescriptor tableDescriptor = builder.build();
82+
ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0];
83+
StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build();
84+
StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context);
85+
tracker.persistConfiguration(builder);
86+
}
7887
}

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.hbase.client;
1919

2020
import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows;
21+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL;
2122
import static org.junit.Assert.assertEquals;
2223
import static org.junit.Assert.assertFalse;
2324
import static org.junit.Assert.assertNotEquals;
@@ -39,6 +40,7 @@
3940
import org.apache.hadoop.hbase.TableExistsException;
4041
import org.apache.hadoop.hbase.TableName;
4142
import org.apache.hadoop.hbase.TableNotFoundException;
43+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
4244
import org.apache.hadoop.hbase.testclassification.ClientTests;
4345
import org.apache.hadoop.hbase.testclassification.LargeTests;
4446
import org.apache.hadoop.hbase.util.Bytes;
@@ -423,6 +425,10 @@ private void testCloneTableSchema(final TableName tableName, final TableName new
423425
assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
424426
assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
425427
assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
428+
// HBASE-26246 introduced persist of store file tracker into table descriptor
429+
tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL,
430+
StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()).
431+
build();
426432
TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
427433

428434
if (preserveSplits) {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818
package org.apache.hadoop.hbase.client;
1919

20+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL;
2021
import static org.junit.Assert.assertEquals;
2122
import static org.junit.Assert.assertFalse;
2223
import static org.junit.Assert.assertTrue;
@@ -34,6 +35,7 @@
3435
import org.apache.hadoop.hbase.TableNotDisabledException;
3536
import org.apache.hadoop.hbase.TableNotEnabledException;
3637
import org.apache.hadoop.hbase.TableNotFoundException;
38+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3739
import org.apache.hadoop.hbase.testclassification.ClientTests;
3840
import org.apache.hadoop.hbase.testclassification.LargeTests;
3941
import org.apache.hadoop.hbase.util.Bytes;
@@ -236,6 +238,10 @@ public void testGetTableDescriptor() throws IOException {
236238
ADMIN.createTable(htd);
237239
Table table = TEST_UTIL.getConnection().getTable(htd.getTableName());
238240
TableDescriptor confirmedHtd = table.getDescriptor();
241+
//HBASE-26246 introduced persist of store file tracker into table descriptor
242+
htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL,
243+
StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()).
244+
build();
239245
assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd));
240246
MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
241247
table.close();

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.hbase.client;
1919

2020
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
21+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL;
2122
import static org.junit.Assert.assertEquals;
2223
import static org.junit.Assert.assertFalse;
2324
import static org.junit.Assert.assertTrue;
@@ -34,6 +35,7 @@
3435
import org.apache.hadoop.hbase.TableExistsException;
3536
import org.apache.hadoop.hbase.TableName;
3637
import org.apache.hadoop.hbase.TableNotFoundException;
38+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3739
import org.apache.hadoop.hbase.testclassification.ClientTests;
3840
import org.apache.hadoop.hbase.testclassification.LargeTests;
3941
import org.apache.hadoop.hbase.util.Bytes;
@@ -372,6 +374,10 @@ private void testCloneTableSchema(final TableName tableName,
372374
assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
373375
assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
374376
assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
377+
//HBASE-26246 introduced persist of store file tracker into table descriptor
378+
tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL,
379+
StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()).
380+
build();
375381
TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
376382

377383
if (preserveSplits) {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.hbase.client;
1919

2020
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
21+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL;
2122
import static org.hamcrest.CoreMatchers.instanceOf;
2223
import static org.hamcrest.MatcherAssert.assertThat;
2324
import static org.junit.Assert.assertEquals;
@@ -35,6 +36,7 @@
3536
import org.apache.hadoop.hbase.HBaseClassTestRule;
3637
import org.apache.hadoop.hbase.HRegionLocation;
3738
import org.apache.hadoop.hbase.TableName;
39+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3840
import org.apache.hadoop.hbase.testclassification.ClientTests;
3941
import org.apache.hadoop.hbase.testclassification.LargeTests;
4042
import org.apache.hadoop.hbase.util.Bytes;
@@ -147,6 +149,10 @@ public void testGetTableDescriptor() throws Exception {
147149
TableDescriptor desc = builder.build();
148150
admin.createTable(desc).join();
149151
TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get();
152+
//HBASE-26246 introduced persist of store file tracker into table descriptor
153+
desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL,
154+
StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()).
155+
build();
150156
assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd));
151157
}
152158

0 commit comments

Comments
 (0)