Skip to content

Commit c76c083

Browse files
Apache9apurtell
Apache9
authored and
apurtell
committed
HBASE-26640 Reimplement master local region initialization to better work with SFT (#4111)
Signed-off-by: Josh Elser <elserj@apache.org> Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
1 parent 4c0786e commit c76c083

File tree

9 files changed

+371
-37
lines changed

9 files changed

+371
-37
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -781,7 +781,7 @@ private void tryMigrateMetaLocationsFromZooKeeper() throws IOException, KeeperEx
781781
// done with in a one row put, which means if we have data in catalog family then we can
782782
// make sure that the migration is done.
783783
LOG.info("The {} family in master local region already has data in it, skip migrating...",
784-
HConstants.CATALOG_FAMILY);
784+
HConstants.CATALOG_FAMILY_STR);
785785
return;
786786
}
787787
}
@@ -4081,7 +4081,7 @@ public MetaLocationSyncer getMetaLocationSyncer() {
40814081

40824082
@RestrictedApi(explanation = "Should only be called in tests", link = "",
40834083
allowedOnPath = ".*/src/test/.*")
4084-
MasterRegion getMasterRegion() {
4084+
public MasterRegion getMasterRegion() {
40854085
return masterRegion;
40864086
}
40874087

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java

Lines changed: 117 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,34 @@
2020
import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
2121

2222
import java.io.IOException;
23+
import java.util.List;
2324
import org.apache.hadoop.conf.Configuration;
2425
import org.apache.hadoop.fs.FileStatus;
2526
import org.apache.hadoop.fs.FileSystem;
2627
import org.apache.hadoop.fs.Path;
2728
import org.apache.hadoop.hbase.HBaseIOException;
2829
import org.apache.hadoop.hbase.Server;
2930
import org.apache.hadoop.hbase.TableName;
31+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
3032
import org.apache.hadoop.hbase.client.Get;
3133
import org.apache.hadoop.hbase.client.RegionInfo;
3234
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
3335
import org.apache.hadoop.hbase.client.Result;
3436
import org.apache.hadoop.hbase.client.ResultScanner;
3537
import org.apache.hadoop.hbase.client.Scan;
3638
import org.apache.hadoop.hbase.client.TableDescriptor;
39+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3740
import org.apache.hadoop.hbase.regionserver.HRegion;
3841
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
3942
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
4043
import org.apache.hadoop.hbase.regionserver.RegionScanner;
44+
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
45+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
46+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
4147
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
4248
import org.apache.hadoop.hbase.util.Bytes;
4349
import org.apache.hadoop.hbase.util.CommonFSUtils;
50+
import org.apache.hadoop.hbase.util.FSTableDescriptors;
4451
import org.apache.hadoop.hbase.util.FSUtils;
4552
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
4653
import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
@@ -92,6 +99,10 @@ public final class MasterRegion {
9299

93100
private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
94101

102+
static final String INITIALIZING_FLAG = ".initializing";
103+
104+
static final String INITIALIZED_FLAG = ".initialized";
105+
95106
private static final int REGION_ID = 1;
96107

97108
private final WALFactory walFactory;
@@ -196,32 +207,39 @@ private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRol
196207

197208
private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
198209
Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
199-
MasterRegionWALRoller walRoller, String serverName) throws IOException {
210+
MasterRegionWALRoller walRoller, String serverName, boolean touchInitializingFlag)
211+
throws IOException {
200212
TableName tn = td.getTableName();
201213
RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
202-
Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
203-
TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
204-
if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
205-
throw new IOException("Can not delete partial created proc region " + tmpTableDir);
206-
}
207-
HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
208214
Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
209-
if (!fs.rename(tmpTableDir, tableDir)) {
210-
throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
215+
// persist table descriptor
216+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, td, true);
217+
HRegion.createHRegion(conf, regionInfo, fs, tableDir, td).close();
218+
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
219+
if (!fs.mkdirs(initializedFlag)) {
220+
throw new IOException("Can not touch initialized flag: " + initializedFlag);
221+
}
222+
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
223+
if (!fs.delete(initializingFlag, true)) {
224+
LOG.warn("failed to clean up initializing flag: " + initializingFlag);
211225
}
212226
WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
213227
return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
214228
}
215229

216-
private static HRegion open(Configuration conf, TableDescriptor td, FileSystem fs, Path rootDir,
217-
FileSystem walFs, Path walRootDir, WALFactory walFactory, MasterRegionWALRoller walRoller,
218-
String serverName) throws IOException {
219-
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
220-
Path regionDir =
221-
fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
222-
.getPath();
223-
RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
230+
private static RegionInfo loadRegionInfo(FileSystem fs, Path tableDir) throws IOException {
231+
// on branch-2, the RegionInfo.isEncodedRegionName will returns true for .initializing and
232+
// .initialized, see HBASE-25368. Since RegionInfo is IA.Public, changing the implementation may
233+
// raise compatibility concerns, so here we just skip them by our own.
234+
Path regionDir = fs.listStatus(tableDir, p -> !p.getName().startsWith(".")
235+
&& RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0].getPath();
236+
return HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
237+
}
224238

239+
private static HRegion open(Configuration conf, TableDescriptor td, RegionInfo regionInfo,
240+
FileSystem fs, Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
241+
MasterRegionWALRoller walRoller, String serverName) throws IOException {
242+
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
225243
Path walRegionDir = FSUtils.getRegionDirFromRootDir(walRootDir, regionInfo);
226244
Path replayEditsDir = new Path(walRegionDir, REPLAY_EDITS_DIR);
227245
if (!walFs.exists(replayEditsDir) && !walFs.mkdirs(replayEditsDir)) {
@@ -287,6 +305,39 @@ private static void replayWALs(Configuration conf, FileSystem walFs, Path walRoo
287305
}
288306
}
289307

308+
private static void tryMigrate(Configuration conf, FileSystem fs, Path tableDir,
309+
RegionInfo regionInfo, TableDescriptor oldTd, TableDescriptor newTd) throws IOException {
310+
Class<? extends StoreFileTracker> oldSft =
311+
StoreFileTrackerFactory.getTrackerClass(oldTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
312+
Class<? extends StoreFileTracker> newSft =
313+
StoreFileTrackerFactory.getTrackerClass(newTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
314+
if (oldSft.equals(newSft)) {
315+
LOG.debug("old store file tracker {} is the same with new store file tracker, skip migration",
316+
StoreFileTrackerFactory.getStoreFileTrackerName(oldSft));
317+
if (!oldTd.equals(newTd)) {
318+
// we may change other things such as adding a new family, so here we still need to persist
319+
// the new table descriptor
320+
LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
321+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
322+
}
323+
return;
324+
}
325+
LOG.info("Migrate store file tracker from {} to {}", oldSft.getSimpleName(),
326+
newSft.getSimpleName());
327+
HRegionFileSystem hfs =
328+
HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false);
329+
for (ColumnFamilyDescriptor oldCfd : oldTd.getColumnFamilies()) {
330+
StoreFileTracker oldTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
331+
StoreFileTracker newTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
332+
List<StoreFileInfo> files = oldTracker.load();
333+
LOG.debug("Store file list for {}: {}", oldCfd.getNameAsString(), files);
334+
newTracker.set(oldTracker.load());
335+
}
336+
// persist the new table descriptor after migration
337+
LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
338+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
339+
}
340+
290341
public static MasterRegion create(MasterRegionParams params) throws IOException {
291342
TableDescriptor td = params.tableDescriptor();
292343
LOG.info("Create or load local region for table " + td);
@@ -321,16 +372,58 @@ public static MasterRegion create(MasterRegionParams params) throws IOException
321372

322373
WALFactory walFactory = new WALFactory(conf, server.getServerName().toString());
323374
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
375+
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
376+
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
324377
HRegion region;
325-
if (fs.exists(tableDir)) {
326-
// load the existing region.
327-
region = open(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
328-
server.getServerName().toString());
329-
} else {
330-
// bootstrapping...
378+
if (!fs.exists(tableDir)) {
379+
// bootstrap, no doubt
380+
if (!fs.mkdirs(initializedFlag)) {
381+
throw new IOException("Can not touch initialized flag");
382+
}
331383
region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
332-
server.getServerName().toString());
384+
server.getServerName().toString(), true);
385+
} else {
386+
if (!fs.exists(initializedFlag)) {
387+
if (!fs.exists(initializingFlag)) {
388+
// should be old style, where we do not have the initializing or initialized file, persist
389+
// the table descriptor, touch the initialized flag and then open the region.
390+
// the store file tracker must be DEFAULT
391+
LOG.info("No {} or {} file, try upgrading", INITIALIZING_FLAG, INITIALIZED_FLAG);
392+
TableDescriptor oldTd =
393+
TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL,
394+
StoreFileTrackerFactory.Trackers.DEFAULT.name()).build();
395+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, oldTd, true);
396+
if (!fs.mkdirs(initializedFlag)) {
397+
throw new IOException("Can not touch initialized flag: " + initializedFlag);
398+
}
399+
RegionInfo regionInfo = loadRegionInfo(fs, tableDir);
400+
tryMigrate(conf, fs, tableDir, regionInfo, oldTd, td);
401+
region = open(conf, td, regionInfo, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
402+
server.getServerName().toString());
403+
} else {
404+
// delete all contents besides the initializing flag, here we can make sure tableDir
405+
// exists(unless someone delete it manually...), so we do not do null check here.
406+
for (FileStatus status : fs.listStatus(tableDir)) {
407+
if (!status.getPath().getName().equals(INITIALIZING_FLAG)) {
408+
fs.delete(status.getPath(), true);
409+
}
410+
}
411+
region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
412+
server.getServerName().toString(), false);
413+
}
414+
} else {
415+
if (fs.exists(initializingFlag) && !fs.delete(initializingFlag, true)) {
416+
LOG.warn("failed to clean up initializing flag: " + initializingFlag);
417+
}
418+
// open it, make sure to load the table descriptor from fs
419+
TableDescriptor oldTd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
420+
RegionInfo regionInfo = loadRegionInfo(fs, tableDir);
421+
tryMigrate(conf, fs, tableDir, regionInfo, oldTd, td);
422+
region = open(conf, td, regionInfo, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
423+
server.getServerName().toString());
424+
}
333425
}
426+
334427
Path globalArchiveDir = HFileArchiveUtil.getArchivePath(baseConf);
335428
MasterRegionFlusherAndCompactor flusherAndCompactor = new MasterRegionFlusherAndCompactor(conf,
336429
server, region, params.flushSize(), params.flushPerChanges(), params.flushIntervalMs(),

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@
2828
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
2929
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
3030
import org.apache.hadoop.hbase.regionserver.BloomType;
31+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
32+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3133
import org.apache.hadoop.hbase.util.Bytes;
34+
import org.apache.hadoop.hbase.util.ReflectionUtils;
3235
import org.apache.yetus.audience.InterfaceAudience;
3336

3437
/**
@@ -78,6 +81,8 @@ public final class MasterRegionFactory {
7881

7982
private static final int DEFAULT_RING_BUFFER_SLOT_COUNT = 128;
8083

84+
public static final String TRACKER_IMPL = "hbase.master.store.region.file-tracker.impl";
85+
8186
public static final TableName TABLE_NAME = TableName.valueOf("master:store");
8287

8388
public static final byte[] PROC_FAMILY = Bytes.toBytes("proc");
@@ -89,10 +94,23 @@ public final class MasterRegionFactory {
8994
.setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
9095
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(PROC_FAMILY)).build();
9196

97+
private static TableDescriptor withTrackerConfigs(Configuration conf) {
98+
String trackerImpl = conf.get(TRACKER_IMPL, conf.get(StoreFileTrackerFactory.TRACKER_IMPL,
99+
StoreFileTrackerFactory.Trackers.DEFAULT.name()));
100+
Class<? extends StoreFileTracker> trackerClass =
101+
StoreFileTrackerFactory.getTrackerClass(trackerImpl);
102+
if (StoreFileTrackerFactory.isMigration(trackerClass)) {
103+
throw new IllegalArgumentException("Should not set store file tracker to " +
104+
StoreFileTrackerFactory.Trackers.MIGRATION.name() + " for master local region");
105+
}
106+
StoreFileTracker tracker = ReflectionUtils.newInstance(trackerClass, conf, true, null);
107+
return tracker.updateWithTrackerConfigs(TableDescriptorBuilder.newBuilder(TABLE_DESC)).build();
108+
}
109+
92110
public static MasterRegion create(Server server) throws IOException {
93-
MasterRegionParams params = new MasterRegionParams().server(server)
94-
.regionDirName(MASTER_STORE_DIR).tableDescriptor(TABLE_DESC);
95111
Configuration conf = server.getConfiguration();
112+
MasterRegionParams params = new MasterRegionParams().server(server)
113+
.regionDirName(MASTER_STORE_DIR).tableDescriptor(withTrackerConfigs(conf));
96114
long flushSize = conf.getLong(FLUSH_SIZE_KEY, DEFAULT_FLUSH_SIZE);
97115
long flushPerChanges = conf.getLong(FLUSH_PER_CHANGES_KEY, DEFAULT_FLUSH_PER_CHANGES);
98116
long flushIntervalMs = conf.getLong(FLUSH_INTERVAL_MS_KEY, DEFAULT_FLUSH_INTERVAL_MS);

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public static String getStoreFileTrackerName(Configuration conf) {
8484
return conf.get(TRACKER_IMPL, Trackers.DEFAULT.name());
8585
}
8686

87-
static String getStoreFileTrackerName(Class<? extends StoreFileTracker> clazz) {
87+
public static String getStoreFileTrackerName(Class<? extends StoreFileTracker> clazz) {
8888
Trackers name = CLASS_TO_ENUM.get(clazz);
8989
return name != null ? name.name() : clazz.getName();
9090
}
@@ -184,4 +184,8 @@ public static TableDescriptor updateWithTrackerConfigs(Configuration conf,
184184
}
185185
return descriptor;
186186
}
187+
188+
public static boolean isMigration(Class<?> clazz) {
189+
return MigrationStoreFileTracker.class.isAssignableFrom(clazz);
190+
}
187191
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,13 @@ private static Optional<Pair<FileStatus, TableDescriptor>> getTableDescriptorFro
511511
return td != null ? Optional.of(Pair.newPair(descFile, td)) : Optional.empty();
512512
}
513513

514+
@RestrictedApi(explanation = "Should only be called in tests", link = "",
515+
allowedOnPath = ".*/src/test/.*")
516+
public static void deleteTableDescriptors(FileSystem fs, Path tableDir) throws IOException {
517+
Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
518+
deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE);
519+
}
520+
514521
/**
515522
* Deletes files matching the table info file pattern within the given directory whose sequenceId
516523
* is at most the given max sequenceId.

‎hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/MasterRegionTestBase.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3535
import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
3636
import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
37+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3738
import org.apache.hadoop.hbase.util.Bytes;
3839
import org.apache.hadoop.hbase.util.CommonFSUtils;
3940
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -85,19 +86,24 @@ public void setUp() throws IOException {
8586
/**
8687
* Creates a new MasterRegion using an existing {@code htu} on this class.
8788
*/
88-
protected void createMasterRegion() throws IOException {
89-
configure(htu.getConfiguration());
89+
protected final void createMasterRegion() throws IOException {
90+
Configuration conf = htu.getConfiguration();
91+
configure(conf);
9092
choreService = new ChoreService(getClass().getSimpleName());
9193
cleanerPool = new DirScanPool(htu.getConfiguration());
9294
Server server = mock(Server.class);
93-
when(server.getConfiguration()).thenReturn(htu.getConfiguration());
95+
when(server.getConfiguration()).thenReturn(conf);
9496
when(server.getServerName())
9597
.thenReturn(ServerName.valueOf("localhost", 12345, EnvironmentEdgeManager.currentTime()));
9698
when(server.getChoreService()).thenReturn(choreService);
9799
Path testDir = htu.getDataTestDir();
98-
CommonFSUtils.setRootDir(htu.getConfiguration(), testDir);
100+
CommonFSUtils.setRootDir(conf, testDir);
99101
MasterRegionParams params = new MasterRegionParams();
100-
params.server(server).regionDirName(REGION_DIR_NAME).tableDescriptor(TD)
102+
TableDescriptor td = TableDescriptorBuilder
103+
.newBuilder(TD).setValue(StoreFileTrackerFactory.TRACKER_IMPL, conf
104+
.get(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()))
105+
.build();
106+
params.server(server).regionDirName(REGION_DIR_NAME).tableDescriptor(td)
101107
.flushSize(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE).flushPerChanges(1_000_000)
102108
.flushIntervalMs(TimeUnit.MINUTES.toMillis(15)).compactMin(4).maxWals(32).useHsync(false)
103109
.ringBufferSlotCount(16).rollPeriodMs(TimeUnit.MINUTES.toMillis(15))

0 commit comments

Comments
 (0)