Skip to content

HBASE-22899 logging improvements for snapshot operations w/large manifests #547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 14, 2019
Merged

HBASE-22899 logging improvements for snapshot operations w/large manifests #547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
47c06ce
HBASE-22899 logging improvements for snapshot operations w/large mani…
rabi-kumar Aug 26, 2019
ce8b14a
HBASE-22899 logging improvements for snapshot operations w/large mani…
rabi-kumar Aug 27, 2019
0519f67
HBASE-22899 logging improvements for snapshot operations w/large mani…
rabi-kumar Aug 27, 2019
8bc487b
HBASE-22899 logging improvements for snapshot operations w/large mani…
rabi-kumar Aug 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,7 @@ private void restoreRegion(final RegionInfo regionInfo,
getRegionHFileReferences(regionManifest);

String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();

// Restore families present in the table
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
Expand All @@ -523,20 +524,21 @@ private void restoreRegion(final RegionInfo regionInfo,
// Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName);
LOG.trace("Removing hfile=" + hfileName +
LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName+
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
}

// Restore Missing files
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
LOG.debug("Restoring missing HFileLink " + storeFile.getName() +
" of snapshot=" + snapshotName+
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
LOG.trace("Removing family=" + Bytes.toString(family) +
LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName +
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
fs.delete(familyDir, true);
Expand All @@ -552,7 +554,8 @@ private void restoreRegion(final RegionInfo regionInfo,
}

for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName()
+ " of snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
Expand Down Expand Up @@ -584,6 +587,7 @@ private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
if (regions == null || regions.isEmpty()) return null;

final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size());
final String snapshotName = snapshotDesc.getName();

// clone region info (change embedded tableName with the new one)
RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
Expand All @@ -596,7 +600,8 @@ private RegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
String snapshotRegionName = snapshotRegionInfo.getEncodedName();
String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName +
" in snapshot " + snapshotName);

// Add mapping between cloned region name and snapshot region info
snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
Expand Down Expand Up @@ -640,10 +645,12 @@ private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> region
private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo,
final SnapshotRegionManifest manifest) throws IOException {
final String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region "
+ "in snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
}
}
Expand Down
9 changes: 6 additions & 3 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,8 @@ public void addMobRegion(RegionInfo regionInfo) throws IOException {
@VisibleForTesting
protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
final String snapshotName = desc.getName();
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();

Expand Down Expand Up @@ -232,7 +233,8 @@ public void addRegion(final HRegion region) throws IOException {
@VisibleForTesting
protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing '" + region + "' region-info for snapshot.");
final String snapshotName = desc.getName();
LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();

Expand All @@ -256,7 +258,8 @@ protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOE
monitor.rethrowException();

// create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath());
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath() +
" for snapshot=" + snapshotName);
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);
Expand Down