Skip to content

Commit 2bb7beb

Browse files
taklwutaklwu
authored
Backport "HBASE-25249 Adding StoreContext" to branch-2 (#2869)
Co-authored-by: Abhishek Khanna <akkhanna@amazon.com> Signed-off-by: Zach York <zyork@apache.org>
1 parent 1612b9e commit 2bb7beb

File tree

11 files changed

+373
-164
lines changed

11 files changed

+373
-164
lines changed

‎hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
import org.apache.hadoop.hbase.regionserver.BloomType;
7171
import org.apache.hadoop.hbase.regionserver.HStore;
7272
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
73+
import org.apache.hadoop.hbase.regionserver.StoreUtils;
7374
import org.apache.hadoop.hbase.util.BloomFilterUtil;
7475
import org.apache.hadoop.hbase.util.Bytes;
7576
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -386,16 +387,15 @@ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration
386387
DataBlockEncoding encoding = overriddenEncoding;
387388
encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding;
388389
encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
389-
HFileContextBuilder contextBuilder = new HFileContextBuilder()
390-
.withCompression(compression).withChecksumType(HStore.getChecksumType(conf))
391-
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize)
390+
HFileContextBuilder contextBuilder = new HFileContextBuilder().withCompression(compression)
391+
.withDataBlockEncoding(encoding).withChecksumType(StoreUtils.getChecksumType(conf))
392+
.withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blockSize)
392393
.withColumnFamily(family).withTableName(tableName);
393394

394395
if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
395396
contextBuilder.withIncludesTags(true);
396397
}
397398

398-
contextBuilder.withDataBlockEncoding(encoding);
399399
HFileContext hFileContext = contextBuilder.build();
400400
if (null == favoredNodes) {
401401
wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)

‎hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,9 @@
6464
import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartitionId;
6565
import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
6666
import org.apache.hadoop.hbase.regionserver.BloomType;
67-
import org.apache.hadoop.hbase.regionserver.HStore;
6867
import org.apache.hadoop.hbase.regionserver.HStoreFile;
6968
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
69+
import org.apache.hadoop.hbase.regionserver.StoreUtils;
7070
import org.apache.hadoop.hbase.util.Bytes;
7171
import org.apache.hadoop.hbase.util.ChecksumType;
7272
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -581,7 +581,7 @@ public static StoreFileWriter createRefFileWriter(Configuration conf, FileSystem
581581
return createWriter(conf, fs, family,
582582
new Path(basePath, UUID.randomUUID().toString().replaceAll("-", "")), maxKeyCount,
583583
family.getCompactionCompressionType(), cacheConfig, cryptoContext,
584-
HStore.getChecksumType(conf), HStore.getBytesPerChecksum(conf), family.getBlocksize(),
584+
StoreUtils.getChecksumType(conf), StoreUtils.getBytesPerChecksum(conf), family.getBlocksize(),
585585
family.getBloomFilterType(), isCompaction);
586586
}
587587

@@ -658,7 +658,7 @@ public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
658658
throws IOException {
659659
return createWriter(conf, fs, family,
660660
new Path(basePath, mobFileName.getFileName()), maxKeyCount, compression, cacheConfig,
661-
cryptoContext, HStore.getChecksumType(conf), HStore.getBytesPerChecksum(conf),
661+
cryptoContext, StoreUtils.getChecksumType(conf), StoreUtils.getBytesPerChecksum(conf),
662662
family.getBlocksize(), BloomType.NONE, isCompaction);
663663
}
664664

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ public class HMobStore extends HStore {
9393
private AtomicLong mobFlushedCellsSize = new AtomicLong();
9494
private AtomicLong mobScanCellsCount = new AtomicLong();
9595
private AtomicLong mobScanCellsSize = new AtomicLong();
96-
private ColumnFamilyDescriptor family;
9796
private Map<String, List<Path>> map = new ConcurrentHashMap<>();
9897
private final IdLock keyLock = new IdLock();
9998
// When we add a MOB reference cell to the HFile, we will add 2 tags along with it
@@ -107,11 +106,10 @@ public class HMobStore extends HStore {
107106
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family,
108107
final Configuration confParam, boolean warmup) throws IOException {
109108
super(region, family, confParam, warmup);
110-
this.family = family;
111109
this.mobFileCache = region.getMobFileCache();
112110
this.homePath = MobUtils.getMobHome(conf);
113111
this.mobFamilyPath = MobUtils.getMobFamilyPath(conf, this.getTableName(),
114-
family.getNameAsString());
112+
family.getNameAsString());
115113
List<Path> locations = new ArrayList<>(2);
116114
locations.add(mobFamilyPath);
117115
TableName tn = region.getTableDescriptor().getTableName();
@@ -248,9 +246,11 @@ public StoreFileWriter createWriterInTmp(String date, Path basePath, long maxKey
248246
public StoreFileWriter createWriterInTmp(MobFileName mobFileName, Path basePath,
249247
long maxKeyCount, Compression.Algorithm compression,
250248
boolean isCompaction) throws IOException {
251-
return MobUtils.createWriter(conf, region.getFilesystem(), family,
252-
new Path(basePath, mobFileName.getFileName()), maxKeyCount, compression, cacheConf,
253-
cryptoContext, checksumType, bytesPerChecksum, blocksize, BloomType.NONE, isCompaction);
249+
return MobUtils.createWriter(conf, getFileSystem(), getColumnFamilyDescriptor(),
250+
new Path(basePath, mobFileName.getFileName()), maxKeyCount, compression, getCacheConfig(),
251+
getStoreContext().getEncryptionContext(), StoreUtils.getChecksumType(conf),
252+
StoreUtils.getBytesPerChecksum(conf), getStoreContext().getBlockSize(), BloomType.NONE,
253+
isCompaction);
254254
}
255255

256256
/**
@@ -268,10 +268,10 @@ public void commitFile(final Path sourceFile, Path targetPath) throws IOExceptio
268268
String msg = "Renaming flushed file from " + sourceFile + " to " + dstPath;
269269
LOG.info(msg);
270270
Path parent = dstPath.getParent();
271-
if (!region.getFilesystem().exists(parent)) {
272-
region.getFilesystem().mkdirs(parent);
271+
if (!getFileSystem().exists(parent)) {
272+
getFileSystem().mkdirs(parent);
273273
}
274-
if (!region.getFilesystem().rename(sourceFile, dstPath)) {
274+
if (!getFileSystem().rename(sourceFile, dstPath)) {
275275
throw new IOException("Failed rename of " + sourceFile + " to " + dstPath);
276276
}
277277
}
@@ -284,7 +284,7 @@ public void commitFile(final Path sourceFile, Path targetPath) throws IOExceptio
284284
private void validateMobFile(Path path) throws IOException {
285285
HStoreFile storeFile = null;
286286
try {
287-
storeFile = new HStoreFile(region.getFilesystem(), path, conf, this.cacheConf,
287+
storeFile = new HStoreFile(getFileSystem(), path, conf, getCacheConfig(),
288288
BloomType.NONE, isPrimaryReplicaStore());
289289
storeFile.initReader();
290290
} catch (IOException e) {
@@ -335,9 +335,9 @@ public MobCell resolve(Cell reference, boolean cacheBlocks, long readPt,
335335
if (locations == null) {
336336
locations = new ArrayList<>(2);
337337
TableName tn = TableName.valueOf(tableNameString);
338-
locations.add(MobUtils.getMobFamilyPath(conf, tn, family.getNameAsString()));
338+
locations.add(MobUtils.getMobFamilyPath(conf, tn, getColumnFamilyName()));
339339
locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn,
340-
MobUtils.getMobRegionInfo(tn).getEncodedName(), family.getNameAsString()));
340+
MobUtils.getMobRegionInfo(tn).getEncodedName(), getColumnFamilyName()));
341341
map.put(tableNameString, locations);
342342
}
343343
} finally {
@@ -390,7 +390,7 @@ private MobCell readCell(List<Path> locations, String fileName, Cell search,
390390
MobFile file = null;
391391
Path path = new Path(location, fileName);
392392
try {
393-
file = mobFileCache.openFile(fs, path, cacheConf);
393+
file = mobFileCache.openFile(fs, path, getCacheConfig());
394394
return readPt != -1 ? file.readCell(search, cacheMobBlocks, readPt)
395395
: file.readCell(search, cacheMobBlocks);
396396
} catch (IOException e) {

0 commit comments

Comments
 (0)