Skip to content

Commit 1ad48c1

Browse files
gjacoby126apurtell
gjacoby126
authored and
apurtell
committed
HBASE-22622 WALKey Extended Attributes (#352)
Signed-off-by: Andrew Purtell <apurtell@apache.org>
1 parent 48c623c commit 1ad48c1

File tree

6 files changed

+159
-30
lines changed

6 files changed

+159
-30
lines changed

‎hbase-protocol-shaded/src/main/protobuf/WAL.proto

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,20 @@ message WALKey {
6363
optional uint64 nonceGroup = 9;
6464
optional uint64 nonce = 10;
6565
optional uint64 orig_sequence_number = 11;
66+
repeated Attribute extended_attributes = 12;
6667

67-
/*
68-
optional CustomEntryType custom_entry_type = 9;
68+
/*
69+
optional CustomEntryType custom_entry_type = 9;
6970
70-
enum CustomEntryType {
71-
COMPACTION = 0;
72-
}
73-
*/
71+
enum CustomEntryType {
72+
COMPACTION = 0;
73+
}
74+
*/
75+
}
76+
77+
message Attribute {
78+
required string key = 1;
79+
required bytes value = 2;
7480
}
7581

7682
enum ScopeType {

‎hbase-protocol/src/main/protobuf/WAL.proto

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ message WALKey {
6262
optional uint64 nonceGroup = 9;
6363
optional uint64 nonce = 10;
6464
optional uint64 orig_sequence_number = 11;
65-
65+
repeated Attribute extended_attributes = 12;
6666
/*
6767
optional CustomEntryType custom_entry_type = 9;
6868
@@ -71,6 +71,10 @@ message WALKey {
7171
}
7272
*/
7373
}
74+
message Attribute {
75+
required string key = 1;
76+
required bytes value = 2;
77+
}
7478

7579
enum ScopeType {
7680
REPLICATION_SCOPE_LOCAL = 0;

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import java.io.IOException;
2323
import java.util.ArrayList;
24+
import java.util.Map;
2425
import java.util.NavigableMap;
2526
import java.util.function.Function;
2627
import org.apache.hadoop.conf.Configuration;
@@ -71,7 +72,7 @@ public static WALKeyImpl writeCompactionMarker(WAL wal,
7172
MultiVersionConcurrencyControl mvcc)
7273
throws IOException {
7374
WALKeyImpl walKey =
74-
writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc);
75+
writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc, null);
7576
if (LOG.isTraceEnabled()) {
7677
LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
7778
}
@@ -87,7 +88,7 @@ public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap<byte[], Integer>
8788
RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc)
8889
throws IOException {
8990
WALKeyImpl walKey = doFullAppendTransaction(wal, replicationScope, hri,
90-
WALEdit.createFlushWALEdit(hri, f), mvcc, sync);
91+
WALEdit.createFlushWALEdit(hri, f), mvcc, null, sync);
9192
if (LOG.isTraceEnabled()) {
9293
LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
9394
}
@@ -103,7 +104,7 @@ public static WALKeyImpl writeRegionEventMarker(WAL wal,
103104
final RegionEventDescriptor r, final MultiVersionConcurrencyControl mvcc)
104105
throws IOException {
105106
WALKeyImpl walKey = writeMarker(wal, replicationScope, hri,
106-
WALEdit.createRegionEventWALEdit(hri, r), mvcc);
107+
WALEdit.createRegionEventWALEdit(hri, r), mvcc, null);
107108
if (LOG.isTraceEnabled()) {
108109
LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
109110
}
@@ -125,19 +126,23 @@ public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal,
125126
final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc)
126127
throws IOException {
127128
WALKeyImpl walKey =
128-
writeMarker(wal, replicationScope, hri, WALEdit.createBulkLoadEvent(hri, desc), mvcc);
129+
writeMarker(wal, replicationScope, hri, WALEdit.createBulkLoadEvent(hri, desc), mvcc, null);
129130
if (LOG.isTraceEnabled()) {
130131
LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc));
131132
}
132133
return walKey;
133134
}
134135

135136
private static WALKeyImpl writeMarker(final WAL wal,
136-
final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri,
137-
final WALEdit edit, final MultiVersionConcurrencyControl mvcc)
137+
final NavigableMap<byte[], Integer> replicationScope,
138+
final RegionInfo hri,
139+
final WALEdit edit,
140+
final MultiVersionConcurrencyControl mvcc,
141+
final Map<String, byte[]> extendedAttributes)
138142
throws IOException {
139143
// If sync == true in below, then timeout is not used; safe to pass UNSPECIFIED_TIMEOUT
140-
return doFullAppendTransaction(wal, replicationScope, hri, edit, mvcc, true);
144+
return doFullAppendTransaction(wal, replicationScope, hri,
145+
edit, mvcc, extendedAttributes, true);
141146
}
142147

143148
/**
@@ -150,11 +155,12 @@ private static WALKeyImpl writeMarker(final WAL wal,
150155
*/
151156
public static WALKeyImpl doFullAppendTransaction(final WAL wal,
152157
final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri,
153-
final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync)
158+
final WALEdit edit, final MultiVersionConcurrencyControl mvcc,
159+
final Map<String, byte[]> extendedAttributes, final boolean sync)
154160
throws IOException {
155161
// TODO: Pass in current time to use?
156162
WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(),
157-
System.currentTimeMillis(), mvcc, replicationScope);
163+
System.currentTimeMillis(), mvcc, replicationScope, extendedAttributes);
158164
long trx = MultiVersionConcurrencyControl.NONE;
159165
try {
160166
trx = wal.append(hri, walKey, edit, false);

‎hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,21 @@ default long getNonce() {
8686
*/
8787
long getOrigLogSeqNum();
8888

89+
/**
90+
* Return a named String value injected into the WALKey during processing, such as by a
91+
* coprocessor
92+
* @param attributeKey The key of a key / value pair
93+
*/
94+
default byte[] getExtendedAttribute(String attributeKey){
95+
return null;
96+
}
97+
98+
/**
99+
* Returns a map of all extended attributes injected into this WAL key.
100+
*/
101+
default Map<String, byte[]> getExtendedAttributes() {
102+
return new HashMap<>();
103+
}
89104
/**
90105
* Produces a string map for this key. Useful for programmatic use and
91106
* manipulation of the data stored in an WALKeyImpl, for example, printing
@@ -98,6 +113,12 @@ default Map<String, Object> toStringMap() {
98113
stringMap.put("table", getTableName());
99114
stringMap.put("region", Bytes.toStringBinary(getEncodedRegionName()));
100115
stringMap.put("sequence", getSequenceId());
116+
Map<String, byte[]> extendedAttributes = getExtendedAttributes();
117+
if (extendedAttributes != null){
118+
for (Map.Entry<String, byte[]> entry : extendedAttributes.entrySet()){
119+
stringMap.put(entry.getKey(), Bytes.toStringBinary(entry.getValue()));
120+
}
121+
}
101122
return stringMap;
102123
}
103124
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java

Lines changed: 71 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@
1919

2020
import java.io.IOException;
2121
import java.util.ArrayList;
22+
import java.util.HashMap;
2223
import java.util.List;
2324
import java.util.Map;
2425
import java.util.NavigableMap;
2526
import java.util.TreeMap;
2627
import java.util.UUID;
28+
2729
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
2830
import org.apache.hadoop.hbase.HConstants;
2931
import org.apache.hadoop.hbase.TableName;
@@ -116,14 +118,16 @@ public void setWriteEntry(MultiVersionConcurrencyControl.WriteEntry writeEntry)
116118
*/
117119
private MultiVersionConcurrencyControl.WriteEntry writeEntry;
118120

121+
private Map<String, byte[]> extendedAttributes;
122+
119123
public WALKeyImpl() {
120124
init(null, null, 0L, HConstants.LATEST_TIMESTAMP,
121-
new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null);
125+
new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null, null);
122126
}
123127

124128
public WALKeyImpl(final NavigableMap<byte[], Integer> replicationScope) {
125129
init(null, null, 0L, HConstants.LATEST_TIMESTAMP,
126-
new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, replicationScope);
130+
new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, replicationScope, null);
127131
}
128132

129133
@VisibleForTesting
@@ -132,7 +136,7 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, lon
132136
List<UUID> clusterIds = new ArrayList<>(1);
133137
clusterIds.add(clusterId);
134138
init(encodedRegionName, tablename, logSeqNum, now, clusterIds, HConstants.NO_NONCE,
135-
HConstants.NO_NONCE, null, null);
139+
HConstants.NO_NONCE, null, null, null);
136140
}
137141

138142
@VisibleForTesting
@@ -141,7 +145,7 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, lon
141145
List<UUID> clusterIds = new ArrayList<>(1);
142146
clusterIds.add(clusterId);
143147
init(encodedRegionName, tablename, logSeqNum, now, clusterIds, HConstants.NO_NONCE,
144-
HConstants.NO_NONCE, mvcc, null);
148+
HConstants.NO_NONCE, mvcc, null, null);
145149
}
146150

147151
// TODO: Fix being able to pass in sequenceid.
@@ -153,20 +157,28 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, fin
153157
EMPTY_UUIDS,
154158
HConstants.NO_NONCE,
155159
HConstants.NO_NONCE,
156-
null, null);
160+
null, null, null);
157161
}
158162

159163
// TODO: Fix being able to pass in sequenceid.
160164
public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now,
161165
final NavigableMap<byte[], Integer> replicationScope) {
162166
init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE,
163-
HConstants.NO_NONCE, null, replicationScope);
167+
HConstants.NO_NONCE, null, replicationScope, null);
164168
}
165169

166170
public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now,
167171
MultiVersionConcurrencyControl mvcc, final NavigableMap<byte[], Integer> replicationScope) {
168172
init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE,
169-
HConstants.NO_NONCE, mvcc, replicationScope);
173+
HConstants.NO_NONCE, mvcc, replicationScope, null);
174+
}
175+
176+
public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now,
177+
MultiVersionConcurrencyControl mvcc,
178+
final NavigableMap<byte[], Integer> replicationScope,
179+
Map<String, byte[]> extendedAttributes) {
180+
init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE,
181+
HConstants.NO_NONCE, mvcc, replicationScope, extendedAttributes);
170182
}
171183

172184
public WALKeyImpl(final byte[] encodedRegionName,
@@ -180,7 +192,7 @@ public WALKeyImpl(final byte[] encodedRegionName,
180192
EMPTY_UUIDS,
181193
HConstants.NO_NONCE,
182194
HConstants.NO_NONCE,
183-
mvcc, null);
195+
mvcc, null, null);
184196
}
185197

186198
/**
@@ -206,7 +218,7 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, lon
206218
final long now, List<UUID> clusterIds, long nonceGroup, long nonce,
207219
MultiVersionConcurrencyControl mvcc, final NavigableMap<byte[], Integer> replicationScope) {
208220
init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc,
209-
replicationScope);
221+
replicationScope, null);
210222
}
211223

212224
/**
@@ -231,7 +243,8 @@ public WALKeyImpl(final byte[] encodedRegionName,
231243
long nonceGroup,
232244
long nonce,
233245
MultiVersionConcurrencyControl mvcc) {
234-
init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, null);
246+
init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup,
247+
nonce, mvcc, null, null);
235248
}
236249

237250
/**
@@ -252,7 +265,7 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename,
252265
final long now, List<UUID> clusterIds, long nonceGroup,
253266
final long nonce, final MultiVersionConcurrencyControl mvcc) {
254267
init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc,
255-
null);
268+
null, null);
256269
}
257270

258271
/**
@@ -275,7 +288,7 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename,
275288
final long nonce, final MultiVersionConcurrencyControl mvcc,
276289
NavigableMap<byte[], Integer> replicationScope) {
277290
init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc,
278-
replicationScope);
291+
replicationScope, null);
279292
}
280293

281294
/**
@@ -304,7 +317,22 @@ public WALKeyImpl(final byte[] encodedRegionName,
304317
EMPTY_UUIDS,
305318
nonceGroup,
306319
nonce,
307-
mvcc, null);
320+
mvcc, null, null);
321+
}
322+
323+
public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename,
324+
final long now, List<UUID> clusterIds, long nonceGroup,
325+
final long nonce, final MultiVersionConcurrencyControl mvcc,
326+
NavigableMap<byte[], Integer> replicationScope,
327+
Map<String, byte[]> extendedAttributes){
328+
init(encodedRegionName,
329+
tablename,
330+
NO_SEQUENCE_ID,
331+
now,
332+
clusterIds,
333+
nonceGroup,
334+
nonce,
335+
mvcc, replicationScope, extendedAttributes);
308336
}
309337

310338
@InterfaceAudience.Private
@@ -316,7 +344,8 @@ protected void init(final byte[] encodedRegionName,
316344
long nonceGroup,
317345
long nonce,
318346
MultiVersionConcurrencyControl mvcc,
319-
NavigableMap<byte[], Integer> replicationScope) {
347+
NavigableMap<byte[], Integer> replicationScope,
348+
Map<String, byte[]> extendedAttributes) {
320349
this.sequenceId = logSeqNum;
321350
this.writeTime = now;
322351
this.clusterIds = clusterIds;
@@ -329,6 +358,7 @@ protected void init(final byte[] encodedRegionName,
329358
setSequenceId(logSeqNum);
330359
}
331360
this.replicationScope = replicationScope;
361+
this.extendedAttributes = extendedAttributes;
332362
}
333363

334364
// For deserialization. DO NOT USE. See setWriteEntry below.
@@ -434,6 +464,17 @@ public UUID getOriginatingClusterId(){
434464
return clusterIds.isEmpty()? HConstants.DEFAULT_CLUSTER_ID: clusterIds.get(0);
435465
}
436466

467+
@Override
468+
public byte[] getExtendedAttribute(String attributeKey){
469+
return extendedAttributes != null ? extendedAttributes.get(attributeKey) : null;
470+
}
471+
472+
@Override
473+
public Map<String, byte[]> getExtendedAttributes(){
474+
return extendedAttributes != null ? new HashMap<String, byte[]>(extendedAttributes) :
475+
new HashMap<String, byte[]>();
476+
}
477+
437478
@Override
438479
public String toString() {
439480
return tablename + "/" + Bytes.toString(encodedRegionName) + "/" + sequenceId;
@@ -539,6 +580,14 @@ public WALProtos.WALKey.Builder getBuilder(WALCellCodec.ByteStringCompressor com
539580
.setScopeType(ScopeType.forNumber(e.getValue())));
540581
}
541582
}
583+
if (extendedAttributes != null){
584+
for (Map.Entry<String, byte[]> e : extendedAttributes.entrySet()){
585+
WALProtos.Attribute attr = WALProtos.Attribute.newBuilder().
586+
setKey(e.getKey()).setValue(compressor.compress(e.getValue(),
587+
CompressionContext.DictionaryIndex.TABLE)).build();
588+
builder.addExtendedAttributes(attr);
589+
}
590+
}
542591
return builder;
543592
}
544593

@@ -573,6 +622,14 @@ public void readFieldsFromPb(WALProtos.WALKey walKey,
573622
if (walKey.hasOrigSequenceNumber()) {
574623
this.origLogSeqNum = walKey.getOrigSequenceNumber();
575624
}
625+
if (walKey.getExtendedAttributesCount() > 0){
626+
this.extendedAttributes = new HashMap<>(walKey.getExtendedAttributesCount());
627+
for (WALProtos.Attribute attr : walKey.getExtendedAttributesList()){
628+
byte[] value =
629+
uncompressor.uncompress(attr.getValue(), CompressionContext.DictionaryIndex.TABLE);
630+
extendedAttributes.put(attr.getKey(), value);
631+
}
632+
}
576633
}
577634

578635
@Override

0 commit comments

Comments
 (0)