Skip to content

Commit f99664d

Browse files
author
Ray Mattingly
committed
add Scan to slow log payload
1 parent 143e9b4 commit f99664d

File tree

8 files changed

+227
-18
lines changed

8 files changed

+227
-18
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/OnlineLogRecord.java

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,20 @@
1717
*/
1818
package org.apache.hadoop.hbase.client;
1919

20+
import java.io.IOException;
21+
import java.util.Optional;
2022
import org.apache.commons.lang3.builder.EqualsBuilder;
2123
import org.apache.commons.lang3.builder.HashCodeBuilder;
2224
import org.apache.commons.lang3.builder.ToStringBuilder;
2325
import org.apache.hadoop.hbase.util.GsonUtil;
2426
import org.apache.yetus.audience.InterfaceAudience;
2527
import org.apache.yetus.audience.InterfaceStability;
28+
import org.slf4j.Logger;
29+
import org.slf4j.LoggerFactory;
2630

2731
import org.apache.hbase.thirdparty.com.google.gson.Gson;
2832
import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
33+
import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
2934
import org.apache.hbase.thirdparty.com.google.gson.JsonSerializer;
3035

3136
/**
@@ -36,6 +41,8 @@
3641
@InterfaceStability.Evolving
3742
final public class OnlineLogRecord extends LogEntry {
3843

44+
private static final Logger LOG = LoggerFactory.getLogger(OnlineLogRecord.class.getName());
45+
3946
// used to convert object to pretty printed format
4047
// used by toJsonPrettyPrint()
4148
private static final Gson GSON =
@@ -52,6 +59,15 @@ final public class OnlineLogRecord extends LogEntry {
5259
if (slowLogPayload.getMultiServiceCalls() == 0) {
5360
jsonObj.remove("multiServiceCalls");
5461
}
62+
if (slowLogPayload.getScan().isPresent()) {
63+
try {
64+
jsonObj.add("scan", JsonParser.parseString(slowLogPayload.getScan().get().toJSON()));
65+
} catch (IOException e) {
66+
LOG.warn("Failed to serialize scan {}", slowLogPayload.getScan().get(), e);
67+
}
68+
} else {
69+
jsonObj.remove("scan");
70+
}
5571
return jsonObj;
5672
}).create();
5773

@@ -72,6 +88,7 @@ final public class OnlineLogRecord extends LogEntry {
7288
private final int multiGetsCount;
7389
private final int multiMutationsCount;
7490
private final int multiServiceCalls;
91+
private final Optional<Scan> scan;
7592

7693
public long getStartTime() {
7794
return startTime;
@@ -136,11 +153,20 @@ public int getMultiServiceCalls() {
136153
return multiServiceCalls;
137154
}
138155

139-
private OnlineLogRecord(final long startTime, final int processingTime, final int queueTime,
156+
/**
157+
* If {@value org.apache.hadoop.hbase.HConstants#SLOW_LOG_SCAN_PAYLOAD_ENABLED} is enabled then
158+
* this value may be present and should represent the Scan that produced the given
159+
* {@link OnlineLogRecord}
160+
*/
161+
public Optional<Scan> getScan() {
162+
return scan;
163+
}
164+
165+
protected OnlineLogRecord(final long startTime, final int processingTime, final int queueTime,
140166
final long responseSize, final long blockBytesScanned, final String clientAddress,
141167
final String serverClass, final String methodName, final String callDetails, final String param,
142168
final String regionName, final String userName, final int multiGetsCount,
143-
final int multiMutationsCount, final int multiServiceCalls) {
169+
final int multiMutationsCount, final int multiServiceCalls, final Optional<Scan> scan) {
144170
this.startTime = startTime;
145171
this.processingTime = processingTime;
146172
this.queueTime = queueTime;
@@ -156,6 +182,7 @@ private OnlineLogRecord(final long startTime, final int processingTime, final in
156182
this.multiGetsCount = multiGetsCount;
157183
this.multiMutationsCount = multiMutationsCount;
158184
this.multiServiceCalls = multiServiceCalls;
185+
this.scan = scan;
159186
}
160187

161188
public static class OnlineLogRecordBuilder {
@@ -174,6 +201,7 @@ public static class OnlineLogRecordBuilder {
174201
private int multiGetsCount;
175202
private int multiMutationsCount;
176203
private int multiServiceCalls;
204+
private Optional<Scan> scan = Optional.empty();
177205

178206
public OnlineLogRecordBuilder setStartTime(long startTime) {
179207
this.startTime = startTime;
@@ -253,10 +281,15 @@ public OnlineLogRecordBuilder setMultiServiceCalls(int multiServiceCalls) {
253281
return this;
254282
}
255283

284+
public OnlineLogRecordBuilder setScan(Scan scan) {
285+
this.scan = Optional.of(scan);
286+
return this;
287+
}
288+
256289
public OnlineLogRecord build() {
257290
return new OnlineLogRecord(startTime, processingTime, queueTime, responseSize,
258291
blockBytesScanned, clientAddress, serverClass, methodName, callDetails, param, regionName,
259-
userName, multiGetsCount, multiMutationsCount, multiServiceCalls);
292+
userName, multiGetsCount, multiMutationsCount, multiServiceCalls, scan);
260293
}
261294
}
262295

@@ -280,15 +313,17 @@ public boolean equals(Object o) {
280313
.append(multiServiceCalls, that.multiServiceCalls).append(clientAddress, that.clientAddress)
281314
.append(serverClass, that.serverClass).append(methodName, that.methodName)
282315
.append(callDetails, that.callDetails).append(param, that.param)
283-
.append(regionName, that.regionName).append(userName, that.userName).isEquals();
316+
.append(regionName, that.regionName).append(userName, that.userName).append(scan, that.scan)
317+
.isEquals();
284318
}
285319

286320
@Override
287321
public int hashCode() {
288322
return new HashCodeBuilder(17, 37).append(startTime).append(processingTime).append(queueTime)
289323
.append(responseSize).append(blockBytesScanned).append(clientAddress).append(serverClass)
290324
.append(methodName).append(callDetails).append(param).append(regionName).append(userName)
291-
.append(multiGetsCount).append(multiMutationsCount).append(multiServiceCalls).toHashCode();
325+
.append(multiGetsCount).append(multiMutationsCount).append(multiServiceCalls).append(scan)
326+
.toHashCode();
292327
}
293328

294329
@Override
@@ -305,7 +340,7 @@ public String toString() {
305340
.append("methodName", methodName).append("callDetails", callDetails).append("param", param)
306341
.append("regionName", regionName).append("userName", userName)
307342
.append("multiGetsCount", multiGetsCount).append("multiMutationsCount", multiMutationsCount)
308-
.append("multiServiceCalls", multiServiceCalls).toString();
343+
.append("multiServiceCalls", multiServiceCalls).append("scan", scan).toString();
309344
}
310345

311346
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/SlowLogParams.java

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import org.apache.commons.lang3.builder.ToStringBuilder;
2424
import org.apache.yetus.audience.InterfaceAudience;
2525

26+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
27+
2628
/**
2729
* SlowLog params object that contains detailed info as params and region name : to be used for
2830
* filter purpose
@@ -32,15 +34,24 @@ public class SlowLogParams {
3234

3335
private final String regionName;
3436
private final String params;
37+
private final ClientProtos.Scan scan;
38+
39+
public SlowLogParams(String regionName, String params, ClientProtos.Scan scan) {
40+
this.regionName = regionName;
41+
this.params = params;
42+
this.scan = scan;
43+
}
3544

3645
public SlowLogParams(String regionName, String params) {
3746
this.regionName = regionName;
3847
this.params = params;
48+
this.scan = null;
3949
}
4050

4151
public SlowLogParams(String params) {
4252
this.regionName = StringUtils.EMPTY;
4353
this.params = params;
54+
this.scan = null;
4455
}
4556

4657
public String getRegionName() {
@@ -51,10 +62,14 @@ public String getParams() {
5162
return params;
5263
}
5364

65+
public ClientProtos.Scan getScan() {
66+
return scan;
67+
}
68+
5469
@Override
5570
public String toString() {
5671
return new ToStringBuilder(this).append("regionName", regionName).append("params", params)
57-
.toString();
72+
.append("scan", scan).toString();
5873
}
5974

6075
@Override
@@ -67,11 +82,11 @@ public boolean equals(Object o) {
6782
}
6883
SlowLogParams that = (SlowLogParams) o;
6984
return new EqualsBuilder().append(regionName, that.regionName).append(params, that.params)
70-
.isEquals();
85+
.append("scan", scan).isEquals();
7186
}
7287

7388
@Override
7489
public int hashCode() {
75-
return new HashCodeBuilder(17, 37).append(regionName).append(params).toHashCode();
90+
return new HashCodeBuilder(17, 37).append(regionName).append(params).append(scan).toHashCode();
7691
}
7792
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@
130130
import org.apache.hadoop.hbase.util.VersionInfo;
131131
import org.apache.hadoop.ipc.RemoteException;
132132
import org.apache.yetus.audience.InterfaceAudience;
133+
import org.slf4j.Logger;
134+
import org.slf4j.LoggerFactory;
133135

134136
import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams;
135137
import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
@@ -231,6 +233,8 @@
231233
@InterfaceAudience.Private // TODO: some clients (Hive, etc) use this class
232234
public final class ProtobufUtil {
233235

236+
private static final Logger LOG = LoggerFactory.getLogger(ProtobufUtil.class.getName());
237+
234238
private ProtobufUtil() {
235239
}
236240

@@ -2144,15 +2148,19 @@ private static String getStringForByteString(ByteString bs) {
21442148
* @param message Message object {@link Message}
21452149
* @return SlowLogParams with regionName(for filter queries) and params
21462150
*/
2147-
public static SlowLogParams getSlowLogParams(Message message) {
2151+
public static SlowLogParams getSlowLogParams(Message message, boolean slowLogScanPayloadEnabled) {
21482152
if (message == null) {
21492153
return null;
21502154
}
21512155
if (message instanceof ScanRequest) {
21522156
ScanRequest scanRequest = (ScanRequest) message;
21532157
String regionName = getStringForByteString(scanRequest.getRegion().getValue());
21542158
String params = TextFormat.shortDebugString(message);
2155-
return new SlowLogParams(regionName, params);
2159+
if (slowLogScanPayloadEnabled) {
2160+
return new SlowLogParams(regionName, params, scanRequest.getScan());
2161+
} else {
2162+
return new SlowLogParams(regionName, params);
2163+
}
21562164
} else if (message instanceof MutationProto) {
21572165
MutationProto mutationProto = (MutationProto) message;
21582166
String params = "type= " + mutationProto.getMutateType().toString();
@@ -3367,7 +3375,7 @@ public static Set<String> toCompactedStoreFiles(byte[] bytes) throws IOException
33673375
* @return SlowLog Payload for client usecase
33683376
*/
33693377
private static LogEntry getSlowLogRecord(final TooSlowLog.SlowLogPayload slowLogPayload) {
3370-
OnlineLogRecord onlineLogRecord =
3378+
OnlineLogRecord.OnlineLogRecordBuilder onlineLogRecord =
33713379
new OnlineLogRecord.OnlineLogRecordBuilder().setCallDetails(slowLogPayload.getCallDetails())
33723380
.setClientAddress(slowLogPayload.getClientAddress())
33733381
.setMethodName(slowLogPayload.getMethodName())
@@ -3379,8 +3387,15 @@ private static LogEntry getSlowLogRecord(final TooSlowLog.SlowLogPayload slowLog
33793387
.setResponseSize(slowLogPayload.getResponseSize())
33803388
.setBlockBytesScanned(slowLogPayload.getBlockBytesScanned())
33813389
.setServerClass(slowLogPayload.getServerClass()).setStartTime(slowLogPayload.getStartTime())
3382-
.setUserName(slowLogPayload.getUserName()).build();
3383-
return onlineLogRecord;
3390+
.setUserName(slowLogPayload.getUserName());
3391+
if (slowLogPayload.hasScan()) {
3392+
try {
3393+
onlineLogRecord.setScan(ProtobufUtil.toScan(slowLogPayload.getScan()));
3394+
} catch (Exception e) {
3395+
LOG.warn("Failed to convert Scan proto {}", slowLogPayload.getScan(), e);
3396+
}
3397+
}
3398+
return onlineLogRecord.build();
33843399
}
33853400

33863401
/**

‎hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.client;
19+
20+
import java.util.Optional;
21+
import org.apache.hadoop.hbase.HBaseClassTestRule;
22+
import org.apache.hadoop.hbase.testclassification.ClientTests;
23+
import org.apache.hadoop.hbase.testclassification.SmallTests;
24+
import org.apache.hadoop.hbase.util.Bytes;
25+
import org.junit.Assert;
26+
import org.junit.ClassRule;
27+
import org.junit.Test;
28+
import org.junit.experimental.categories.Category;
29+
30+
@Category({ ClientTests.class, SmallTests.class })
31+
public class TestOnlineLogRecord {
32+
33+
@ClassRule
34+
public static final HBaseClassTestRule CLASS_RULE =
35+
HBaseClassTestRule.forClass(TestOnlineLogRecord.class);
36+
37+
@Test
38+
public void itSerializesScan() {
39+
Scan scan = new Scan();
40+
scan.withStartRow(Bytes.toBytes(123));
41+
scan.withStopRow(Bytes.toBytes(456));
42+
String expectedOutput = "{\n" + " \"startTime\": 1,\n" + " \"processingTime\": 2,\n"
43+
+ " \"queueTime\": 3,\n" + " \"responseSize\": 4,\n" + " \"blockBytesScanned\": 5,\n"
44+
+ " \"multiGetsCount\": 6,\n" + " \"multiMutationsCount\": 7,\n" + " \"scan\": {\n"
45+
+ " \"startRow\": \"\\\\x00\\\\x00\\\\x00{\",\n"
46+
+ " \"stopRow\": \"\\\\x00\\\\x00\\\\x01\\\\xC8\",\n" + " \"batch\": -1,\n"
47+
+ " \"cacheBlocks\": true,\n" + " \"totalColumns\": 0,\n"
48+
+ " \"maxResultSize\": \"-1\",\n" + " \"families\": {},\n" + " \"caching\": -1,\n"
49+
+ " \"maxVersions\": 1,\n" + " \"timeRange\": [\n" + " \"0\",\n"
50+
+ " \"9223372036854775807\"\n" + " ]\n" + " }\n" + "}";
51+
OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null,
52+
6, 7, 0, Optional.of(scan));
53+
String actualOutput = o.toJsonPrettyPrint();
54+
Assert.assertEquals(actualOutput, expectedOutput);
55+
}
56+
}

‎hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1569,6 +1569,9 @@ public enum OperationStatusCode {
15691569
// Default 10 mins.
15701570
public static final int DEFAULT_SLOW_LOG_SYS_TABLE_CHORE_DURATION = 10 * 60 * 1000;
15711571

1572+
public static final String SLOW_LOG_SCAN_PAYLOAD_ENABLED = "hbase.slowlog.scan.payload.enabled";
1573+
public static final boolean SLOW_LOG_SCAN_PAYLOAD_ENABLED_DEFAULT = false;
1574+
15721575
public static final String SHELL_TIMESTAMP_FORMAT_EPOCH_KEY =
15731576
"hbase.shell.timestamp.format.epoch";
15741577

‎hbase-protocol-shaded/src/main/protobuf/server/region/TooSlowLog.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ option java_outer_classname = "TooSlowLog";
2727
option java_generate_equals_and_hash = true;
2828
option optimize_for = SPEED;
2929

30+
import "client/Client.proto";
31+
3032
message SlowLogPayload {
3133
required int64 start_time = 1;
3234
required int32 processing_time = 2;
@@ -45,6 +47,7 @@ message SlowLogPayload {
4547
required Type type = 15;
4648

4749
optional int64 block_bytes_scanned = 16;
50+
optional Scan scan = 17;
4851

4952
// SLOW_LOG is RPC call slow in nature whereas LARGE_LOG is RPC call quite large.
5053
// Majority of times, slow logs are also large logs and hence, ALL is combination of

0 commit comments

Comments
 (0)