Skip to content

Commit ab4351a

Browse files
authored
HBASE-26869 RSRpcServices.scan should deep clone cells when RpcCallCo… (#4249)
1 parent b67c16a commit ab4351a

File tree

4 files changed

+252
-7
lines changed

4 files changed

+252
-7
lines changed

hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import java.io.IOException;
2626
import java.nio.ByteBuffer;
27+
import java.util.ArrayList;
2728
import java.util.Arrays;
2829
import java.util.Iterator;
2930
import java.util.List;
@@ -849,4 +850,18 @@ public final static int compareColumns(Cell left, byte[] right, int rfoffset, in
849850
if (diff != 0) return diff;
850851
return compareQualifiers(left, right, rqoffset, rqlength);
851852
}
853+
854+
public static void cloneIfNecessary(ArrayList<Cell> cells) {
855+
if (cells == null || cells.isEmpty()) {
856+
return;
857+
}
858+
for (int i = 0; i < cells.size(); i++) {
859+
Cell cell = cells.get(i);
860+
cells.set(i, cloneIfNecessary(cell));
861+
}
862+
}
863+
864+
public static Cell cloneIfNecessary(Cell cell) {
865+
return (cell instanceof ByteBufferExtendedCell ? KeyValueUtil.copyToNewKeyValue(cell) : cell);
866+
}
852867
}

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@
7878
import org.apache.hadoop.fs.FileSystem;
7979
import org.apache.hadoop.fs.LocatedFileStatus;
8080
import org.apache.hadoop.fs.Path;
81-
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
8281
import org.apache.hadoop.hbase.Cell;
8382
import org.apache.hadoop.hbase.CellBuilderType;
8483
import org.apache.hadoop.hbase.CellComparator;
@@ -94,7 +93,6 @@
9493
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
9594
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
9695
import org.apache.hadoop.hbase.KeyValue;
97-
import org.apache.hadoop.hbase.KeyValueUtil;
9896
import org.apache.hadoop.hbase.MetaCellComparator;
9997
import org.apache.hadoop.hbase.NamespaceDescriptor;
10098
import org.apache.hadoop.hbase.NotServingRegionException;
@@ -7871,8 +7869,8 @@ private List<Cell> getInternal(Get get, boolean withCoprocessor, long nonceGroup
78717869
// This can be an EXPENSIVE call. It may make an extra copy from offheap to onheap buffers.
78727870
// See more details in HBASE-26036.
78737871
for (Cell cell : tmp) {
7874-
results.add(cell instanceof ByteBufferExtendedCell ?
7875-
KeyValueUtil.copyToNewKeyValue(cell) : cell);
7872+
results.add(
7873+
CellUtil.cloneIfNecessary(cell));
78767874
}
78777875
}
78787876

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3307,7 +3307,7 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan
33073307
// This is cells inside a row. Default size is 10 so if many versions or many cfs,
33083308
// then we'll resize. Resizings show in profiler. Set it higher than 10. For now
33093309
// arbitrary 32. TODO: keep record of general size of results being returned.
3310-
List<Cell> values = new ArrayList<>(32);
3310+
ArrayList<Cell> values = new ArrayList<>(32);
33113311
region.startRegionOperation(Operation.SCAN);
33123312
long before = EnvironmentEdgeManager.currentTime();
33133313
// Used to check if we've matched the row limit set on the Scan
@@ -3368,9 +3368,16 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan
33683368
// reset the batch progress between nextRaw invocations since we don't want the
33693369
// batch progress from previous calls to affect future calls
33703370
scannerContext.setBatchProgress(0);
3371+
assert values.isEmpty();
33713372

33723373
// Collect values to be returned here
33733374
moreRows = scanner.nextRaw(values, scannerContext);
3375+
if (context == null) {
3376+
// When there is no RpcCallContext,copy EC to heap, then the scanner would close,
3377+
// This can be an EXPENSIVE call. It may make an extra copy from offheap to onheap
3378+
// buffers.See more details in HBASE-26036.
3379+
CellUtil.cloneIfNecessary(values);
3380+
}
33743381
numOfNextRawCalls++;
33753382

33763383
if (!values.isEmpty()) {
@@ -3727,14 +3734,25 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque
37273734
if (context != null) {
37283735
context.setCallBack(rsh.shippedCallback);
37293736
} else {
3730-
// When context != null, adding back the lease will be done in callback set above.
3731-
addScannerLeaseBack(lease);
3737+
// If context is null,here we call rsh.shippedCallback directly to reuse the logic in
3738+
// rsh.shippedCallback to release the internal resources in rsh,and lease is also added
3739+
// back to regionserver's LeaseManager in rsh.shippedCallback.
3740+
runShippedCallback(rsh);
37323741
}
37333742
}
37343743
quota.close();
37353744
}
37363745
}
37373746

3747+
private void runShippedCallback(RegionScannerHolder rsh) throws ServiceException {
3748+
assert rsh.shippedCallback != null;
3749+
try {
3750+
rsh.shippedCallback.run();
3751+
} catch (IOException ioe) {
3752+
throw new ServiceException(ioe);
3753+
}
3754+
}
3755+
37383756
private void closeScanner(HRegion region, RegionScanner scanner, String scannerName,
37393757
RpcCallContext context) throws IOException {
37403758
if (region.getCoprocessorHost() != null) {
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.regionserver;
19+
20+
import static org.junit.Assert.assertNull;
21+
import static org.junit.Assert.assertTrue;
22+
23+
import java.io.IOException;
24+
import java.util.Optional;
25+
import java.util.concurrent.atomic.AtomicReference;
26+
import org.apache.hadoop.conf.Configuration;
27+
import org.apache.hadoop.hbase.HBaseClassTestRule;
28+
import org.apache.hadoop.hbase.HBaseTestingUtil;
29+
import org.apache.hadoop.hbase.HConstants;
30+
import org.apache.hadoop.hbase.SingleProcessHBaseCluster.MiniHBaseClusterRegionServer;
31+
import org.apache.hadoop.hbase.TableName;
32+
import org.apache.hadoop.hbase.client.Admin;
33+
import org.apache.hadoop.hbase.client.Put;
34+
import org.apache.hadoop.hbase.client.Result;
35+
import org.apache.hadoop.hbase.client.ResultScanner;
36+
import org.apache.hadoop.hbase.client.Scan;
37+
import org.apache.hadoop.hbase.client.Table;
38+
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
39+
import org.apache.hadoop.hbase.io.DeallocateRewriteByteBuffAllocator;
40+
import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
41+
import org.apache.hadoop.hbase.ipc.RpcCall;
42+
import org.apache.hadoop.hbase.ipc.RpcServer;
43+
import org.apache.hadoop.hbase.testclassification.LargeTests;
44+
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
45+
import org.apache.hadoop.hbase.util.Bytes;
46+
import org.junit.AfterClass;
47+
import org.junit.BeforeClass;
48+
import org.junit.ClassRule;
49+
import org.junit.Rule;
50+
import org.junit.Test;
51+
import org.junit.experimental.categories.Category;
52+
import org.junit.rules.TestName;
53+
54+
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
55+
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
56+
57+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest;
58+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse;
59+
60+
@Category({ RegionServerTests.class, LargeTests.class })
61+
public class TestRegionServerScan {
62+
@ClassRule
63+
public static final HBaseClassTestRule CLASS_RULE =
64+
HBaseClassTestRule.forClass(TestRegionServerScan.class);
65+
66+
@Rule
67+
public TestName name = new TestName();
68+
69+
private static final byte[] CF = Bytes.toBytes("CF");
70+
private static final byte[] CQ = Bytes.toBytes("CQ");
71+
private static final byte[] VALUE = new byte[1200];
72+
73+
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
74+
private static final Configuration conf = TEST_UTIL.getConfiguration();
75+
private static Admin admin = null;
76+
static final TableName tableName = TableName.valueOf("TestRegionServerScan");
77+
static final byte[] r0 = Bytes.toBytes("row-0");
78+
static final byte[] r1 = Bytes.toBytes("row-1");
79+
static final byte[] r2 = Bytes.toBytes("row-2");
80+
81+
@BeforeClass
82+
public static void setupBeforeClass() throws Exception {
83+
/**
84+
* Use {@link DeallocateRewriteByteBuffAllocator} to rewrite the bytebuffers right after
85+
* released.
86+
*/
87+
conf.set(ByteBuffAllocator.BYTEBUFF_ALLOCATOR_CLASS,
88+
DeallocateRewriteByteBuffAllocator.class.getName());
89+
conf.setBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, true);
90+
conf.setInt(ByteBuffAllocator.MIN_ALLOCATE_SIZE_KEY, 0);
91+
conf.setInt(BlockCacheFactory.BUCKET_CACHE_WRITER_THREADS_KEY, 20);
92+
conf.setInt(ByteBuffAllocator.BUFFER_SIZE_KEY, 2048);
93+
conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
94+
conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 64);
95+
conf.setStrings(HConstants.REGION_SERVER_IMPL, MyRegionServer.class.getName());
96+
conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 30 * 60 * 1000);
97+
conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 30 * 60 * 1000);
98+
99+
conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 60 * 60 * 1000);
100+
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
101+
conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 10000);
102+
conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 1024 * 1024 * 1024);
103+
TEST_UTIL.startMiniCluster(1);
104+
admin = TEST_UTIL.getAdmin();
105+
}
106+
107+
@AfterClass
108+
public static void tearDownAfterClass() throws Exception {
109+
TEST_UTIL.shutdownMiniCluster();
110+
}
111+
112+
@Test
113+
public void testScannWhenRpcCallContextNull() throws Exception {
114+
ResultScanner resultScanner = null;
115+
Table table = null;
116+
try {
117+
table =
118+
TEST_UTIL.createTable(tableName, new byte[][] { CF }, 1, 1024, null);
119+
putToTable(table, r0);
120+
putToTable(table, r1);
121+
putToTable(table, r2);
122+
123+
admin.flush(table.getName());
124+
125+
Scan scan = new Scan();
126+
scan.setCaching(2);
127+
scan.withStartRow(r0, true).withStopRow(r2, true);
128+
129+
MyRSRpcServices.inTest = true;
130+
resultScanner = table.getScanner(scan);
131+
Result result = resultScanner.next();
132+
byte[] rowKey = result.getRow();
133+
assertTrue(Bytes.equals(r0, rowKey));
134+
135+
result = resultScanner.next();
136+
rowKey = result.getRow();
137+
assertTrue(Bytes.equals(r1, rowKey));
138+
139+
result = resultScanner.next();
140+
rowKey = result.getRow();
141+
assertTrue(Bytes.equals(r2, rowKey));
142+
assertNull(resultScanner.next());
143+
assertTrue(MyRSRpcServices.exceptionRef.get() == null);
144+
} finally {
145+
MyRSRpcServices.inTest = false;
146+
if (resultScanner != null) {
147+
resultScanner.close();
148+
}
149+
if (table != null) {
150+
table.close();
151+
}
152+
}
153+
}
154+
155+
private static void putToTable(Table table, byte[] rowkey) throws IOException {
156+
Put put = new Put(rowkey);
157+
put.addColumn(CF, CQ, VALUE);
158+
table.put(put);
159+
}
160+
161+
private static class MyRegionServer extends MiniHBaseClusterRegionServer {
162+
public MyRegionServer(Configuration conf) throws IOException, InterruptedException {
163+
super(conf);
164+
}
165+
166+
@Override
167+
protected RSRpcServices createRpcServices() throws IOException {
168+
return new MyRSRpcServices(this);
169+
}
170+
}
171+
172+
private static class MyRSRpcServices extends RSRpcServices {
173+
private static AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>(null);
174+
private static volatile boolean inTest = false;
175+
176+
public MyRSRpcServices(HRegionServer rs) throws IOException {
177+
super(rs);
178+
}
179+
180+
@Override
181+
public ScanResponse scan(RpcController controller, ScanRequest request)
182+
throws ServiceException {
183+
try {
184+
if (!inTest) {
185+
return super.scan(controller, request);
186+
}
187+
188+
HRegion region = null;
189+
if (request.hasRegion()) {
190+
region = this.getRegion(request.getRegion());
191+
}
192+
193+
if (region != null
194+
&& !tableName.equals(region.getTableDescriptor().getTableName())) {
195+
return super.scan(controller, request);
196+
}
197+
198+
ScanResponse result = null;
199+
//Simulate RpcCallContext is null for test.
200+
Optional<RpcCall> rpcCall = RpcServer.unsetCurrentCall();
201+
try {
202+
result = super.scan(controller, request);
203+
} finally {
204+
rpcCall.ifPresent(RpcServer::setCurrentCall);
205+
}
206+
return result;
207+
} catch (Throwable e) {
208+
exceptionRef.set(e);
209+
throw new ServiceException(e);
210+
}
211+
}
212+
}
213+
214+
}

0 commit comments

Comments
 (0)