Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-25215 TestClientOperationTimeout.testScanTimeout is flaky #2583

Merged
merged 1 commit into from
Oct 26, 2020
Merged

HBASE-25215 TestClientOperationTimeout.testScanTimeout is flaky #2583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@
public class RSRpcServices implements HBaseRPCErrorHandler,
AdminService.BlockingInterface, ClientService.BlockingInterface, PriorityFunction,
ConfigurationObserver {
protected static final Logger LOG = LoggerFactory.getLogger(RSRpcServices.class);
private static final Logger LOG = LoggerFactory.getLogger(RSRpcServices.class);

/** RPC scheduler to use for the region server. */
public static final String REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS =
Expand Down
95 changes: 54 additions & 41 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/TestClientOperationTimeout.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,35 +17,40 @@
*/
package org.apache.hadoop.hbase;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.ipc.CallTimeoutException;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
Expand All @@ -69,6 +74,8 @@
@Category({ ClientTests.class, MediumTests.class })
public class TestClientOperationTimeout {

private static final Logger LOG = LoggerFactory.getLogger(TestClientOperationTimeout.class);

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestClientOperationTimeout.class);
Expand All @@ -91,7 +98,7 @@ public class TestClientOperationTimeout {
private static Table TABLE;

@BeforeClass
public static void setUpClass() throws Exception {
public static void setUp() throws Exception {
// Set RegionServer class and use default values for other options.
StartMiniClusterOption option =
StartMiniClusterOption.builder().rsClass(DelayedRegionServer.class).build();
Expand All @@ -108,21 +115,21 @@ public static void setUpClass() throws Exception {
TABLE = CONN.getTable(TABLE_NAME);
}

@Before
public void setUp() throws Exception {
DELAY_GET = 0;
DELAY_SCAN = 0;
DELAY_MUTATE = 0;
DELAY_BATCH_MUTATE = 0;
}

@AfterClass
public static void tearDown() throws Exception {
Closeables.close(TABLE, true);
Closeables.close(CONN, true);
UTIL.shutdownMiniCluster();
}

@Before
public void setUpBeforeTest() throws Exception {
DELAY_GET = 0;
DELAY_SCAN = 0;
DELAY_MUTATE = 0;
DELAY_BATCH_MUTATE = 0;
}

/**
* Tests that a get on a table throws {@link RetriesExhaustedException} when the operation takes
* longer than 'hbase.client.operation.timeout'.
Expand All @@ -132,10 +139,11 @@ public void testGetTimeout() {
DELAY_GET = 600;
try {
TABLE.get(new Get(ROW));
Assert.fail("should not reach here");
fail("should not reach here");
} catch (Exception e) {
Assert.assertTrue(
e instanceof RetriesExhaustedException && e.getCause() instanceof CallTimeoutException);
LOG.info("Got exception for get", e);
assertThat(e, instanceOf(RetriesExhaustedException.class));
assertThat(e.getCause(), instanceOf(CallTimeoutException.class));
}
}

Expand All @@ -150,10 +158,11 @@ public void testPutTimeout() {
put.addColumn(FAMILY, QUALIFIER, VALUE);
try {
TABLE.put(put);
Assert.fail("should not reach here");
fail("should not reach here");
} catch (Exception e) {
Assert.assertTrue(
e instanceof RetriesExhaustedException && e.getCause() instanceof CallTimeoutException);
LOG.info("Got exception for put", e);
assertThat(e, instanceOf(RetriesExhaustedException.class));
assertThat(e.getCause(), instanceOf(CallTimeoutException.class));
}
}

Expand All @@ -164,20 +173,17 @@ public void testPutTimeout() {
@Test
public void testMultiPutsTimeout() {
DELAY_BATCH_MUTATE = 600;
Put put1 = new Put(ROW);
put1.addColumn(FAMILY, QUALIFIER, VALUE);
Put put2 = new Put(ROW);
put2.addColumn(FAMILY, QUALIFIER, VALUE);
List<Put> puts = new ArrayList<>();
puts.add(put1);
puts.add(put2);
Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
Put put2 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
List<Put> puts = Arrays.asList(put1, put2);
try {
TABLE.batch(puts, new Object[2]);
Assert.fail("should not reach here");
fail("should not reach here");
} catch (Exception e) {
Assert.assertTrue(
e instanceof RetriesExhaustedException && e.getCause() instanceof RetriesExhaustedException
&& e.getCause().getCause() instanceof CallTimeoutException);
LOG.info("Got exception for batch", e);
assertThat(e, instanceOf(RetriesExhaustedException.class));
assertThat(e.getCause(), instanceOf(RetriesExhaustedException.class));
assertThat(e.getCause().getCause(), instanceOf(CallTimeoutException.class));
}
}

Expand All @@ -186,19 +192,26 @@ public void testMultiPutsTimeout() {
* longer than 'hbase.client.scanner.timeout.period'.
*/
@Test
public void testScanTimeout() {
public void testScanTimeout() throws IOException, InterruptedException {
// cache the region location.
try (RegionLocator locator = TABLE.getRegionLocator()) {
locator.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY);
}
// sleep a bit to make sure the location has been cached as it is an async operation.
Thread.sleep(100);
DELAY_SCAN = 600;
try {
ResultScanner scanner = TABLE.getScanner(new Scan());
try (ResultScanner scanner = TABLE.getScanner(new Scan())) {
scanner.next();
Assert.fail("should not reach here");
fail("should not reach here");
} catch (Exception e) {
Assert.assertTrue(
e instanceof RetriesExhaustedException && e.getCause() instanceof TimeoutIOException);
LOG.info("Got exception for scan", e);
assertThat(e, instanceOf(RetriesExhaustedException.class));
assertThat(e.getCause(), instanceOf(CallTimeoutException.class));
}
}

private static class DelayedRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
public static final class DelayedRegionServer
extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
public DelayedRegionServer(Configuration conf) throws IOException, InterruptedException {
super(conf);
}
Expand All @@ -212,14 +225,14 @@ protected RSRpcServices createRpcServices() throws IOException {
/**
* This {@link RSRpcServices} class injects delay for Rpc calls and after executes super methods.
*/
public static class DelayedRSRpcServices extends RSRpcServices {
private static final class DelayedRSRpcServices extends RSRpcServices {
DelayedRSRpcServices(HRegionServer rs) throws IOException {
super(rs);
}

@Override
public ClientProtos.GetResponse get(RpcController controller, ClientProtos.GetRequest request)
throws ServiceException {
throws ServiceException {
try {
Thread.sleep(DELAY_GET);
} catch (InterruptedException e) {
Expand All @@ -230,7 +243,7 @@ public ClientProtos.GetResponse get(RpcController controller, ClientProtos.GetRe

@Override
public ClientProtos.MutateResponse mutate(RpcController rpcc,
ClientProtos.MutateRequest request) throws ServiceException {
ClientProtos.MutateRequest request) throws ServiceException {
try {
Thread.sleep(DELAY_MUTATE);
} catch (InterruptedException e) {
Expand All @@ -241,7 +254,7 @@ public ClientProtos.MutateResponse mutate(RpcController rpcc,

@Override
public ClientProtos.ScanResponse scan(RpcController controller,
ClientProtos.ScanRequest request) throws ServiceException {
ClientProtos.ScanRequest request) throws ServiceException {
try {
Thread.sleep(DELAY_SCAN);
} catch (InterruptedException e) {
Expand All @@ -252,7 +265,7 @@ public ClientProtos.ScanResponse scan(RpcController controller,

@Override
public ClientProtos.MultiResponse multi(RpcController rpcc, ClientProtos.MultiRequest request)
throws ServiceException {
throws ServiceException {
try {
Thread.sleep(DELAY_BATCH_MUTATE);
} catch (InterruptedException e) {
Expand Down