Skip to content

Commit fcc0071

Browse files
ddupginfraio
ddupg
authored and
infraio
committed
HBASE-23345 Table need to replication unless all of cfs are excluded
Signed-off-by: Guanghao Zhang <zghao@apache.org>
1 parent 0cd1fa9 commit fcc0071

File tree

10 files changed

+287
-357
lines changed

10 files changed

+287
-357
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -366,22 +366,31 @@ public String toString() {
366366
* @return true if the table need replicate to the peer cluster
367367
*/
368368
public boolean needToReplicate(TableName table) {
369+
String namespace = table.getNamespaceAsString();
369370
if (replicateAllUserTables) {
370-
if (excludeNamespaces != null && excludeNamespaces.contains(table.getNamespaceAsString())) {
371+
// replicate all user tables, but filter by exclude namespaces and table-cfs config
372+
if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
371373
return false;
372374
}
373-
if (excludeTableCFsMap != null && excludeTableCFsMap.containsKey(table)) {
374-
return false;
375+
// trap here, must check existence first since HashMap allows null value.
376+
if (excludeTableCFsMap == null || !excludeTableCFsMap.containsKey(table)) {
377+
return true;
375378
}
376-
return true;
379+
Collection<String> cfs = excludeTableCFsMap.get(table);
380+
// if cfs is null or empty then we can make sure that we do not need to replicate this table,
381+
// otherwise, we may still need to replicate the table but filter out some families.
382+
return cfs != null && !cfs.isEmpty();
377383
} else {
378-
if (namespaces != null && namespaces.contains(table.getNamespaceAsString())) {
379-
return true;
384+
// Not replicate all user tables, so filter by namespaces and table-cfs config
385+
if (namespaces == null && tableCFsMap == null) {
386+
return false;
380387
}
381-
if (tableCFsMap != null && tableCFsMap.containsKey(table)) {
388+
// First filter by namespaces config
389+
// If table's namespace in peer config, all the tables data are applicable for replication
390+
if (namespaces != null && namespaces.contains(namespace)) {
382391
return true;
383392
}
384-
return false;
393+
return tableCFsMap != null && tableCFsMap.containsKey(table);
385394
}
386395
}
387396
}

‎hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,17 @@
1717
*/
1818
package org.apache.hadoop.hbase.replication;
1919

20+
import java.util.HashMap;
21+
import java.util.HashSet;
22+
import java.util.List;
23+
import java.util.Map;
24+
import java.util.Set;
2025
import org.apache.hadoop.hbase.HBaseClassTestRule;
26+
import org.apache.hadoop.hbase.TableName;
2127
import org.apache.hadoop.hbase.testclassification.ClientTests;
2228
import org.apache.hadoop.hbase.testclassification.SmallTests;
2329
import org.apache.hadoop.hbase.util.BuilderStyleTest;
30+
import org.junit.Assert;
2431
import org.junit.ClassRule;
2532
import org.junit.Test;
2633
import org.junit.experimental.categories.Category;
@@ -32,6 +39,9 @@ public class TestReplicationPeerConfig {
3239
public static final HBaseClassTestRule CLASS_RULE =
3340
HBaseClassTestRule.forClass(TestReplicationPeerConfig.class);
3441

42+
private static TableName TABLE_A = TableName.valueOf("replication", "testA");
43+
private static TableName TABLE_B = TableName.valueOf("replication", "testB");
44+
3545
@Test
3646
public void testClassMethodsAreBuilderStyle() {
3747
/* ReplicationPeerConfig should have a builder style setup where setXXX/addXXX methods
@@ -48,4 +58,196 @@ public void testClassMethodsAreBuilderStyle() {
4858

4959
BuilderStyleTest.assertClassesAreBuilderStyle(ReplicationPeerConfig.class);
5060
}
61+
62+
@Test
63+
public void testNeedToReplicateWithReplicatingAll() {
64+
ReplicationPeerConfig peerConfig;
65+
ReplicationPeerConfig.ReplicationPeerConfigBuilderImpl builder =
66+
new ReplicationPeerConfig.ReplicationPeerConfigBuilderImpl();
67+
Map<TableName, List<String>> tableCfs = new HashMap<>();
68+
Set<String> namespaces = new HashSet<>();
69+
70+
// 1. replication_all flag is true, no namespaces and table-cfs config
71+
builder.setReplicateAllUserTables(true);
72+
peerConfig = builder.build();
73+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
74+
75+
// 2. replicate_all flag is true, and config in excludedTableCfs
76+
builder.setExcludeNamespaces(null);
77+
// empty map
78+
tableCfs = new HashMap<>();
79+
builder.setReplicateAllUserTables(true);
80+
builder.setExcludeTableCFsMap(tableCfs);
81+
peerConfig = builder.build();
82+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
83+
84+
// table testB
85+
tableCfs = new HashMap<>();
86+
tableCfs.put(TABLE_B, null);
87+
builder.setReplicateAllUserTables(true);
88+
builder.setExcludeTableCFsMap(tableCfs);
89+
peerConfig = builder.build();
90+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
91+
92+
// table testA
93+
tableCfs = new HashMap<>();
94+
tableCfs.put(TABLE_A, null);
95+
builder.setReplicateAllUserTables(true);
96+
builder.setExcludeTableCFsMap(tableCfs);
97+
peerConfig = builder.build();
98+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
99+
100+
// 3. replicate_all flag is true, and config in excludeNamespaces
101+
builder.setExcludeTableCFsMap(null);
102+
// empty set
103+
namespaces = new HashSet<>();
104+
builder.setReplicateAllUserTables(true);
105+
builder.setExcludeNamespaces(namespaces);
106+
peerConfig = builder.build();
107+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
108+
109+
// namespace default
110+
namespaces = new HashSet<>();
111+
namespaces.add("default");
112+
builder.setReplicateAllUserTables(true);
113+
builder.setExcludeNamespaces(namespaces);
114+
peerConfig = builder.build();
115+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
116+
117+
// namespace replication
118+
namespaces = new HashSet<>();
119+
namespaces.add("replication");
120+
builder.setReplicateAllUserTables(true);
121+
builder.setExcludeNamespaces(namespaces);
122+
peerConfig = builder.build();
123+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
124+
125+
// 4. replicate_all flag is true, and config excludeNamespaces and excludedTableCfs both
126+
// Namespaces config doesn't conflict with table-cfs config
127+
namespaces = new HashSet<>();
128+
tableCfs = new HashMap<>();
129+
namespaces.add("replication");
130+
tableCfs.put(TABLE_A, null);
131+
builder.setReplicateAllUserTables(true);
132+
builder.setExcludeTableCFsMap(tableCfs);
133+
builder.setExcludeNamespaces(namespaces);
134+
peerConfig = builder.build();
135+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
136+
137+
// Namespaces config conflicts with table-cfs config
138+
namespaces = new HashSet<>();
139+
tableCfs = new HashMap<>();
140+
namespaces.add("default");
141+
tableCfs.put(TABLE_A, null);
142+
builder.setReplicateAllUserTables(true);
143+
builder.setExcludeTableCFsMap(tableCfs);
144+
builder.setExcludeNamespaces(namespaces);
145+
peerConfig = builder.build();
146+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
147+
148+
namespaces = new HashSet<>();
149+
tableCfs = new HashMap<>();
150+
namespaces.add("replication");
151+
tableCfs.put(TABLE_B, null);
152+
builder.setReplicateAllUserTables(true);
153+
builder.setExcludeTableCFsMap(tableCfs);
154+
builder.setExcludeNamespaces(namespaces);
155+
peerConfig = builder.build();
156+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
157+
158+
}
159+
160+
@Test
161+
public void testNeedToReplicateWithoutReplicatingAll() {
162+
ReplicationPeerConfig peerConfig;
163+
ReplicationPeerConfig.ReplicationPeerConfigBuilderImpl builder =
164+
new ReplicationPeerConfig.ReplicationPeerConfigBuilderImpl();
165+
Map<TableName, List<String>> tableCfs = new HashMap<>();
166+
Set<String> namespaces = new HashSet<>();
167+
168+
// 1. replication_all flag is false, no namespaces and table-cfs config
169+
builder.setReplicateAllUserTables(false);
170+
peerConfig = builder.build();
171+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
172+
173+
// 2. replicate_all flag is false, and only config table-cfs in peer
174+
// empty map
175+
builder.setReplicateAllUserTables(false);
176+
builder.setTableCFsMap(tableCfs);
177+
peerConfig = builder.build();
178+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
179+
180+
// table testB
181+
tableCfs = new HashMap<>();
182+
tableCfs.put(TABLE_B, null);
183+
builder.setReplicateAllUserTables(false);
184+
builder.setTableCFsMap(tableCfs);
185+
peerConfig = builder.build();
186+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
187+
188+
// table testA
189+
tableCfs = new HashMap<>();
190+
tableCfs.put(TABLE_A, null);
191+
builder.setReplicateAllUserTables(false);
192+
builder.setTableCFsMap(tableCfs);
193+
peerConfig = builder.build();
194+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
195+
196+
// 3. replication_all flag is false, and only config namespace in peer
197+
builder.setTableCFsMap(null);
198+
// empty set
199+
builder.setReplicateAllUserTables(false);
200+
builder.setNamespaces(namespaces);
201+
peerConfig = builder.build();
202+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
203+
204+
// namespace default
205+
namespaces = new HashSet<>();
206+
namespaces.add("default");
207+
builder.setReplicateAllUserTables(false);
208+
builder.setNamespaces(namespaces);
209+
peerConfig = builder.build();
210+
Assert.assertFalse(peerConfig.needToReplicate(TABLE_A));
211+
212+
// namespace replication
213+
namespaces = new HashSet<>();
214+
namespaces.add("replication");
215+
builder.setReplicateAllUserTables(false);
216+
builder.setNamespaces(namespaces);
217+
peerConfig = builder.build();
218+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
219+
220+
// 4. replicate_all flag is false, and config namespaces and table-cfs both
221+
// Namespaces config doesn't conflict with table-cfs config
222+
namespaces = new HashSet<>();
223+
tableCfs = new HashMap<>();
224+
namespaces.add("replication");
225+
tableCfs.put(TABLE_A, null);
226+
builder.setReplicateAllUserTables(false);
227+
builder.setTableCFsMap(tableCfs);
228+
builder.setNamespaces(namespaces);
229+
peerConfig = builder.build();
230+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
231+
232+
// Namespaces config conflicts with table-cfs config
233+
namespaces = new HashSet<>();
234+
tableCfs = new HashMap<>();
235+
namespaces.add("default");
236+
tableCfs.put(TABLE_A, null);
237+
builder.setReplicateAllUserTables(false);
238+
builder.setTableCFsMap(tableCfs);
239+
builder.setNamespaces(namespaces);
240+
peerConfig = builder.build();
241+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
242+
243+
namespaces = new HashSet<>();
244+
tableCfs = new HashMap<>();
245+
namespaces.add("replication");
246+
tableCfs.put(TABLE_B, null);
247+
builder.setReplicateAllUserTables(false);
248+
builder.setTableCFsMap(tableCfs);
249+
builder.setNamespaces(namespaces);
250+
peerConfig = builder.build();
251+
Assert.assertTrue(peerConfig.needToReplicate(TABLE_A));
252+
}
51253
}

‎hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java

Lines changed: 9 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -135,40 +135,16 @@ public static boolean isReplicationForBulkLoadDataEnabled(final Configuration c)
135135
}
136136

137137
/**
138-
* Returns whether we should replicate the given table.
138+
* Get the adaptive timeout value when performing a retry
139139
*/
140-
public static boolean contains(ReplicationPeerConfig peerConfig, TableName tableName) {
141-
String namespace = tableName.getNamespaceAsString();
142-
if (peerConfig.replicateAllUserTables()) {
143-
// replicate all user tables, but filter by exclude namespaces and table-cfs config
144-
Set<String> excludeNamespaces = peerConfig.getExcludeNamespaces();
145-
if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
146-
return false;
147-
}
148-
Map<TableName, List<String>> excludedTableCFs = peerConfig.getExcludeTableCFsMap();
149-
// trap here, must check existence first since HashMap allows null value.
150-
if (excludedTableCFs == null || !excludedTableCFs.containsKey(tableName)) {
151-
return true;
152-
}
153-
List<String> cfs = excludedTableCFs.get(tableName);
154-
// if cfs is null or empty then we can make sure that we do not need to replicate this table,
155-
// otherwise, we may still need to replicate the table but filter out some families.
156-
return cfs != null && !cfs.isEmpty();
157-
} else {
158-
// Not replicate all user tables, so filter by namespaces and table-cfs config
159-
Set<String> namespaces = peerConfig.getNamespaces();
160-
Map<TableName, List<String>> tableCFs = peerConfig.getTableCFsMap();
161-
162-
if (namespaces == null && tableCFs == null) {
163-
return false;
164-
}
165-
166-
// First filter by namespaces config
167-
// If table's namespace in peer config, all the tables data are applicable for replication
168-
if (namespaces != null && namespaces.contains(namespace)) {
169-
return true;
170-
}
171-
return tableCFs != null && tableCFs.containsKey(tableName);
140+
public static int getAdaptiveTimeout(final int initialValue, final int retries) {
141+
int ntries = retries;
142+
if (ntries >= HConstants.RETRY_BACKOFF.length) {
143+
ntries = HConstants.RETRY_BACKOFF.length - 1;
144+
}
145+
if (ntries < 0) {
146+
ntries = 0;
172147
}
148+
return initialValue * HConstants.RETRY_BACKOFF[ntries];
173149
}
174150
}

0 commit comments

Comments
 (0)