Skip to content

Commit 0763a74

Browse files
chandrasekhar-188kchandrasekhar-188k
authored
HBASE-28124 Missing fields in Scan.toJSON (#5678)
Signed-off-by: Pankaj Kumar < pankajkumar@apache.org> Signed-off-by: Rajeshbabu Chintaguntla <rajeshbabu@apache.org> Signed-off-by: Duo Zhang <zhangduo@apache.org>
1 parent 0c1224c commit 0763a74

File tree

3 files changed

+144
-11
lines changed

3 files changed

+144
-11
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.NavigableSet;
2626
import java.util.TreeMap;
2727
import java.util.TreeSet;
28+
import java.util.stream.Collectors;
2829
import org.apache.hadoop.hbase.HConstants;
2930
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
3031
import org.apache.hadoop.hbase.filter.Filter;
@@ -747,7 +748,7 @@ public Map<String, Object> getFingerprint() {
747748
*/
748749
@Override
749750
public Map<String, Object> toMap(int maxCols) {
750-
// start with the fingerpring map and build on top of it
751+
// start with the fingerprint map and build on top of it
751752
Map<String, Object> map = getFingerprint();
752753
// map from families to column list replaces fingerprint's list of families
753754
Map<String, List<String>> familyColumns = new HashMap<>();
@@ -795,6 +796,34 @@ public Map<String, Object> toMap(int maxCols) {
795796
if (getId() != null) {
796797
map.put("id", getId());
797798
}
799+
map.put("includeStartRow", includeStartRow);
800+
map.put("includeStopRow", includeStopRow);
801+
map.put("allowPartialResults", allowPartialResults);
802+
map.put("storeLimit", storeLimit);
803+
map.put("storeOffset", storeOffset);
804+
map.put("reversed", reversed);
805+
if (null != asyncPrefetch) {
806+
map.put("asyncPrefetch", asyncPrefetch);
807+
}
808+
map.put("mvccReadPoint", mvccReadPoint);
809+
map.put("limit", limit);
810+
map.put("readType", readType);
811+
map.put("needCursorResult", needCursorResult);
812+
map.put("targetReplicaId", targetReplicaId);
813+
map.put("consistency", consistency);
814+
if (!colFamTimeRangeMap.isEmpty()) {
815+
Map<String, List<Long>> colFamTimeRangeMapStr = colFamTimeRangeMap.entrySet().stream()
816+
.collect(Collectors.toMap((e) -> Bytes.toStringBinary(e.getKey()), e -> {
817+
TimeRange value = e.getValue();
818+
List<Long> rangeList = new ArrayList<>();
819+
rangeList.add(value.getMin());
820+
rangeList.add(value.getMax());
821+
return rangeList;
822+
}));
823+
824+
map.put("colFamTimeRangeMap", colFamTimeRangeMapStr);
825+
}
826+
map.put("priority", getPriority());
798827
return map;
799828
}
800829

‎hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,26 @@ public void itSerializesScan() {
4444
Scan scan = new Scan();
4545
scan.withStartRow(Bytes.toBytes(123));
4646
scan.withStopRow(Bytes.toBytes(456));
47-
String expectedOutput = "{\n" + " \"startTime\": 1,\n" + " \"processingTime\": 2,\n"
48-
+ " \"queueTime\": 3,\n" + " \"responseSize\": 4,\n" + " \"blockBytesScanned\": 5,\n"
49-
+ " \"fsReadTime\": 6,\n" + " \"multiGetsCount\": 6,\n" + " \"multiMutationsCount\": 7,\n"
50-
+ " \"scan\": {\n" + " \"startRow\": \"\\\\x00\\\\x00\\\\x00{\",\n"
51-
+ " \"stopRow\": \"\\\\x00\\\\x00\\\\x01\\\\xC8\",\n" + " \"batch\": -1,\n"
52-
+ " \"cacheBlocks\": true,\n" + " \"totalColumns\": 0,\n"
53-
+ " \"maxResultSize\": -1,\n" + " \"families\": {},\n" + " \"caching\": -1,\n"
54-
+ " \"maxVersions\": 1,\n" + " \"timeRange\": [\n" + " 0,\n"
55-
+ " 9223372036854775807\n" + " ]\n" + " }\n" + "}";
47+
String expectedOutput =
48+
"{\n" + " \"startTime\": 1,\n" + " \"processingTime\": 2,\n" + " \"queueTime\": 3,\n"
49+
+ " \"responseSize\": 4,\n" + " \"blockBytesScanned\": 5,\n" + " \"fsReadTime\": 6,\n"
50+
+ " \"multiGetsCount\": 6,\n" + " \"multiMutationsCount\": 7,\n" + " \"scan\": {\n"
51+
+ " \"startRow\": \"\\\\x00\\\\x00\\\\x00{\",\n" + " \"targetReplicaId\": -1,\n"
52+
+ " \"batch\": -1,\n" + " \"totalColumns\": 0,\n" + " \"maxResultSize\": -1,\n"
53+
+ " \"families\": {},\n" + " \"priority\": -1,\n" + " \"caching\": -1,\n"
54+
+ " \"includeStopRow\": false,\n" + " \"consistency\": \"STRONG\",\n"
55+
+ " \"maxVersions\": 1,\n" + " \"storeOffset\": 0,\n" + " \"mvccReadPoint\": -1,\n"
56+
+ " \"includeStartRow\": true,\n" + " \"needCursorResult\": false,\n"
57+
+ " \"stopRow\": \"\\\\x00\\\\x00\\\\x01\\\\xC8\",\n" + " \"storeLimit\": -1,\n"
58+
+ " \"limit\": -1,\n" + " \"cacheBlocks\": true,\n"
59+
+ " \"readType\": \"DEFAULT\",\n" + " \"allowPartialResults\": false,\n"
60+
+ " \"reversed\": false,\n" + " \"timeRange\": [\n" + " 0,\n"
61+
+ " 9223372036854775807\n" + " ]\n" + " }\n" + "}";
5662
OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null,
5763
null, 6, 7, 0, scan, Collections.emptyMap(), Collections.emptyMap());
5864
String actualOutput = o.toJsonPrettyPrint();
5965
System.out.println(actualOutput);
60-
Assert.assertEquals(actualOutput, expectedOutput);
66+
Assert.assertEquals(expectedOutput, actualOutput);
6167
}
6268

6369
@Test

‎hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@
6969

7070
import org.apache.hbase.thirdparty.com.google.common.reflect.TypeToken;
7171
import org.apache.hbase.thirdparty.com.google.gson.Gson;
72+
import org.apache.hbase.thirdparty.com.google.gson.GsonBuilder;
73+
import org.apache.hbase.thirdparty.com.google.gson.LongSerializationPolicy;
74+
import org.apache.hbase.thirdparty.com.google.gson.ToNumberPolicy;
7275

7376
/**
7477
* Run tests that use the functionality of the Operation superclass for Puts, Gets, Deletes, Scans,
@@ -345,6 +348,101 @@ public void testOperationJSON() throws IOException {
345348
kvMap.get("qualifier"));
346349
}
347350

351+
/**
352+
* Test the client Scan Operations' JSON encoding to ensure that produced JSON is parseable and
353+
* that the details are present and not corrupted.
354+
* @throws IOException if the JSON conversion fails
355+
*/
356+
@Test
357+
public void testScanOperationToJSON() throws IOException {
358+
// produce a Scan Operation
359+
Scan scan = new Scan().withStartRow(ROW, true);
360+
scan.addColumn(FAMILY, QUALIFIER);
361+
scan.withStopRow(ROW, true);
362+
scan.readVersions(5);
363+
scan.setBatch(10);
364+
scan.setAllowPartialResults(true);
365+
scan.setMaxResultsPerColumnFamily(3);
366+
scan.setRowOffsetPerColumnFamily(8);
367+
scan.setCaching(20);
368+
scan.setMaxResultSize(50);
369+
scan.setCacheBlocks(true);
370+
scan.setReversed(true);
371+
scan.setTimeRange(1000, 2000);
372+
scan.setAsyncPrefetch(true);
373+
scan.setMvccReadPoint(123);
374+
scan.setLimit(5);
375+
scan.setReadType(Scan.ReadType.PREAD);
376+
scan.setNeedCursorResult(true);
377+
scan.setFilter(SCV_FILTER);
378+
scan.setReplicaId(1);
379+
scan.setConsistency(Consistency.STRONG);
380+
scan.setLoadColumnFamiliesOnDemand(true);
381+
scan.setColumnFamilyTimeRange(FAMILY, 2000, 3000);
382+
scan.setPriority(10);
383+
384+
// get its JSON representation, and parse it
385+
String json = scan.toJSON();
386+
Type typeOfHashMap = new TypeToken<Map<String, Object>>() {
387+
}.getType();
388+
Gson gson = new GsonBuilder().setLongSerializationPolicy(LongSerializationPolicy.STRING)
389+
.setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create();
390+
Map<String, Object> parsedJSON = gson.fromJson(json, typeOfHashMap);
391+
// check for the row
392+
assertEquals("startRow incorrect in Scan.toJSON()", Bytes.toStringBinary(ROW),
393+
parsedJSON.get("startRow"));
394+
// check for the family and the qualifier.
395+
List familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY));
396+
assertNotNull("Family absent in Scan.toJSON()", familyInfo);
397+
assertEquals("Qualifier absent in Scan.toJSON()", 1, familyInfo.size());
398+
assertEquals("Qualifier incorrect in Scan.toJSON()", Bytes.toStringBinary(QUALIFIER),
399+
familyInfo.get(0));
400+
assertEquals("stopRow incorrect in Scan.toJSON()", Bytes.toStringBinary(ROW),
401+
parsedJSON.get("stopRow"));
402+
assertEquals("includeStartRow incorrect in Scan.toJSON()", true,
403+
parsedJSON.get("includeStartRow"));
404+
assertEquals("includeStopRow incorrect in Scan.toJSON()", true,
405+
parsedJSON.get("includeStopRow"));
406+
assertEquals("maxVersions incorrect in Scan.toJSON()", 5L, parsedJSON.get("maxVersions"));
407+
assertEquals("batch incorrect in Scan.toJSON()", 10L, parsedJSON.get("batch"));
408+
assertEquals("allowPartialResults incorrect in Scan.toJSON()", true,
409+
parsedJSON.get("allowPartialResults"));
410+
assertEquals("storeLimit incorrect in Scan.toJSON()", 3L, parsedJSON.get("storeLimit"));
411+
assertEquals("storeOffset incorrect in Scan.toJSON()", 8L, parsedJSON.get("storeOffset"));
412+
assertEquals("caching incorrect in Scan.toJSON()", 20L, parsedJSON.get("caching"));
413+
assertEquals("maxResultSize incorrect in Scan.toJSON()", "50", parsedJSON.get("maxResultSize"));
414+
assertEquals("cacheBlocks incorrect in Scan.toJSON()", true, parsedJSON.get("cacheBlocks"));
415+
assertEquals("reversed incorrect in Scan.toJSON()", true, parsedJSON.get("reversed"));
416+
List trList = (List) parsedJSON.get("timeRange");
417+
assertEquals("timeRange incorrect in Scan.toJSON()", 2, trList.size());
418+
assertEquals("timeRange incorrect in Scan.toJSON()", "1000", trList.get(0));
419+
assertEquals("timeRange incorrect in Scan.toJSON()", "2000", trList.get(1));
420+
421+
assertEquals("asyncPrefetch incorrect in Scan.toJSON()", true, parsedJSON.get("asyncPrefetch"));
422+
assertEquals("mvccReadPoint incorrect in Scan.toJSON()", "123",
423+
parsedJSON.get("mvccReadPoint"));
424+
assertEquals("limit incorrect in Scan.toJSON()", 5L, parsedJSON.get("limit"));
425+
assertEquals("readType incorrect in Scan.toJSON()", "PREAD", parsedJSON.get("readType"));
426+
assertEquals("needCursorResult incorrect in Scan.toJSON()", true,
427+
parsedJSON.get("needCursorResult"));
428+
429+
Map colFamTimeRange = (Map) parsedJSON.get("colFamTimeRangeMap");
430+
assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", 1L, colFamTimeRange.size());
431+
List testFamily = (List) colFamTimeRange.get("testFamily");
432+
assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", 2L, testFamily.size());
433+
assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", "2000", testFamily.get(0));
434+
assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", "3000", testFamily.get(1));
435+
436+
assertEquals("targetReplicaId incorrect in Scan.toJSON()", 1L,
437+
parsedJSON.get("targetReplicaId"));
438+
assertEquals("consistency incorrect in Scan.toJSON()", "STRONG", parsedJSON.get("consistency"));
439+
assertEquals("loadColumnFamiliesOnDemand incorrect in Scan.toJSON()", true,
440+
parsedJSON.get("loadColumnFamiliesOnDemand"));
441+
442+
assertEquals("priority incorrect in Scan.toJSON()", 10L, parsedJSON.get("priority"));
443+
444+
}
445+
348446
@Test
349447
public void testPutCreationWithByteBuffer() {
350448
Put p = new Put(ROW);

0 commit comments

Comments
 (0)