HBASE-23015 : Moving from Jackson2 to shaded Gson (#616)

* Backport HBASE-20587
* moving to shaded gson with jdk7 compatibility
* Include jackson-mapper-asl for testing-util because of Hadoop
* Update shaded jar check to allow hbase-thirdparty libs

Signed-off-by: Sean Busbey <busbey@apache.org>
Co-authored-by: Duo Zhang <zhangduo@apache.org>

Author: 2 people

hbase-client/pom.xml

@@ -252,6 +252,10 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-shaded-gson</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>

hbase-client/src/main/java/org/apache/hadoop/hbase/util/JsonMapper.java

@@ -18,12 +18,12 @@
  */
 package org.apache.hadoop.hbase.util;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.util.Map;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hbase.thirdparty.com.google.gson.Gson;
 
 /**
  * Utility class for converting objects to JSON
@@ -34,12 +34,13 @@ public final class JsonMapper {
   private JsonMapper() {
   }
 
-  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final Gson GSON = GsonUtil.createGson().create();
 
-  public static String writeMapAsString(Map<String, Object> map) throws IOException { 
+  public static String writeMapAsString(Map<String, Object> map) throws IOException {
     return writeObjectAsString(map);
   }
-  public static String writeObjectAsString(Object object) throws IOException { 
-    return MAPPER.writeValueAsString(object);
+
+  public static String writeObjectAsString(Object object) throws IOException {
+    return GSON.toJson(object);
   }
 }

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java

@@ -27,11 +27,10 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
+import java.lang.reflect.Type;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -62,9 +61,9 @@
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
 import org.apache.hadoop.hbase.util.BuilderStyleTest;
 import org.apache.hadoop.hbase.util.Bytes;
-
-import org.junit.Assert;
-import org.junit.Test;
+import org.apache.hadoop.hbase.util.GsonUtil;
+import org.apache.hbase.thirdparty.com.google.gson.Gson;
+import org.apache.hbase.thirdparty.com.google.gson.reflect.TypeToken;
 import org.junit.experimental.categories.Category;
 
 /**
@@ -78,7 +77,7 @@ public class TestOperation {
   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
   private static byte [] VALUE = Bytes.toBytes("testValue");
 
-  private static ObjectMapper mapper = new ObjectMapper();
+  private static Gson GSON = GsonUtil.createGson().create();
 
   private static List<Long> TS_LIST = Arrays.asList(2L, 3L, 5L);
   private static TimestampsFilter TS_FILTER = new TimestampsFilter(TS_LIST);
@@ -304,7 +303,9 @@ public void testOperationJSON() throws IOException {
     scan.addColumn(FAMILY, QUALIFIER);
     // get its JSON representation, and parse it
     String json = scan.toJSON();
-    Map<String, Object> parsedJSON = mapper.readValue(json, HashMap.class);
+    Type typeOfHashMap = new TypeToken<Map<String, Object>>() {
+    }.getType();
+    Map<String, Object> parsedJSON = GSON.fromJson(json, typeOfHashMap);
     // check for the row
     assertEquals("startRow incorrect in Scan.toJSON()",
         Bytes.toStringBinary(ROW), parsedJSON.get("startRow"));
@@ -322,7 +323,7 @@ public void testOperationJSON() throws IOException {
     get.addColumn(FAMILY, QUALIFIER);
     // get its JSON representation, and parse it
     json = get.toJSON();
-    parsedJSON = mapper.readValue(json, HashMap.class);
+    parsedJSON = GSON.fromJson(json, typeOfHashMap);
     // check for the row
     assertEquals("row incorrect in Get.toJSON()",
         Bytes.toStringBinary(ROW), parsedJSON.get("row"));
@@ -340,7 +341,7 @@ public void testOperationJSON() throws IOException {
     put.add(FAMILY, QUALIFIER, VALUE);
     // get its JSON representation, and parse it
     json = put.toJSON();
-    parsedJSON = mapper.readValue(json, HashMap.class);
+    parsedJSON = GSON.fromJson(json, typeOfHashMap);
     // check for the row
     assertEquals("row absent in Put.toJSON()",
         Bytes.toStringBinary(ROW), parsedJSON.get("row"));
@@ -354,14 +355,14 @@ public void testOperationJSON() throws IOException {
         Bytes.toStringBinary(QUALIFIER),
         kvMap.get("qualifier"));
     assertEquals("Value length incorrect in Put.toJSON()",
-        VALUE.length, kvMap.get("vlen"));
+      VALUE.length, ((Number) kvMap.get("vlen")).intValue());
 
     // produce a Delete operation
     Delete delete = new Delete(ROW);
     delete.deleteColumn(FAMILY, QUALIFIER);
     // get its JSON representation, and parse it
     json = delete.toJSON();
-    parsedJSON = mapper.readValue(json, HashMap.class);
+    parsedJSON = GSON.fromJson(json, typeOfHashMap);
     // check for the row
     assertEquals("row absent in Delete.toJSON()",
         Bytes.toStringBinary(ROW), parsedJSON.get("row"));

hbase-common/pom.xml

@@ -277,6 +277,10 @@
       <artifactId>jackson-mapper-asl</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-shaded-gson</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.jaxrs</groupId>
       <artifactId>jackson-jaxrs-json-provider</artifactId>

hbase-common/src/main/java/org/apache/hadoop/hbase/util/GsonUtil.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.gson.GsonBuilder;
+import org.apache.hbase.thirdparty.com.google.gson.LongSerializationPolicy;
+
+/**
+ * Helper class for gson.
+ */
+@InterfaceAudience.Private
+public final class GsonUtil {
+
+  private GsonUtil() {
+  }
+
+  /**
+   * Create a builder which is used to create a Gson instance.
+   * <p/>
+   * Will set some common configs for the builder.
+   */
+  public static GsonBuilder createGson() {
+    return new GsonBuilder().setLongSerializationPolicy(LongSerializationPolicy.STRING);
+  }
+}

hbase-resource-bundle/src/main/resources/supplemental-models.xml

@@ -1665,6 +1665,25 @@ Copyright 2005 Sun Microsystems, Inc. and portions Copyright Apache Software Fou
       </licenses>
     </project>
   </supplement>
+  <supplement>
+    <project>
+      <groupId>com.google.errorprone</groupId>
+      <artifactId>error_prone_annotations</artifactId>
+
+      <organization>
+        <name>Google</name>
+        <url>http://www.google.com</url>
+      </organization>
+      <licenses>
+        <license>
+          <!-- It has been incorrectly called Apache 2.0 in the original pom-->
+          <name>Apache License, Version 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </project>
+  </supplement>
   <supplement>
     <project>
       <groupId>org.jamon</groupId>

hbase-server/pom.xml

@@ -484,6 +484,10 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-math</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-shaded-gson</artifactId>
+    </dependency>
     <dependency>
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>

hbase-server/src/main/java/org/apache/hadoop/hbase/http/jmx/JMXJsonServlet.java

@@ -163,14 +163,14 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
       try {
         jsonpcb = checkCallbackName(request.getParameter(CALLBACK_PARAM));
         writer = response.getWriter();
-        beanWriter = this.jsonBeanWriter.open(writer);
         // "callback" parameter implies JSONP outpout
         if (jsonpcb != null) {
           response.setContentType("application/javascript; charset=utf8");
           writer.write(jsonpcb + "(");
         } else {
           response.setContentType("application/json; charset=utf8");
         }
+        beanWriter = this.jsonBeanWriter.open(writer);
         // Should we output description on each attribute and bean?
         String tmpStr = request.getParameter(INCLUDE_DESCRIPTION);
         boolean description = tmpStr != null && tmpStr.length() > 0;
@@ -204,9 +204,11 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
           response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
         }
       } finally {
-        if (beanWriter != null) beanWriter.close();
+        if (beanWriter != null) {
+          beanWriter.close();
+        }
         if (jsonpcb != null) {
-           writer.write(");");
+          writer.write(");");
         }
         if (writer != null) {
           writer.close();

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java

@@ -25,11 +25,10 @@
  * Snapshot of block cache age in cache.
  * This object is preferred because we can control how it is serialized out when JSON'ing.
  */
-@JsonIgnoreProperties({"ageHistogram", "snapshot"})
 public class AgeSnapshot {
 
-  private final FastLongHistogram ageHistogram;
-  private final long[] quantiles;
+  private transient final FastLongHistogram ageHistogram;
+  private transient final long[] quantiles;
 
   AgeSnapshot(final FastLongHistogram ageHistogram) {
     this.ageHistogram = ageHistogram;

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java

@@ -17,11 +17,6 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import java.io.IOException;
 import java.util.NavigableMap;
 import java.util.NavigableSet;
@@ -31,6 +26,13 @@
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.util.FastLongHistogram;
+import org.apache.hadoop.hbase.util.GsonUtil;
+
+
+import org.apache.hbase.thirdparty.com.google.gson.Gson;
+import org.apache.hbase.thirdparty.com.google.gson.TypeAdapter;
+import org.apache.hbase.thirdparty.com.google.gson.stream.JsonReader;
+import org.apache.hbase.thirdparty.com.google.gson.stream.JsonWriter;
 
 /**
  * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches.
@@ -41,12 +43,29 @@ public class BlockCacheUtil {
   /**
    * Needed generating JSON.
    */
-  private static final ObjectMapper MAPPER = new ObjectMapper();
-  static {
-    MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
-    MAPPER.configure(SerializationFeature.FLUSH_AFTER_WRITE_VALUE, true);
-    MAPPER.configure(SerializationFeature.INDENT_OUTPUT, true);
-  }
+  private static final Gson GSON = GsonUtil.createGson()
+    .registerTypeAdapter(FastLongHistogram.class, new TypeAdapter<FastLongHistogram>() {
+
+      @Override
+      public void write(JsonWriter out, FastLongHistogram value) throws IOException {
+        AgeSnapshot snapshot = new AgeSnapshot(value);
+        out.beginObject();
+        out.name("mean").value(snapshot.getMean());
+        out.name("min").value(snapshot.getMin());
+        out.name("max").value(snapshot.getMax());
+        out.name("75thPercentile").value(snapshot.get75thPercentile());
+        out.name("95thPercentile").value(snapshot.get95thPercentile());
+        out.name("98thPercentile").value(snapshot.get98thPercentile());
+        out.name("99thPercentile").value(snapshot.get99thPercentile());
+        out.name("999thPercentile").value(snapshot.get999thPercentile());
+        out.endObject();
+      }
+
+      @Override
+      public FastLongHistogram read(JsonReader in) throws IOException {
+        throw new UnsupportedOperationException();
+      }
+    }).setPrettyPrinting().create();
 
   /**
    * @param cb
@@ -93,15 +112,10 @@ public String getFilename() {
   }
 
   /**
-   * @param filename
-   * @param blocks
    * @return A JSON String of <code>filename</code> and counts of <code>blocks</code>
-   * @throws JsonGenerationException
-   * @throws JsonMappingException
-   * @throws IOException
    */
   public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
-  throws JsonGenerationException, JsonMappingException, IOException {
+      throws IOException {
     CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
     for (CachedBlock cb: blocks) {
       counts.count++;
@@ -112,31 +126,21 @@ public static String toJSON(final String filename, final NavigableSet<CachedBloc
         counts.sizeData += cb.getSize();
       }
     }
-    return MAPPER.writeValueAsString(counts);
+    return GSON.toJson(counts);
   }
 
   /**
-   * @param cbsbf
    * @return JSON string of <code>cbsf</code> aggregated
-   * @throws JsonGenerationException
-   * @throws JsonMappingException
-   * @throws IOException
    */
-  public static String toJSON(final CachedBlocksByFile cbsbf)
-  throws JsonGenerationException, JsonMappingException, IOException {
-    return MAPPER.writeValueAsString(cbsbf);
+  public static String toJSON(final CachedBlocksByFile cbsbf) throws IOException {
+    return GSON.toJson(cbsbf);
   }
 
   /**
-   * @param bc
    * @return JSON string of <code>bc</code> content.
-   * @throws JsonGenerationException
-   * @throws JsonMappingException
-   * @throws IOException
    */
-  public static String toJSON(final BlockCache bc)
-  throws JsonGenerationException, JsonMappingException, IOException {
-    return MAPPER.writeValueAsString(bc);
+  public static String toJSON(final BlockCache bc) throws IOException {
+    return GSON.toJson(bc);
   }
 
   /**
@@ -172,7 +176,6 @@ public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration
    * This is different than metrics in that it is stats on current state of a cache.
    * See getLoadedCachedBlocksByFile
    */
-  @JsonIgnoreProperties({"cachedBlockStatsByFile"})
   public static class CachedBlocksByFile {
     private int count;
     private int dataBlockCount;
@@ -200,7 +203,7 @@ public static class CachedBlocksByFile {
     /**
      * Map by filename. use concurent utils because we want our Map and contained blocks sorted.
      */
-    private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
+    private transient NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
       new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
     FastLongHistogram hist = new FastLongHistogram();