KYLIN-3644 Fix SparkFactDistinct step NumberFormatException

JinfengWu98 · Nov 1, 2018 · 6544143 · 6544143
1 parent 3fabbe6
commit 6544143
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -56,7 +56,7 @@ private void initNullStrings() {
         }
     }
 
-    protected boolean isNull(String v) {
+    public boolean isNull(String v) {
         return nullStrs.contains(v);
     }
 

diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -63,6 +63,7 @@
 import org.apache.kylin.cube.cuboid.CuboidUtil;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.cube.util.KeyValueBuilder;
 import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.IDictionaryBuilder;
 import org.apache.kylin.engine.EngineFactory;
@@ -250,6 +251,7 @@ static class FlatOutputFucntion implements PairFlatMapFunction<Iterator<String[]
         private Map<Integer, DimensionRangeInfo> dimensionRangeInfoMap;
         private transient ByteBuffer tmpbuf;
         private LongAccumulator bytesWritten;
+        private KeyValueBuilder keyValueBuilder;
 
         public FlatOutputFucntion(String cubeName, String segmentId, String metaurl, SerializableConfiguration conf,
                 int samplingPercent, LongAccumulator bytesWritten) {
@@ -272,6 +274,7 @@ private void init() {
                 CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(
                         EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
 
+                keyValueBuilder = new KeyValueBuilder(intermediateTableDesc);
                 reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance);
                 tmpbuf = ByteBuffer.allocate(4096);
 
@@ -317,7 +320,7 @@ public Iterator<Tuple2<SelfDefineSortableKey, Text>> call(Iterator<String[]> row
 
                 for (int i = 0; i < allCols.size(); i++) {
                     String fieldValue = row[columnIndex[i]];
-                    if (fieldValue == null)
+                    if (fieldValue == null || keyValueBuilder.isNull(fieldValue))
                         continue;
 
                     final DataType type = allCols.get(i).getType();
-Original file line number
+Diff line change
@@ Expand Up / @@ -56,7 +56,7 @@ private void initNullStrings() { @@
             }
         }
-        protected boolean isNull(String v) {
+        public boolean isNull(String v) {
             return nullStrs.contains(v);
         }
@@ Expand Down @@