Add comment on filtering null semantic

hvanhovell · hvanhovell · commit 80b2166c62b4 · 2016-09-27T11:49:57.000-07:00
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -65,6 +65,8 @@ abstract class Collect extends ImperativeAggregate {
   }
 
   override def update(b: MutableRow, input: InternalRow): Unit = {
+    // Do not allow null values. We follow the semantics of Hive's collect_list/collect_set here.
+    // See: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMkCollectionEvaluator
     val value = child.eval(input)
     if (value != null) {
       buffer += value

Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,8 @@ abstract class Collect extends ImperativeAggregate {`
`65`	`65`	`}`
`66`	`66`
`67`	`67`	`override def update(b: MutableRow, input: InternalRow): Unit = {`
	`68`	`+ // Do not allow null values. We follow the semantics of Hive's collect_list/collect_set here.`
	`69`	`+ // See: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMkCollectionEvaluator`
`68`	`70`	`val value = child.eval(input)`
`69`	`71`	`if (value != null) {`
`70`	`72`	`buffer += value`