Skip to content

Commit 62aca29

Browse files
committed
Revert createDataFrame for Java Beans.
- because the Encoder for Java Beans needs not only getters but also setters.
1 parent 036d5fd commit 62aca29

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,14 @@ class SparkSession private(
326326
* @since 2.0.0
327327
*/
328328
def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
329-
val encoder = Encoders.bean(beanClass).asInstanceOf[Encoder[AnyRef]]
330-
Dataset.ofRows(self, ExistingRDD(rdd.asInstanceOf[RDD[AnyRef]])(self)(encoder))
329+
val attributeSeq: Seq[AttributeReference] = getSchema(beanClass)
330+
val className = beanClass.getName
331+
val rowRdd = rdd.mapPartitions { iter =>
332+
// BeanInfo is not serializable so we must rediscover it remotely for each partition.
333+
val localBeanInfo = Introspector.getBeanInfo(Utils.classForName(className))
334+
SQLContext.beansToRows(iter, localBeanInfo, attributeSeq)
335+
}
336+
Dataset.ofRows(self, LogicalRDD(attributeSeq, rowRdd)(self))
331337
}
332338

333339
/**

0 commit comments

Comments
 (0)