Skip to content

Commit 1a3b63d

Browse files
cafreemanDavies Liu
cafreeman
authored and
Davies Liu
committed
new version of CreateDF
New version uses takes a `StructType` from R and creates a DataFrame. Commented out the `tojson` version since we don't currently use it.
1 parent 836c4bf commit 1a3b63d

File tree

1 file changed

+9
-0
lines changed

1 file changed

+9
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ private[r] object SQLUtils {
6666
StructField(name, dtObj, nullable)
6767
}
6868

69+
// cfreeman: I don't think we need this anymore since we can pass structType from R
70+
// def createDF(rdd: RDD[Array[Byte]], schemaString: String, sqlContext: SQLContext): DataFrame = {
71+
// val schema = DataType.fromJson(schemaString).asInstanceOf[StructType]
72+
// val num = schema.fields.size
73+
// val rowRDD = rdd.map(bytesToRow)
74+
// sqlContext.createDataFrame(rowRDD, schema)
75+
// }
76+
77+
def createDF(rdd: RDD[Array[Byte]], schema: StructType, sqlContext: SQLContext): DataFrame = {
6978
val num = schema.fields.size
7079
val rowRDD = rdd.map(bytesToRow)
7180
sqlContext.createDataFrame(rowRDD, schema)

0 commit comments

Comments
 (0)