use cache (#988)

Jacob-Eliat-Eliat · web-flow · commit 98413556b512 · 2024-11-14T15:12:02.000Z
diff --git a/src/main/scala/cognite/spark/v1/FileContentRelation.scala b/src/main/scala/cognite/spark/v1/FileContentRelation.scala
@@ -97,7 +97,7 @@ class FileContentRelation(config: RelationConfig, fileExternalId: String, inferS
         Array(CdfPartition(0))
     }
     import sqlContext.sparkSession.implicits._
-    val dataset = rdd.toDS()
+    val dataset = rdd.cache().toDS()
 
     if (inferSchema) {
       sqlContext.sparkSession.read.json(dataset)

Original file line number	Diff line number	Diff line change
`@@ -97,7 +97,7 @@ class FileContentRelation(config: RelationConfig, fileExternalId: String, inferS`
`97`	`97`	`Array(CdfPartition(0))`
`98`	`98`	`}`
`99`	`99`	`import sqlContext.sparkSession.implicits._`
`100`		`- val dataset = rdd.toDS()`
	`100`	`+ val dataset = rdd.cache().toDS()`
`101`	`101`
`102`	`102`	`if (inferSchema) {`
`103`	`103`	`sqlContext.sparkSession.read.json(dataset)`