Skip to content

Commit 7fdfc35

Browse files
committed
[SPARK-23305][SQL][TEST] Add spark.sql.files.ignoreMissingFiles test case for ORC
1 parent ec63e2d commit 7fdfc35

File tree

1 file changed

+31
-0
lines changed

1 file changed

+31
-0
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,4 +655,35 @@ class OrcQuerySuite extends OrcQueryTest with SharedSQLContext {
655655
}
656656
}
657657
}
658+
659+
testQuietly("Enabling/disabling ignoreMissingFiles") {
660+
def testIgnoreMissingFiles(): Unit = {
661+
withTempDir { dir =>
662+
val basePath = dir.getCanonicalPath
663+
spark.range(1).toDF("a").write.orc(new Path(basePath, "first").toString)
664+
spark.range(1, 2).toDF("a").write.orc(new Path(basePath, "second").toString)
665+
val thirdPath = new Path(basePath, "third")
666+
spark.range(2, 3).toDF("a").write.orc(thirdPath.toString)
667+
val df = spark.read.orc(
668+
new Path(basePath, "first").toString,
669+
new Path(basePath, "second").toString,
670+
new Path(basePath, "third").toString)
671+
672+
val fs = thirdPath.getFileSystem(spark.sparkContext.hadoopConfiguration)
673+
fs.delete(thirdPath, true)
674+
checkAnswer(df, Seq(Row(0), Row(1)))
675+
}
676+
}
677+
678+
withSQLConf(SQLConf.IGNORE_MISSING_FILES.key -> "true") {
679+
testIgnoreMissingFiles()
680+
}
681+
682+
withSQLConf(SQLConf.IGNORE_MISSING_FILES.key -> "false") {
683+
val exception = intercept[SparkException] {
684+
testIgnoreMissingFiles()
685+
}
686+
assert(exception.getMessage().contains("does not exist"))
687+
}
688+
}
658689
}

0 commit comments

Comments
 (0)