-
Notifications
You must be signed in to change notification settings - Fork 253
Closed
Labels
bugSomething isn't workingSomething isn't working
Description
Describe the bug
ParquetEncryptionITCase fails with native_iceberg_compat.
Error: ParquetEncryptionITCase>SparkFunSuite.run:69->SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run:69->AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run:1563->AnyFunSuite.runTests:1563->SparkFunSuite.runTest:69->SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest:69->SparkFunSuite.withFixture:227->SparkFunSuite.$anonfun$test$2:155->SparkFunSuite.failAfter:69->$anonfun$test$2:144->withSQLConf:44->org$apache$spark$sql$test$SQLTestUtilsBase$$super$withSQLConf:44->$anonfun$test$3:144->$anonfun$new$1:55->$anonfun$new$2$adapted:55->$anonfun$new$2:56->withTempDir:44->org$apache$spark$sql$test$SQLTestUtils$$super$withTempDir:44->SparkFunSuite.withTempDir:245->$anonfun$new$3$adapted:56->$anonfun$new$3:62->withSQLConf:44->org$apache$spark$sql$test$SQLTestUtilsBase$$super$withSQLConf:44->$anonfun$new$4:81->QueryTest.checkAnswer:159->QueryTest.checkAnswer:151 ? Spark Job aborted due to stage failure: Task 0 in stage 7.0 failed 1 times, most recent failure: Lost task 0.0 in stage 7.0 (TID 23) (3eee3659fd80 executor driver): org.apache.spark.SparkException: Encountered error while reading file file:///tmp/spark-6f6947fd-892b-44eb-9bf1-7fe510dc88b6/parquet/part-00003-e9818b1e-fcea-4fa8-9fd8-aead581d4d25-c000.snappy.parquet. Details:
at org.apache.spark.sql.errors.QueryExecutionErrors$.cannotReadFilesError(QueryExecutionErrors.scala:864)
at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:296)
at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:131)
at org.apache.spark.sql.comet.CometScanExec$$anon$1.hasNext(CometScanExec.scala:257)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.cometcolumnartorow_nextBatch_0$(Unknown Source)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:43)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.util.Iterators$.size(Iterators.scala:29)
at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1787)
at org.apache.spark.rdd.RDD.$anonfun$count$1(RDD.scala:1296)
at org.apache.spark.rdd.RDD.$anonfun$count$1$adapted(RDD.scala:1296)
at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2433)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
at org.apache.spark.scheduler.Task.run(Task.scala:141)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:620)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:623)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: org.apache.comet.CometNativeException: Parquet error: Invalid Parquet file. Corrupt footer
at org.apache.comet.parquet.Native.readNextRecordBatch(Native Method)
at org.apache.comet.parquet.NativeBatchReader.loadNextBatch(NativeBatchReader.java:469)
at org.apache.comet.parquet.NativeBatchReader.nextBatch(NativeBatchReader.java:412)
at org.apache.comet.parquet.NativeBatchReader.nextKeyValue(NativeBatchReader.java:373)
at org.apache.spark.sql.execution.datasources.RecordReaderIterator.hasNext(RecordReaderIterator.scala:39)
at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:131)
at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:286)
Steps to reproduce
No response
Expected behavior
No response
Additional context
No response
Metadata
Metadata
Assignees
Labels
bugSomething isn't workingSomething isn't working