Revert "Spark 3.4, 3.5: Iceberg / DataFusion Comet integration (apach…

…e#12147)" This reverts commit c5822c4.
pvary · Feb 3, 2025 · 2dcb297 · 2dcb297
1 parent 507e2a9
commit 2dcb297
Show file tree

Hide file tree

Showing 21 changed files with 28 additions and 941 deletions.
diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
@@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
       exclude group: 'org.roaringbitmap'
     }
 
-    compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0"
+    compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.3.0"
 
     implementation libs.parquet.column
     implementation libs.parquet.hadoop
@@ -185,7 +185,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
     testImplementation libs.avro.avro
     testImplementation libs.parquet.hadoop
     testImplementation libs.junit.vintage.engine
-    testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0"
+    testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.3.0"
 
     // Required because we remove antlr plugin dependencies from the compile configuration, see note above
     runtimeOnly libs.antlr.runtime

diff --git a/...spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/...spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java
@@ -53,8 +53,7 @@ private static class DeleteColumnReader extends MetadataColumnReader {
           DataTypes.BooleanType,
           TypeUtil.convertToParquet(
               new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())),
-          false /* useDecimal128 = false */,
-          false /* isConstant */);
+          false /* useDecimal128 = false */);
       this.isDeleted = new boolean[0];
     }
 

diff --git a/...ark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/...ark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java
@@ -42,11 +42,7 @@ private static class PositionColumnReader extends MetadataColumnReader {
     private long position;
 
     PositionColumnReader(ColumnDescriptor descriptor) {
-      super(
-          DataTypes.LongType,
-          descriptor,
-          false /* useDecimal128 = false */,
-          false /* isConstant */);
+      super(DataTypes.LongType, descriptor, false /* useDecimal128 = false */);
     }
 
     @Override

diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle
@@ -52,8 +52,6 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
   dependencies {
     implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
     api project(':iceberg-api')
-    annotationProcessor libs.immutables.value
-    compileOnly libs.immutables.value
     implementation project(':iceberg-common')
     implementation project(':iceberg-core')
     implementation project(':iceberg-data')
@@ -75,8 +73,6 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
       exclude group: 'org.roaringbitmap'
     }
 
-    compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0"
-
     implementation libs.parquet.column
     implementation libs.parquet.hadoop
 
@@ -183,7 +179,6 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
     testImplementation libs.avro.avro
     testImplementation libs.parquet.hadoop
     testImplementation libs.awaitility
-    testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0"
 
     // Required because we remove antlr plugin dependencies from the compile configuration, see note above
     runtimeOnly libs.antlr.runtime

diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/OrcBatchReadConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/OrcBatchReadConf.java
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ParquetBatchReadConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ParquetBatchReadConf.java
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ParquetReaderType.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ParquetReaderType.java
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -355,12 +355,4 @@ public boolean reportColumnStats() {
         .defaultValue(SparkSQLProperties.REPORT_COLUMN_STATS_DEFAULT)
         .parse();
   }
-
-  public ParquetReaderType parquetReaderType() {
-    return confParser
-        .enumConf(ParquetReaderType::fromString)
-        .sessionConf(SparkSQLProperties.PARQUET_READER_TYPE)
-        .defaultValue(SparkSQLProperties.PARQUET_READER_TYPE_DEFAULT)
-        .parse();
-  }
 }
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
@@ -27,9 +27,6 @@ private SparkSQLProperties() {}
   // Controls whether vectorized reads are enabled
   public static final String VECTORIZATION_ENABLED = "spark.sql.iceberg.vectorization.enabled";
 
-  // Controls which Parquet reader implementation to use
-  public static final String PARQUET_READER_TYPE = "spark.sql.iceberg.parquet.reader-type";
-  public static final ParquetReaderType PARQUET_READER_TYPE_DEFAULT = ParquetReaderType.ICEBERG;
   // Controls whether to perform the nullability check during writes
   public static final String CHECK_NULLABILITY = "spark.sql.iceberg.check-nullability";
   public static final boolean CHECK_NULLABILITY_DEFAULT = true;

diff --git a/.../v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/.../v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java