Build: Support building with Java 21 (apache#10474)

.github/workflows/delta-conversion-ci.yml

@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -100,7 +100,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/flink-ci.yml

@@ -73,12 +73,15 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
         flink: ['1.17', '1.18', '1.19']
         exclude:
           # Flink 1.17 does not support Java 17.
           - jvm: 17
             flink: '1.17'
+          # Flink 1.17 does not support Java 21.
+          - jvm: 21
+            flink: '1.17'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/hive-ci.yml

@@ -69,7 +69,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/java-ci.yml

@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
@@ -107,7 +107,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4

.github/workflows/spark-ci.yml

@@ -71,9 +71,16 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [8, 11, 17, 21]
         spark: ['3.3', '3.4', '3.5']
         scala: ['2.12', '2.13']
+        exclude:
+          # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
+          # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
+          - jvm: 21
+            spark: '3.3'
+          - jvm: 21
+            spark: '3.4'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

README.md

@@ -51,7 +51,7 @@ Community discussions happen primarily on the [dev mailing list][dev-list] or on
 
 ### Building
 
-Iceberg is built using Gradle with Java 8, 11, or 17.
+Iceberg is built using Gradle with Java 8, 11, 17, or 21.
 
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`

baseline.gradle

@@ -45,7 +45,16 @@ subprojects {
   apply plugin: 'com.palantir.baseline-reproducibility'
   apply plugin: 'com.palantir.baseline-exact-dependencies'
   apply plugin: 'com.palantir.baseline-release-compatibility'
-  apply plugin: 'com.diffplug.spotless'
+  // We need to update Google Java Format to 1.17.0+ to run spotless on JDK 8, but that requires dropping support for JDK 8.
+  if (JavaVersion.current() == JavaVersion.VERSION_21) {
+    task spotlessApply {
+      doLast {
+        throw new GradleException("Spotless plugin is currently disabled when running on JDK 21 (until we drop JDK 8). To run spotlessApply please use a different JDK version.")
+      }
+    }
+  } else {
+    apply plugin: 'com.diffplug.spotless'
+  }
 
   pluginManager.withPlugin('com.palantir.baseline-checkstyle') {
     checkstyle {

build.gradle

@@ -62,8 +62,8 @@ if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
 } else if (JavaVersion.current() == JavaVersion.VERSION_11) {
   project.ext.jdkVersion = '11'
   project.ext.extraJvmArgs = []
-} else if (JavaVersion.current() == JavaVersion.VERSION_17) {
-  project.ext.jdkVersion = '17'
+} else if (JavaVersion.current() == JavaVersion.VERSION_17 || JavaVersion.current() == JavaVersion.VERSION_21) {
+  project.ext.jdkVersion = JavaVersion.current().getMajorVersion().toString()
   project.ext.extraJvmArgs = ["--add-opens", "java.base/java.io=ALL-UNNAMED",
                               "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED",
                               "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED",
@@ -86,7 +86,7 @@ if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
                               "--add-opens", "java.base/sun.security.action=ALL-UNNAMED",
                               "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED"]
 } else {
-  throw new GradleException("This build must be run with JDK 8 or 11 or 17 but was executed with JDK " + JavaVersion.current())
+  throw new GradleException("This build must be run with JDK 8 or 11 or 17 or 21 but was executed with JDK " + JavaVersion.current())
 }
 
 tasks.withType(AbstractArchiveTask).configureEach {

jmh.gradle

@@ -17,8 +17,8 @@
  * under the License.
  */
 
-if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17') {
-  throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17")
+if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17' && jdkVersion != '21') {
+  throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17 or JDK 21")
 }
 
 def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",")

site/docs/contribute.md

@@ -84,7 +84,7 @@ settle disagreements or to force a decision.
 
 ## Building the Project Locally
 
-Iceberg is built using Gradle with Java 8, 11, or 17.
+Iceberg is built using Gradle with Java 8, 11, 17, or 21.
 
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`

spark/v3.3/build.gradle

@@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
     implementation project(':iceberg-parquet')
     implementation project(':iceberg-arrow')
     implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
 
     compileOnly libs.errorprone.annotations
     compileOnly libs.avro.avro
@@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
 
   dependencies {
     implementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
     implementation libs.roaringbitmap
 
     compileOnly "org.scala-lang:scala-library"

spark/v3.4/build.gradle

@@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
     implementation project(':iceberg-parquet')
     implementation project(':iceberg-arrow')
     implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
 
     compileOnly libs.errorprone.annotations
     compileOnly libs.avro.avro
@@ -138,6 +142,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
 
   dependencies {
     implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
     implementation libs.roaringbitmap
 
     compileOnly "org.scala-lang:scala-library"

spark/v3.5/build.gradle

@@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
     implementation project(':iceberg-parquet')
     implementation project(':iceberg-arrow')
     implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
 
     compileOnly libs.errorprone.annotations
     compileOnly libs.avro.avro
@@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
 
   dependencies {
     implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
+    if (scalaVersion == '2.12') {
+      // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support
+      implementation 'org.scala-lang:scala-library:2.12.18'
+    }
     implementation libs.roaringbitmap
 
     compileOnly "org.scala-lang:scala-library"