Revert [SPARK-43836][SPARK-43845][SPARK-43858][SPARK-43858] to keep S…

…cala 2.12 for Spark 3.x ### What changes were proposed in this pull request? This PR aims to revert the following 4 commits effectively. SPARK-43836 Make Scala 2.13 as default in Spark 3.5 SPARK-43845 Setup Scala 2.12 Daily GitHub Action Job SPARK-43858 Make benchmark Github Action task use Scala 2.13 as default SPARK-43858 Restore the Scala version to 2.13 after run benchmark ### Why are the changes needed? According to the dev mailing list discussion, we keep the Scala 2.12 for Spark 3.x. - https://lists.apache.org/thread/xz7x0nkjn1d13yl9on59lfttm6f0yd45 We open a new thread, "Apache Spark 4.0 Timeframe?", for Spark 4 planning. - https://lists.apache.org/thread/xhkgj60j361gdpywoxxz7qspp2w80ry6 ### Does this PR introduce _any_ user-facing change? No. This is a recovery to the existing behavior. ### How was this patch tested? Pass the CIs. Closes apache#41416 from dongjoon-hyun/revert-2.13. Lead-authored-by: Dongjoon Hyun <dongjoon@apache.org> Co-authored-by: yangjie01 <yangjie01@baidu.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
panbingkun · Jun 1, 2023 · c8c6b0c · c8c6b0c
1 parent c02c8be
commit c8c6b0c
Show file tree

Hide file tree

Showing 47 changed files with 213 additions and 207 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -33,7 +33,7 @@ on:
       scala:
         description: 'Scala version: 2.12 or 2.13'
         required: true
-        default: '2.13'
+        default: '2.12'
       failfast:
         description: 'Failfast: true or false'
         required: true
@@ -182,7 +182,7 @@ jobs:
           "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
           "${{ github.event.inputs.class }}"
         # Revert to default Scala version to clean up unnecessary git diff
-        dev/change-scala-version.sh 2.13
+        dev/change-scala-version.sh 2.12
         # To keep the directory structure and file permissions, tar them
         # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
         echo "Preparing the benchmark results:"

diff --git a/.github/workflows/build_scala212.yml → .github/workflows/build_scala213.yml b/.github/workflows/build_scala212.yml → .github/workflows/build_scala213.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.12, Hadoop 3, JDK 8)"
+name: "Build (master, Scala 2.13, Hadoop 3, JDK 8)"
 
 on:
   schedule:
@@ -36,7 +36,7 @@ jobs:
       hadoop: hadoop3
       envs: >-
         {
-          "SCALA_PROFILE": "scala2.12"
+          "SCALA_PROFILE": "scala2.13"
         }
       jobs: >-
         {

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-assembly_2.13</artifactId>
+  <artifactId>spark-assembly_2.12</artifactId>
   <name>Spark Project Assembly</name>
   <url>https://spark.apache.org/</url>
   <packaging>pom</packaging>

diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-kvstore_2.13</artifactId>
+  <artifactId>spark-kvstore_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Local DB</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-common_2.13</artifactId>
+  <artifactId>spark-network-common_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Networking</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-shuffle_2.13</artifactId>
+  <artifactId>spark-network-shuffle_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Shuffle Streaming Service</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-network-yarn_2.13</artifactId>
+  <artifactId>spark-network-yarn_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project YARN Shuffle Service</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-sketch_2.13</artifactId>
+  <artifactId>spark-sketch_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Sketch</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/tags/pom.xml b/common/tags/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-tags_2.13</artifactId>
+  <artifactId>spark-tags_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Tags</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-unsafe_2.13</artifactId>
+  <artifactId>spark-unsafe_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Unsafe</name>
   <url>https://spark.apache.org/</url>

diff --git a/common/utils/pom.xml b/common/utils/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-common-utils_2.13</artifactId>
+  <artifactId>spark-common-utils_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Common Utils</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-avro_2.13</artifactId>
+  <artifactId>spark-avro_2.12</artifactId>
   <properties>
     <sbt.project.name>avro</sbt.project.name>
   </properties>
@@ -70,12 +70,12 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #if scala-2.13 -->
+    <!-- #if scala-2.13 --><!--
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #endif scala-2.13 -->
+    --><!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.tukaani</groupId>
       <artifactId>xz</artifactId>

diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-connect-client-jvm_2.13</artifactId>
+  <artifactId>spark-connect-client-jvm_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Connect Client</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
@@ -21,12 +21,12 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.spark</groupId>
-        <artifactId>spark-parent_2.13</artifactId>
+        <artifactId>spark-parent_2.12</artifactId>
         <version>3.5.0-SNAPSHOT</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>spark-connect-common_2.13</artifactId>
+    <artifactId>spark-connect-common_2.12</artifactId>
     <packaging>jar</packaging>
     <name>Spark Project Connect Common</name>
     <url>https://spark.apache.org/</url>

diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-connect_2.13</artifactId>
+  <artifactId>spark-connect_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Connect Server</name>
   <url>https://spark.apache.org/</url>
@@ -156,12 +156,12 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- #if scala-2.13 -->
+    <!-- #if scala-2.13 --><!--
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #endif scala-2.13 -->
+    --><!-- #endif scala-2.13 -->
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>

diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml
@@ -21,12 +21,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-docker-integration-tests_2.13</artifactId>
+  <artifactId>spark-docker-integration-tests_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Docker Integration Tests</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kafka-0-10-assembly_2.13</artifactId>
+  <artifactId>spark-streaming-kafka-0-10-assembly_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Integration for Kafka 0.10 Assembly</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml
@@ -20,13 +20,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.spark</groupId>
-  <artifactId>spark-sql-kafka-0-10_2.13</artifactId>
+  <artifactId>spark-sql-kafka-0-10_2.12</artifactId>
   <properties>
     <sbt.project.name>sql-kafka-0-10</sbt.project.name>
   </properties>
@@ -74,12 +74,12 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <!-- #if scala-2.13 -->
+    <!-- #if scala-2.13 --><!--
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #endif scala-2.13 -->
+    --><!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>

diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml
@@ -20,13 +20,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.spark</groupId>
-  <artifactId>spark-token-provider-kafka-0-10_2.13</artifactId>
+  <artifactId>spark-token-provider-kafka-0-10_2.12</artifactId>
   <properties>
     <sbt.project.name>token-provider-kafka-0-10</sbt.project.name>
   </properties>

diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kafka-0-10_2.13</artifactId>
+  <artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
   <properties>
     <sbt.project.name>streaming-kafka-0-10</sbt.project.name>
   </properties>
@@ -59,12 +59,12 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-    <!-- #if scala-2.13 -->
+    <!-- #if scala-2.13 --><!--
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
     </dependency>
-    <!-- #endif scala-2.13 -->
+    --><!-- #endif scala-2.13 -->
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>

diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml
@@ -20,12 +20,12 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-streaming-kinesis-asl-assembly_2.13</artifactId>
+  <artifactId>spark-streaming-kinesis-asl-assembly_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Project Kinesis Assembly</name>
   <url>https://spark.apache.org/</url>

diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml
@@ -19,13 +19,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.13</artifactId>
+    <artifactId>spark-parent_2.12</artifactId>
     <version>3.5.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
   <!-- Kinesis integration is not included by default due to ASL-licensed code. -->
-  <artifactId>spark-streaming-kinesis-asl_2.13</artifactId>
+  <artifactId>spark-streaming-kinesis-asl_2.12</artifactId>
   <packaging>jar</packaging>
   <name>Spark Kinesis Integration</name>