From c8c6b0c33084814a566a9dd98109453291f0f098 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 31 May 2023 22:37:03 -0700 Subject: [PATCH] Revert [SPARK-43836][SPARK-43845][SPARK-43858][SPARK-43858] to keep Scala 2.12 for Spark 3.x ### What changes were proposed in this pull request? This PR aims to revert the following 4 commits effectively. SPARK-43836 Make Scala 2.13 as default in Spark 3.5 SPARK-43845 Setup Scala 2.12 Daily GitHub Action Job SPARK-43858 Make benchmark Github Action task use Scala 2.13 as default SPARK-43858 Restore the Scala version to 2.13 after run benchmark ### Why are the changes needed? According to the dev mailing list discussion, we keep the Scala 2.12 for Spark 3.x. - https://lists.apache.org/thread/xz7x0nkjn1d13yl9on59lfttm6f0yd45 We open a new thread, "Apache Spark 4.0 Timeframe?", for Spark 4 planning. - https://lists.apache.org/thread/xhkgj60j361gdpywoxxz7qspp2w80ry6 ### Does this PR introduce _any_ user-facing change? No. This is a recovery to the existing behavior. ### How was this patch tested? Pass the CIs. Closes #41416 from dongjoon-hyun/revert-2.13. Lead-authored-by: Dongjoon Hyun Co-authored-by: yangjie01 Signed-off-by: Dongjoon Hyun --- .github/workflows/benchmark.yml | 4 +- ...{build_scala212.yml => build_scala213.yml} | 4 +- assembly/pom.xml | 4 +- common/kvstore/pom.xml | 4 +- common/network-common/pom.xml | 4 +- common/network-shuffle/pom.xml | 4 +- common/network-yarn/pom.xml | 4 +- common/sketch/pom.xml | 4 +- common/tags/pom.xml | 4 +- common/unsafe/pom.xml | 4 +- common/utils/pom.xml | 4 +- connector/avro/pom.xml | 8 +- connector/connect/client/jvm/pom.xml | 4 +- connector/connect/common/pom.xml | 4 +- connector/connect/server/pom.xml | 8 +- connector/docker-integration-tests/pom.xml | 4 +- connector/kafka-0-10-assembly/pom.xml | 4 +- connector/kafka-0-10-sql/pom.xml | 8 +- connector/kafka-0-10-token-provider/pom.xml | 4 +- connector/kafka-0-10/pom.xml | 8 +- connector/kinesis-asl-assembly/pom.xml | 4 +- connector/kinesis-asl/pom.xml | 4 +- connector/protobuf/pom.xml | 8 +- connector/spark-ganglia-lgpl/pom.xml | 4 +- core/pom.xml | 8 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 43 +++--- dev/mima | 8 +- dev/test-dependencies.sh | 2 +- docs/_plugins/copy_api_dirs.rb | 14 +- examples/pom.xml | 4 +- graphx/pom.xml | 4 +- hadoop-cloud/pom.xml | 4 +- launcher/pom.xml | 4 +- mllib-local/pom.xml | 4 +- mllib/pom.xml | 8 +- pom.xml | 137 ++++++++++-------- repl/pom.xml | 4 +- resource-managers/kubernetes/core/pom.xml | 4 +- .../kubernetes/integration-tests/pom.xml | 4 +- resource-managers/mesos/pom.xml | 4 +- resource-managers/yarn/pom.xml | 4 +- sql/catalyst/pom.xml | 8 +- sql/core/pom.xml | 8 +- sql/hive-thriftserver/pom.xml | 8 +- sql/hive/pom.xml | 8 +- streaming/pom.xml | 8 +- tools/pom.xml | 4 +- 47 files changed, 213 insertions(+), 207 deletions(-) rename .github/workflows/{build_scala212.yml => build_scala213.yml} (93%) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f9699ac80f24f..8671cff054bb8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,7 +33,7 @@ on: scala: description: 'Scala version: 2.12 or 2.13' required: true - default: '2.13' + default: '2.12' failfast: description: 'Failfast: true or false' required: true @@ -182,7 +182,7 @@ jobs: "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \ "${{ github.event.inputs.class }}" # Revert to default Scala version to clean up unnecessary git diff - dev/change-scala-version.sh 2.13 + dev/change-scala-version.sh 2.12 # To keep the directory structure and file permissions, tar them # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files echo "Preparing the benchmark results:" diff --git a/.github/workflows/build_scala212.yml b/.github/workflows/build_scala213.yml similarity index 93% rename from .github/workflows/build_scala212.yml rename to .github/workflows/build_scala213.yml index 89d22db72f0f8..cae0981ee1e8a 100644 --- a/.github/workflows/build_scala212.yml +++ b/.github/workflows/build_scala213.yml @@ -17,7 +17,7 @@ # under the License. # -name: "Build (master, Scala 2.12, Hadoop 3, JDK 8)" +name: "Build (master, Scala 2.13, Hadoop 3, JDK 8)" on: schedule: @@ -36,7 +36,7 @@ jobs: hadoop: hadoop3 envs: >- { - "SCALA_PROFILE": "scala2.12" + "SCALA_PROFILE": "scala2.13" } jobs: >- { diff --git a/assembly/pom.xml b/assembly/pom.xml index d4d7a1db4a29e..09d6bd8a33f79 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-assembly_2.13 + spark-assembly_2.12 Spark Project Assembly https://spark.apache.org/ pom diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 69f9f186e0889..bef8303874b20 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-kvstore_2.13 + spark-kvstore_2.12 jar Spark Project Local DB https://spark.apache.org/ diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 9f90d12216e69..8a63e999c53cd 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-network-common_2.13 + spark-network-common_2.12 jar Spark Project Networking https://spark.apache.org/ diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 864f1cc2d3715..a8bde14a259f0 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-network-shuffle_2.13 + spark-network-shuffle_2.12 jar Spark Project Shuffle Streaming Service https://spark.apache.org/ diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index c19ac33afa5cd..671d5cb7e0178 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-network-yarn_2.13 + spark-network-yarn_2.12 jar Spark Project YARN Shuffle Service https://spark.apache.org/ diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 6cf1a4fb83e56..4cc597519c3dd 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-sketch_2.13 + spark-sketch_2.12 jar Spark Project Sketch https://spark.apache.org/ diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 1eb8352e32df3..9a44c847d8a03 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-tags_2.13 + spark-tags_2.12 jar Spark Project Tags https://spark.apache.org/ diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index e37f1b80a5045..a61f00084eb64 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-unsafe_2.13 + spark-unsafe_2.12 jar Spark Project Unsafe https://spark.apache.org/ diff --git a/common/utils/pom.xml b/common/utils/pom.xml index ee10a60618297..36cfceed931e0 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-common-utils_2.13 + spark-common-utils_2.12 jar Spark Project Common Utils https://spark.apache.org/ diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml index 7087fdbccd04d..597e3c2235f7a 100644 --- a/connector/avro/pom.xml +++ b/connector/avro/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-avro_2.13 + spark-avro_2.12 avro @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - + + --> org.tukaani xz diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 8dc4c3b7a19dd..4d0a437932964 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../../../pom.xml - spark-connect-client-jvm_2.13 + spark-connect-client-jvm_2.12 jar Spark Project Connect Client https://spark.apache.org/ diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index fcd72480e0ef3..06076646df7a5 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../../pom.xml - spark-connect-common_2.13 + spark-connect-common_2.12 jar Spark Project Connect Common https://spark.apache.org/ diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index f3b43e14cdab2..f9dc40ce85a44 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../../pom.xml - spark-connect_2.13 + spark-connect_2.12 jar Spark Project Connect Server https://spark.apache.org/ @@ -156,12 +156,12 @@ - + + --> com.google.guava guava diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml index e40269838522d..cc549487a8b57 100644 --- a/connector/docker-integration-tests/pom.xml +++ b/connector/docker-integration-tests/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-docker-integration-tests_2.13 + spark-docker-integration-tests_2.12 jar Spark Project Docker Integration Tests https://spark.apache.org/ diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index f339e8c2e4fd2..340974cc789bd 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10-assembly_2.13 + spark-streaming-kafka-0-10-assembly_2.12 jar Spark Integration for Kafka 0.10 Assembly https://spark.apache.org/ diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index f5a12b61c2bea..fdd1196cd446a 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-sql-kafka-0-10_2.13 + spark-sql-kafka-0-10_2.12 sql-kafka-0-10 @@ -74,12 +74,12 @@ test-jar test - + + --> org.apache.kafka kafka-clients diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml index b3c0889d9475a..3256130c50f3b 100644 --- a/connector/kafka-0-10-token-provider/pom.xml +++ b/connector/kafka-0-10-token-provider/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-token-provider-kafka-0-10_2.13 + spark-token-provider-kafka-0-10_2.12 token-provider-kafka-0-10 diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml index f1820bb595a2d..706eb2dd2c399 100644 --- a/connector/kafka-0-10/pom.xml +++ b/connector/kafka-0-10/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10_2.13 + spark-streaming-kafka-0-10_2.12 streaming-kafka-0-10 @@ -59,12 +59,12 @@ test-jar test - + + --> org.apache.kafka kafka-clients diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml index 2cba2668f049a..cd5c0393f6f84 100644 --- a/connector/kinesis-asl-assembly/pom.xml +++ b/connector/kinesis-asl-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl-assembly_2.13 + spark-streaming-kinesis-asl-assembly_2.12 jar Spark Project Kinesis Assembly https://spark.apache.org/ diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml index af9cd4b7ec96e..c70a073e73407 100644 --- a/connector/kinesis-asl/pom.xml +++ b/connector/kinesis-asl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl_2.13 + spark-streaming-kinesis-asl_2.12 jar Spark Kinesis Integration diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index c99b9a0608745..e85f07841dfab 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-protobuf_2.13 + spark-protobuf_2.12 protobuf @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - + + --> com.google.protobuf protobuf-java diff --git a/connector/spark-ganglia-lgpl/pom.xml b/connector/spark-ganglia-lgpl/pom.xml index 00f4769fd60ad..c0dcde1355849 100644 --- a/connector/spark-ganglia-lgpl/pom.xml +++ b/connector/spark-ganglia-lgpl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-ganglia-lgpl_2.13 + spark-ganglia-lgpl_2.12 jar Spark Ganglia Integration diff --git a/core/pom.xml b/core/pom.xml index 79bf8a2163554..6519b46d96e31 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-core_2.13 + spark-core_2.12 jar Spark Project Core https://spark.apache.org/ @@ -35,12 +35,12 @@ - + + --> org.apache.avro avro diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 99b627af34079..830c391046c92 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -5,7 +5,7 @@ RoaringBitmap/0.9.44//RoaringBitmap-0.9.44.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar -algebra_2.13/2.8.0//algebra_2.13-2.8.0.jar +algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar @@ -30,11 +30,11 @@ azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar azure-storage/7.0.1//azure-storage-7.0.1.jar blas/3.0.3//blas-3.0.3.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar -breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar -breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar -cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar +breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar +breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar +cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar chill-java/0.10.0//chill-java-0.10.0.jar -chill_2.13/0.10.0//chill_2.13-0.10.0.jar +chill_2.12/0.10.0//chill_2.12-0.10.0.jar commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar @@ -104,7 +104,7 @@ jackson-databind/2.15.1//jackson-databind-2.15.1.jar jackson-dataformat-cbor/2.15.1//jackson-dataformat-cbor-2.15.1.jar jackson-dataformat-yaml/2.15.1//jackson-dataformat-yaml-2.15.1.jar jackson-datatype-jsr310/2.15.1//jackson-datatype-jsr310-2.15.1.jar -jackson-module-scala_2.13/2.15.1//jackson-module-scala_2.13-2.15.1.jar +jackson-module-scala_2.12/2.15.1//jackson-module-scala_2.12-2.15.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar @@ -129,16 +129,14 @@ jettison/1.5.3//jettison-1.5.3.jar jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar jline/2.14.6//jline-2.14.6.jar -jline/3.21.0//jline-3.21.0.jar -jna/5.9.0//jna-5.9.0.jar joda-time/2.12.5//joda-time-2.12.5.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar -json4s-ast_2.13/3.7.0-M11//json4s-ast_2.13-3.7.0-M11.jar -json4s-core_2.13/3.7.0-M11//json4s-core_2.13-3.7.0-M11.jar -json4s-jackson_2.13/3.7.0-M11//json4s-jackson_2.13-3.7.0-M11.jar -json4s-scalap_2.13/3.7.0-M11//json4s-scalap_2.13-3.7.0-M11.jar +json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar +json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar +json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar +json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/2.0.7//jul-to-slf4j-2.0.7.jar @@ -226,22 +224,21 @@ pickle/1.3//pickle-1.3.jar py4j/0.10.9.7//py4j-0.10.9.7.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar rocksdbjni/8.1.1.1//rocksdbjni-8.1.1.1.jar -scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar -scala-compiler/2.13.8//scala-compiler-2.13.8.jar -scala-library/2.13.8//scala-library-2.13.8.jar -scala-parallel-collections_2.13/1.0.4//scala-parallel-collections_2.13-1.0.4.jar -scala-parser-combinators_2.13/2.2.0//scala-parser-combinators_2.13-2.2.0.jar -scala-reflect/2.13.8//scala-reflect-2.13.8.jar -scala-xml_2.13/2.1.0//scala-xml_2.13-2.1.0.jar +scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar +scala-compiler/2.12.17//scala-compiler-2.12.17.jar +scala-library/2.12.17//scala-library-2.12.17.jar +scala-parser-combinators_2.12/2.2.0//scala-parser-combinators_2.12-2.2.0.jar +scala-reflect/2.12.17//scala-reflect-2.12.17.jar +scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar shims/0.9.44//shims-0.9.44.jar slf4j-api/2.0.7//slf4j-api-2.0.7.jar snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar snakeyaml/2.0//snakeyaml-2.0.jar snappy-java/1.1.10.0//snappy-java-1.1.10.0.jar -spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar -spire-platform_2.13/0.18.0//spire-platform_2.13-0.18.0.jar -spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar -spire_2.13/0.18.0//spire_2.13-0.18.0.jar +spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar +spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar +spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar +spire_2.12/0.17.0//spire_2.12-0.17.0.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar super-csv/2.2.0//super-csv-2.2.0.jar diff --git a/dev/mima b/dev/mima index 32c3718e4ccca..4a9e343b0a78f 100755 --- a/dev/mima +++ b/dev/mima @@ -24,9 +24,9 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} -TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} +TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" +OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* @@ -42,7 +42,7 @@ $JAVA_CMD \ -cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \ org.apache.spark.tools.GenerateMIMAIgnore -echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" +echo -e "q\n" | build/sbt -mem 5120 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" ret_val=$? if [ $ret_val != 0 ]; then diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 07da6497b97d2..d7967ac3afa90 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -62,7 +62,7 @@ SCALA_BINARY_VERSION=$($MVN -q \ -Dexec.args='${scala.binary.version}' \ --non-recursive \ org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+') -if [[ "$SCALA_BINARY_VERSION" != "2.13" ]]; then +if [[ "$SCALA_BINARY_VERSION" != "2.12" ]]; then echo "Skip dependency testing on $SCALA_BINARY_VERSION" exit 0 fi diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 9cb073ef1e00c..28d5e0d82c93a 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -26,8 +26,8 @@ curr_dir = pwd cd("..") - puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") + puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") puts "Moving back into docs dir." cd("docs") @@ -37,7 +37,7 @@ # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.13/unidoc" + source = "../target/scala-2.12/unidoc" dest = "api/scala" puts "Making directory " + dest @@ -119,8 +119,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark doc generation failed") + puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt clean package -Phive") || raise("PySpark doc generation failed") puts "Moving back into docs dir." cd("docs") @@ -165,8 +165,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc generation failed") + puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt clean package -Phive") || raise("SQL doc generation failed") puts "Moving back into docs dir." cd("docs") diff --git a/examples/pom.xml b/examples/pom.xml index 57e41724bdca4..e8f22b995fded 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-examples_2.13 + spark-examples_2.12 jar Spark Project Examples https://spark.apache.org/ diff --git a/graphx/pom.xml b/graphx/pom.xml index 5d01dd06c0ecb..48baeb9a87560 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-graphx_2.13 + spark-graphx_2.12 graphx diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 21c1e0fee1ddf..02e7675df286c 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-hadoop-cloud_2.13 + spark-hadoop-cloud_2.12 jar Spark Project Hadoop Cloud Integration diff --git a/launcher/pom.xml b/launcher/pom.xml index 0bc3ae20ee183..aba7ee82d53cf 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-launcher_2.13 + spark-launcher_2.12 jar Spark Project Launcher https://spark.apache.org/ diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 83ca643f43bce..00c16a8b6a544 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-mllib-local_2.13 + spark-mllib-local_2.12 mllib-local diff --git a/mllib/pom.xml b/mllib/pom.xml index 07290124273f2..73af83c758688 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-mllib_2.13 + spark-mllib_2.12 mllib @@ -91,12 +91,12 @@ test-jar test - + + --> org.scalanlp breeze_${scala.binary.version} diff --git a/pom.xml b/pom.xml index acac91f8d33c1..8bf01db85a950 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 18 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT pom Spark Project Parent POM @@ -167,8 +167,8 @@ 3.2.2 4.4 - 2.13.8 - 2.13 + 2.12.17 + 2.12 2.2.0 4.8.0 @@ -422,13 +422,13 @@ ${project.version} test-jar - + + --> com.twitter chill_${scala.binary.version} @@ -1073,7 +1073,7 @@ org.scala-lang.modules - scala-xml_2.13 + scala-xml_2.12 @@ -2866,49 +2866,9 @@ -feature -explaintypes -target:jvm-1.8 - -Wconf:cat=deprecation:wv,any:e - -Wunused:imports - - -Wconf:cat=scaladoc:wv - -Wconf:cat=lint-multiarg-infix:wv - -Wconf:cat=other-nullary-override:wv - -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv - - -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s - -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s - -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s - -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s - -Wconf:msg=method without a parameter list overrides a method with a single empty one:s - - -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e - - -Wconf:cat=unchecked&msg=outer reference:s - -Wconf:cat=unchecked&msg=eliminated by erasure:s - -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s - - -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBase.scala:s - -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBaseOps.scala:s + -Xfatal-warnings + -Ywarn-unused:imports + -P:silencer:globalFilters=.*deprecated.* -Xss128m @@ -2924,6 +2884,13 @@ ${java.version} -Xlint:all,-serial,-path,-try + + + com.github.ghik + silencer-plugin_${scala.version} + 1.7.10 + + @@ -3574,13 +3541,6 @@ - - scala-2.13 - - 2.13.8 - - - scala-2.12 @@ -3590,6 +3550,20 @@ --> 2.12.17 + + + + + + + + + + scala-2.13 + + 2.13.8 + 2.13 + @@ -3603,16 +3577,51 @@ -feature -explaintypes -target:jvm-1.8 - -Xfatal-warnings - -Ywarn-unused:imports - -P:silencer:globalFilters=.*deprecated.* + -Wconf:cat=deprecation:wv,any:e + -Wunused:imports + + -Wconf:cat=scaladoc:wv + -Wconf:cat=lint-multiarg-infix:wv + -Wconf:cat=other-nullary-override:wv + -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv + + -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s + -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s + -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s + -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s + -Wconf:msg=method without a parameter list overrides a method with a single empty one:s + + -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e + + -Wconf:cat=unchecked&msg=outer reference:s + -Wconf:cat=unchecked&msg=eliminated by erasure:s + -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s + + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBase.scala:s + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBaseOps.scala:s - - com.github.ghik - silencer-plugin_${scala.version} - 1.7.10 - diff --git a/repl/pom.xml b/repl/pom.xml index 74ac775100cb8..8c0f9f989c170 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-repl_2.13 + spark-repl_2.12 jar Spark Project REPL https://spark.apache.org/ diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 72c7f1f12f42d..9dab5496184e2 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../../pom.xml - spark-kubernetes_2.13 + spark-kubernetes_2.12 jar Spark Project Kubernetes diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 3e25e7053707a..02894f82eec9d 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../../pom.xml - spark-kubernetes-integration-tests_2.13 + spark-kubernetes-integration-tests_2.12 kubernetes-integration-tests diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 267d6c7d84f21..7510ecac3e7fc 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-mesos_2.13 + spark-mesos_2.12 jar Spark Project Mesos diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 2cda552a9c47c..dcc7bcdd1af38 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -19,12 +19,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-yarn_2.13 + spark-yarn_2.12 jar Spark Project YARN diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index cf01c0af4a452..9057cc8a20efe 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-catalyst_2.13 + spark-catalyst_2.12 jar Spark Project Catalyst https://spark.apache.org/ @@ -87,12 +87,12 @@ spark-sketch_${scala.binary.version} ${project.version} - + + --> org.scalacheck scalacheck_${scala.binary.version} diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 5d4f7572d0022..7f4c2a4cfa54d 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-sql_2.13 + spark-sql_2.12 jar Spark Project SQL https://spark.apache.org/ @@ -89,12 +89,12 @@ test - + + --> org.apache.orc orc-core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 4bbb92d1376a0..ad7fc0d2ac4bd 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-hive-thriftserver_2.13 + spark-hive-thriftserver_2.12 jar Spark Project Hive Thrift Server https://spark.apache.org/ @@ -61,12 +61,12 @@ test-jar test - + + --> com.google.guava guava diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index b267830a3ad5f..16d915c233ee4 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../../pom.xml - spark-hive_2.13 + spark-hive_2.12 jar Spark Project Hive https://spark.apache.org/ @@ -79,12 +79,12 @@ test-jar test - + + --> ${hive.group} hive-common diff --git a/streaming/pom.xml b/streaming/pom.xml index a36370a1e8b61..bebfd3abcce39 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.13 + spark-parent_2.12 3.5.0-SNAPSHOT ../pom.xml - spark-streaming_2.13 + spark-streaming_2.12 streaming @@ -50,12 +50,12 @@ org.apache.spark spark-tags_${scala.binary.version} - + + -->