[HADP-59723] Add celeborn plugins (apache#848)

Author: turboFei

pom.xml

@@ -313,7 +313,7 @@
     <!-- Needed for consistent times -->
     <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
 
-    <celeborn.client.version>0.6.0.0.3.2</celeborn.client.version>
+    <celeborn.client.version>0.6.1.0.0.0</celeborn.client.version>
     <iceberg.version>1.6.1.1.9.0</iceberg.version>
 
     <!-- SPARK-36796 for JDK-17 test-->

sql/ebay-etl/pom.xml

@@ -71,6 +71,10 @@
             <artifactId>spark-hive_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.celeborn</groupId>
+            <artifactId>celeborn-client-spark-3-shaded_${scala.binary.version}</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

sql/ebay-etl/src/main/java/org/apache/celeborn/client/spark/ebay/ShuffleMapPartitionsFallbackPolicy.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.client.spark.ebay;
+
+import org.apache.spark.ShuffleDependency;
+import org.apache.spark.shuffle.celeborn.ShuffleFallbackPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.celeborn.client.LifecycleManager;
+import org.apache.celeborn.common.CelebornConf;
+
+public class ShuffleMapPartitionsFallbackPolicy implements ShuffleFallbackPolicy {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(ShuffleMapPartitionsFallbackPolicy.class);
+  public static final String SPARK_SHUFFLE_FALLBACK_MAP_PARTITION_THRESHOLD =
+      "celeborn.client.spark.shuffle.fallback.numMapPartitionsThreshold";
+
+  @Override
+  public boolean needFallback(
+      ShuffleDependency<?, ?, ?> shuffleDependency,
+      CelebornConf celebornConf,
+      LifecycleManager lifecycleManager) {
+    Integer shuffleMapPartitions = shuffleDependency.rdd().getNumPartitions();
+    Integer largeMapPartitionsThreshold =
+        celebornConf.getInt(SPARK_SHUFFLE_FALLBACK_MAP_PARTITION_THRESHOLD, Integer.MAX_VALUE);
+    boolean needFallback = shuffleMapPartitions > largeMapPartitionsThreshold;
+    if (needFallback) {
+      LOG.warn(
+          "Shuffle map partition number {} exceeds threshold {}, fallback to spark built-in shuffle implementation.",
+          shuffleMapPartitions,
+          largeMapPartitionsThreshold);
+    }
+    return needFallback;
+  }
+}

sql/ebay-etl/src/main/resources/META-INF/services/org.apache.spark.shuffle.celeborn.ShuffleFallbackPolicy

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.celeborn.client.spark.ebay.ShuffleMapPartitionsFallbackPolicy

sql/ebay-etl/src/main/scala/org/apache/spark/sql/ebay/celeborn/ApplicationInfoProviderImpl.scala

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.ebay.celeborn
+
+import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.client.ApplicationInfoProvider
+
+import org.apache.spark.SparkContext
+
+class ApplicationInfoProviderImpl(conf: CelebornConf) extends ApplicationInfoProvider(conf) {
+  override def provide(): Map[String, String] = {
+    SparkContext.getActive.map { sc =>
+      Map("queue" -> sc.getConf.get("spark.yarn.queue", "default"))
+    }.getOrElse(Map.empty[String, String])
+  }
+}