Switch to Kyuubi Shaded Zookeeper

.github/workflows/master.yml

@@ -57,17 +57,17 @@ jobs:
         include:
           - java: 8
             spark: '3.3'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
             comment: 'verify-on-spark-3.1-binary'
           - java: 8
             spark: '3.3'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
             comment: 'verify-on-spark-3.2-binary'
           - java: 8
             spark: '3.3'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
             comment: 'verify-on-spark-3.4-binary'
     env:

LICENSE-binary

@@ -212,9 +212,6 @@ com.google.android:annotations
 commons-lang:commons-lang
 commons-logging:commons-logging
 org.apache.commons:commons-lang3
-org.apache.curator:curator-client
-org.apache.curator:curator-framework
-org.apache.curator:curator-recipes
 org.apache.derby:derby
 com.google.errorprone:error_prone_annotations
 net.jodah:failsafe
@@ -318,7 +315,6 @@ io.swagger.core.v3:swagger-jaxrs2
 io.swagger.core.v3:swagger-models
 io.vertx:vertx-core
 io.vertx:vertx-grpc
-org.apache.zookeeper:zookeeper
 com.squareup.retrofit2:retrofit
 com.squareup.okhttp3:okhttp
 org.apache.kafka:kafka-clients

NOTICE-binary

@@ -92,15 +92,6 @@ Copyright 2001-2020 The Apache Software Foundation
 Apache Commons Logging
 Copyright 2003-2013 The Apache Software Foundation
 
-Curator Client
-Copyright 2011-2017 The Apache Software Foundation
-
-Curator Framework
-Copyright 2011-2017 The Apache Software Foundation
-
-Curator Recipes
-Copyright 2011-2017 The Apache Software Foundation
-
 =========================================================================
 ==  NOTICE file corresponding to section 4(d) of the Apache License,
 ==  Version 2.0, in this case for the Apache Derby distribution.

bin/kyuubi-zk-cli

@@ -17,7 +17,7 @@
 #
 
 ## Zookeeper Shell Client Entrance
-CLASS="org.apache.zookeeper.ZooKeeperMain"
+CLASS="org.apache.kyuubi.shaded.zookeeper.ZooKeeperMain"
 
 export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
 

dev/dependencyList

@@ -32,9 +32,6 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-lang/2.6//commons-lang-2.6.jar
 commons-lang3/3.12.0//commons-lang3-3.12.0.jar
 commons-logging/1.1.3//commons-logging-1.1.3.jar
-curator-client/2.12.0//curator-client-2.12.0.jar
-curator-framework/2.12.0//curator-framework-2.12.0.jar
-curator-recipes/2.12.0//curator-recipes-2.12.0.jar
 derby/10.14.2.0//derby-10.14.2.0.jar
 error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar
 failsafe/2.4.4//failsafe-2.4.4.jar
@@ -195,5 +192,4 @@ units/1.6//units-1.6.jar
 vertx-core/4.3.2//vertx-core-4.3.2.jar
 vertx-grpc/4.3.2//vertx-grpc-4.3.2.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
 zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar

externals/kyuubi-flink-sql-engine/pom.xml

@@ -186,20 +186,15 @@
                     <shadedArtifactAttached>false</shadedArtifactAttached>
                     <artifactSet>
                         <includes>
-                            <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
                             <include>com.fasterxml.jackson.core:*</include>
                             <include>com.fasterxml.jackson.module:*</include>
                             <include>com.google.guava:failureaccess</include>
                             <include>com.google.guava:guava</include>
                             <include>commons-codec:commons-codec</include>
                             <include>org.apache.commons:commons-lang3</include>
-                            <include>org.apache.curator:curator-client</include>
-                            <include>org.apache.curator:curator-framework</include>
-                            <include>org.apache.curator:curator-recipes</include>
                             <include>org.apache.hive:hive-service-rpc</include>
                             <include>org.apache.thrift:*</include>
-                            <include>org.apache.zookeeper:*</include>
+                            <include>org.apache.kyuubi:*</include>
                         </includes>
                     </artifactSet>
                     <filters>
@@ -228,13 +223,6 @@
                                 <include>com.fasterxml.jackson.**</include>
                             </includes>
                         </relocation>
-                        <relocation>
-                            <pattern>org.apache.curator</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
-                            <includes>
-                                <include>org.apache.curator.**</include>
-                            </includes>
-                        </relocation>
                         <relocation>
                             <pattern>com.google.common</pattern>
                             <shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
@@ -278,20 +266,6 @@
                                 <include>org.apache.thrift.**</include>
                             </includes>
                         </relocation>
-                        <relocation>
-                            <pattern>org.apache.jute</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
-                            <includes>
-                                <include>org.apache.jute.**</include>
-                            </includes>
-                        </relocation>
-                        <relocation>
-                            <pattern>org.apache.zookeeper</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
-                            <includes>
-                                <include>org.apache.zookeeper.**</include>
-                            </includes>
-                        </relocation>
                     </relocations>
                     <transformers>
                         <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>

externals/kyuubi-hive-sql-engine/pom.xml

@@ -179,12 +179,7 @@
                             <include>com.fasterxml.jackson.core:jackson-core</include>
                             <include>com.fasterxml.jackson.core:jackson-databind</include>
                             <include>com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
-                            <include>org.apache.curator:curator-client</include>
-                            <include>org.apache.curator:curator-framework</include>
-                            <include>org.apache.curator:curator-recipes</include>
+                            <include>org.apache.kyuubi:*</include>
                         </includes>
                     </artifactSet>
                     <filters>
@@ -205,15 +200,6 @@
                             </excludes>
                         </filter>
                     </filters>
-                    <relocations>
-                        <relocation>
-                            <pattern>org.apache.curator</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
-                            <includes>
-                                <include>org.apache.curator.**</include>
-                            </includes>
-                        </relocation>
-                    </relocations>
                 </configuration>
                 <executions>
                     <execution>

externals/kyuubi-spark-sql-engine/pom.xml

@@ -194,15 +194,9 @@
                             <include>io.perfmark:perfmark-api</include>
                             <include>io.vertx:*</include>
                             <include>net.jodah:failsafe</include>
-                            <include>org.apache.curator:curator-client</include>
-                            <include>org.apache.curator:curator-framework</include>
-                            <include>org.apache.curator:curator-recipes</include>
                             <include>org.apache.hive:hive-service-rpc</include>
-                            <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
-                            <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
+                            <include>org.apache.kyuubi:*</include>
                             <include>org.apache.thrift:*</include>
-                            <include>org.apache.zookeeper:zookeeper</include>
                             <include>org.checkerframework:checker-qual</include>
                             <include>org.codehaus.mojo:animal-sniffer-annotations</include>
                         </includes>
@@ -227,27 +221,6 @@
                         </filter>
                     </filters>
                     <relocations>
-                        <relocation>
-                            <pattern>org.apache.curator</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
-                            <includes>
-                                <include>org.apache.curator.**</include>
-                            </includes>
-                        </relocation>
-                        <relocation>
-                            <pattern>org.apache.zookeeper</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
-                            <includes>
-                                <include>org.apache.zookeeper.**</include>
-                            </includes>
-                        </relocation>
-                        <relocation>
-                            <pattern>org.apache.jute</pattern>
-                            <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
-                            <includes>
-                                <include>org.apache.jute.**</include>
-                            </includes>
-                        </relocation>
                         <relocation>
                             <pattern>org.apache.hive.service.rpc.thrift</pattern>
                             <shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>

kyuubi-assembly/pom.xml

@@ -69,28 +69,18 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-client-api</artifactId>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-client-runtime</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-framework</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-client</artifactId>
+            <artifactId>hadoop-client-api</artifactId>
         </dependency>
 
         <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-recipes</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-runtime</artifactId>
         </dependency>
 
         <dependency>

kyuubi-ctl/pom.xml

@@ -48,6 +48,11 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client-api</artifactId>
@@ -60,16 +65,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-framework</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-recipes</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>com.github.scopt</groupId>
             <artifactId>scopt_${scala.binary.version}</artifactId>
@@ -86,11 +81,6 @@
             <version>${snakeyaml.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.zookeeper</groupId>
-            <artifactId>zookeeper</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.kyuubi</groupId>
             <artifactId>kyuubi-common_${scala.binary.version}</artifactId>

kyuubi-ha/pom.xml

@@ -38,18 +38,8 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-framework</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-recipes</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.zookeeper</groupId>
-            <artifactId>zookeeper</artifactId>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
         </dependency>
 
         <dependency>

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala

@@ -17,7 +17,7 @@
 
 package org.apache.kyuubi.ha.client
 
-import org.apache.curator.utils.ZKPaths
+import org.apache.kyuubi.shaded.curator.utils.ZKPaths
 
 object DiscoveryPaths {
   def makePath(parent: String, firstChild: String, restChildren: String*): String = {

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala

@@ -17,13 +17,12 @@
 
 package org.apache.kyuubi.ha.client.zookeeper
 
-import org.apache.curator.framework.api.ACLProvider
-import org.apache.zookeeper.ZooDefs
-import org.apache.zookeeper.data.ACL
-
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.AuthTypes
+import org.apache.kyuubi.shaded.curator.framework.api.ACLProvider
+import org.apache.kyuubi.shaded.zookeeper.ZooDefs
+import org.apache.kyuubi.shaded.zookeeper.data.ACL
 
 class ZookeeperACLProvider(conf: KyuubiConf) extends ACLProvider {
 

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala

@@ -24,8 +24,6 @@ import javax.security.auth.login.Configuration
 import scala.util.Random
 
 import com.google.common.annotations.VisibleForTesting
-import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
-import org.apache.curator.retry._
 import org.apache.hadoop.security.UserGroupInformation
 
 import org.apache.kyuubi.Logging
@@ -34,6 +32,8 @@ import org.apache.kyuubi.ha.HighAvailabilityConf._
 import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies}
 import org.apache.kyuubi.ha.client.RetryPolicies._
 import org.apache.kyuubi.reflection.DynConstructors
+import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
+import org.apache.kyuubi.shaded.curator.retry._
 import org.apache.kyuubi.util.KyuubiHadoopUtils
 
 object ZookeeperClientProvider extends Logging {

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala

@@ -25,23 +25,6 @@ import java.util.concurrent.atomic.AtomicBoolean
 import scala.collection.JavaConverters._
 
 import com.google.common.annotations.VisibleForTesting
-import org.apache.curator.framework.CuratorFramework
-import org.apache.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
-import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex
-import org.apache.curator.framework.recipes.nodes.PersistentNode
-import org.apache.curator.framework.state.ConnectionState
-import org.apache.curator.framework.state.ConnectionState.CONNECTED
-import org.apache.curator.framework.state.ConnectionState.LOST
-import org.apache.curator.framework.state.ConnectionState.RECONNECTED
-import org.apache.curator.framework.state.ConnectionStateListener
-import org.apache.curator.retry.RetryForever
-import org.apache.curator.utils.ZKPaths
-import org.apache.zookeeper.CreateMode
-import org.apache.zookeeper.CreateMode.PERSISTENT
-import org.apache.zookeeper.KeeperException
-import org.apache.zookeeper.KeeperException.NodeExistsException
-import org.apache.zookeeper.WatchedEvent
-import org.apache.zookeeper.Watcher
 
 import org.apache.kyuubi.KYUUBI_VERSION
 import org.apache.kyuubi.KyuubiException
@@ -58,6 +41,23 @@ import org.apache.kyuubi.ha.client.ServiceNodeInfo
 import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.buildZookeeperClient
 import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.getGracefulStopThreadDelay
 import org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient.connectionChecker
+import org.apache.kyuubi.shaded.curator.framework.CuratorFramework
+import org.apache.kyuubi.shaded.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
+import org.apache.kyuubi.shaded.curator.framework.recipes.locks.InterProcessSemaphoreMutex
+import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.CONNECTED
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.LOST
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.RECONNECTED
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionStateListener
+import org.apache.kyuubi.shaded.curator.retry.RetryForever
+import org.apache.kyuubi.shaded.curator.utils.ZKPaths
+import org.apache.kyuubi.shaded.zookeeper.CreateMode
+import org.apache.kyuubi.shaded.zookeeper.CreateMode.PERSISTENT
+import org.apache.kyuubi.shaded.zookeeper.KeeperException
+import org.apache.kyuubi.shaded.zookeeper.KeeperException.NodeExistsException
+import org.apache.kyuubi.shaded.zookeeper.WatchedEvent
+import org.apache.kyuubi.shaded.zookeeper.Watcher
 import org.apache.kyuubi.util.ThreadUtils
 
 class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient {