Merge branch 'apache:master' into master

.github/workflows/style.yml

@@ -38,6 +38,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Setup JDK 8
         uses: actions/setup-java@v3
         with:
@@ -104,10 +106,32 @@ jobs:
           echo "---------------------------------------------------------------------------------"
 
   shellcheck:
-    name: Shellcheck
+    name: Super Linter and Shellcheck
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
+      - name: Super Linter Checks
+        uses: github/super-linter/slim@v4
+        env:
+          CREATE_LOG_FILE: true
+          ERROR_ON_MISSING_EXEC_BIT: true
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IGNORE_GENERATED_FILES: true
+          IGNORE_GITIGNORED_FILES: true
+          LINTER_RULES_PATH: /
+          LOG_LEVEL: NOTICE
+          SUPPRESS_POSSUM: true
+          VALIDATE_BASH_EXEC: true
+          VALIDATE_ENV: true
+          VALIDATE_JSONC: true
+          VALIDATE_POWERSHELL: true
+          VALIDATE_XML: true
+      - name: Upload Super Linter logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: super-linter-log
+          path: super-linter.log
       - name: check bin directory
         uses: ludeeus/action-shellcheck@1.1.0
         with:

.gitignore

@@ -58,8 +58,7 @@ metastore_db
 derby.log
 rest-audit.log
 **/dependency-reduced-pom.xml
-metrics/report.json
-metrics/.report.json.crc
+metrics/
 /kyuubi-ha/embedded_zookeeper/
 embedded_zookeeper/
 /externals/kyuubi-spark-sql-engine/operation_logs/

LICENSE-binary

@@ -319,6 +319,8 @@ io.swagger.core.v3:swagger-models
 io.vertx:vertx-core
 io.vertx:vertx-grpc
 org.apache.zookeeper:zookeeper
+com.squareup.retrofit2:retrofit
+com.squareup.okhttp3:okhttp
 
 BSD
 ------------
@@ -356,6 +358,9 @@ org.codehaus.mojo:animal-sniffer-annotations
 org.slf4j:slf4j-api
 org.slf4j:jcl-over-slf4j
 org.slf4j:jul-over-slf4j
+com.theokanning.openai-gpt3-java:api
+com.theokanning.openai-gpt3-java:client
+com.theokanning.openai-gpt3-java:service
 
 kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/*
 kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css

build/dist

@@ -256,6 +256,7 @@ mkdir -p "$DISTDIR/externals/engines/spark"
 mkdir -p "$DISTDIR/externals/engines/trino"
 mkdir -p "$DISTDIR/externals/engines/hive"
 mkdir -p "$DISTDIR/externals/engines/jdbc"
+mkdir -p "$DISTDIR/externals/engines/chat"
 echo "Kyuubi $VERSION $GITREVSTRING built for" > "$DISTDIR/RELEASE"
 echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE"
 echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE"
@@ -313,6 +314,18 @@ for jar in $(ls "$DISTDIR/jars/"); do
   fi
 done
 
+# Copy chat engines
+cp "$KYUUBI_HOME/externals/kyuubi-chat-engine/target/kyuubi-chat-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/chat/"
+cp -r "$KYUUBI_HOME"/externals/kyuubi-chat-engine/target/scala-$SCALA_VERSION/jars/*.jar "$DISTDIR/externals/engines/chat/"
+
+# Share the jars w/ server to reduce binary size
+# shellcheck disable=SC2045
+for jar in $(ls "$DISTDIR/jars/"); do
+  if [[ -f "$DISTDIR/externals/engines/chat/$jar" ]]; then
+    (cd $DISTDIR/externals/engines/chat; ln -snf "../../../jars/$jar" "$DISTDIR/externals/engines/chat/$jar")
+  fi
+done
+
 # Copy kyuubi tools
 if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then
   mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes"

build/mvnd

@@ -25,7 +25,7 @@ _CALLING_DIR="$(pwd)"
 _COMPILE_JVM_OPTS="-Xms2g -Xmx2g -XX:ReservedCodeCacheSize=1g -Xss128m"
 
 if [ "$CI" ]; then
-  export MAVEN_CLI_OPTS="-Dmvnd.minThreads=8 --no-transfer-progress --errors --fail-fast -Dstyle.color=always"
+  export MAVEN_CLI_OPTS="-Dmvnd.minThreads=4 --no-transfer-progress --errors --fail-fast -Dstyle.color=always"
 fi
 
 # Installs any application tarball given a URL, the expected tarball name,

docker/playground/.env

@@ -19,7 +19,7 @@ AWS_JAVA_SDK_VERSION=1.12.239
 HADOOP_VERSION=3.3.1
 HIVE_VERSION=2.3.9
 ICEBERG_VERSION=1.1.0
-KYUUBI_VERSION=1.6.1-incubating
+KYUUBI_VERSION=1.7.0
 KYUUBI_HADOOP_VERSION=3.3.4
 POSTGRES_VERSION=12
 POSTGRES_JDBC_VERSION=42.3.4

docker/playground/compose.yml

@@ -68,6 +68,7 @@ services:
     ports:
       - 4040-4050:4040-4050
       - 10009:10009
+      - 10099:10099
     volumes:
       - ./conf/core-site.xml:/etc/hadoop/conf/core-site.xml
       - ./conf/hive-site.xml:/etc/hive/conf/hive-site.xml

docker/playground/conf/kyuubi-defaults.conf

@@ -18,8 +18,10 @@
 ## Kyuubi Configurations
 
 kyuubi.authentication=NONE
-kyuubi.frontend.thrift.binary.bind.host=0.0.0.0
+kyuubi.frontend.bind.host=0.0.0.0
+kyuubi.frontend.protocols=THRIFT_BINARY,REST
 kyuubi.frontend.thrift.binary.bind.port=10009
+kyuubi.frontend.rest.bind.port=10099
 kyuubi.ha.addresses=zookeeper:2181
 kyuubi.session.engine.idle.timeout=PT5M
 kyuubi.operation.incremental.collect=true

docs/client/rest/rest_api.md

@@ -89,13 +89,9 @@ Create a session
 
 #### Request Parameters
 
-| Name            | Description                              | Type   |
-|:----------------|:-----------------------------------------|:-------|
-| protocolVersion | The protocol version of Hive CLI service | Int    |
-| user            | The user name                            | String |
-| password        | The user password                        | String |
-| ipAddr          | The user client IP address               | String |
-| configs         | The configuration of the session         | Map    |
+| Name    | Description                      | Type |
+|:--------|:---------------------------------|:-----|
+| configs | The configuration of the session | Map  |
 
 #### Response Body
 

docs/deployment/migration-guide.md

@@ -17,14 +17,18 @@
 
 # Kyuubi Migration Guide
 
+## Upgrading from Kyuubi 1.7.0 to 1.7.1
+
+* Since Kyuubi 1.7.1, `protocolVersion` is removed from the request parameters of the REST API `Open(create) a session`. All removed or unknown parameters will be silently ignored and affects nothing.
+
 ## Upgrading from Kyuubi 1.6 to 1.7
 
 * In Kyuubi 1.7, `kyuubi.ha.zookeeper.engine.auth.type` does not fallback to `kyuubi.ha.zookeeper.auth.type`.  
   When Kyuubi engine does Kerberos authentication with Zookeeper, user needs to explicitly set `kyuubi.ha.zookeeper.engine.auth.type` to `KERBEROS`.
 * Since Kyuubi 1.7, Kyuubi returns engine's information for `GetInfo` request instead of server. To restore the previous behavior, set `kyuubi.server.info.provider` to `SERVER`.
 * Since Kyuubi 1.7, Kyuubi session type `SQL` is refactored to `INTERACTIVE`, because Kyuubi supports not only `SQL` session, but also `SCALA` and `PYTHON` sessions.
   User need to use `INTERACTIVE` sessionType to look up the session event.
-* Since Kyuubi 1.7, the REST API of `Open(create) a session` will not contains parameters `user` `password` and `IpAddr`. User and password should be set in `Authorization` of http request if needed.
+* Since Kyuubi 1.7, the REST API of `Open(create) a session` will not contain parameters `user` `password` and `IpAddr`. User and password should be set in `Authorization` of http request if needed.
 
 ## Upgrading from Kyuubi 1.6.0 to 1.6.1