[test](jdbc) add jdbc and hive regression test (apache#13143)

1. Modify default behavior of `build.sh`
    The `BUILD_JAVA_UDF` is default ON, so that jvm is needed for compilation and runtime.

2. Add docker-compose for MySQL 5.7, PostgreSQL 14 and Hive 2
   See `docker/thirdparties/docker-compose`.

3. Add some regression test cases for jdbc query on MySQL, PG and Hive Catalog
   The default is `false`, if set to true, you need first start docker for MySQL/PG/Hive.

4. Support `if not exists` and `if exists` for create/drop resource and create/drop encryptkey

.licenserc.yaml

@@ -70,5 +70,7 @@ header:
     - "docs/package-lock.json"
     - "regression-test/script/README"
     - "regression-test/suites/load_p0/stream_load/data"
+    - "docker/thirdparties/docker-compose/hive/scripts/README"
+    - "docker/thirdparties/docker-compose/hive/scripts/create.hql"
 
   comment: on-failure

build.sh

@@ -39,14 +39,14 @@ usage() {
 Usage: $0 <options>
   Optional options:
      [no option]        build all components
-     --fe               build Frontend and Spark DPP application
-     --be               build Backend
-     --meta-tool        build Backend meta tool
-     --broker           build Broker
-     --audit            build audit loader
-     --spark-dpp        build Spark DPP application
-     --hive-udf         build Hive UDF library for Spark Load
-     --java-udf         build Java UDF library
+     --fe               build Frontend and Spark DPP application. Default ON.
+     --be               build Backend. Default ON.
+     --meta-tool        build Backend meta tool. Default OFF.
+     --broker           build Broker. Default ON.
+     --audit            build audit loader. Default ON.
+     --spark-dpp        build Spark DPP application. Default ON.
+     --hive-udf         build Hive UDF library for Spark Load. Default ON.
+     --java-udf         build Java UDF library. Default ON.
      --clean            clean and build target
      -j                 build Backend parallel
 
@@ -145,7 +145,7 @@ if [[ "$#" == 1 ]]; then
     BUILD_AUDIT=1
     BUILD_META_TOOL='OFF'
     BUILD_SPARK_DPP=1
-    BUILD_JAVA_UDF=0 # TODO: open it when ready
+    BUILD_JAVA_UDF=1
     BUILD_HIVE_UDF=1
     CLEAN=0
 else
@@ -154,10 +154,13 @@ else
         --fe)
             BUILD_FE=1
             BUILD_SPARK_DPP=1
+            BUILD_JAVA_UDF=1
+            BUILD_HIVE_UDF=1
             shift
             ;;
         --be)
             BUILD_BE=1
+            BUILD_JAVA_UDF=1
             shift
             ;;
         --broker)

docker/thirdparties/docker-compose/hive/gen_env.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+####################################################################
+# This script will generate hadoop-hive.env from hadoop-hive.env.tpl
+####################################################################
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+FS_PORT=8120
+HMS_PORT=9183
+
+cp "${ROOT}"/hadoop-hive.env.tpl "${ROOT}"/hadoop-hive.env
+# Need to set hostname of container to same as host machine's.
+# Otherwise, the doris process can not connect to namenode directly.
+HOST_NAME=$(hostname)
+
+{
+    echo "FS_PORT=${FS_PORT}"
+    echo "HMS_PORT=${HMS_PORT}"
+    echo "CORE_CONF_fs_defaultFS=hdfs://${HOST_NAME}:${FS_PORT}"
+    echo "HOST_NAME=${HOST_NAME}"
+} >>"${ROOT}"/hadoop-hive.env

docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://doris--hive-metastore-postgresql/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
+HIVE_SITE_CONF_hive_metastore_uris=thrift://doris--hive-metastore:9083
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
+HIVE_SITE_CONF_hive_server2_thrift_port=10000
+
+CORE_CONF_hadoop_http_staticuser_user=root
+CORE_CONF_hadoop_proxyuser_hue_hosts=*
+CORE_CONF_hadoop_proxyuser_hue_groups=*
+CORE_CONF_hadoop_proxyuser_hive_hosts=*
+
+HDFS_CONF_dfs_webhdfs_enabled=true
+HDFS_CONF_dfs_permissions_enabled=false
+
+YARN_CONF_yarn_log___aggregation___enable=true
+YARN_CONF_yarn_resourcemanager_recovery_enabled=true
+YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
+YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
+YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
+YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
+YARN_CONF_yarn_timeline___service_enabled=true
+YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
+YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
+YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
+YARN_CONF_yarn_timeline___service_hostname=historyserver
+YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
+YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
+

docker/thirdparties/docker-compose/hive/hive-2x.yaml

@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+
+services:
+  doris--namenode:
+    image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
+    environment:
+      - CLUSTER_NAME=test
+    env_file:
+      - ./hadoop-hive.env
+    hostname: ${HOST_NAME}
+    expose:
+      - "50070"
+      - "${FS_PORT}"
+    ports:
+    - target: ${FS_PORT}
+      published: ${FS_PORT}
+      protocol: tcp
+      mode: host
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:50070/" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    networks:
+      - doris--hive
+  doris--datanode:
+    image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
+    env_file:
+      - ./hadoop-hive.env
+    environment:
+      SERVICE_PRECONDITION: "doris--namenode:50070"
+    expose:
+      - "50075"
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:50075" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    networks:
+      - doris--hive
+  doris--hive-server:
+    image: bde2020/hive:2.3.2-postgresql-metastore
+    env_file:
+      - ./hadoop-hive.env
+    environment:
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://doris--hive-metastore-postgresql:5432/metastore"
+      SERVICE_PRECONDITION: "doris--hive-metastore:9083"
+    expose:
+      - "10000"
+    depends_on:
+      doris--datanode:
+        condition: service_healthy
+      doris--namenode:
+        condition: service_healthy
+    healthcheck:
+      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    networks:
+      - doris--hive
+  doris--hive-metastore:
+    image: bde2020/hive:2.3.2-postgresql-metastore
+    env_file:
+      - ./hadoop-hive.env
+    command: ["sh","-c","/mnt/scripts/hive-metastore.sh"]
+    environment:
+      SERVICE_PRECONDITION: "doris--namenode:50070 doris--datanode:50075 doris--hive-metastore-postgresql:5432"
+    expose:
+      - "9083"
+    ports:
+      - ${HMS_PORT}:9083 
+    volumes:
+      - ./scripts:/mnt/scripts
+    depends_on:
+      doris--hive-metastore-postgresql:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    networks:
+      - doris--hive
+  doris--hive-metastore-postgresql:
+    image: bde2020/hive-metastore-postgresql:2.3.0
+    expose:
+      - "5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    networks:
+      - doris--hive
+  hello-world:
+    image: hello-world
+    depends_on:
+      doris--hive-metastore:
+        condition: service_healthy 
+    networks:
+      - doris--hive
+
+networks:
+  doris--hive:

docker/thirdparties/docker-compose/hive/scripts/README

@@ -0,0 +1,4 @@
+1. Download parquet file packages from:
+    https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/tpch1_parquet/tpch1.db.tar.gz
+
+2. Unzip and name it to "tpch1.db"