feat: remove flink 1.13, add 1.18 and fix commons-jexl3 (DataLinkDC#2446

) * feat: remove flink 1.13 and add 1.18 * fix: revert some files * fix: commons-jexl3 package error * feat: remove 1.13 add 1.18 Signed-off-by: licho <lecho.sun@gmail.com> * fix: default version Signed-off-by: licho <lecho.sun@gmail.com> * Spotless Apply --------- Signed-off-by: licho <lecho.sun@gmail.com> Co-authored-by: leechor <leechor@users.noreply.github.com>
zhmingyong · Oct 27, 2023 · 4d9e70d · 4d9e70d
1 parent 9fa6d42
commit 4d9e70d
Show file tree

Hide file tree

Showing 125 changed files with 1,288 additions and 13,297 deletions.
diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml
@@ -89,7 +89,7 @@ jobs:
       fail-fast: true
       matrix:
         jdk: [8, 11]
-        flink: [1.13, 1.14, 1.15, 1.16, 1.17]
+        flink: [1.14, 1.15, 1.16, 1.17, 1.18]
 
     timeout-minutes: 30
     env:

diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml
@@ -105,7 +105,6 @@ jobs:
       fail-fast: true
       matrix:
         url: [ registry.cn-hangzhou.aliyuncs.com ,docker.io ]
-#        FLINK_VERSION: [1.11.6 , 1.12.7 , 1.13.6 , 1.14.6 , 1.15.2 , 1.16.0, 1.17.0]
         include:
           - url: registry.cn-hangzhou.aliyuncs.com
             namespace: dinky
@@ -154,14 +153,12 @@ jobs:
       fail-fast: true
       matrix:
         url: [ registry.cn-hangzhou.aliyuncs.com ,docker.io ]
-        FLINK_VERSION: [1.13.6 , 1.14.6 , 1.15.2 , 1.16.0, 1.17.0]
+        FLINK_VERSION: [1.14.6 , 1.15.2 , 1.16.0, 1.17.0, 1.18.0]
         include:
           - url: registry.cn-hangzhou.aliyuncs.com
             namespace: dinky
           - url: docker.io
             namespace: dinkydocker
-          - FLINK_VERSION: 1.13.6
-            FLINK_BIG_VERSION: 1.13
           - FLINK_VERSION: 1.14.6
             FLINK_BIG_VERSION: 1.14
           - FLINK_VERSION: 1.15.4
@@ -170,6 +167,8 @@ jobs:
             FLINK_BIG_VERSION: 1.16
           - FLINK_VERSION : 1.17.1
             FLINK_BIG_VERSION: 1.17
+          - FLINK_VERSION: 1.18.0
+            FLINK_BIG_VERSION: 1.18
     steps:
       - uses: actions/checkout@v3
       - name: Move Dockerfile

diff --git a/dinky-app/dinky-app-1.13/pom.xml → dinky-app/dinky-app-1.18/pom.xml b/dinky-app/dinky-app-1.13/pom.xml → dinky-app/dinky-app-1.18/pom.xml
@@ -1,20 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
@@ -24,11 +8,11 @@
         <version>${revision}</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
-    <artifactId>dinky-app-1.13</artifactId>
+    <artifactId>dinky-app-1.18</artifactId>
 
     <packaging>jar</packaging>
 
-    <name>Dinky : App 1.13</name>
+    <name>Dinky : App 1.18</name>
 
     <properties>
         <mainClass>org.dinky.app.MainApp</mainClass>
@@ -39,26 +23,15 @@
             <groupId>org.dinky</groupId>
             <artifactId>dinky-app-base</artifactId>
         </dependency>
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-1.13</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-flink-1.13</artifactId>
+            <artifactId>dinky-client-${dinky.flink.version}</artifactId>
             <scope>${scope.runtime}</scope>
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-client-base</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.dinky</groupId>
-            <artifactId>dinky-executor</artifactId>
+            <artifactId>dinky-flink-${dinky.flink.version}</artifactId>
+            <scope>${scope.runtime}</scope>
         </dependency>
     </dependencies>
 

diff --git a/.../src/main/java/org/dinky/app/MainApp.java → .../src/main/java/org/dinky/app/MainApp.java b/.../src/main/java/org/dinky/app/MainApp.java → .../src/main/java/org/dinky/app/MainApp.java
diff --git a/dinky-app/dinky-app-base/pom.xml b/dinky-app/dinky-app-base/pom.xml
@@ -71,56 +71,56 @@
             <dependencies>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-client-1.13</artifactId>
+                    <artifactId>dinky-client-1.14</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-flink-1.13</artifactId>
+                    <artifactId>dinky-flink-1.14</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
 
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-client-1.14</artifactId>
+                    <artifactId>dinky-client-1.15</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-flink-1.14</artifactId>
+                    <artifactId>dinky-flink-1.15</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
 
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-client-1.15</artifactId>
+                    <artifactId>dinky-client-1.16</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-flink-1.15</artifactId>
+                    <artifactId>dinky-flink-1.16</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
 
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-client-1.16</artifactId>
+                    <artifactId>dinky-client-1.17</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-flink-1.16</artifactId>
+                    <artifactId>dinky-flink-1.17</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
 
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-client-1.17</artifactId>
+                    <artifactId>dinky-client-1.18</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.dinky</groupId>
-                    <artifactId>dinky-flink-1.17</artifactId>
+                    <artifactId>dinky-flink-1.18</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
             </dependencies>

diff --git a/dinky-app/pom.xml b/dinky-app/pom.xml
@@ -27,11 +27,11 @@
         <profile>
             <id>flink-all</id>
             <modules>
-                <module>dinky-app-1.13</module>
                 <module>dinky-app-1.14</module>
                 <module>dinky-app-1.15</module>
                 <module>dinky-app-1.16</module>
                 <module>dinky-app-1.17</module>
+                <module>dinky-app-1.18</module>
             </modules>
         </profile>
 

diff --git a/dinky-assembly/src/main/assembly/package.xml b/dinky-assembly/src/main/assembly/package.xml
@@ -59,15 +59,6 @@
                 <include>upgrade/*/*/*.sql</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>${project.parent.basedir}/build/extends/</directory>
-            <outputDirectory>plugins/flink1.13</outputDirectory>
-            <includes>
-                <include>dinky-catalog-mysql-1.13-${project.version}.jar</include>
-                <include>dinky-client-1.13-${project.version}.jar</include>
-                <include>dinky-connector-jdbc-1.13-${project.version}.jar</include>
-            </includes>
-        </fileSet>
         <fileSet>
             <directory>${project.parent.basedir}/build/extends/</directory>
             <outputDirectory>plugins/flink1.14</outputDirectory>
@@ -101,6 +92,14 @@
                 <include>dinky-client-1.17-${project.version}.jar</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/build/extends/</directory>
+            <outputDirectory>plugins/flink1.18</outputDirectory>
+            <includes>
+                <include>dinky-catalog-mysql-1.18-${project.version}.jar</include>
+                <include>dinky-client-1.18-${project.version}.jar</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>${project.parent.basedir}/build/extends/</directory>
             <outputDirectory>plugins</outputDirectory>

diff --git a/...og-mysql/dinky-catalog-mysql-1.13/pom.xml → ...og-mysql/dinky-catalog-mysql-1.18/pom.xml b/...og-mysql/dinky-catalog-mysql-1.13/pom.xml → ...og-mysql/dinky-catalog-mysql-1.18/pom.xml
@@ -1,20 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
@@ -24,10 +8,11 @@
         <version>${revision}</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
+    <artifactId>dinky-catalog-mysql-1.18</artifactId>
 
-    <artifactId>dinky-catalog-mysql-1.13</artifactId>
     <packaging>jar</packaging>
-    <name>Dinky : Catalog : Mysql 1.13</name>
+
+    <name>Dinky : Catalog : Mysql 1.18</name>
 
     <dependencies>
         <dependency>
@@ -36,7 +21,7 @@
         </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
-            <artifactId>dinky-flink-1.13</artifactId>
+            <artifactId>dinky-flink-1.18</artifactId>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

diff --git a/...inky/flink/catalog/DinkyMysqlCatalog.java → ...inky/flink/catalog/DinkyMysqlCatalog.java b/...inky/flink/catalog/DinkyMysqlCatalog.java → ...inky/flink/catalog/DinkyMysqlCatalog.java
@@ -99,6 +99,7 @@ public class DinkyMysqlCatalog extends AbstractCatalog {
 
     /** 对象类型，例如 库、表、视图等 */
     protected static class ObjectType {
+
         /** 数据库 */
         public static final String DATABASE = "database";
 
@@ -111,6 +112,7 @@ protected static class ObjectType {
 
     /** 对象类型，例如 库、表、视图等 */
     protected static class ColumnType {
+
         /** 物理字段 */
         public static final String PHYSICAL = "physical";
 
@@ -251,7 +253,7 @@ public List<String> listDatabases() throws CatalogException {
 
     @Override
     public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException {
-        String querySql = "SELECT id, database_name,description FROM metadata_database where database_name=?";
+        String querySql = "SELECT id, database_name,description " + " FROM metadata_database where database_name=?";
         Connection conn = getConnection();
         try (PreparedStatement ps = conn.prepareStatement(querySql)) {
             ps.setString(1, databaseName);
@@ -263,7 +265,7 @@ public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistE
 
                 Map<String, String> map = new HashMap<>();
 
-                String sql = "select `key`,`value` from metadata_database_property where database_id=? ";
+                String sql = "select `key`,`value` " + "from metadata_database_property " + "where database_id=? ";
                 try (PreparedStatement pStat = conn.prepareStatement(sql)) {
                     pStat.setInt(1, id);
                     ResultSet prs = pStat.executeQuery();
@@ -341,7 +343,7 @@ public void createDatabase(String databaseName, CatalogDatabase db, boolean igno
                         && db.getProperties().size() > 0) {
                     int id = idRs.getInt(1);
                     String propInsertSql =
-                            "insert into metadata_database_property(database_id, `key`,`value`)" + " values (?,?,?)";
+                            "insert into metadata_database_property(database_id, " + "`key`,`value`) values (?,?,?)";
                     PreparedStatement pstat = conn.prepareStatement(propInsertSql);
                     for (Map.Entry<String, String> entry : db.getProperties().entrySet()) {
                         pstat.setInt(1, id);
@@ -555,6 +557,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep
                     String dataType = crs.getString("data_type");
 
                     builder.column(colName, dataType);
+                    String cDesc = crs.getString("description");
+                    if (null != cDesc && cDesc.length() > 0) {
+                        builder.withComment(cDesc);
+                    }
                 }
                 cStat.close();
                 // 3、取出query
@@ -767,7 +773,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig
                             colIStat.setString(2, ColumnType.PHYSICAL);
                             colIStat.setString(3, dataType.getLogicalType().asSerializableString());
                             colIStat.setObject(4, null);
-                            colIStat.setString(5, "");
+                            colIStat.setString(5, pCol.getComment().orElse(""));
                             colIStat.setInt(6, id);
                             colIStat.setObject(7, null); // view没有主键
                             colIStat.addBatch();
@@ -1050,7 +1056,7 @@ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
         }
 
         Connection conn = getConnection();
-        String insertSql = "delete from metadata_function where id=?";
+        String insertSql = "delete from metadata_function " + " where id=?";
         try (PreparedStatement ps = conn.prepareStatement(insertSql)) {
             ps.setInt(1, id);
             ps.executeUpdate();
@@ -1069,12 +1075,11 @@ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath)
         if (!tableExists(tablePath)) {
             throw new TableNotExistException(getName(), tablePath);
         }
-        /*if (!isPartitionedTable(tablePath)) {
-            CatalogTableStatistics result = tableStats.get(tablePath);
-            return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN;
-        } else {
-            return CatalogTableStatistics.UNKNOWN;
-        }*/
+        /*
+         * if (!isPartitionedTable(tablePath)) { CatalogTableStatistics result = tableStats.get(tablePath); return
+         * result != null ? result.copy() : CatalogTableStatistics.UNKNOWN; } else { return
+         * CatalogTableStatistics.UNKNOWN; }
+         */
         return CatalogTableStatistics.UNKNOWN;
     }
 

diff --git a/...log/factory/DinkyMysqlCatalogFactory.java → ...log/factory/DinkyMysqlCatalogFactory.java b/...log/factory/DinkyMysqlCatalogFactory.java → ...log/factory/DinkyMysqlCatalogFactory.java
diff --git a/...tory/DinkyMysqlCatalogFactoryOptions.java → ...tory/DinkyMysqlCatalogFactoryOptions.java b/...tory/DinkyMysqlCatalogFactoryOptions.java → ...tory/DinkyMysqlCatalogFactoryOptions.java
diff --git a/.../org.apache.flink.table.factories.Factory → .../org.apache.flink.table.factories.Factory b/.../org.apache.flink.table.factories.Factory → .../org.apache.flink.table.factories.Factory
diff --git a/dinky-catalog/dinky-catalog-mysql/pom.xml b/dinky-catalog/dinky-catalog-mysql/pom.xml
@@ -40,11 +40,11 @@
         <profile>
             <id>flink-all</id>
             <modules>
-                <module>dinky-catalog-mysql-1.13</module>
                 <module>dinky-catalog-mysql-1.14</module>
                 <module>dinky-catalog-mysql-1.15</module>
                 <module>dinky-catalog-mysql-1.16</module>
                 <module>dinky-catalog-mysql-1.17</module>
+                <module>dinky-catalog-mysql-1.18</module>
             </modules>
         </profile>
     </profiles>