refactor: metadata, a little adjustitive on code (DataLinkDC#2055)

* refactor: test * refactor: * refactor: * Spotless Apply * fix: test Signed-off-by: licho <lecho.sun@gmail.com> * fix: test Signed-off-by: licho <lecho.sun@gmail.com> * Spotless Apply * refactor: add AbstractTypeConvert * Spotless Apply * chore: format * refactor: starRoocksTypeConvert Signed-off-by: Licho <lecho.sun@gmail.com> * Spotless Apply * refactor: SqlServerTypeConvert * Spotless Apply * refactor: Signed-off-by: Licho <lecho.sun@gmail.com> * Spotless Apply * refactor: revert application.yml Signed-off-by: Licho <lecho.sun@gmail.com> * refactor: add register * Spotless Apply * refactor: remove unused * refactor: all typeconvert * Spotless Apply * fix: HiveTypeConvert * fix: ClickHouseTypeConvert * Spotless Apply --------- Signed-off-by: licho <lecho.sun@gmail.com> Signed-off-by: Licho <lecho.sun@gmail.com> Co-authored-by: leechor <leechor@users.noreply.github.com>
hugh018 · Jun 12, 2023 · ae619b9 · ae619b9
1 parent 411adc8
commit ae619b9
Show file tree

Hide file tree

Showing 19 changed files with 650 additions and 792 deletions.
diff --git a/dinky-common/src/main/java/org/dinky/data/enums/ColumnType.java b/dinky-common/src/main/java/org/dinky/data/enums/ColumnType.java
@@ -55,8 +55,8 @@ public enum ColumnType {
     T("T[]", "ARRAY"),
     MAP("java.util.Map<K, V>", "MAP");
 
-    private String javaType;
-    private String flinkType;
+    private final String javaType;
+    private final String flinkType;
 
     ColumnType(String javaType, String flinkType) {
         this.javaType = javaType;

diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java
@@ -121,7 +121,8 @@ public String getFlinkDDL(String flinkConfig, String tableName) {
                                         comment =
                                                 String.format(
                                                         " COMMENT '%s'",
-                                                        column.getComment().replaceAll("\"|'", ""));
+                                                        column.getComment()
+                                                                .replaceAll("[\"']", ""));
                                     }
                                     return String.format(
                                             "    `%s` %s%s",
@@ -144,17 +145,15 @@ public String getFlinkDDL(String flinkConfig, String tableName) {
                                         Collectors.joining(
                                                 ",", ",\n    PRIMARY KEY ( ", " ) NOT ENFORCED\n"));
 
-        String result =
-                MessageFormat.format(
-                        "CREATE TABLE IF NOT EXISTS {0} (\n{1}{2}) WITH (\n{3})\n",
-                        tableName, columnStrs, primaryKeyStr, flinkConfig);
-        return result;
+        return MessageFormat.format(
+                "CREATE TABLE IF NOT EXISTS {0} (\n{1}{2}) WITH (\n{3})\n",
+                tableName, columnStrs, primaryKeyStr, flinkConfig);
     }
 
     @Transient
     public String getFlinkTableSql(String catalogName, String flinkConfig) {
         String createSql = getFlinkDDL(getFlinkTableWith(flinkConfig), name);
-        return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql);
+        return String.format("DROP TABLE IF EXISTS %s;%n%s", name, createSql);
     }
 
     @Override
@@ -167,52 +166,4 @@ public Object clone() {
         }
         return table;
     }
-
-    @Transient
-    public String getFlinkTableSql(String flinkConfig) {
-        return getFlinkDDL(flinkConfig, name);
-    }
-
-    @Transient
-    public String getSqlSelect(String catalogName) {
-        StringBuilder sb = new StringBuilder("SELECT\n");
-        for (int i = 0; i < columns.size(); i++) {
-            sb.append("    ");
-            if (i > 0) {
-                sb.append(",");
-            }
-            String columnComment = columns.get(i).getComment();
-            if (Asserts.isNotNullString(columnComment)) {
-                if (columnComment.contains("\'") | columnComment.contains("\"")) {
-                    columnComment = columnComment.replaceAll("\"|'", "");
-                }
-                sb.append("`" + columns.get(i).getName() + "`  --  " + columnComment + " \n");
-            } else {
-                sb.append("`" + columns.get(i).getName() + "` \n");
-            }
-        }
-        if (Asserts.isNotNullString(comment)) {
-            sb.append(" FROM " + schema + "." + name + ";" + " -- " + comment + "\n");
-        } else {
-            sb.append(" FROM " + schema + "." + name + ";\n");
-        }
-        return sb.toString();
-    }
-
-    @Transient
-    public String getCDCSqlInsert(String targetName, String sourceName) {
-        StringBuilder sb = new StringBuilder("INSERT INTO ");
-        sb.append(targetName);
-        sb.append(" SELECT\n");
-        for (int i = 0; i < columns.size(); i++) {
-            sb.append("    ");
-            if (i > 0) {
-                sb.append(",");
-            }
-            sb.append("`" + columns.get(i).getName() + "` \n");
-        }
-        sb.append(" FROM ");
-        sb.append(sourceName);
-        return sb.toString();
-    }
 }
diff --git a/dinky-metadata/dinky-metadata-base/pom.xml b/dinky-metadata/dinky-metadata-base/pom.xml
@@ -43,6 +43,58 @@
             <artifactId>lombok</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>org.junit.vintage</groupId>
+            <artifactId>junit-vintage-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <type>jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <type>jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <type>jar</type>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.mockito</groupId>
+                    <artifactId>mockito-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <type>jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/...ata/dinky-metadata-base/src/main/java/org/dinky/metadata/convert/AbstractTypeConvert.java b/...ata/dinky-metadata-base/src/main/java/org/dinky/metadata/convert/AbstractTypeConvert.java
@@ -0,0 +1,85 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.metadata.convert;
+
+import org.dinky.assertion.Asserts;
+import org.dinky.data.enums.ColumnType;
+import org.dinky.data.model.Column;
+import org.dinky.metadata.driver.DriverConfig;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiFunction;
+
+public class AbstractTypeConvert implements ITypeConvert {
+    protected Map<String, BiFunction<Column, DriverConfig, Optional<ColumnType>>> convertMap =
+            new LinkedHashMap<>();
+
+    @Override
+    public String convertToDB(ColumnType columnType) {
+        return null;
+    }
+
+    @Override
+    public ColumnType convert(Column column) {
+        return convert(column, null);
+    }
+
+    @Override
+    public ColumnType convert(Column column, DriverConfig driverConfig) {
+        if (Asserts.isNull(column)) {
+            return ColumnType.STRING;
+        }
+        for (Map.Entry<String, BiFunction<Column, DriverConfig, Optional<ColumnType>>> entry :
+                convertMap.entrySet()) {
+            if (column.getType().contains(entry.getKey())) {
+                Optional<ColumnType> columnType = entry.getValue().apply(column, driverConfig);
+                if (columnType.isPresent()) {
+                    return columnType.get();
+                }
+            }
+        }
+
+        return ColumnType.STRING;
+    }
+
+    protected void register(String type, ColumnType columnType) {
+        this.convertMap.put(type, (c, d) -> getColumnType(c, columnType, columnType));
+    }
+
+    protected void register(String type, ColumnType notNullType, ColumnType nullType) {
+        this.convertMap.put(type, (c, d) -> getColumnType(c, notNullType, nullType));
+    }
+
+    protected void register(
+            String type, BiFunction<Column, DriverConfig, Optional<ColumnType>> func) {
+        this.convertMap.put(type, func);
+    }
+
+    private static Optional<ColumnType> getColumnType(
+            Column column, ColumnType notNullType, ColumnType nullType) {
+        boolean isNullable = !column.isKeyFlag() && column.isNullable();
+        if (isNullable) {
+            return Optional.of(nullType);
+        }
+        return Optional.of(notNullType);
+    }
+}
diff --git a/...-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractDriver.java b/...-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractDriver.java
@@ -102,26 +102,23 @@ public String getSqlSelect(Table table) {
             }
             String columnComment = columns.get(i).getComment();
             if (Asserts.isNotNullString(columnComment)) {
-                if (columnComment.contains("\'") | columnComment.contains("\"")) {
-                    columnComment = columnComment.replaceAll("\"|'", "");
+                if (columnComment.contains("'") || columnComment.contains("\"")) {
+                    columnComment = columnComment.replaceAll("[\"']", "");
                 }
-                sb.append("`" + columns.get(i).getName() + "`  --  " + columnComment + " \n");
+                sb.append(
+                        String.format("`%s`  --  %s %n", columns.get(i).getName(), columnComment));
             } else {
-                sb.append("`" + columns.get(i).getName() + "` \n");
+                sb.append(String.format("`%s` %%n", columns.get(i).getName()));
             }
         }
+
         if (Asserts.isNotNullString(table.getComment())) {
             sb.append(
-                    " FROM "
-                            + table.getSchema()
-                            + "."
-                            + table.getName()
-                            + ";"
-                            + " -- "
-                            + table.getComment()
-                            + "\n");
+                    String.format(
+                            " FROM %s.%s; -- %s%n",
+                            table.getSchema(), table.getName(), table.getComment()));
         } else {
-            sb.append(" FROM " + table.getSchema() + "." + table.getName() + ";\n");
+            sb.append(String.format(" FROM %s.%s;%n", table.getSchema(), table.getName()));
         }
         return sb.toString();
     }

diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/Driver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/Driver.java
@@ -58,6 +58,7 @@ static Driver build(DriverConfig config) {
         if (DriverPool.exist(key)) {
             return getHealthDriver(key);
         }
+
         synchronized (Driver.class) {
             Optional<Driver> optionalDriver = Driver.get(config);
             if (!optionalDriver.isPresent()) {
@@ -71,7 +72,6 @@ static Driver build(DriverConfig config) {
     }
 
     static Driver buildUnconnected(DriverConfig config) {
-        String key = config.getName();
         synchronized (Driver.class) {
             Optional<Driver> optionalDriver = Driver.get(config);
             if (!optionalDriver.isPresent()) {
@@ -114,6 +114,7 @@ static Driver build(String connector, String url, String username, String passwo
                 type = "Greenplum";
             }
         }
+
         if (Asserts.isNull(type)) {
             throw new MetaDataException("缺少数据源类型:【" + connector + "】");
         }

diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/DriverPool.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/DriverPool.java
@@ -32,10 +32,7 @@ public class DriverPool {
     private static volatile Map<String, Driver> driverMap = new ConcurrentHashMap<>();
 
     public static boolean exist(String key) {
-        if (driverMap.containsKey(key)) {
-            return true;
-        }
-        return false;
+        return driverMap.containsKey(key);
     }
 
     public static Integer push(String key, Driver gainer) {