Merge pull request #277 from apecloud/feat/pg_to_starrocks

add "0x" prefix to column values when converted from MySQL blob/binar…
apecloud · Dec 6, 2024 · d5d1501 · d5d1501
2 parents 5f23f9e + 95b8586
commit d5d1501
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -18,12 +18,12 @@
 - Supports starting ape-dts as an HTTP server to pull data from the source, allowing users to use any language's HTTP client to retrieve and consume the data.
 
 ## Supported task types
-|  | mysql -> mysql | pg -> pg | mongo -> mongo | redis -> redis | mysql -> kafka | pg -> kafka| mysql -> starrocks | mysql -> clickhouse | mysql -> tidb |
-| :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- |
-| Snapshot | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
-| CDC | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
-| Data check/revise/review | &#10004; | &#10004; | &#10004; | | | | | | &#10004; |
-| Structure migration | &#10004; | &#10004; | | | | | &#10004; | &#10004; | &#10004; |
+|  | mysql -> mysql | pg -> pg | mongo -> mongo | redis -> redis | mysql -> kafka | pg -> kafka| mysql -> starrocks | mysql -> clickhouse | mysql -> tidb | pg -> starrocks |
+| :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- |
+| Snapshot | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
+| CDC | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
+| Data check/revise/review | &#10004; | &#10004; | &#10004; | | | | | | &#10004; | |
+| Structure migration | &#10004; | &#10004; | | | | | &#10004; | &#10004; | &#10004; | &#10004; |
 
 
 # Quick starts

diff --git a/README_ZH.md b/README_ZH.md
@@ -21,12 +21,12 @@
 
 <br/>
 
-|  | mysql -> mysql | pg -> pg | mongo -> mongo | redis -> redis | mysql -> kafka | pg -> kafka| mysql -> starrocks | mysql -> clickhouse | mysql -> tidb |
-| :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- |
-| 全量迁移 | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
-| 增量同步 | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
-| 数据校验/订正/复查 | &#10004; | &#10004; | &#10004; | | | | | | &#10004; |
-| 库表结构迁移 | &#10004; | &#10004; | | | | | &#10004; | &#10004; | &#10004; |
+|  | mysql -> mysql | pg -> pg | mongo -> mongo | redis -> redis | mysql -> kafka | pg -> kafka| mysql -> starrocks | mysql -> clickhouse | mysql -> tidb | pg -> starrocks |
+| :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- | :-------- |
+| 全量迁移 | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
+| 增量同步 | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; | &#10004; |
+| 数据校验/订正/复查 | &#10004; | &#10004; | &#10004; | | | | | | &#10004; | |
+| 库表结构迁移 | &#10004; | &#10004; | | | | | &#10004; | &#10004; | &#10004; | &#10004; |
 
 # 快速上手
 

diff --git a/docs/en/tutorial/mysql_to_starrocks.md b/docs/en/tutorial/mysql_to_starrocks.md
@@ -22,6 +22,8 @@ docker run -itd --name some-starrocks \
 # Migrate structures
 ## Prepare source data
 ```
+mysql -h127.0.0.1 -uroot -p123456 -P3307
+
 CREATE DATABASE test_db;
 CREATE TABLE test_db.tb_1(id int, value int, primary key(id));
 ```
@@ -317,10 +319,10 @@ Refer to [config](/docs/en/config.md) for other common configurations
 | varchar | VARCHAR |
 | binary | VARBINARY |
 | varbinary | VARBINARY |
-| tinytext | CHAR/VARCHAR/STRING/TEXT |
-| text | CHAR/VARCHAR/STRING/TEXT |
-| mediumtext | CHAR/VARCHAR/STRING/TEXT |
-| longtext | CHAR/VARCHAR/STRING/TEXT |
+| tinytext | STRING |
+| text | STRING |
+| mediumtext | STRING |
+| longtext | STRING |
 | tinyblob | VARBINARY |
 | blob | VARBINARY |
 | mediumblob | VARBINARY |

diff --git a/dt-connector/src/sinker/clickhouse/clickhouse_sinker.rs b/dt-connector/src/sinker/clickhouse/clickhouse_sinker.rs
@@ -122,12 +122,22 @@ impl ClickhouseSinker {
         let mut new_col_values: HashMap<String, ColValue> = HashMap::new();
         for (col, col_value) in col_values.iter() {
             match col_value {
-                ColValue::Blob(v) | ColValue::RawString(v) => {
-                    new_col_values.insert(
-                        col.to_owned(),
-                        ColValue::String(SqlUtil::binary_to_str(v).0),
-                    );
+                ColValue::RawString(v) => {
+                    let (str, is_hex) = SqlUtil::binary_to_str(v);
+                    if is_hex {
+                        new_col_values
+                            .insert(col.to_owned(), ColValue::String(format!("0x{}", str)));
+                    } else {
+                        new_col_values.insert(col.to_owned(), ColValue::String(str));
+                    }
                 }
+
+                ColValue::Blob(v) => {
+                    let hex_str = hex::encode(v);
+                    new_col_values
+                        .insert(col.to_owned(), ColValue::String(format!("0x{}", hex_str)));
+                }
+
                 _ => {}
             }
         }