From fdb1c30c80124a8437ad3a1e96ce38e431f2a293 Mon Sep 17 00:00:00 2001 From: "shicai.xu" Date: Tue, 15 Aug 2023 19:02:22 +0800 Subject: [PATCH] 1, sink mysql json columns. 2, sink redis rdb data by rewrite besides restore. 3, sink basic ddls for mysql --- Cargo.toml | 3 +- dt-common/Cargo.toml | 3 +- dt-common/src/config/config_enums.rs | 6 +- dt-common/src/config/extractor_config.rs | 2 +- dt-common/src/config/sinker_config.rs | 3 +- dt-common/src/config/task_config.rs | 34 +- dt-common/src/error.rs | 84 +-- dt-common/src/lib.rs | 2 - dt-common/src/utils/rdb_filter.rs | 19 +- .../extractor/mongo/mongo_cdc_extractor.rs | 4 - .../mongo/mongo_snapshot_extractor.rs | 4 - .../extractor/mysql/mysql_cdc_extractor.rs | 91 +++- .../extractor/mysql/mysql_check_extractor.rs | 4 - .../extractor/mysql/mysql_struct_extractor.rs | 5 +- .../src/extractor/pg/pg_cdc_client.rs | 7 +- .../src/extractor/pg/pg_cdc_extractor.rs | 10 +- .../src/extractor/pg/pg_check_extractor.rs | 4 - .../src/extractor/pg/pg_struct_extractor.rs | 5 +- .../redis/rdb/entry_parser/hash_parser.rs | 13 +- .../redis/rdb/entry_parser/list_parser.rs | 14 +- .../redis/rdb/entry_parser/module2_parser.rs | 20 +- .../redis/rdb/entry_parser/set_parser.rs | 15 +- .../redis/rdb/entry_parser/stream_parser.rs | 21 +- .../redis/rdb/entry_parser/zset_parser.rs | 14 +- .../src/extractor/redis/rdb/rdb_loader.rs | 4 +- .../src/extractor/redis/rdb/reader/length.rs | 18 +- .../extractor/redis/rdb/reader/list_pack.rs | 13 +- .../src/extractor/redis/rdb/reader/string.rs | 14 +- .../extractor/redis/rdb/reader/zip_list.rs | 21 +- .../extractor/redis/redis_cdc_extractor.rs | 7 +- .../src/extractor/redis/redis_client.rs | 4 +- .../extractor/redis/redis_psync_extractor.rs | 18 +- dt-connector/src/lib.rs | 4 + dt-connector/src/rdb_query_builder.rs | 22 +- dt-connector/src/sinker/foxlake_sinker.rs | 4 - dt-connector/src/sinker/kafka/kafka_router.rs | 7 +- dt-connector/src/sinker/kafka/kafka_sinker.rs | 10 +- dt-connector/src/sinker/mongo/mongo_sinker.rs | 10 +- dt-connector/src/sinker/mysql/mysql_sinker.rs | 38 +- dt-connector/src/sinker/open_faas_sinker.rs | 10 +- dt-connector/src/sinker/pg/pg_sinker.rs | 5 - .../src/sinker/pg/pg_struct_sinker.rs | 2 +- dt-connector/src/sinker/rdb_router.rs | 7 +- .../src/sinker/redis/entry_rewriter.rs | 15 +- dt-connector/src/sinker/redis/redis_sinker.rs | 89 ++- .../src/adaptor/mysql_col_value_convertor.rs | 37 +- dt-meta/src/adaptor/sqlx_ext.rs | 1 + dt-meta/src/col_value.rs | 3 + dt-meta/src/ddl_data.rs | 1 + dt-meta/src/ddl_type.rs | 2 + dt-meta/src/mysql/mysql_meta_manager.rs | 22 +- dt-meta/src/pg/pg_meta_manager.rs | 7 +- dt-meta/src/rdb_meta_manager.rs | 6 +- dt-meta/src/redis/mod.rs | 1 + dt-meta/src/redis/redis_object.rs | 8 + dt-meta/src/redis/redis_write_method.rs | 23 + dt-meta/src/sql_parser/ddl_parser.rs | 137 ++++- dt-pipeline/src/pipeline.rs | 13 +- dt-task/src/sinker_util.rs | 16 +- dt-task/src/task_runner.rs | 9 +- dt-task/src/task_util.rs | 16 +- .../mysql_to_mysql/cdc/ddl_test/dst_ddl.sql | 20 + .../mysql_to_mysql/cdc/ddl_test/src_ddl.sql | 20 + .../mysql_to_mysql/cdc/ddl_test/src_dml.sql | 51 ++ .../cdc/ddl_test/task_config.ini | 36 ++ .../mysql_to_mysql/cdc/json_test/dst_ddl.sql | 5 + .../mysql_to_mysql/cdc/json_test/src_ddl.sql | 6 + .../mysql_to_mysql/cdc/json_test/src_dml.sql | 134 +++++ .../cdc/json_test/task_config.ini | 36 ++ dt-tests/tests/mysql_to_mysql/cdc_tests.rs | 12 + .../snapshot/json_test/dst_ddl.sql | 5 + .../snapshot/json_test/src_ddl.sql | 6 + .../snapshot/json_test/src_dml.sql | 127 +++++ .../snapshot/json_test/task_config.ini | 33 ++ .../tests/mysql_to_mysql/snapshot_tests.rs | 9 + dt-tests/tests/pg_to_pg/snapshot_tests.rs | 3 + .../cdc/2_8/cmds_test/src_dml.sql | 2 +- .../cdc/4_0/cmds_test/src_dml.sql | 2 +- .../cdc/5_0/cmds_test/src_dml.sql | 2 +- .../cdc/6_0/cmds_test/src_dml.sql | 2 +- .../cdc/6_2/cmds_test/src_dml.sql | 2 +- .../cdc/7_0/cmds_test/src_dml.sql | 2 +- .../snapshot/2_8/cmds_test/src_dml.sql | 2 +- .../snapshot/4_0/cmds_test/src_dml.sql | 2 +- .../snapshot/5_0/cmds_test/src_dml.sql | 2 +- .../snapshot/6_0/cmds_test/src_dml.sql | 2 +- .../snapshot/6_2/cmds_test/src_dml.sql | 2 +- .../snapshot/7_0/cmds_test/src_dml.sql | 2 +- .../7_0/rewrite_stream_test/dst_ddl.sql | 1 + .../7_0/rewrite_stream_test/src_ddl.sql | 1 + .../7_0/rewrite_stream_test/src_dml.sql | 61 +++ .../7_0/rewrite_stream_test/task_config.ini | 35 ++ .../snapshot/7_0/rewrite_test/dst_ddl.sql | 1 + .../snapshot/7_0/rewrite_test/src_ddl.sql | 1 + .../snapshot/7_0/rewrite_test/src_dml.sql | 510 ++++++++++++++++++ .../snapshot/7_0/rewrite_test/task_config.ini | 35 ++ .../snapshot/7_0/stream_test/src_dml.sql | 35 +- .../redis_to_redis/snapshot_7_0_tests.rs | 14 + dt-tests/tests/test_runner/rdb_test_runner.rs | 305 ++++++----- .../tests/test_runner/redis_test_runner.rs | 8 +- dt-tests/tests/test_runner/test_base.rs | 23 +- 101 files changed, 2005 insertions(+), 563 deletions(-) create mode 100644 dt-meta/src/redis/redis_write_method.rs create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/ddl_test/dst_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_dml.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/ddl_test/task_config.ini create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/json_test/dst_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/json_test/src_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/json_test/src_dml.sql create mode 100644 dt-tests/tests/mysql_to_mysql/cdc/json_test/task_config.ini create mode 100644 dt-tests/tests/mysql_to_mysql/snapshot/json_test/dst_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_ddl.sql create mode 100644 dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_dml.sql create mode 100644 dt-tests/tests/mysql_to_mysql/snapshot/json_test/task_config.ini create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/dst_ddl.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_ddl.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_dml.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/task_config.ini create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/dst_ddl.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_ddl.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_dml.sql create mode 100644 dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/task_config.ini diff --git a/Cargo.toml b/Cargo.toml index 67f6a9c9..5fe23c32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,4 +52,5 @@ uuid = { version = "1.3.1", features = ["v4"] } nom = "7.1.3" mongodb = { version = "2.5.0" } dotenv = "0.15.0" -redis = {git = "https://github.com/qianyiwen2019/redis-rs", features = ["tokio-comp"]} \ No newline at end of file +redis = { version = "0.23.1", features = ["tokio-comp"] } +thiserror = "1.0.44" \ No newline at end of file diff --git a/dt-common/Cargo.toml b/dt-common/Cargo.toml index f9bc7b32..310c9694 100644 --- a/dt-common/Cargo.toml +++ b/dt-common/Cargo.toml @@ -22,4 +22,5 @@ futures = { workspace = true } serde_yaml = { workspace = true } regex = { workspace = true } nom = { workspace = true } -tokio = { workspace = true } \ No newline at end of file +tokio = { workspace = true } +thiserror = { workspace = true } \ No newline at end of file diff --git a/dt-common/src/config/config_enums.rs b/dt-common/src/config/config_enums.rs index 75b41c87..7bfafa64 100644 --- a/dt-common/src/config/config_enums.rs +++ b/dt-common/src/config/config_enums.rs @@ -4,7 +4,7 @@ use strum::{Display, EnumString, IntoStaticStr}; use crate::error::Error; -#[derive(Clone, Display, EnumString, IntoStaticStr, Debug)] +#[derive(Clone, Display, EnumString, IntoStaticStr, Debug, PartialEq, Eq)] pub enum DbType { #[strum(serialize = "mysql")] Mysql, @@ -36,7 +36,7 @@ pub enum ExtractType { Basic, } -#[derive(EnumString, IntoStaticStr)] +#[derive(Display, EnumString, IntoStaticStr)] pub enum SinkType { #[strum(serialize = "write")] Write, @@ -73,7 +73,7 @@ pub enum RouteType { Tb, } -#[derive(Clone, IntoStaticStr)] +#[derive(Clone, Debug, IntoStaticStr)] pub enum ConflictPolicyEnum { #[strum(serialize = "ignore")] Ignore, diff --git a/dt-common/src/config/extractor_config.rs b/dt-common/src/config/extractor_config.rs index 158a057e..7538df3a 100644 --- a/dt-common/src/config/extractor_config.rs +++ b/dt-common/src/config/extractor_config.rs @@ -1,6 +1,6 @@ use super::config_enums::DbType; -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum ExtractorConfig { Basic { url: String, diff --git a/dt-common/src/config/sinker_config.rs b/dt-common/src/config/sinker_config.rs index 0a53c1bc..9346d525 100644 --- a/dt-common/src/config/sinker_config.rs +++ b/dt-common/src/config/sinker_config.rs @@ -1,6 +1,6 @@ use super::config_enums::{ConflictPolicyEnum, DbType}; -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum SinkerConfig { Basic { url: String, @@ -69,6 +69,7 @@ pub enum SinkerConfig { Redis { url: String, batch_size: usize, + method: String, }, } diff --git a/dt-common/src/config/task_config.rs b/dt-common/src/config/task_config.rs index 8b00688a..0eb0d5e4 100644 --- a/dt-common/src/config/task_config.rs +++ b/dt-common/src/config/task_config.rs @@ -157,9 +157,10 @@ impl TaskConfig { ExtractType::Basic => Ok(ExtractorConfig::Basic { url, db_type }), - t => Err(Error::Unexpected { - error: format!("extract type: {} not supported", t), - }), + extract_type => Err(Error::ConfigError(format!( + "extract type: {} not supported", + extract_type + ))), }, DbType::Redis => { @@ -179,15 +180,17 @@ impl TaskConfig { now_db_id: ini.getint(EXTRACTOR, "now_db_id").unwrap().unwrap(), }), - t => Err(Error::Unexpected { - error: format!("extract type: {} not supported", t), - }), + extract_type => Err(Error::ConfigError(format!( + "extract type: {} not supported", + extract_type + ))), } } - _ => Err(Error::Unexpected { - error: "extractor db type not supported".to_string(), - }), + db_type => Err(Error::ConfigError(format!( + "extractor db type: {} not supported", + db_type + ))), } } @@ -240,9 +243,10 @@ impl TaskConfig { SinkType::Basic => Ok(SinkerConfig::Basic { url, db_type }), - _ => Err(Error::Unexpected { - error: "sinker db type not supported".to_string(), - }), + db_type => Err(Error::ConfigError(format!( + "sinker db type: {} not supported", + db_type + ))), }, DbType::Kafka => Ok(SinkerConfig::Kafka { @@ -267,7 +271,11 @@ impl TaskConfig { root_dir: ini.get(SINKER, "root_dir").unwrap(), }), - DbType::Redis => Ok(SinkerConfig::Redis { url, batch_size }), + DbType::Redis => Ok(SinkerConfig::Redis { + url, + batch_size, + method: Self::get_optional_value(ini, SINKER, "method"), + }), } } diff --git a/dt-common/src/error.rs b/dt-common/src/error.rs index f6931779..c9bd6931 100644 --- a/dt-common/src/error.rs +++ b/dt-common/src/error.rs @@ -1,70 +1,40 @@ -#[derive(Debug)] +use thiserror::Error; + +#[derive(Error, Debug)] pub enum Error { - ConfigError { - error: String, - }, + #[error("config error: {0}")] + ConfigError(String), - BinlogError { - error: mysql_binlog_connector_rust::binlog_error::BinlogError, - }, + #[error("extractor error: {0}")] + ExtractorError(String), - SqlxError { - error: sqlx::Error, - }, + #[error("sinker error: {0}")] + SinkerError(String), - Unexpected { - error: String, - }, + #[error("pull mysql binlog error: {0}")] + BinlogError(#[from] mysql_binlog_connector_rust::binlog_error::BinlogError), - MetadataError { - error: String, - }, + #[error("sqlx error: {0}")] + SqlxError(#[from] sqlx::Error), - IoError { - error: std::io::Error, - }, + #[error("unexpected error: {0}")] + Unexpected(String), - YamlError { - error: serde_yaml::Error, - }, + #[error("parse redis rdb error: {0}")] + RedisRdbError(String), - EnvVarError { - error: std::env::VarError, - }, + #[error("metadata error: {0}")] + MetadataError(String), - StructError { - error: String, - }, + #[error("io error: {0}")] + IoError(#[from] std::io::Error), - ColumnNotMatch, -} + #[error("yaml error: {0}")] + YamlError(#[from] serde_yaml::Error), -impl From for Error { - fn from(err: mysql_binlog_connector_rust::binlog_error::BinlogError) -> Self { - Self::BinlogError { error: err } - } -} - -impl From for Error { - fn from(err: sqlx::Error) -> Self { - Self::SqlxError { error: err } - } -} - -impl From for Error { - fn from(err: std::io::Error) -> Self { - Self::IoError { error: err } - } -} - -impl From for Error { - fn from(err: serde_yaml::Error) -> Self { - Self::YamlError { error: err } - } -} + #[error("from utf8 error: {0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), -impl From for Error { - fn from(err: std::env::VarError) -> Self { - Self::EnvVarError { error: err } - } + #[error("struct error: {0}")] + StructError(String), } diff --git a/dt-common/src/lib.rs b/dt-common/src/lib.rs index 5e5ecc21..a6e075e0 100644 --- a/dt-common/src/lib.rs +++ b/dt-common/src/lib.rs @@ -1,5 +1,3 @@ -// extern crate dt-meta; - pub mod config; pub mod constants; pub mod error; diff --git a/dt-common/src/utils/rdb_filter.rs b/dt-common/src/utils/rdb_filter.rs index 1426dbda..02e11805 100644 --- a/dt-common/src/utils/rdb_filter.rs +++ b/dt-common/src/utils/rdb_filter.rs @@ -21,6 +21,8 @@ pub struct RdbFilter { pub cache: HashMap<(String, String), bool>, } +const DDL: &str = "ddl"; + impl RdbFilter { pub fn from_config(config: &FilterConfig, db_type: DbType) -> Result { match config { @@ -80,12 +82,20 @@ impl RdbFilter { } pub fn filter_event(&mut self, db: &str, tb: &str, row_type: &str) -> bool { - if self.do_events.is_empty() || !self.do_events.contains(&row_type.to_string()) { + if self.do_events.is_empty() || !self.do_events.contains(row_type) { return false; } self.filter_tb(db, tb) } + pub fn filter_ddl(&mut self) -> bool { + // filter ddl by default + if self.do_events.is_empty() { + return true; + } + !self.do_events.contains(DDL) + } + fn contain_tb( set: &HashSet<(String, String)>, db: &str, @@ -175,9 +185,10 @@ impl RdbFilter { let tokens = ConfigTokenParser::parse(config_str, &delimiters, escape_pairs); for token in tokens.iter() { if !SqlUtil::is_valid_token(token, db_type, escape_pairs) { - return Err(Error::ConfigError { - error: format!("invalid filter config, check error near: {}", token), - }); + return Err(Error::ConfigError(format!( + "invalid filter config, check error near: {}", + token + ))); } } Ok(tokens) diff --git a/dt-connector/src/extractor/mongo/mongo_cdc_extractor.rs b/dt-connector/src/extractor/mongo/mongo_cdc_extractor.rs index b060d9bb..387737a5 100644 --- a/dt-connector/src/extractor/mongo/mongo_cdc_extractor.rs +++ b/dt-connector/src/extractor/mongo/mongo_cdc_extractor.rs @@ -40,10 +40,6 @@ impl Extractor for MongoCdcExtractor { ); self.extract_internal().await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl MongoCdcExtractor { diff --git a/dt-connector/src/extractor/mongo/mongo_snapshot_extractor.rs b/dt-connector/src/extractor/mongo/mongo_snapshot_extractor.rs index 2a6fa419..a1af89be 100644 --- a/dt-connector/src/extractor/mongo/mongo_snapshot_extractor.rs +++ b/dt-connector/src/extractor/mongo/mongo_snapshot_extractor.rs @@ -40,10 +40,6 @@ impl Extractor for MongoSnapshotExtractor { ); self.extract_internal().await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl MongoSnapshotExtractor { diff --git a/dt-connector/src/extractor/mysql/mysql_cdc_extractor.rs b/dt-connector/src/extractor/mysql/mysql_cdc_extractor.rs index 9de91e05..22fb0190 100644 --- a/dt-connector/src/extractor/mysql/mysql_cdc_extractor.rs +++ b/dt-connector/src/extractor/mysql/mysql_cdc_extractor.rs @@ -16,14 +16,14 @@ use dt_meta::{ use mysql_binlog_connector_rust::{ binlog_client::BinlogClient, event::{ - event_data::EventData, event_header::EventHeader, row_event::RowEvent, - table_map_event::TableMapEvent, + event_data::EventData, event_header::EventHeader, query_event::QueryEvent, + row_event::RowEvent, table_map_event::TableMapEvent, }, }; use dt_common::{ error::Error, - log_info, + log_error, log_info, utils::{position_util::PositionUtil, rdb_filter::RdbFilter}, }; @@ -52,10 +52,6 @@ impl Extractor for MysqlCdcExtractor { ); self.extract_internal().await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl MysqlCdcExtractor { @@ -170,23 +166,7 @@ impl MysqlCdcExtractor { } EventData::Query(query) => { - let mut ddl_data = DdlData { - schema: query.schema, - query: query.query.clone(), - ddl_type: DdlType::Unknown, - meta: None, - }; - - if query.query != QUERY_BEGIN { - if let Ok((ddl_type, schema, _)) = DdlParser::parse(&query.query) { - ddl_data.ddl_type = ddl_type; - if let Some(schema) = schema { - ddl_data.schema = schema; - } - } - BaseExtractor::push_dt_data(self.buffer.as_ref(), DtData::Ddl { ddl_data }) - .await?; - } + self.handle_query_event(query).await?; } EventData::Xid(xid) => { @@ -226,7 +206,9 @@ impl MysqlCdcExtractor { .await?; if included_columns.len() != event.column_values.len() { - return Err(Error::ColumnNotMatch); + return Err(Error::ExtractorError( + "included_columns not match column_values in binlog".into(), + )); } let mut data = HashMap::new(); @@ -239,9 +221,66 @@ impl MysqlCdcExtractor { let col_type = tb_meta.col_type_map.get(key).unwrap(); let raw_value = event.column_values.remove(i); - let value = MysqlColValueConvertor::from_binlog(col_type, raw_value); + let value = MysqlColValueConvertor::from_binlog(col_type, raw_value)?; data.insert(key.clone(), value); } Ok(data) } + + async fn handle_query_event(&mut self, query: QueryEvent) -> Result<(), Error> { + if query.query == QUERY_BEGIN { + return Ok(()); + } + + log_info!("received ddl: {:?}", query); + let mut ddl_data = DdlData { + schema: query.schema, + tb: String::new(), + query: query.query.clone(), + ddl_type: DdlType::Unknown, + meta: None, + }; + + let parse_result = DdlParser::parse(&query.query); + if let Err(error) = parse_result { + // clear all metadata cache + self.meta_manager.invalidate_cache("", ""); + log_error!( + "failed to parse ddl, will try ignore it, please execute the ddl manually in target, sql: {}, error: {}", + ddl_data.query, + error + ); + return Ok(()); + } + + // case 1, execute: use db_1; create table tb_1(id int); + // binlog query.schema == db_1, schema from DdlParser == None + // case 2, execute: create table db_1.tb_1(id int); + // binlog query.schema == empty, schema from DdlParser == db_1 + // case 3, execute: use db_1; create table db_2.tb_1(id int); + // binlog query.schema == db_1, schema from DdlParser == db_2 + let (ddl_type, schema, tb) = parse_result.unwrap(); + ddl_data.ddl_type = ddl_type; + if let Some(schema) = schema { + ddl_data.schema = schema; + } + if let Some(tb) = tb { + ddl_data.tb = tb; + } + + // invalidate metadata cache + self.meta_manager + .invalidate_cache(&ddl_data.schema, &ddl_data.tb); + + let filter = if ddl_data.tb.is_empty() { + self.filter.filter_db(&ddl_data.schema) + } else { + self.filter.filter_tb(&ddl_data.schema, &ddl_data.tb) + }; + + if !self.filter.filter_ddl() && !filter { + BaseExtractor::push_dt_data(self.buffer.as_ref(), DtData::Ddl { ddl_data }).await?; + } + Ok(()) + } } diff --git a/dt-connector/src/extractor/mysql/mysql_check_extractor.rs b/dt-connector/src/extractor/mysql/mysql_check_extractor.rs index ef905201..a7277cfa 100644 --- a/dt-connector/src/extractor/mysql/mysql_check_extractor.rs +++ b/dt-connector/src/extractor/mysql/mysql_check_extractor.rs @@ -50,10 +50,6 @@ impl Extractor for MysqlCheckExtractor { base_check_extractor.extract(self).await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } #[async_trait] diff --git a/dt-connector/src/extractor/mysql/mysql_struct_extractor.rs b/dt-connector/src/extractor/mysql/mysql_struct_extractor.rs index 910bf1bc..51eecc6f 100644 --- a/dt-connector/src/extractor/mysql/mysql_struct_extractor.rs +++ b/dt-connector/src/extractor/mysql/mysql_struct_extractor.rs @@ -28,10 +28,6 @@ impl Extractor for MysqlStructExtractor { log_info!("MysqlStructExtractor starts, schema: {}", self.db,); self.extract_internal().await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl MysqlStructExtractor { @@ -60,6 +56,7 @@ impl MysqlStructExtractor { pub async fn push_dt_data(&mut self, meta: &StructModel) { let ddl_data = DdlData { schema: self.db.clone(), + tb: String::new(), query: String::new(), meta: Some(meta.to_owned()), ddl_type: DdlType::Unknown, diff --git a/dt-connector/src/extractor/pg/pg_cdc_client.rs b/dt-connector/src/extractor/pg/pg_cdc_client.rs index 66000d57..e136a323 100644 --- a/dt-connector/src/extractor/pg/pg_cdc_client.rs +++ b/dt-connector/src/extractor/pg/pg_cdc_client.rs @@ -81,9 +81,10 @@ impl PgCdcClient { start_lsn = if let Row(row) = &res[0] { row.get("consistent_point").unwrap().to_string() } else { - return Err(Error::MetadataError { - error: format!("failed in: {}", query), - }); + return Err(Error::ExtractorError(format!( + "failed to create replication slot by query: {}", + query + ))); }; } diff --git a/dt-connector/src/extractor/pg/pg_cdc_extractor.rs b/dt-connector/src/extractor/pg/pg_cdc_extractor.rs index e495ae5a..c811aa2d 100644 --- a/dt-connector/src/extractor/pg/pg_cdc_extractor.rs +++ b/dt-connector/src/extractor/pg/pg_cdc_extractor.rs @@ -67,10 +67,6 @@ impl Extractor for PgCdcExtractor { self.extract_internal().await.unwrap(); Ok(()) } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl PgCdcExtractor { @@ -335,9 +331,9 @@ impl PgCdcExtractor { } TupleData::UnchangedToast => { - return Err(Error::Unexpected { - error: "unexpected UnchangedToast value received".to_string(), - }) + return Err(Error::ExtractorError( + "unexpected UnchangedToast value received".into(), + )) } } } diff --git a/dt-connector/src/extractor/pg/pg_check_extractor.rs b/dt-connector/src/extractor/pg/pg_check_extractor.rs index 682f1378..9c67a3c9 100644 --- a/dt-connector/src/extractor/pg/pg_check_extractor.rs +++ b/dt-connector/src/extractor/pg/pg_check_extractor.rs @@ -54,10 +54,6 @@ impl Extractor for PgCheckExtractor { base_check_extractor.extract(self).await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } #[async_trait] diff --git a/dt-connector/src/extractor/pg/pg_struct_extractor.rs b/dt-connector/src/extractor/pg/pg_struct_extractor.rs index 5920e012..0bc98a75 100644 --- a/dt-connector/src/extractor/pg/pg_struct_extractor.rs +++ b/dt-connector/src/extractor/pg/pg_struct_extractor.rs @@ -29,10 +29,6 @@ impl Extractor for PgStructExtractor { log_info!("PgStructExtractor starts, schema: {}", self.db,); self.extract_internal().await } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl PgStructExtractor { @@ -77,6 +73,7 @@ impl PgStructExtractor { pub async fn push_dt_data(&mut self, meta: &StructModel) { let ddl_data = DdlData { schema: self.db.clone(), + tb: String::new(), query: String::new(), meta: Some(meta.to_owned()), ddl_type: DdlType::Unknown, diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/hash_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/hash_parser.rs index 1a36432e..aff68f87 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/hash_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/hash_parser.rs @@ -20,9 +20,10 @@ impl HashLoader { super::RDB_TYPE_HASH_ZIP_LIST => Self::read_hash_zip_list(&mut obj, reader)?, super::RDB_TYPE_HASH_LIST_PACK => Self::read_hash_list_pack(&mut obj, reader)?, _ => { - return Err(Error::Unexpected { - error: format!("unknown hash type. type_byte=[{}]", type_byte), - }) + return Err(Error::RedisRdbError(format!( + "unknown hash type. type_byte=[{}]", + type_byte + ))) } } Ok(obj) @@ -39,9 +40,9 @@ impl HashLoader { } fn read_hash_zip_map(_obj: &mut HashObject, _reader: &mut RdbReader) -> Result<(), Error> { - Err(Error::Unexpected { - error: "not implemented rdb_type_zip_map".to_string(), - }) + Err(Error::RedisRdbError( + "not implemented rdb_type_zip_map".to_string(), + )) } fn read_hash_zip_list(obj: &mut HashObject, reader: &mut RdbReader) -> Result<(), Error> { diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/list_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/list_parser.rs index 2ede2df1..38a35620 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/list_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/list_parser.rs @@ -23,9 +23,10 @@ impl ListLoader { super::RDB_TYPE_LIST_QUICK_LIST => Self::read_quick_list(&mut obj, reader)?, super::RDB_TYPE_LIST_QUICK_LIST_2 => Self::read_quick_list_2(&mut obj, reader)?, _ => { - return Err(Error::Unexpected { - error: format!("unknown list type {}", type_byte), - }) + return Err(Error::RedisRdbError(format!( + "unknown list type {}", + type_byte + ))) } } Ok(obj) @@ -66,9 +67,10 @@ impl ListLoader { } _ => { - return Err(Error::Unexpected { - error: format!("unknown quicklist container {}", container), - }); + return Err(Error::RedisRdbError(format!( + "unknown quicklist container {}", + container + ))); } } } diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/module2_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/module2_parser.rs index 310a8bc2..5deafa80 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/module2_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/module2_parser.rs @@ -14,12 +14,10 @@ impl ModuleLoader { let obj = ModuleObject::new(); if type_byte == super::RDB_TYPE_MODULE { - return Err(Error::Unexpected { - error: format!( - "module type with version 1 is not supported, key=[{}]", - String::from(key) - ), - }); + return Err(Error::RedisRdbError(format!( + "module type with version 1 is not supported, key=[{}]", + String::from(key) + ))); } let module_id = reader.read_length()?; @@ -44,12 +42,10 @@ impl ModuleLoader { } _ => { - return Err(Error::Unexpected { - error: format!( - "unknown module opcode=[{}], module name=[{}]", - op_code, module_name - ), - }); + return Err(Error::RedisRdbError(format!( + "unknown module opcode=[{}], module name=[{}]", + op_code, module_name + ))); } } op_code = reader.read_byte()?; diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/set_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/set_parser.rs index 1cfab014..1e042976 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/set_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/set_parser.rs @@ -21,9 +21,10 @@ impl SetLoader { super::RDB_TYPE_SET => Self::read_str_set(&mut obj, reader)?, super::RDB_TYPE_SET_INT_SET => Self::read_int_set(&mut obj, reader)?, _ => { - return Err(Error::Unexpected { - error: format!("unknown set type. type_byte=[{}]", type_byte).to_string(), - }) + return Err(Error::RedisRdbError(format!( + "unknown set type. type_byte=[{}]", + type_byte + ))) } } Ok(obj) @@ -50,10 +51,10 @@ impl SetLoader { 4 => LittleEndian::read_i32(&buf).to_string(), 8 => LittleEndian::read_i64(&buf).to_string(), _ => { - return Err(Error::Unexpected { - error: format!("unknown int encoding type: {:x}", encoding_type) - .to_string(), - }); + return Err(Error::RedisRdbError(format!( + "unknown int encoding type: {:x}", + encoding_type + ))); } }; obj.elements.push(RedisString::from(int_str)); diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/stream_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/stream_parser.rs index dbc038f2..e2116926 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/stream_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/stream_parser.rs @@ -44,10 +44,10 @@ impl StreamLoader { // master entry end by zero let last_entry = String::from(Self::next(&mut inx, &elements).clone()); if last_entry != "0" { - return Err(Error::Unexpected { - error: format!("master entry not ends by zero. lastEntry=[{}]", last_entry) - .to_string(), - }); + return Err(Error::RedisRdbError(format!( + "master entry not ends by zero. lastEntry=[{}]", + last_entry + ))); } // Parse entries @@ -71,9 +71,9 @@ impl StreamLoader { } else { // get field by lp.Next() let num = Self::next_integer(&mut inx, &elements) as usize; - let _ = elements[inx..inx + num * 2] - .iter() - .map(|i| cmd.add_redis_arg(i)); + for ele in elements[inx..inx + num * 2].iter() { + cmd.add_redis_arg(ele); + } inx += num * 2; } @@ -100,7 +100,7 @@ impl StreamLoader { // the key we are serializing is an empty string, which is possible // for the Stream type. let mut cmd = RedisCmd::new(); - cmd.add_str_arg("xadd"); + cmd.add_str_arg("XADD"); cmd.add_redis_arg(&master_key); cmd.add_str_arg("MAXLEN"); cmd.add_str_arg("0"); @@ -113,7 +113,7 @@ impl StreamLoader { // Append XSETID after XADD, make sure lastid is correct, // in case of XDEL lastid. let mut cmd = RedisCmd::new(); - cmd.add_str_arg("xsetid"); + cmd.add_str_arg("XSETID"); cmd.add_redis_arg(&master_key); cmd.add_str_arg(&last_id); obj.cmds.push(cmd); @@ -146,6 +146,7 @@ impl StreamLoader { /* Create Group */ let mut cmd = RedisCmd::new(); + cmd.add_str_arg("XGROUP"); cmd.add_str_arg("CREATE"); cmd.add_redis_arg(&master_key); cmd.add_redis_arg(&group_name); @@ -199,7 +200,7 @@ impl StreamLoader { /* Send */ let mut cmd = RedisCmd::new(); - cmd.add_str_arg("xclaim"); + cmd.add_str_arg("XCLAIM"); cmd.add_redis_arg(&master_key); cmd.add_redis_arg(&group_name); cmd.add_redis_arg(&consumer_name); diff --git a/dt-connector/src/extractor/redis/rdb/entry_parser/zset_parser.rs b/dt-connector/src/extractor/redis/rdb/entry_parser/zset_parser.rs index bbd2189f..a98c4ebe 100644 --- a/dt-connector/src/extractor/redis/rdb/entry_parser/zset_parser.rs +++ b/dt-connector/src/extractor/redis/rdb/entry_parser/zset_parser.rs @@ -20,9 +20,10 @@ impl ZsetLoader { super::RDB_TYPE_ZSET_ZIP_LIST => Self::read_zset_zip_list(&mut obj, reader)?, super::RDB_TYPE_ZSET_LIST_PACK => Self::read_zset_list_pack(&mut obj, reader)?, _ => { - return Err(Error::Unexpected { - error: format!("unknown zset type. type_byte=[{}]", type_byte), - }); + return Err(Error::RedisRdbError(format!( + "unknown zset type. type_byte=[{}]", + type_byte + ))); } } Ok(obj) @@ -64,9 +65,10 @@ impl ZsetLoader { fn parse_zset_result(obj: &mut ZsetObject, list: Vec) -> Result<(), Error> { let size = list.len(); if size % 2 != 0 { - return Err(Error::Unexpected { - error: format!("zset list pack size is not even. size=[{}]", size), - }); + return Err(Error::RedisRdbError(format!( + "zset list pack size is not even. size=[{}]", + size + ))); } for i in (0..size).step_by(2) { diff --git a/dt-connector/src/extractor/redis/rdb/rdb_loader.rs b/dt-connector/src/extractor/redis/rdb/rdb_loader.rs index 7f837cd2..6d071609 100644 --- a/dt-connector/src/extractor/redis/rdb/rdb_loader.rs +++ b/dt-connector/src/extractor/redis/rdb/rdb_loader.rs @@ -35,9 +35,7 @@ impl RdbLoader<'_> { let mut buf = self.reader.read_raw(5)?; let magic = String::from_utf8(buf).unwrap(); if magic != "REDIS" { - return Err(Error::Unexpected { - error: "invalid rdb format".to_string(), - }); + return Err(Error::RedisRdbError("invalid rdb format".to_string())); } // version diff --git a/dt-connector/src/extractor/redis/rdb/reader/length.rs b/dt-connector/src/extractor/redis/rdb/reader/length.rs index 611b03a3..595affe0 100644 --- a/dt-connector/src/extractor/redis/rdb/reader/length.rs +++ b/dt-connector/src/extractor/redis/rdb/reader/length.rs @@ -15,9 +15,7 @@ impl RdbReader<'_> { pub fn read_length(&mut self) -> Result { let (len, special) = self.read_encoded_length()?; if special { - Err(Error::Unexpected { - error: format!("illegal length special=true").to_string(), - }) + Err(Error::RedisRdbError("illegal length special=true".into())) } else { Ok(len) } @@ -51,9 +49,10 @@ impl RdbReader<'_> { Ok((len, false)) } - _ => Err(Error::Unexpected { - error: format!("illegal length encoding: {:x}", first_byte).to_string(), - }), + _ => Err(Error::RedisRdbError(format!( + "illegal length encoding: {:x}", + first_byte + ))), }, RDB_SPECIAL_LEN => { @@ -61,9 +60,10 @@ impl RdbReader<'_> { Ok((len, true)) } - _ => Err(Error::Unexpected { - error: format!("illegal length encoding: {:x}", first_byte).to_string(), - }), + _ => Err(Error::RedisRdbError(format!( + "illegal length encoding: {:x}", + first_byte + ))), } } } diff --git a/dt-connector/src/extractor/redis/rdb/reader/list_pack.rs b/dt-connector/src/extractor/redis/rdb/reader/list_pack.rs index 8b8aaca1..4cbdc4e0 100644 --- a/dt-connector/src/extractor/redis/rdb/reader/list_pack.rs +++ b/dt-connector/src/extractor/redis/rdb/reader/list_pack.rs @@ -51,9 +51,9 @@ impl RdbReader<'_> { let last_byte = reader.read_u8()?; if last_byte != 0xFF { - return Err(Error::Unexpected { - error: "read_listpack: last byte is not 0xFF".to_string(), - }); + return Err(Error::RedisRdbError( + "read_listpack: last byte is not 0xFF".into(), + )); } Ok(elements) } @@ -130,9 +130,10 @@ impl RdbReader<'_> { // uval = 12345678900000000 + uint64(fireByte) // negstart = math.MaxUint64 // negmax = 0 - return Err(Error::Unexpected { - error: format!("unknown encoding: {}", first_byte).to_string(), - }); + return Err(Error::RedisRdbError(format!( + "unknown encoding: {}", + first_byte + ))); } // We reach this code path only for integer encodings. diff --git a/dt-connector/src/extractor/redis/rdb/reader/string.rs b/dt-connector/src/extractor/redis/rdb/reader/string.rs index 3070fc5b..961722d3 100644 --- a/dt-connector/src/extractor/redis/rdb/reader/string.rs +++ b/dt-connector/src/extractor/redis/rdb/reader/string.rs @@ -29,9 +29,10 @@ impl RdbReader<'_> { } _ => { - return Err(Error::Unexpected { - error: format!("Unknown string encode type {}", len).to_string(), - }) + return Err(Error::RedisRdbError(format!( + "Unknown string encode type {}", + len + ))) } } } else { @@ -73,9 +74,10 @@ impl RdbReader<'_> { } if o != out_len { - Err(Error::Unexpected { - error: format!("lzf decompress failed: out_len: {}, o: {}", out_len, o).to_string(), - }) + Err(Error::RedisRdbError(format!( + "lzf decompress failed: out_len: {}, o: {}", + out_len, o + ))) } else { Ok(out) } diff --git a/dt-connector/src/extractor/redis/rdb/reader/zip_list.rs b/dt-connector/src/extractor/redis/rdb/reader/zip_list.rs index a2092876..33233e5c 100644 --- a/dt-connector/src/extractor/redis/rdb/reader/zip_list.rs +++ b/dt-connector/src/extractor/redis/rdb/reader/zip_list.rs @@ -52,9 +52,10 @@ impl RdbReader<'_> { let last_byte = reader.read_u8()?; if last_byte != 0xFF { - return Err(Error::Unexpected { - error: format!("invalid zipList lastByte encoding: {}", last_byte), - }); + return Err(Error::RedisRdbError(format!( + "invalid zipList lastByte encoding: {}", + last_byte + ))); } } @@ -129,15 +130,17 @@ impl RdbReader<'_> { if first_byte >> 4 == ZIP_INT_04B { let v = (first_byte & 0x0f) as i8 - 1; if v < 0 || v > 12 { - return Err(Error::Unexpected { - error: format!("invalid zipInt04B encoding: {}", v), - }); + return Err(Error::RedisRdbError(format!( + "invalid zipInt04B encoding: {}", + v + ))); } return Ok(RedisString::from(v.to_string())); } - Err(Error::Unexpected { - error: format!("invalid encoding: {}", first_byte), - }) + Err(Error::RedisRdbError(format!( + "invalid encoding: {}", + first_byte + ))) } } diff --git a/dt-connector/src/extractor/redis/redis_cdc_extractor.rs b/dt-connector/src/extractor/redis/redis_cdc_extractor.rs index ab326841..df768ad2 100644 --- a/dt-connector/src/extractor/redis/redis_cdc_extractor.rs +++ b/dt-connector/src/extractor/redis/redis_cdc_extractor.rs @@ -117,9 +117,10 @@ impl RedisCdcExtractor { } } v => { - return Err(Error::Unexpected { - error: format!("received unexpected aof value: {:?}", v), - }); + return Err(Error::RedisRdbError(format!( + "received unexpected aof value: {:?}", + v + ))); } } Ok(cmd) diff --git a/dt-connector/src/extractor/redis/redis_client.rs b/dt-connector/src/extractor/redis/redis_client.rs index 1d6a2014..cc185f28 100644 --- a/dt-connector/src/extractor/redis/redis_client.rs +++ b/dt-connector/src/extractor/redis/redis_client.rs @@ -42,9 +42,7 @@ impl RedisClient { return Ok(me); } } - return Err(Error::Unexpected { - error: format!("can't cnnect redis: {}", url), - }); + return Err(Error::Unexpected(format!("can't cnnect redis: {}", url))); } Ok(me) diff --git a/dt-connector/src/extractor/redis/redis_psync_extractor.rs b/dt-connector/src/extractor/redis/redis_psync_extractor.rs index e801ae74..f2214695 100644 --- a/dt-connector/src/extractor/redis/redis_psync_extractor.rs +++ b/dt-connector/src/extractor/redis/redis_psync_extractor.rs @@ -44,9 +44,9 @@ impl RedisPsyncExtractor<'_> { self.conn.send(&repl_cmd).await.unwrap(); if let Value::Okay = self.conn.read().await.unwrap() { } else { - return Err(Error::Unexpected { - error: "replconf listening-port response is not Ok".to_string(), - }); + return Err(Error::ExtractorError( + "replconf listening-port response is not Ok".into(), + )); } let full_sync = self.run_id.is_empty() && self.repl_offset == 0; @@ -68,14 +68,14 @@ impl RedisPsyncExtractor<'_> { self.run_id = tokens[1].to_string(); self.repl_offset = tokens[2].parse::().unwrap(); } else if s != "CONTINUE" { - return Err(Error::Unexpected { - error: "PSYNC command response is NOT CONTINUE".to_string(), - }); + return Err(Error::ExtractorError( + "PSYNC command response is NOT CONTINUE".into(), + )); } } else { - return Err(Error::Unexpected { - error: "PSYNC command response is NOT status".to_string(), - }); + return Err(Error::ExtractorError( + "PSYNC command response is NOT status".into(), + )); }; if full_sync { diff --git a/dt-connector/src/lib.rs b/dt-connector/src/lib.rs index 841f2517..28d48e9c 100644 --- a/dt-connector/src/lib.rs +++ b/dt-connector/src/lib.rs @@ -26,6 +26,10 @@ pub trait Sinker { async fn sink_raw(&mut self, mut _data: Vec, _batch: bool) -> Result<(), Error> { Ok(()) } + + async fn refresh_meta(&mut self, _data: Vec) -> Result<(), Error> { + Ok(()) + } } #[async_trait] diff --git a/dt-connector/src/rdb_query_builder.rs b/dt-connector/src/rdb_query_builder.rs index b788a4a3..0e3a5c03 100644 --- a/dt-connector/src/rdb_query_builder.rs +++ b/dt-connector/src/rdb_query_builder.rs @@ -125,12 +125,10 @@ impl RdbQueryBuilder<'_> { cols.push(col.clone()); let col_value = before.get(col); if *col_value.unwrap() == ColValue::None { - return Err(Error::Unexpected { - error: format!( + return Err(Error::Unexpected(format!( "db: {}, tb: {}, where col: {} is NULL, which should not happen in batch delete", self.rdb_tb_meta.schema, self.rdb_tb_meta.tb, col - ), - }); + ))); } binds.push(col_value); } @@ -252,12 +250,10 @@ impl RdbQueryBuilder<'_> { } if set_pairs.is_empty() { - return Err(Error::Unexpected { - error: format!( - "db: {}, tb: {}, no cols in after, which should not happen in update", - self.rdb_tb_meta.schema, self.rdb_tb_meta.tb - ), - }); + return Err(Error::Unexpected(format!( + "db: {}, tb: {}, no cols in after, which should not happen in update", + self.rdb_tb_meta.schema, self.rdb_tb_meta.tb + ))); } let (where_sql, not_null_cols) = self.get_where_info(placeholder_index, before)?; @@ -336,12 +332,10 @@ impl RdbQueryBuilder<'_> { cols.push(col.clone()); let col_value = after.get(col); if *col_value.unwrap() == ColValue::None { - return Err(Error::Unexpected { - error: format!( + return Err(Error::Unexpected(format!( "db: {}, tb: {}, where col: {} is NULL, which should not happen in batch select", self.rdb_tb_meta.schema, self.rdb_tb_meta.tb, col - ), - }); + ))); } binds.push(col_value); } diff --git a/dt-connector/src/sinker/foxlake_sinker.rs b/dt-connector/src/sinker/foxlake_sinker.rs index 42959e9c..ffb6f67d 100644 --- a/dt-connector/src/sinker/foxlake_sinker.rs +++ b/dt-connector/src/sinker/foxlake_sinker.rs @@ -56,10 +56,6 @@ impl Sinker for FoxlakeSinker { self.put_to_file(&key, &content).await.unwrap(); Ok(()) } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl FoxlakeSinker { diff --git a/dt-connector/src/sinker/kafka/kafka_router.rs b/dt-connector/src/sinker/kafka/kafka_router.rs index bd0d4436..01ecd4c4 100644 --- a/dt-connector/src/sinker/kafka/kafka_router.rs +++ b/dt-connector/src/sinker/kafka/kafka_router.rs @@ -51,9 +51,10 @@ impl KafkaRouter { let tokens: Vec<&str> = name.split(':').collect(); if tokens.len() != 2 { - return Err(Error::ConfigError { - error: format!("invalid router config, check error near: {}", name), - }); + return Err(Error::ConfigError(format!( + "invalid router config, check error near: {}", + name + ))); } map.insert( tokens.first().unwrap().to_string(), diff --git a/dt-connector/src/sinker/kafka/kafka_sinker.rs b/dt-connector/src/sinker/kafka/kafka_sinker.rs index bee01860..4935b479 100644 --- a/dt-connector/src/sinker/kafka/kafka_sinker.rs +++ b/dt-connector/src/sinker/kafka/kafka_sinker.rs @@ -4,7 +4,7 @@ use crate::{call_batch_fn, Sinker}; use dt_common::error::Error; -use dt_meta::{ddl_data::DdlData, row_data::RowData}; +use dt_meta::row_data::RowData; use kafka::producer::{Producer, Record}; @@ -22,14 +22,6 @@ impl Sinker for KafkaSinker { call_batch_fn!(self, data, Self::send); Ok(()) } - - async fn sink_ddl(&mut self, _data: Vec, _batch: bool) -> Result<(), Error> { - Ok(()) - } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl KafkaSinker { diff --git a/dt-connector/src/sinker/mongo/mongo_sinker.rs b/dt-connector/src/sinker/mongo/mongo_sinker.rs index 6ccadd03..40c302c2 100644 --- a/dt-connector/src/sinker/mongo/mongo_sinker.rs +++ b/dt-connector/src/sinker/mongo/mongo_sinker.rs @@ -7,7 +7,7 @@ use mongodb::{ use dt_common::{constants::MongoConstants, error::Error, log_error}; -use dt_meta::{col_value::ColValue, ddl_data::DdlData, row_data::RowData, row_type::RowType}; +use dt_meta::{col_value::ColValue, row_data::RowData, row_type::RowType}; use crate::{call_batch_fn, sinker::rdb_router::RdbRouter, Sinker}; @@ -40,14 +40,6 @@ impl Sinker for MongoSinker { } Ok(()) } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } - - async fn sink_ddl(&mut self, _data: Vec, _batch: bool) -> Result<(), Error> { - Ok(()) - } } impl MongoSinker { diff --git a/dt-connector/src/sinker/mysql/mysql_sinker.rs b/dt-connector/src/sinker/mysql/mysql_sinker.rs index 2e3af821..a0701273 100644 --- a/dt-connector/src/sinker/mysql/mysql_sinker.rs +++ b/dt-connector/src/sinker/mysql/mysql_sinker.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use crate::{ call_batch_fn, close_conn_pool, rdb_query_builder::RdbQueryBuilder, @@ -5,21 +7,26 @@ use crate::{ Sinker, }; -use dt_common::{error::Error, log_error}; +use dt_common::{error::Error, log_error, log_info}; use dt_meta::{ ddl_data::DdlData, + ddl_type::DdlType, mysql::{mysql_meta_manager::MysqlMetaManager, mysql_tb_meta::MysqlTbMeta}, row_data::RowData, row_type::RowType, }; -use sqlx::{MySql, Pool}; +use sqlx::{ + mysql::{MySqlConnectOptions, MySqlPoolOptions}, + MySql, Pool, +}; use async_trait::async_trait; #[derive(Clone)] pub struct MysqlSinker { + pub url: String, pub conn_pool: Pool, pub meta_manager: MysqlMetaManager, pub router: RdbRouter, @@ -53,7 +60,32 @@ impl Sinker for MysqlSinker { return close_conn_pool!(self); } - async fn sink_ddl(&mut self, _data: Vec, _batch: bool) -> Result<(), Error> { + async fn sink_ddl(&mut self, data: Vec, _batch: bool) -> Result<(), Error> { + for ddl_data in data.iter() { + log_info!("sink ddl: {}", ddl_data.query); + let query = sqlx::query(&ddl_data.query); + + // create a tmp connection with databse since sqlx conn pool does NOT support `USE db` + let mut conn_options = MySqlConnectOptions::from_str(&self.url).unwrap(); + if !ddl_data.schema.is_empty() && ddl_data.ddl_type != DdlType::CreateDatabase { + conn_options = conn_options.database(&ddl_data.schema); + } + + let conn_pool = MySqlPoolOptions::new() + .max_connections(1) + .connect_with(conn_options) + .await + .unwrap(); + query.execute(&conn_pool).await.unwrap(); + } + Ok(()) + } + + async fn refresh_meta(&mut self, data: Vec) -> Result<(), Error> { + for ddl_data in data.iter() { + self.meta_manager + .invalidate_cache(&ddl_data.schema, &ddl_data.tb); + } Ok(()) } } diff --git a/dt-connector/src/sinker/open_faas_sinker.rs b/dt-connector/src/sinker/open_faas_sinker.rs index 977d2dd1..db6df6d7 100644 --- a/dt-connector/src/sinker/open_faas_sinker.rs +++ b/dt-connector/src/sinker/open_faas_sinker.rs @@ -1,7 +1,7 @@ use crate::{call_batch_fn, Sinker}; use async_trait::async_trait; use dt_common::{error::Error, log_error}; -use dt_meta::{ddl_data::DdlData, row_data::RowData}; +use dt_meta::row_data::RowData; use reqwest::Client; use serde_json::json; @@ -17,14 +17,6 @@ impl Sinker for OpenFaasSinker { call_batch_fn!(self, data, Self::invoke); Ok(()) } - - async fn sink_ddl(&mut self, _data: Vec, _batch: bool) -> Result<(), Error> { - Ok(()) - } - - async fn close(&mut self) -> Result<(), Error> { - Ok(()) - } } impl OpenFaasSinker { diff --git a/dt-connector/src/sinker/pg/pg_sinker.rs b/dt-connector/src/sinker/pg/pg_sinker.rs index d2d0b501..503c3656 100644 --- a/dt-connector/src/sinker/pg/pg_sinker.rs +++ b/dt-connector/src/sinker/pg/pg_sinker.rs @@ -10,7 +10,6 @@ use sqlx::{Pool, Postgres}; use dt_meta::{ col_value::ColValue, - ddl_data::DdlData, pg::{pg_meta_manager::PgMetaManager, pg_tb_meta::PgTbMeta}, row_data::RowData, row_type::RowType, @@ -52,10 +51,6 @@ impl Sinker for PgSinker { async fn close(&mut self) -> Result<(), Error> { return close_conn_pool!(self); } - - async fn sink_ddl(&mut self, _data: Vec, _batch: bool) -> Result<(), Error> { - Ok(()) - } } impl PgSinker { diff --git a/dt-connector/src/sinker/pg/pg_struct_sinker.rs b/dt-connector/src/sinker/pg/pg_struct_sinker.rs index 93a32a23..523dcc28 100644 --- a/dt-connector/src/sinker/pg/pg_struct_sinker.rs +++ b/dt-connector/src/sinker/pg/pg_struct_sinker.rs @@ -110,7 +110,7 @@ impl PgStructSinker { if constraint_type == &ConstraintTypeEnum::Foregin.to_charval().unwrap() { let msg = format!("foreign key is not supported yet, schema:[{}], table:[{}], constraint_name:[{}]", schema_name, table_name, constraint_name); println!("{}", msg); - return Err(Error::StructError { error: msg }); + return Err(Error::StructError(msg)); } let sql = format!( "ALTER TABLE \"{}\".\"{}\" ADD CONSTRAINT \"{}\" {}", diff --git a/dt-connector/src/sinker/rdb_router.rs b/dt-connector/src/sinker/rdb_router.rs index a72190f5..8f89f3d2 100644 --- a/dt-connector/src/sinker/rdb_router.rs +++ b/dt-connector/src/sinker/rdb_router.rs @@ -62,9 +62,10 @@ impl RdbRouter { || !Self::is_valid_name(tokens[0], &name_type) || !Self::is_valid_name(tokens[1], &name_type) { - return Err(Error::ConfigError { - error: format!("invalid router config, check error near: {}", name), - }); + return Err(Error::ConfigError(format!( + "invalid router config, check error near: {}", + name + ))); } map.insert( tokens.first().unwrap().to_string(), diff --git a/dt-connector/src/sinker/redis/entry_rewriter.rs b/dt-connector/src/sinker/redis/entry_rewriter.rs index 38479666..387c9a98 100644 --- a/dt-connector/src/sinker/redis/entry_rewriter.rs +++ b/dt-connector/src/sinker/redis/entry_rewriter.rs @@ -1,9 +1,8 @@ -use dt_common::{error::Error, log_warn}; +use dt_common::error::Error; use dt_meta::redis::{ redis_entry::RedisEntry, redis_object::{ - HashObject, ListObject, ModuleObject, RedisCmd, SetObject, StreamObject, StringObject, - ZsetObject, + HashObject, ListObject, ModuleObject, RedisCmd, SetObject, StringObject, ZsetObject, }, }; @@ -295,9 +294,7 @@ impl EntryRewriter { } pub fn rewrite_module(_obj: &mut ModuleObject) -> Result, Error> { - Err(Error::Unexpected { - error: "module rewrite not implemented".to_string(), - }) + Err(Error::Unexpected("module rewrite not implemented".into())) } pub fn rewrite_set(obj: &mut SetObject) -> Result, Error> { @@ -312,11 +309,6 @@ impl EntryRewriter { Ok(cmds) } - pub fn rewrite_stream(obj: &mut StreamObject) -> Result, Error> { - // Ok(obj.cmds.drain(..).collect()) - Ok(obj.cmds.clone()) - } - pub fn rewrite_string(obj: &mut StringObject) -> Result, Error> { let mut cmd = RedisCmd::new(); cmd.add_str_arg("set"); @@ -346,7 +338,6 @@ impl EntryRewriter { cmd.add_str_arg(&entry.expire_ms.to_string()); cmd.add_arg(value); if version >= 3.0 { - log_warn!("RDB restore command behavior is rewrite, but target redis version is {}, not support REPLACE modifier", version); cmd.add_str_arg("replace"); } Ok(cmd) diff --git a/dt-connector/src/sinker/redis/redis_sinker.rs b/dt-connector/src/sinker/redis/redis_sinker.rs index 60665108..f5f74310 100644 --- a/dt-connector/src/sinker/redis/redis_sinker.rs +++ b/dt-connector/src/sinker/redis/redis_sinker.rs @@ -2,6 +2,8 @@ use async_trait::async_trait; use dt_common::error::Error; use dt_meta::dt_data::DtData; use dt_meta::redis::redis_object::RedisCmd; +use dt_meta::redis::redis_object::RedisObject; +use dt_meta::redis::redis_write_method::RedisWriteMethod; use redis::Connection; use redis::ConnectionLike; @@ -16,12 +18,17 @@ pub struct RedisSinker { pub conn: Connection, pub now_db_id: i64, pub version: f32, + pub method: RedisWriteMethod, } #[async_trait] impl Sinker for RedisSinker { async fn sink_raw(&mut self, mut data: Vec, _batch: bool) -> Result<(), Error> { - call_batch_fn!(self, data, Self::batch_sink); + if self.batch_size > 1 { + call_batch_fn!(self, data, Self::batch_sink); + } else { + self.serial_sink(&mut data).await?; + } Ok(()) } } @@ -33,41 +40,85 @@ impl RedisSinker { start_index: usize, batch_size: usize, ) -> Result<(), Error> { + let mut cmds = Vec::new(); + for dt_data in data.iter_mut().skip(start_index).take(batch_size) { + cmds.extend_from_slice(&self.rewrite_entry(dt_data)?); + } + let mut packed_cmds = Vec::new(); - for dt_data in data.iter().skip(start_index).take(batch_size) { - packed_cmds.extend_from_slice(&self.pack_entry(dt_data)?); + for cmd in cmds { + packed_cmds.extend_from_slice(&CmdEncoder::encode(&cmd)); } - // TODO, check the result and add retry logic, if write failed, self.db_id should also be reset - let _ = self - .conn - .req_packed_commands(&packed_cmds, 0, batch_size) - .unwrap(); + let result = self.conn.req_packed_commands(&packed_cmds, 0, batch_size); + if let Err(error) = result { + return Err(Error::SinkerError(format!( + "batch sink failed, error: {:?}", + error + ))); + } Ok(()) } - fn pack_entry(&mut self, dt_data: &DtData) -> Result, Error> { - let mut packed_cmds = Vec::new(); + async fn serial_sink(&mut self, data: &mut [DtData]) -> Result<(), Error> { + for dt_data in data.iter_mut() { + let cmds = self.rewrite_entry(dt_data)?; + for cmd in cmds { + let result = self.conn.req_packed_command(&CmdEncoder::encode(&cmd)); + if let Err(error) = result { + return Err(Error::SinkerError(format!( + "serial sink failed, error: {:?}, cmd: {}", + error, + cmd.to_string() + ))); + } + } + } + Ok(()) + } + + fn rewrite_entry(&mut self, dt_data: &mut DtData) -> Result, Error> { + let mut cmds = Vec::new(); match dt_data { - DtData::Redis { entry } => { + DtData::Redis { ref mut entry } => { if entry.db_id != self.now_db_id { let db_id = &entry.db_id.to_string(); let args = vec!["SELECT", db_id]; let cmd = RedisCmd::from_str_args(&args); - - packed_cmds.extend_from_slice(&CmdEncoder::encode(&cmd)); + cmds.push(cmd); self.now_db_id = entry.db_id; } - if entry.is_raw() { - let cmd = EntryRewriter::rewrite_as_restore(&entry, self.version)?; - packed_cmds.extend_from_slice(&CmdEncoder::encode(&cmd)); - } else { - packed_cmds.extend_from_slice(&CmdEncoder::encode(&entry.cmd)); + match self.method { + RedisWriteMethod::Restore => { + if entry.is_raw() { + let cmd = EntryRewriter::rewrite_as_restore(&entry, self.version)?; + cmds.push(cmd); + } else { + cmds.push(entry.cmd.clone()); + } + } + + RedisWriteMethod::Rewrite => { + let rewrite_cmds = match entry.value { + RedisObject::String(ref mut obj) => EntryRewriter::rewrite_string(obj), + RedisObject::List(ref mut obj) => EntryRewriter::rewrite_list(obj), + RedisObject::Set(ref mut obj) => EntryRewriter::rewrite_set(obj), + RedisObject::Hash(ref mut obj) => EntryRewriter::rewrite_hash(obj), + RedisObject::Zset(ref mut obj) => EntryRewriter::rewrite_zset(obj), + RedisObject::Stream(ref mut obj) => Ok(obj.cmds.drain(..).collect()), + RedisObject::Module(_) => { + let cmd = EntryRewriter::rewrite_as_restore(&entry, self.version)?; + Ok(vec![cmd]) + } + _ => return Err(Error::SinkerError("rewrite not implemented".into())), + }?; + cmds.extend(rewrite_cmds); + } } } _ => {} } - Ok(packed_cmds) + Ok(cmds) } } diff --git a/dt-meta/src/adaptor/mysql_col_value_convertor.rs b/dt-meta/src/adaptor/mysql_col_value_convertor.rs index c79cc083..1dd07417 100644 --- a/dt-meta/src/adaptor/mysql_col_value_convertor.rs +++ b/dt-meta/src/adaptor/mysql_col_value_convertor.rs @@ -3,7 +3,9 @@ use std::io::{Cursor, Seek, SeekFrom}; use byteorder::{LittleEndian, ReadBytesExt}; use chrono::{TimeZone, Utc}; use dt_common::error::Error; -use mysql_binlog_connector_rust::column::column_value::ColumnValue; +use mysql_binlog_connector_rust::column::{ + column_value::ColumnValue, json::json_binary::JsonBinary, +}; use sqlx::{mysql::MySqlRow, Row}; use crate::{col_value::ColValue, mysql::mysql_col_type::MysqlColType}; @@ -69,8 +71,8 @@ impl MysqlColValueConvertor { } } - pub fn from_binlog(col_type: &MysqlColType, value: ColumnValue) -> ColValue { - match value { + pub fn from_binlog(col_type: &MysqlColType, value: ColumnValue) -> Result { + let col_value = match value { ColumnValue::Tiny(v) => { if *col_type == MysqlColType::UnsignedTiny { ColValue::UnsignedTiny(v as u8) @@ -133,7 +135,7 @@ impl MysqlColValueConvertor { if length as usize > v.len() { let pad_v: Vec = vec![0; length as usize - v.len()]; let final_v = [v, pad_v].concat(); - return ColValue::Blob(final_v); + return Ok(ColValue::Blob(final_v)); } } ColValue::Blob(v) @@ -143,10 +145,15 @@ impl MysqlColValueConvertor { ColumnValue::Bit(v) => ColValue::Bit(v), ColumnValue::Set(v) => ColValue::Set(v), ColumnValue::Enum(v) => ColValue::Enum(v), - ColumnValue::Json(v) => ColValue::Json(v), + ColumnValue::Json(v) => { + let v = JsonBinary::parse_as_string(&v)?; + ColValue::Json2(v) + } _ => ColValue::None, - } + }; + + Ok(col_value) } pub fn from_str(col_type: &MysqlColType, value_str: &str) -> Result { @@ -218,10 +225,13 @@ impl MysqlColValueConvertor { MysqlColType::Set => ColValue::String(value_str), MysqlColType::Enum => ColValue::String(value_str), + MysqlColType::Json => ColValue::Json2(value_str), + _ => { - return Err(Error::Unexpected { - error: format!("unsupported column type, column type: {:?}", col_type), - }) + return Err(Error::Unexpected(format!( + "unsupported column type: {:?}", + col_type + ))) } }; @@ -335,8 +345,13 @@ impl MysqlColValueConvertor { return Ok(ColValue::Enum2(value)); } MysqlColType::Json => { - let value: Vec = row.get_unchecked(col); - return Ok(ColValue::Json(value)); + let value: serde_json::Value = row.try_get(col)?; + // TODO, decimal will lose precision when insert into target mysql as string. + // insert into json_table(id, json_col) values(1, "212765.700000000010000"); the result will be: + // +-----+--------------------------+ + // | id | json_col | + // | 1 | 212765.7 | + return Ok(ColValue::Json2(value.to_string())); } _ => {} } diff --git a/dt-meta/src/adaptor/sqlx_ext.rs b/dt-meta/src/adaptor/sqlx_ext.rs index 9c98fc30..1679acdd 100644 --- a/dt-meta/src/adaptor/sqlx_ext.rs +++ b/dt-meta/src/adaptor/sqlx_ext.rs @@ -111,6 +111,7 @@ impl<'q> SqlxMysqlExt<'q> for Query<'q, MySql, MySqlArguments> { ColValue::Enum(v) => self.bind(v), ColValue::Enum2(v) => self.bind(v), ColValue::Json(v) => self.bind(v), + ColValue::Json2(v) => self.bind(v), _ => { let none: Option = Option::None; self.bind(none) diff --git a/dt-meta/src/col_value.rs b/dt-meta/src/col_value.rs index 81db22e7..0a657f1a 100644 --- a/dt-meta/src/col_value.rs +++ b/dt-meta/src/col_value.rs @@ -35,6 +35,7 @@ pub enum ColValue { Set2(String), Enum2(String), Json(Vec), + Json2(String), MongoDoc(Document), } @@ -77,6 +78,7 @@ impl ColValue { ColValue::Enum2(v) => Some(v.to_string()), // TODO: support JSON ColValue::Json(v) => Some(format!("{:?}", v)), + ColValue::Json2(v) => Some(v.to_string()), _ => Option::None, } } @@ -113,6 +115,7 @@ impl Serialize for ColValue { ColValue::Enum2(v) => serializer.serialize_str(v), // TODO: support JSON ColValue::Json(v) => serializer.serialize_bytes(v), + ColValue::Json2(v) => serializer.serialize_str(v), _ => serializer.serialize_none(), } } diff --git a/dt-meta/src/ddl_data.rs b/dt-meta/src/ddl_data.rs index 9671e042..bb01b545 100644 --- a/dt-meta/src/ddl_data.rs +++ b/dt-meta/src/ddl_data.rs @@ -6,6 +6,7 @@ use super::{ddl_type::DdlType, struct_meta::database_model::StructModel}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DdlData { pub schema: String, + pub tb: String, pub query: String, #[serde(skip)] pub meta: Option, diff --git a/dt-meta/src/ddl_type.rs b/dt-meta/src/ddl_type.rs index 444f44de..4c6decfe 100644 --- a/dt-meta/src/ddl_type.rs +++ b/dt-meta/src/ddl_type.rs @@ -19,6 +19,8 @@ pub enum DdlType { AlterDatabase, #[strum(serialize = "alter_table")] AlterTable, + #[strum(serialize = "create_index")] + CreateIndex, #[strum(serialize = "unknown")] Unknown, } diff --git a/dt-meta/src/mysql/mysql_meta_manager.rs b/dt-meta/src/mysql/mysql_meta_manager.rs index b58d5b5b..93d4d1f0 100644 --- a/dt-meta/src/mysql/mysql_meta_manager.rs +++ b/dt-meta/src/mysql/mysql_meta_manager.rs @@ -36,6 +36,16 @@ impl<'a> MysqlMetaManager { Ok(self) } + pub fn invalidate_cache(&mut self, schema: &str, tb: &str) { + if !schema.is_empty() && !tb.is_empty() { + let full_name = format!("{}.{}", schema, tb); + self.cache.remove(&full_name); + } else { + // clear all cache is always safe + self.cache.clear(); + } + } + pub async fn get_tb_meta(&mut self, schema: &str, tb: &str) -> Result { let full_name = format!("{}.{}", schema, tb); if let Some(tb_meta) = self.cache.get(&full_name) { @@ -92,9 +102,10 @@ impl<'a> MysqlMetaManager { } if cols.is_empty() { - return Err(Error::MetadataError { - error: format!("failed to get table metadata for: `{}`.`{}`", schema, tb), - }); + return Err(Error::MetadataError(format!( + "failed to get table metadata for: `{}`.`{}`", + schema, tb + ))); } Ok((cols, col_type_map)) } @@ -234,9 +245,6 @@ impl<'a> MysqlMetaManager { self.version = row.try_get(0)?; return Ok(()); } - - Err(Error::MetadataError { - error: "failed to init mysql version".to_string(), - }) + Err(Error::MetadataError("failed to init mysql version".into())) } } diff --git a/dt-meta/src/pg/pg_meta_manager.rs b/dt-meta/src/pg/pg_meta_manager.rs index ca1507df..574f2154 100644 --- a/dt-meta/src/pg/pg_meta_manager.rs +++ b/dt-meta/src/pg/pg_meta_manager.rs @@ -185,8 +185,9 @@ impl PgMetaManager { return Ok(oid); } - Err(Error::MetadataError { - error: format!("failed in get_oid for {}", tb), - }) + Err(Error::MetadataError(format!( + "failed to get oid for: {} by query: {}", + tb, sql + ))) } } diff --git a/dt-meta/src/rdb_meta_manager.rs b/dt-meta/src/rdb_meta_manager.rs index 3e9b076a..c4801d1f 100644 --- a/dt-meta/src/rdb_meta_manager.rs +++ b/dt-meta/src/rdb_meta_manager.rs @@ -38,9 +38,9 @@ impl RdbMetaManager { return Ok(tb_meta.basic); } - Err(Error::Unexpected { - error: "no available meta_manager in partitioner".to_string(), - }) + Err(Error::Unexpected( + "no available meta_manager in partitioner".into(), + )) } pub fn parse_rdb_cols( diff --git a/dt-meta/src/redis/mod.rs b/dt-meta/src/redis/mod.rs index 4dc16856..eeef73a9 100644 --- a/dt-meta/src/redis/mod.rs +++ b/dt-meta/src/redis/mod.rs @@ -1,2 +1,3 @@ pub mod redis_entry; pub mod redis_object; +pub mod redis_write_method; diff --git a/dt-meta/src/redis/redis_object.rs b/dt-meta/src/redis/redis_object.rs index e11de4c7..04fc3a49 100644 --- a/dt-meta/src/redis/redis_object.rs +++ b/dt-meta/src/redis/redis_object.rs @@ -198,4 +198,12 @@ impl RedisCmd { String::from_utf8(self.args[0].clone()).unwrap() } } + + pub fn to_string(&self) -> String { + let mut str_args = Vec::new(); + for arg in self.args.iter() { + str_args.push(String::from_utf8_lossy(arg)); + } + str_args.join(" ") + } } diff --git a/dt-meta/src/redis/redis_write_method.rs b/dt-meta/src/redis/redis_write_method.rs new file mode 100644 index 00000000..a2aad18e --- /dev/null +++ b/dt-meta/src/redis/redis_write_method.rs @@ -0,0 +1,23 @@ +use std::str::FromStr; + +use dt_common::error::Error; +use strum::IntoStaticStr; + +#[derive(Clone, IntoStaticStr, Debug)] +pub enum RedisWriteMethod { + #[strum(serialize = "restore")] + Restore, + + #[strum(serialize = "rewrite")] + Rewrite, +} + +impl FromStr for RedisWriteMethod { + type Err = Error; + fn from_str(str: &str) -> Result { + match str { + "rewrite" => Ok(Self::Rewrite), + _ => Ok(Self::Restore), + } + } +} diff --git a/dt-meta/src/sql_parser/ddl_parser.rs b/dt-meta/src/sql_parser/ddl_parser.rs index 3205eb30..426aa650 100644 --- a/dt-meta/src/sql_parser/ddl_parser.rs +++ b/dt-meta/src/sql_parser/ddl_parser.rs @@ -12,7 +12,10 @@ use nom::{ }; use regex::Regex; -use std::{borrow::Cow, str}; +use std::{ + borrow::Cow, + str::{self}, +}; use crate::ddl_type::DdlType; @@ -30,10 +33,20 @@ impl DdlParser { let sql = Self::remove_comments(sql); let input = sql.trim().as_bytes(); match sql_query(input) { - Ok((_, o)) => Ok(o), - Err(_) => Err(Error::Unexpected { - error: format!("failed to parse sql: {}", sql), - }), + Ok((_, o)) => { + let database = if let Some(db) = o.1 { + Some(String::from_utf8(db)?) + } else { + None + }; + let table = if let Some(tb) = o.2 { + Some(String::from_utf8(tb)?) + } else { + None + }; + Ok((o.0, database, table)) + } + Err(_) => Err(Error::Unexpected(format!("failed to parse sql: {}", sql))), } } @@ -46,7 +59,7 @@ impl DdlParser { /// parse ddl sql and return: (ddl_type, schema, table) #[allow(clippy::type_complexity)] -pub fn sql_query(i: &[u8]) -> IResult<&[u8], (DdlType, Option, Option)> { +pub fn sql_query(i: &[u8]) -> IResult<&[u8], (DdlType, Option>, Option>)> { alt(( map(create_database, |r| { (DdlType::CreateDatabase, Some(r), None) @@ -61,7 +74,7 @@ pub fn sql_query(i: &[u8]) -> IResult<&[u8], (DdlType, Option, Option IResult<&[u8], String> { +pub fn create_database(i: &[u8]) -> IResult<&[u8], Vec> { let (remaining_input, (_, _, _, _, _, database, _)) = tuple(( tag_no_case("create"), multispace1, @@ -71,11 +84,10 @@ pub fn create_database(i: &[u8]) -> IResult<&[u8], String> { sql_identifier, multispace0, ))(i)?; - let database = String::from(str::from_utf8(database).unwrap()); - Ok((remaining_input, database)) + Ok((remaining_input, database.to_vec())) } -pub fn drop_database(i: &[u8]) -> IResult<&[u8], String> { +pub fn drop_database(i: &[u8]) -> IResult<&[u8], Vec> { let (remaining_input, (_, _, _, _, _, database, _)) = tuple(( tag_no_case("drop"), multispace1, @@ -85,11 +97,10 @@ pub fn drop_database(i: &[u8]) -> IResult<&[u8], String> { sql_identifier, multispace0, ))(i)?; - let database = String::from(str::from_utf8(database).unwrap()); - Ok((remaining_input, database)) + Ok((remaining_input, database.to_vec())) } -pub fn alter_database(i: &[u8]) -> IResult<&[u8], String> { +pub fn alter_database(i: &[u8]) -> IResult<&[u8], Vec> { let (remaining_input, (_, _, _, _, database, _)) = tuple(( tag_no_case("alter"), multispace1, @@ -98,11 +109,10 @@ pub fn alter_database(i: &[u8]) -> IResult<&[u8], String> { sql_identifier, multispace1, ))(i)?; - let database = String::from(str::from_utf8(database).unwrap()); - Ok((remaining_input, database)) + Ok((remaining_input, database.to_vec())) } -pub fn create_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn create_table(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { let (remaining_input, (_, _, _, _, _, table, _)) = tuple(( tag_no_case("create"), multispace1, @@ -115,7 +125,7 @@ pub fn create_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { Ok((remaining_input, table)) } -pub fn drop_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn drop_table(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { let (remaining_input, (_, _, _, _, _, table, _)) = tuple(( tag_no_case("drop"), multispace1, @@ -128,7 +138,7 @@ pub fn drop_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { Ok((remaining_input, table)) } -pub fn alter_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn alter_table(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { let (remaining_input, (_, _, _, _, table, _)) = tuple(( tag_no_case("alter"), multispace1, @@ -140,10 +150,22 @@ pub fn alter_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { Ok((remaining_input, table)) } -pub fn truncate_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn truncate_table(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { let (remaining_input, (_, _, _, _, table, _)) = tuple(( tag_no_case("truncate"), multispace1, + opt(tag_no_case("table")), + opt(multispace1), + schema_table_reference, + multispace0, + ))(i)?; + Ok((remaining_input, table)) +} + +pub fn rename_table(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { + let (remaining_input, (_, _, _, _, table, _)) = tuple(( + tag_no_case("rename"), + multispace1, tag_no_case("table"), multispace1, schema_table_reference, @@ -152,7 +174,7 @@ pub fn truncate_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { Ok((remaining_input, table)) } -pub fn rename_table(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn create_index(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { let (remaining_input, (_, _, _, _, table, _)) = tuple(( tag_no_case("rename"), multispace1, @@ -187,7 +209,7 @@ pub fn if_exists(i: &[u8]) -> IResult<&[u8], ()> { } // Parse a reference to a named schema.table, with an optional alias -pub fn schema_table_reference(i: &[u8]) -> IResult<&[u8], (Option, String)> { +pub fn schema_table_reference(i: &[u8]) -> IResult<&[u8], (Option>, Vec)> { map( tuple(( opt(pair(sql_identifier, pair(multispace0, tag(".")))), @@ -195,10 +217,8 @@ pub fn schema_table_reference(i: &[u8]) -> IResult<&[u8], (Option, Strin sql_identifier, )), |tup| { - let name = String::from(str::from_utf8(tup.2).unwrap()); - let schema = tup - .0 - .map(|(schema, _)| String::from(str::from_utf8(schema).unwrap())); + let name = tup.2.to_vec(); + let schema = tup.0.map(|(schema, _)| schema.to_vec()); (schema, name) }, )(i) @@ -206,14 +226,28 @@ pub fn schema_table_reference(i: &[u8]) -> IResult<&[u8], (Option, Strin #[inline] pub fn is_sql_identifier(chr: u8) -> bool { - is_alphanumeric(chr) || chr == b'_' || chr == b'@' + is_alphanumeric(chr) || chr == b'_' +} + +#[inline] +pub fn is_escaped_sql_identifier_1(chr: u8) -> bool { + chr != b'`' +} + +#[inline] +pub fn is_escaped_sql_identifier_2(chr: u8) -> bool { + chr != b'"' } pub fn sql_identifier(i: &[u8]) -> IResult<&[u8], &[u8]> { alt(( preceded(not(peek(sql_keyword)), take_while1(is_sql_identifier)), - delimited(tag("`"), take_while1(is_sql_identifier), tag("`")), - delimited(tag("["), take_while1(is_sql_identifier), tag("]")), + delimited(tag("`"), take_while1(is_escaped_sql_identifier_1), tag("`")), + delimited( + tag("\""), + take_while1(is_escaped_sql_identifier_2), + tag("\""), + ), ))(i) } @@ -262,6 +296,33 @@ mod test { } } + #[test] + fn test_create_table_with_schema_with_special_characters() { + let sqls = vec![ + // mysql + "CREATE TABLE IF NOT EXISTS `test_db_*.*`.bbb(id int);", + "CREATE TABLE IF NOT EXISTS `δΈ­ζ–‡.others*&^%$#@!+_)(&^%#`.`δΈ­ζ–‡!@$#$%^&*&(_+)`(id int);", + // pg + r#"CREATE TABLE IF NOT EXISTS "test_db_*.*".bbb(id int);"#, + r#"CREATE TABLE IF NOT EXISTS "δΈ­ζ–‡.others*&^%$#@!+_)(&^%#"."δΈ­ζ–‡!@$#$%^&*&(_+)"(id int);"#, + ]; + let dbs = vec![ + "test_db_*.*", + "δΈ­ζ–‡.others*&^%$#@!+_)(&^%#", + "test_db_*.*", + "δΈ­ζ–‡.others*&^%$#@!+_)(&^%#", + ]; + let tbs = vec!["bbb", "δΈ­ζ–‡!@$#$%^&*&(_+)", "bbb", "δΈ­ζ–‡!@$#$%^&*&(_+)"]; + + for i in 0..sqls.len() { + let sql = sqls[i]; + let r = DdlParser::parse(sql).unwrap(); + assert_eq!(r.0, DdlType::CreateTable); + assert_eq!(r.1, Some(dbs[i].to_string())); + assert_eq!(r.2, Some(tbs[i].to_string())); + } + } + #[test] fn test_create_table_without_schema() { let sqls = vec![ @@ -421,6 +482,23 @@ mod test { } } + #[test] + fn test_create_database_with_special_characters() { + let sqls = vec![ + "CREATE DATABASE IF NOT EXISTS `test_db_*.*`;", + "CREATE DATABASE IF NOT EXISTS `δΈ­ζ–‡.others*&^%$#@!+_)(&^%#`;", + ]; + let dbs = vec!["test_db_*.*", "δΈ­ζ–‡.others*&^%$#@!+_)(&^%#"]; + + for i in 0..sqls.len() { + let sql = sqls[i]; + let r = DdlParser::parse(sql).unwrap(); + assert_eq!(r.0, DdlType::CreateDatabase); + assert_eq!(r.1, Some(dbs[i].to_string())); + assert_eq!(r.2, None); + } + } + #[test] fn test_drop_database() { let sqls = vec![ @@ -486,6 +564,9 @@ mod test { "truncate /*some comments,*/table/*some comments*/ `aaa`.`bbb`", // escapes + spaces + comments "truncate /*some comments,*/table/*some comments*/ `aaa` . `bbb` ", + // without keyword `table` + "truncate `aaa`.`bbb`", + "truncate /*some comments,*/table/*some comments*/ `aaa`.`bbb`", ]; for sql in sqls { diff --git a/dt-pipeline/src/pipeline.rs b/dt-pipeline/src/pipeline.rs index 5fd4632f..1b1fc003 100644 --- a/dt-pipeline/src/pipeline.rs +++ b/dt-pipeline/src/pipeline.rs @@ -136,9 +136,18 @@ impl Pipeline { let count = data.len(); if count > 0 { self.parallelizer - .sink_ddl(data, &self.sinkers) + .sink_ddl(data.clone(), &self.sinkers) .await - .unwrap() + .unwrap(); + // only part of sinkers will execute sink_ddl, but all sinkers should refresh metadata + for sinker in self.sinkers.iter_mut() { + sinker + .lock() + .await + .refresh_meta(data.clone()) + .await + .unwrap(); + } } Ok((count, last_received_position, last_commit_position)) } diff --git a/dt-task/src/sinker_util.rs b/dt-task/src/sinker_util.rs index 7b08733e..3ab5e44f 100644 --- a/dt-task/src/sinker_util.rs +++ b/dt-task/src/sinker_util.rs @@ -22,7 +22,10 @@ use dt_connector::{ }, Sinker, }; -use dt_meta::{mysql::mysql_meta_manager::MysqlMetaManager, pg::pg_meta_manager::PgMetaManager}; +use dt_meta::{ + mysql::mysql_meta_manager::MysqlMetaManager, pg::pg_meta_manager::PgMetaManager, + redis::redis_write_method::RedisWriteMethod, +}; use kafka::producer::{Producer, RequiredAcks}; use reqwest::Client; use rusoto_core::Region; @@ -178,11 +181,16 @@ impl SinkerUtil { .await? } - SinkerConfig::Redis { url, batch_size } => { + SinkerConfig::Redis { + url, + batch_size, + method, + } => { SinkerUtil::create_redis_sinker( url, task_config.pipeline.parallel_size, *batch_size, + method, ) .await? } @@ -208,6 +216,7 @@ impl SinkerUtil { let mut sub_sinkers: Vec>>> = Vec::new(); for _ in 0..parallel_size { let sinker = MysqlSinker { + url: url.to_string(), conn_pool: conn_pool.clone(), meta_manager: meta_manager.clone(), router: router.clone(), @@ -445,16 +454,19 @@ impl SinkerUtil { url: &str, parallel_size: usize, batch_size: usize, + method: &str, ) -> Result>>>, Error> { let mut sub_sinkers: Vec>>> = Vec::new(); for _ in 0..parallel_size { let mut conn = TaskUtil::create_redis_conn(url).await?; let version = TaskUtil::get_redis_version(&mut conn)?; + let method = RedisWriteMethod::from_str(method).unwrap(); let sinker = RedisSinker { conn, batch_size, now_db_id: -1, version, + method, }; sub_sinkers.push(Arc::new(async_mutex::Mutex::new(Box::new(sinker)))); } diff --git a/dt-task/src/task_runner.rs b/dt-task/src/task_runner.rs index 70f4be2a..a3d36fc4 100644 --- a/dt-task/src/task_runner.rs +++ b/dt-task/src/task_runner.rs @@ -113,10 +113,7 @@ impl TaskRunner { }, _ => { - return Err(Error::Unexpected { - error: "unexpected extractor config type for rdb snapshot task" - .to_string(), - }); + return Err(Error::ConfigError("unsupported extractor config".into())); } }; @@ -377,9 +374,7 @@ impl TaskRunner { } _ => { - return Err(Error::ConfigError { - error: String::from("extractor_config type is not supported."), - }) + return Err(Error::ConfigError("unsupported extractor config".into())); } }; Ok(extractor) diff --git a/dt-task/src/task_util.rs b/dt-task/src/task_util.rs index bb33758a..7a65b50e 100644 --- a/dt-task/src/task_util.rs +++ b/dt-task/src/task_util.rs @@ -81,9 +81,9 @@ impl TaskUtil { let version_str = cap[1].to_string(); let tokens: Vec<&str> = version_str.split(".").collect(); if tokens.is_empty() { - return Err(Error::Unexpected { - error: "can not get redis version by INFO".to_string(), - }); + return Err(Error::Unexpected( + "can not get redis version by INFO".into(), + )); } let mut version = tokens[0].to_string(); @@ -92,9 +92,9 @@ impl TaskUtil { } return Ok(f32::from_str(&version).unwrap()); } - Err(Error::Unexpected { - error: "can not get redis version by INFO".to_string(), - }) + Err(Error::Unexpected( + "can not get redis version by INFO".into(), + )) } pub async fn create_rdb_meta_manager(config: &TaskConfig) -> Result { @@ -111,9 +111,7 @@ impl TaskUtil { } _ => { - return Err(Error::Unexpected { - error: "unexpected sinker type".to_string(), - }); + return Err(Error::ConfigError("unsupported sinker config".into())); } }; Ok(meta_manager) diff --git a/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/dst_ddl.sql b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/dst_ddl.sql new file mode 100644 index 00000000..ef6ad3bd --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/dst_ddl.sql @@ -0,0 +1,20 @@ +DROP DATABASE IF EXISTS test_db_1; +DROP DATABASE IF EXISTS test_db_2; +DROP DATABASE IF EXISTS test_db_3; +DROP DATABASE IF EXISTS test_db_4; +DROP DATABASE IF EXISTS `δΈ­ζ–‡database!@#$%^&*()_+`; +CREATE DATABASE test_db_1; +CREATE DATABASE test_db_2; +CREATE DATABASE test_db_3; + +CREATE TABLE test_db_1.tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE test_db_1.drop_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE test_db_1.truncate_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO test_db_1.truncate_tb_1 VALUES (1, 1); + +CREATE TABLE test_db_1.truncate_tb_2 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO test_db_1.truncate_tb_2 VALUES (1, 1); + +CREATE TABLE test_db_2.truncate_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_ddl.sql b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_ddl.sql new file mode 100644 index 00000000..ef6ad3bd --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_ddl.sql @@ -0,0 +1,20 @@ +DROP DATABASE IF EXISTS test_db_1; +DROP DATABASE IF EXISTS test_db_2; +DROP DATABASE IF EXISTS test_db_3; +DROP DATABASE IF EXISTS test_db_4; +DROP DATABASE IF EXISTS `δΈ­ζ–‡database!@#$%^&*()_+`; +CREATE DATABASE test_db_1; +CREATE DATABASE test_db_2; +CREATE DATABASE test_db_3; + +CREATE TABLE test_db_1.tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE test_db_1.drop_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE test_db_1.truncate_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO test_db_1.truncate_tb_1 VALUES (1, 1); + +CREATE TABLE test_db_1.truncate_tb_2 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO test_db_1.truncate_tb_2 VALUES (1, 1); + +CREATE TABLE test_db_2.truncate_tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_dml.sql b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_dml.sql new file mode 100644 index 00000000..261bcd84 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/src_dml.sql @@ -0,0 +1,51 @@ + +INSERT INTO test_db_1.tb_1 VALUES (1,1); + +-- add column +ALTER TABLE test_db_1.tb_1 ADD COLUMN f_2 smallint DEFAULT NULL; +ALTER TABLE test_db_1.tb_1 ADD COLUMN f_3 smallint DEFAULT NULL; + +INSERT INTO test_db_1.tb_1 VALUES (2,2,2,2); + +-- drop column +ALTER TABLE test_db_1.tb_1 DROP COLUMN f_2; + +INSERT INTO test_db_1.tb_1 VALUES (3,3,3); + +-- truncate table +TRUNCATE test_db_1.truncate_tb_1; +TRUNCATE TABLE test_db_1.truncate_tb_2; + +-- rename table +ALTER TABLE test_db_1.tb_1 RENAME test_db_1.tb_2; +RENAME TABLE test_db_1.tb_2 TO test_db_1.tb_3; + +-- drop table +DROP TABLE test_db_1.drop_tb_1; + +-- drop database +DROP DATABASE test_db_3; + +-- create database +CREATE DATABASE test_db_4; + +-- create table +CREATE TABLE test_db_2.tb_1 ( f_0 tinyint, f_1 smallint DEFAULT NULL, f_2 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +INSERT INTO test_db_2.tb_1 VALUES (1,1,1); + +-- add index +ALTER TABLE test_db_2.tb_1 ADD INDEX idx_f_1 (f_1); + +-- NOT supported ddl +CREATE INDEX idx_f_2 ON test_db_2.tb_1 (f_2); + +-- RENAME TABLE products TO products_old, products_new TO products; + +-- create database with special character +CREATE DATABASE `δΈ­ζ–‡database!@#$%^&*()_+`; + +-- create table with chinese character +CREATE TABLE `δΈ­ζ–‡database!@#$%^&*()_+`.`δΈ­ζ–‡` ( f_0 tinyint, f_1 smallint DEFAULT NULL, f_2 smallint DEFAULT NULL, PRIMARY KEY (f_0) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +INSERT INTO `δΈ­ζ–‡database!@#$%^&*()_+`.`δΈ­ζ–‡` VALUES(1, 1, 1); diff --git a/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/task_config.ini b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/task_config.ini new file mode 100644 index 00000000..eef2e54c --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/ddl_test/task_config.ini @@ -0,0 +1,36 @@ +[extractor] +db_type=mysql +extract_type=cdc +binlog_position=0 +binlog_filename= +server_id=2000 +url=mysql://root:123456@127.0.0.1:3307?ssl-mode=disabled + +[filter] +ignore_dbs= +do_dbs= +do_tbs=*.* +ignore_tbs= +do_events=insert,update,delete,ddl + +[sinker] +db_type=mysql +sink_type=write +batch_size=4 +url=mysql://root:123456@127.0.0.1:3308 + +[router] +tb_map= +field_map= +db_map= + +[pipeline] +parallel_type=rdb_merge +buffer_size=4 +checkpoint_interval_secs=1 +parallel_size=2 + +[runtime] +log_dir=./logs +log_level=info +log4rs_file=./log4rs.yaml \ No newline at end of file diff --git a/dt-tests/tests/mysql_to_mysql/cdc/json_test/dst_ddl.sql b/dt-tests/tests/mysql_to_mysql/cdc/json_test/dst_ddl.sql new file mode 100644 index 00000000..3dc35718 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/json_test/dst_ddl.sql @@ -0,0 +1,5 @@ +DROP DATABASE IF EXISTS test_db_1; + +CREATE DATABASE test_db_1; + +CREATE TABLE test_db_1.json_test(f_0 INT AUTO_INCREMENT, f_1 JSON, PRIMARY KEY(f_0)); diff --git a/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_ddl.sql b/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_ddl.sql new file mode 100644 index 00000000..765f37c7 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_ddl.sql @@ -0,0 +1,6 @@ +DROP DATABASE IF EXISTS test_db_1; + +CREATE DATABASE test_db_1; + +CREATE TABLE test_db_1.json_test(f_0 INT AUTO_INCREMENT, f_1 JSON, PRIMARY KEY(f_0)); + diff --git a/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_dml.sql b/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_dml.sql new file mode 100644 index 00000000..04e9ddab --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/json_test/src_dml.sql @@ -0,0 +1,134 @@ +-- basic json object +INSERT INTO test_db_1.json_test VALUES (NULL, '{"k.1":1,"k.0":0,"k.-1":-1,"k.true":true,"k.false":false,"k.null":null,"k.string":"string","k.true_false":[true,false],"k.32767":32767,"k.32768":32768,"k.-32768":-32768,"k.-32769":-32769,"k.2147483647":2147483647,"k.2147483648":2147483648,"k.-2147483648":-2147483648,"k.-2147483649":-2147483649,"k.18446744073709551615":18446744073709551615,"k.18446744073709551616":18446744073709551616,"k.3.14":3.14,"k.{}":{},"k.[]":[]}') + +-- unicode support +INSERT INTO test_db_1.json_test VALUES (NULL, '{"key":"éééàààà"}') +INSERT INTO test_db_1.json_test VALUES (NULL, '{"δΈ­ζ–‡":"πŸ˜€"}') + +-- multiple nested json object +INSERT INTO test_db_1.json_test VALUES (NULL, '{"literal1":true,"i16":4,"i32":2147483647,"int64":4294967295,"double":1.0001,"string":"abc","time":"2022-01-01 12:34:56.000000","array":[1,2,{"i16":4,"array":[false,true,"abcd"]}],"small_document":{"i16":4,"array":[false,true,3],"small_document":{"i16":4,"i32":2147483647,"int64":4294967295}}}'),(5, '[{"i16":4,"small_document":{"i16":4,"i32":2147483647,"int64":4294967295}},{"i16":4,"array":[false,true,"abcd"]},"abc",10,null,true,false]'); + +-- null +INSERT INTO test_db_1.json_test VALUES (NULL, null) + +-- json with empty key +INSERT INTO test_db_1.json_test VALUES (NULL, '{"bitrate":{"":0}}') + +-- json array +INSERT INTO test_db_1.json_test VALUES (NULL, '[-1,0,1,true,false,null,"string",[true,false],32767,32768,-32768,-32769,2147483647,2147483648,-2147483648,-2147483649,18446744073709551615,18446744073709551616,3.14,{},[]]') + +-- json array nested +INSERT INTO test_db_1.json_test VALUES (NULL, '[-1,["b",["c"]],1]') + +-- scalar string +INSERT INTO test_db_1.json_test VALUES (NULL, '"scalar string"'),(11, '"LONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONG"') + +-- scalar boolean: true +INSERT INTO test_db_1.json_test VALUES (NULL, 'true') + +-- scalar boolean: false +INSERT INTO test_db_1.json_test VALUES (NULL, 'false') + +-- scalar null +INSERT INTO test_db_1.json_test VALUES (NULL, 'null') + +-- scalar negative integer +INSERT INTO test_db_1.json_test VALUES (NULL, '-1') + +-- scalar positive integer +INSERT INTO test_db_1.json_test VALUES (NULL, '1') + +-- scalar max positive int16 +INSERT INTO test_db_1.json_test VALUES (NULL, '32767') + +-- scalar int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '32768') + +-- scalar min negative int16 +INSERT INTO test_db_1.json_test VALUES (NULL, '-32768') + +-- scalar negative int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '-32769') + +-- scalar max_positive int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '2147483647') + +-- scalar positive int64 +INSERT INTO test_db_1.json_test VALUES (NULL, '2147483648') + +-- scalar min negative int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '-2147483648') + +-- scalar negative int64 +INSERT INTO test_db_1.json_test VALUES (NULL, '-2147483649') + +-- scalar uint64 +INSERT INTO test_db_1.json_test VALUES (NULL, '18446744073709551615') + +-- scalar uint64 overflow +INSERT INTO test_db_1.json_test VALUES (NULL, '18446744073709551616') + +-- scalar float +INSERT INTO test_db_1.json_test VALUES (NULL, '3.14') + +-- empty object +INSERT INTO test_db_1.json_test VALUES (NULL, '{}') + +-- empty array +INSERT INTO test_db_1.json_test VALUES (NULL, '[]') + +-- TODO, scalar json objects may lose precision in binlog. +-- for example, INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25' AS TIME) AS JSON)), the result will be: +-- +-----+--------------------------+ +-- | f_0 | f_1 | +-- | 1 | "23:24:25.000000" | +-- but in binlog, we get f_1: 23:24:25 + +-- scalar datetime +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('2015-01-15 23:24:25' AS DATETIME) AS JSON)) + +-- scalar time +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25' AS TIME) AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25.12' AS TIME(3)) AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25.0237' AS TIME(3)) AS JSON)) + +-- scalar timestamp +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25' AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25.12' AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25.0237' AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, (CAST(UNIX_TIMESTAMP(CONVERT_TZ('2015-01-15 23:24:25','GMT',@@session.time_zone)) AS JSON))) + +-- scalar geometry +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(ST_GeomFromText('POINT(1 1)') AS JSON)) + +-- scalar string with charset conversion +INSERT INTO test_db_1.json_test VALUES (NULL, CAST('[]' AS CHAR CHARACTER SET 'ascii')) + +-- scalar binary as base64 +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(x'cafe' AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(x'cafebabe' AS JSON)) + +-- scalar decimal +-- TODO, decimal will lose precision when insert into target mysql as string +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST("212765.700000000010000" AS DECIMAL(21,15)) AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST("111111.11111110000001" AS DECIMAL(24,17)) AS JSON)) + +-- set partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_SET(f_1, '$.addr.detail.ab', '970785C8') + +-- remove partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$.addr.detail.ab') + +-- remove partial update with holes and sparse keys +INSERT INTO test_db_1.json_test VALUES (NULL, '{"17fc9889474028063990914001f6854f6b8b5784":"test_field_for_remove_fields_behaviour_2","1f3a2ea5bc1f60258df20521bee9ac636df69a3a":{"currency":"USD"},"4f4d99a438f334d7dbf83a1816015b361b848b3b":{"currency":"USD"},"9021162291be72f5a8025480f44bf44d5d81d07c":"test_field_for_remove_fields_behaviour_3_will_be_removed","9b0ed11532efea688fdf12b28f142b9eb08a80c5":{"currency":"USD"},"e65ad0762c259b05b4866f7249eabecabadbe577":"test_field_for_remove_fields_behaviour_1_updated","ff2c07edcaa3e987c23fb5cc4fe860bb52becf00":{"currency":"USD"}}') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$."17fc9889474028063990914001f6854f6b8b5784"') + +-- replace partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_REPLACE(f_1, '$.addr.detail.ab', '9707') + +-- remove array value +INSERT INTO test_db_1.json_test VALUES (NULL, '["foo","bar","baz"]') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$[1]') \ No newline at end of file diff --git a/dt-tests/tests/mysql_to_mysql/cdc/json_test/task_config.ini b/dt-tests/tests/mysql_to_mysql/cdc/json_test/task_config.ini new file mode 100644 index 00000000..dc986260 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/cdc/json_test/task_config.ini @@ -0,0 +1,36 @@ +[extractor] +db_type=mysql +extract_type=cdc +binlog_position=0 +binlog_filename= +server_id=2000 +url=mysql://root:123456@127.0.0.1:3307?ssl-mode=disabled + +[filter] +ignore_dbs= +do_dbs= +do_tbs=test_db_1.* +ignore_tbs= +do_events=insert,update,delete + +[sinker] +db_type=mysql +sink_type=write +batch_size=2 +url=mysql://root:123456@127.0.0.1:3308 + +[router] +tb_map= +field_map= +db_map= + +[pipeline] +parallel_type=rdb_merge +buffer_size=4 +checkpoint_interval_secs=1 +parallel_size=2 + +[runtime] +log_dir=./logs +log_level=info +log4rs_file=./log4rs.yaml \ No newline at end of file diff --git a/dt-tests/tests/mysql_to_mysql/cdc_tests.rs b/dt-tests/tests/mysql_to_mysql/cdc_tests.rs index b29d90fd..ccba6735 100644 --- a/dt-tests/tests/mysql_to_mysql/cdc_tests.rs +++ b/dt-tests/tests/mysql_to_mysql/cdc_tests.rs @@ -22,6 +22,18 @@ mod test { TestBase::run_cdc_test("mysql_to_mysql/cdc/charset_test", 3000, 2000).await; } + #[tokio::test] + #[serial] + async fn cdc_json_test() { + TestBase::run_cdc_test("mysql_to_mysql/cdc/json_test", 3000, 2000).await; + } + + #[tokio::test] + #[serial] + async fn cdc_ddl_test() { + TestBase::run_ddl_test("mysql_to_mysql/cdc/ddl_test", 3000, 5000).await; + } + #[tokio::test] #[serial] async fn cdc_timezone_test() { diff --git a/dt-tests/tests/mysql_to_mysql/snapshot/json_test/dst_ddl.sql b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/dst_ddl.sql new file mode 100644 index 00000000..3dc35718 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/dst_ddl.sql @@ -0,0 +1,5 @@ +DROP DATABASE IF EXISTS test_db_1; + +CREATE DATABASE test_db_1; + +CREATE TABLE test_db_1.json_test(f_0 INT AUTO_INCREMENT, f_1 JSON, PRIMARY KEY(f_0)); diff --git a/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_ddl.sql b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_ddl.sql new file mode 100644 index 00000000..765f37c7 --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_ddl.sql @@ -0,0 +1,6 @@ +DROP DATABASE IF EXISTS test_db_1; + +CREATE DATABASE test_db_1; + +CREATE TABLE test_db_1.json_test(f_0 INT AUTO_INCREMENT, f_1 JSON, PRIMARY KEY(f_0)); + diff --git a/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_dml.sql b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_dml.sql new file mode 100644 index 00000000..2c55b56b --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/src_dml.sql @@ -0,0 +1,127 @@ +-- basic json object +INSERT INTO test_db_1.json_test VALUES (NULL, '{"k.1":1,"k.0":0,"k.-1":-1,"k.true":true,"k.false":false,"k.null":null,"k.string":"string","k.true_false":[true,false],"k.32767":32767,"k.32768":32768,"k.-32768":-32768,"k.-32769":-32769,"k.2147483647":2147483647,"k.2147483648":2147483648,"k.-2147483648":-2147483648,"k.-2147483649":-2147483649,"k.18446744073709551615":18446744073709551615,"k.18446744073709551616":18446744073709551616,"k.3.14":3.14,"k.{}":{},"k.[]":[]}') + +-- unicode support +INSERT INTO test_db_1.json_test VALUES (NULL, '{"key":"éééàààà"}') +INSERT INTO test_db_1.json_test VALUES (NULL, '{"δΈ­ζ–‡":"πŸ˜€"}') + +-- multiple nested json object +INSERT INTO test_db_1.json_test VALUES (NULL, '{"literal1":true,"i16":4,"i32":2147483647,"int64":4294967295,"double":1.0001,"string":"abc","time":"2022-01-01 12:34:56.000000","array":[1,2,{"i16":4,"array":[false,true,"abcd"]}],"small_document":{"i16":4,"array":[false,true,3],"small_document":{"i16":4,"i32":2147483647,"int64":4294967295}}}'),(5, '[{"i16":4,"small_document":{"i16":4,"i32":2147483647,"int64":4294967295}},{"i16":4,"array":[false,true,"abcd"]},"abc",10,null,true,false]'); + +-- null +INSERT INTO test_db_1.json_test VALUES (NULL, null) + +-- json with empty key +INSERT INTO test_db_1.json_test VALUES (NULL, '{"bitrate":{"":0}}') + +-- json array +INSERT INTO test_db_1.json_test VALUES (NULL, '[-1,0,1,true,false,null,"string",[true,false],32767,32768,-32768,-32769,2147483647,2147483648,-2147483648,-2147483649,18446744073709551615,18446744073709551616,3.14,{},[]]') + +-- json array nested +INSERT INTO test_db_1.json_test VALUES (NULL, '[-1,["b",["c"]],1]') + +-- scalar string +INSERT INTO test_db_1.json_test VALUES (NULL, '"scalar string"'),(11, '"LONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONGLONG"') + +-- scalar boolean: true +INSERT INTO test_db_1.json_test VALUES (NULL, 'true') + +-- scalar boolean: false +INSERT INTO test_db_1.json_test VALUES (NULL, 'false') + +-- scalar null +INSERT INTO test_db_1.json_test VALUES (NULL, 'null') + +-- scalar negative integer +INSERT INTO test_db_1.json_test VALUES (NULL, '-1') + +-- scalar positive integer +INSERT INTO test_db_1.json_test VALUES (NULL, '1') + +-- scalar max positive int16 +INSERT INTO test_db_1.json_test VALUES (NULL, '32767') + +-- scalar int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '32768') + +-- scalar min negative int16 +INSERT INTO test_db_1.json_test VALUES (NULL, '-32768') + +-- scalar negative int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '-32769') + +-- scalar max_positive int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '2147483647') + +-- scalar positive int64 +INSERT INTO test_db_1.json_test VALUES (NULL, '2147483648') + +-- scalar min negative int32 +INSERT INTO test_db_1.json_test VALUES (NULL, '-2147483648') + +-- scalar negative int64 +INSERT INTO test_db_1.json_test VALUES (NULL, '-2147483649') + +-- scalar uint64 +INSERT INTO test_db_1.json_test VALUES (NULL, '18446744073709551615') + +-- scalar uint64 overflow +INSERT INTO test_db_1.json_test VALUES (NULL, '18446744073709551616') + +-- scalar float +INSERT INTO test_db_1.json_test VALUES (NULL, '3.14') + +-- scalar datetime +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('2015-01-15 23:24:25' AS DATETIME) AS JSON)) + +-- scalar time +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25' AS TIME) AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25.12' AS TIME(3)) AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST('23:24:25.0237' AS TIME(3)) AS JSON)) + +-- scalar timestamp +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25' AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25.12' AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(TIMESTAMP'2015-01-15 23:24:25.0237' AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, (CAST(UNIX_TIMESTAMP(CONVERT_TZ('2015-01-15 23:24:25','GMT',@@session.time_zone)) AS JSON))) + +-- scalar geometry +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(ST_GeomFromText('POINT(1 1)') AS JSON)) + +-- scalar string with charset conversion +INSERT INTO test_db_1.json_test VALUES (NULL, CAST('[]' AS CHAR CHARACTER SET 'ascii')) + +-- scalar binary as base64 +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(x'cafe' AS JSON)) +INSERT INTO test_db_1.json_test VALUES (NULL, CAST(x'cafebabe' AS JSON)) + +-- scalar decimal +-- TODO, decimal will lose precision when insert into target mysql as string +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST("212765.700000000010000" AS DECIMAL(21,15)) AS JSON)) +-- INSERT INTO test_db_1.json_test VALUES (NULL, CAST(CAST("111111.11111110000001" AS DECIMAL(24,17)) AS JSON)) + +-- empty object +INSERT INTO test_db_1.json_test VALUES (NULL, '{}') + +-- empty array +INSERT INTO test_db_1.json_test VALUES (NULL, '[]') + +-- set partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_SET(f_1, '$.addr.detail.ab', '970785C8') + +-- remove partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$.addr.detail.ab') + +-- remove partial update with holes and sparse keys +INSERT INTO test_db_1.json_test VALUES (NULL, '{"17fc9889474028063990914001f6854f6b8b5784":"test_field_for_remove_fields_behaviour_2","1f3a2ea5bc1f60258df20521bee9ac636df69a3a":{"currency":"USD"},"4f4d99a438f334d7dbf83a1816015b361b848b3b":{"currency":"USD"},"9021162291be72f5a8025480f44bf44d5d81d07c":"test_field_for_remove_fields_behaviour_3_will_be_removed","9b0ed11532efea688fdf12b28f142b9eb08a80c5":{"currency":"USD"},"e65ad0762c259b05b4866f7249eabecabadbe577":"test_field_for_remove_fields_behaviour_1_updated","ff2c07edcaa3e987c23fb5cc4fe860bb52becf00":{"currency":"USD"}}') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$."17fc9889474028063990914001f6854f6b8b5784"') + +-- replace partial update with holes +INSERT INTO test_db_1.json_test VALUES (NULL, '{"age":22,"addr":{"code":100,"detail":{"ab":"970785C8-C299"}},"name":"Alice"}') +UPDATE test_db_1.json_test SET f_1 = JSON_REPLACE(f_1, '$.addr.detail.ab', '9707') + +-- remove array value +INSERT INTO test_db_1.json_test VALUES (NULL, '["foo","bar","baz"]') +UPDATE test_db_1.json_test SET f_1 = JSON_REMOVE(f_1, '$[1]') \ No newline at end of file diff --git a/dt-tests/tests/mysql_to_mysql/snapshot/json_test/task_config.ini b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/task_config.ini new file mode 100644 index 00000000..7376a7da --- /dev/null +++ b/dt-tests/tests/mysql_to_mysql/snapshot/json_test/task_config.ini @@ -0,0 +1,33 @@ +[extractor] +db_type=mysql +extract_type=snapshot +url=mysql://root:123456@127.0.0.1:3307?ssl-mode=disabled + +[sinker] +db_type=mysql +sink_type=write +url=mysql://root:123456@127.0.0.1:3308 +batch_size=2 + +[filter] +do_dbs= +ignore_dbs= +do_tbs=test_db_1.* +ignore_tbs= +do_events=insert + +[router] +db_map= +tb_map= +field_map= + +[pipeline] +parallel_type=snapshot +buffer_size=4 +parallel_size=2 +checkpoint_interval_secs=1 + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs \ No newline at end of file diff --git a/dt-tests/tests/mysql_to_mysql/snapshot_tests.rs b/dt-tests/tests/mysql_to_mysql/snapshot_tests.rs index 477d44ca..7ce0b124 100644 --- a/dt-tests/tests/mysql_to_mysql/snapshot_tests.rs +++ b/dt-tests/tests/mysql_to_mysql/snapshot_tests.rs @@ -3,6 +3,7 @@ mod test { use std::collections::HashMap; + use dt_common::config::config_enums::DbType; use serial_test::serial; use crate::test_runner::test_base::TestBase; @@ -54,6 +55,7 @@ mod test { TestBase::run_snapshot_test_and_check_dst_count( "mysql_to_mysql/snapshot/special_character_in_name_test", + &DbType::Mysql, dst_expected_counts, ) .await; @@ -74,11 +76,18 @@ mod test { TestBase::run_snapshot_test_and_check_dst_count( "mysql_to_mysql/snapshot/resume_test", + &DbType::Mysql, dst_expected_counts, ) .await; } + #[tokio::test] + #[serial] + async fn snapshot_json_test() { + TestBase::run_snapshot_test("mysql_to_mysql/snapshot/json_test").await; + } + #[tokio::test] #[serial] async fn snapshot_timezone_test() { diff --git a/dt-tests/tests/pg_to_pg/snapshot_tests.rs b/dt-tests/tests/pg_to_pg/snapshot_tests.rs index 03a42579..f3d9ecc7 100644 --- a/dt-tests/tests/pg_to_pg/snapshot_tests.rs +++ b/dt-tests/tests/pg_to_pg/snapshot_tests.rs @@ -3,6 +3,7 @@ mod test { use std::collections::HashMap; + use dt_common::config::config_enums::DbType; use serial_test::serial; use crate::test_runner::test_base::TestBase; @@ -63,6 +64,7 @@ mod test { TestBase::run_snapshot_test_and_check_dst_count( "pg_to_pg/snapshot/resume_test", + &DbType::Pg, dst_expected_counts, ) .await; @@ -91,6 +93,7 @@ mod test { TestBase::run_snapshot_test_and_check_dst_count( "pg_to_pg/snapshot/special_character_in_name_test", + &DbType::Pg, dst_expected_counts, ) .await; diff --git a/dt-tests/tests/redis_to_redis/cdc/2_8/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/2_8/cmds_test/src_dml.sql index 7b000991..8ba5cf9f 100644 --- a/dt-tests/tests/redis_to_redis/cdc/2_8/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/2_8/cmds_test/src_dml.sql @@ -419,7 +419,7 @@ SUNIONSTORE key 63-1 63-2 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -- version: 5.0.0 --- XADD 66-1 * a 1 +-- XADD 66-1 1538561700640-0 a 1 -- XADD 66-1 * b 2 -- XADD 66-1 * c 3 -- XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/cdc/4_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/4_0/cmds_test/src_dml.sql index 69a1a5c4..cab07c67 100644 --- a/dt-tests/tests/redis_to_redis/cdc/4_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/4_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ UNLINK 64-1 64-2 64-3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -- version: 5.0.0 --- XADD 66-1 * a 1 +-- XADD 66-1 1538561700640-0 a 1 -- XADD 66-1 * b 2 -- XADD 66-1 * c 3 -- XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/cdc/5_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/5_0/cmds_test/src_dml.sql index 2940af58..28967e25 100644 --- a/dt-tests/tests/redis_to_redis/cdc/5_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/5_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/cdc/6_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/6_0/cmds_test/src_dml.sql index 2940af58..28967e25 100644 --- a/dt-tests/tests/redis_to_redis/cdc/6_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/6_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/cdc/6_2/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/6_2/cmds_test/src_dml.sql index 1d0b5550..3f646b99 100644 --- a/dt-tests/tests/redis_to_redis/cdc/6_2/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/6_2/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/cdc/7_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/cdc/7_0/cmds_test/src_dml.sql index ea20c62b..65054f69 100644 --- a/dt-tests/tests/redis_to_redis/cdc/7_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/cdc/7_0/cmds_test/src_dml.sql @@ -404,7 +404,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/2_8/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/2_8/cmds_test/src_dml.sql index 7b000991..8ba5cf9f 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/2_8/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/2_8/cmds_test/src_dml.sql @@ -419,7 +419,7 @@ SUNIONSTORE key 63-1 63-2 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -- version: 5.0.0 --- XADD 66-1 * a 1 +-- XADD 66-1 1538561700640-0 a 1 -- XADD 66-1 * b 2 -- XADD 66-1 * c 3 -- XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/4_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/4_0/cmds_test/src_dml.sql index 69a1a5c4..cab07c67 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/4_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/4_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ UNLINK 64-1 64-2 64-3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -- version: 5.0.0 --- XADD 66-1 * a 1 +-- XADD 66-1 1538561700640-0 a 1 -- XADD 66-1 * b 2 -- XADD 66-1 * c 3 -- XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/5_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/5_0/cmds_test/src_dml.sql index 2940af58..28967e25 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/5_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/5_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/6_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/6_0/cmds_test/src_dml.sql index 2940af58..28967e25 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/6_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/6_0/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/6_2/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/6_2/cmds_test/src_dml.sql index 1d0b5550..3f646b99 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/6_2/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/6_2/cmds_test/src_dml.sql @@ -413,7 +413,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/cmds_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/cmds_test/src_dml.sql index 38764a64..fadeeb11 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/7_0/cmds_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/cmds_test/src_dml.sql @@ -407,7 +407,7 @@ XADD 65-1 * field1 value1 field2 value2 field3 value3 -- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 -- XDEL -XADD 66-1 * a 1 +XADD 66-1 1538561700640-0 a 1 XADD 66-1 * b 2 XADD 66-1 * c 3 XDEL 66-1 1538561700640-0 diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/dst_ddl.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/dst_ddl.sql new file mode 100644 index 00000000..c6cb0c67 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/dst_ddl.sql @@ -0,0 +1 @@ +flushall \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_ddl.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_ddl.sql new file mode 100644 index 00000000..c6cb0c67 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_ddl.sql @@ -0,0 +1 @@ +flushall \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_dml.sql new file mode 100644 index 00000000..1068b3c5 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/src_dml.sql @@ -0,0 +1,61 @@ +-- in redis 7.0, RDB_TYPE_STREAM_LIST_PACKS_2 (19): +-- LP_ENCODING_6BIT_STR: Used for strings less than 64 bytes +-- LP_ENCODING_12BIT_STR: Used for strings less than 4k bytes +-- LP_ENCODING_32BIT_STR: Used for strings less than 4G + +-- RDB_TYPE_STREAM_LIST_PACKS_2 (19) +XADD 1-1 1526919030474-57 f_1 abc f_2 "abc\r\nδΈ­ζ–‡πŸ˜€" + +-- RDB_TYPE_STREAM_LIST_PACKS_2 (19) +XADD 2-1 * f_1 value_1 f_2 value_2 f_3 value_3 + +-- RDB_TYPE_STREAM_LIST_PACKS_2 (19) +-- this will add 2 entries in the same stream +XADD 3-1 * name John age 30 name Lucy age 20 +XADD 3-1 * name Mike age 40 name Jack age 30 + +-- the longest string will be compressed in listpack by: LP_ENCODING_32BIT_STR +XADD 4-1 * f_1 abc f_2 "abc\r\nδΈ­ζ–‡πŸ˜€" f_1 abcdefg f_2 24YLsf3sJ0X7n3docweHrAiyn8OXtzs9BKRgyYx8XqiKQwNa26dCdmIWkXcvhPptBp5MpPYFdg02KZ0OJEjT30DZR9UNIom4WaJWDXONNRliujCFuvWnZTDlx17e9gsFeY6AmfWX1hsStv9DFGJMkxXf6JJNQZ04JWhDIdarm2klt6Y0g4WLt71J2YbJa0Me5g5z82G86wuYNtbSdzBiaKb2uNwXzzT079zk67rwWsgV02k6iJ9UQYILIvT02qKApD1uhq7WpQiTTEYFb7JV2qT4zfQORrl4KbBZ1XPQBUawY6BzGNNfPM4R5bSbxP16fY3IeLQQ87sOYlfmdkQRprwgNZniigKZTqAcqqFJsOiqQdrG1TAykoWAXvOitLVH56XKBXhQbX3Clos31XH7FxWgH9aOxzP5LGEm5q0GRvjOzQvgOCKR2QuDHZiikoihjZcqKtthmDTkifA0aRkv8jAZTvv1ovPCR05YM2y6uWOfirlzv0pxrdjVfKkIg3N5HiyenQzKSOWJrUBM2IGgVCEtkdSC6gV3vjqqh1FbYtldbGqWlT1Amoj41HC6LEjvZAV9eaDTWgZlPKXdXOStrrN8motzLsxK7lMiW8Y1UBe0E3scYcvsQQHIT6hYLfRRVh0Vv6qzxtH19wUzxmnLBSu58LgC5JuCFAkm1WsTCqQv3Zm3jfq8ce2zqGoqFrcmuN4mAfmOOBvPprIf2vhWZvm38befuAwhCNVhsYgbkt1HGF5eVFEBOWwCvZTp65tstM2gJeHEJmvqPPzd2XUy78kgP4Ho8dkP29kCd2gQGCT49QK7Ltc29cm2Px5pO3osFosJ6ICjikrOSqsnqPVxJzz3O5Gf9sZNejHglx1D3pbndftHVyrdlEb24NAPQWOivUK0EVOMoOj78Zu5Ud1C26nbriJiU5DasWQmjtgyaPD2N9ijw69qBWsU5U0NFMfgtbl6XabxZW9GX24mBIcThgWsM4G7nUvJZqrc5uj4trHWliUPlYSZKBu4pjiMzOZXNSGoW3WdzN7LUFGD1SWwbZkTyG7QxtvEioxppr0eBQkyLMOvCj425v82FKhikmEMqBZYJTcSdnuGxHApAfmRiDmZEqc43jD62QBIccQiHLpxMlIkmqobq3JPTwBcURWlW6MnIENWfr9tQWudUeaTNGvB4YqwcImgCZYCX8SBJ1rRvop6AMn5F9TZo2GBQkKKfxWQOZvfXejtOMfqbcUBvEWEuJ4vk8XKxOLy9avhX2MHDYE3sRWCxmYaBBbwfbflvubcLjeXzxJAIC5zApSEnve60WwaqwBBmE9oenBeJSLhW4N3z0SNtWb1KavFV5plWXi3CzkEBSf6zP4R4vwMchXrJ7mlufOfHV0mJRA8hjUYJ9X0212EEWEMxICzH4X7UkCrC5mv7okgZ8XpfPnTcvZpTMDWBWyMct9pVxS5Bx6t2w2a7NUDuz4NpA8Pyhi9L8kkCYawNN2fhqx6N19c24ChZutlNargHF7TvcFaFI3S5CORg87v7h2XuRXSPzrv8FE9RXugY59eI5Ip43qTftS7WtTIVCjVuTFC2SETWmBMrJIFgRZxKgHSgtPPs2CUhngk33XNpXqQF0AH6NPMmXCp2MkOqHOc8QYy4rSnqkOTQ5jFmVO0QNvTAWZ4xEWzx6gCACmPxOAPZUWtLD0QN7oXcGQweYYdwKmAYm1JD1DrJx7Z553VR3pZz5R2kYO786uyRvzK5ROspht1KBfx5iChSf4O1SYwy2gCFmgZs6De6Tx9nAP0TlEPsew7Q7hBdoVwggUhZpY42588YrArvwCuiEu41tRTPJVmu8tbUlFR9emNz24ghsrNeq2kkixT0ERqDErSZDI6MYdqtJd730939NGCi8PfQgsbdaxI0btD9p0sONYTzlPUf4mvwcLb9koPPfTTexQnWVaMhlGjzp9q5xOaPTk8uufnaWslM63UEVAQUhl6RzrFcOVHKOlNWJApezmTFYohVccASaOa9jXTPpuYbfdgeSKzFjDpdSenjv7jOt4AU1gCCiuBxJPBf80EKrjYZaC6o5AKj7B0okqAqOXD1f8nA1QooKWodvMFQ44jl7L1QoQGPd6Qq4mVs95eLju2QGtPOnXYNHY9AtKK9VSygDRGuArZJT4yqjET9G34tibC40bvHu4YopJu2xtFuQzcSZRwlhpYmvwM7tjhCoIHP0VPJ2p5gtxjZZlp7GhWVs6drIzIZl1r8rs0uN4jAQmn4XWDLQHErk0H99Ozd0Pk5xEO65Uovx50ZsxDncMGc3HhwNNh29pmiRT0ee3TcRMCdbq43nRJW1BQIHUzLOTPO5pD2GdpIAx5hyUccNhrAKwDIutfoC5LeBjqRPMEOCXpgYWJo8E8izBmAzzSnpzPSyrt1VLdeIGPe1zoDr8Ztjwx4PRF1KSWXc3SfnOnR24RaR4pFQ0rPzea6Z6Bd3Fiq6HEl80WrXv0U7PnZJqyYbiK2r006UswIaChP08PgdF4tZdBUTcjmblGlX1jks91mtz6r2tLlMSovh21uTuFSxeYQUYdXFSXn6sMY3m6JV6525LvBp2JJzPKGWyJ2rKxtmMQhCA69TmCBQpajWsl5Xuky29KugybABOcG1VUUgea6Wh2JAEbFh8jEECIUJCcaJlVPlmnzNVEd3nnM3aMveKqoGrwA7QrpyEggYNqvSRsYGlLcFRXM00J8kXd8VIY5wC96fNeE82UI1UcDWIBIFgokBFLMLITbCfJHZD6dJjFgye1RtvGDiWqF7E2xGA4qJxpMY06TaG5ySqp6lthtRgPEyODdI8yRdJlkaEQGxVDBkBFebN2EoEg6kfIpYE9LoM4K4zNWrlvfwAwTd5fT7vWBH1hYTU6mISVKrAwvPvhlG2Z482UEoTt6G103gy3qjUmUhSwwiTgb2La9pBqHRUlH8zNYUdFRieeArhWWGrAGbnQ4M4iUjXkLZSjSALSzWZvX3vxK6AbZRLVG9BU2AGGtvSO86jI08rtCbEP8S6Kx786czPZBrAn25uNrZark0FhCqhxI5KAaTlm5ul4TrHXen752g1XcTuKhiNfrurUqSyzb5bgORj82abPbNBTc4U4yYFmTnXsvuraxGv1Aa7gFje6FPta8AOWYRgY3GN06CFvHLYUPjp1EqYKHBUwXJoempJlApDQorKvOuaz1cG9Meblo5l1wnzMSsRCYEKy9VJuuMxzECixxPVh6Zs822rOXtoaR7dOk5bT7XPZFQ8GasSs8n4Wxvn0CwhaluoCBv0gwncXD1z5a08Ewdvvahag3aUV6SEI51zfB05PWL7VKG4H5UAdWawhw6YTor6pOp2A3brY8aCseaCvZ0HAlWCWRISI3k8bHWNuxM5Z8ZezaBal29NxC5TbvOP9ygqspj2mbYrxPfOuUWG0XIWLKd0iQBZYflYlAYvEnjEzDA0o3xEye8s19a9CHe7nLt2hoJWlJgsluiTAdgdMMAmj0jvEk75pqKYAHycBHKURS6omrQSdn0lS2tBPHMAovUaWZlK5d156Mhua9HfoGNbtUiARaXgIb23xnklIDDNZ7LKTNeKL1Uuny39uybzuN8IJAaQMidCmV18tzWQMI4aCiMSCMIuhmPZt0F1SlLsl0d38NNEmR8ninom1ZY7pgobAWUeGjXWtOiNh5b9JgTlluHOmVonuQzqouBqGZmvtjEjP6UrJSuVHFjqBnzrelAu5oRGwX5VTyvaAf2ni1QXVMVMln5XjmbZgcxN4cEVv4m3Jd9avF1YLsqc1zOWoHzBTuYNee4MypodZZmwvlxusHFRgFauwkSYMS4QEMnZJhWxr2v6saSTzBeYlfwUft8IVsshC0mOnAruEJpmiTIZrIkWqG3VHvOOMQLDedOp7hkdr6NrY64HYJzewoGEdDjSZQk2MmsdbWrczLwca7LBxHch16L01uvXJlzmHjwHSnsmdsoQWwhbDaQt4Wm6MTC8tMRmbw9EhvTVbXLsD4gRcwTBswFtmMxjxahgzd3e5GqoLQ48YocJUju4OsbyArkwtz6wXGqgaItgKIwyARPXvpMGInfqQAC8TlcXiAYqHPu3ma66B5jE99gtq9NcsPTPG0AFFJj2MHKXEjF0TwcI0cpDhwhD2Yy1rQBE18gDvkCYl4cmE3XSqyoAbxXiofPveQZYmq7kEZnMrtMXJEAM1qPmpyC7sok6bCWzE71eywctLP6pp1Rnx96uG7Ni4U6uR4GU3IYhPqWoT4LbgFaoDUBX1hjO2hfUJeFa5gcGyC1LGiH7f0OTTY0gYI6ZB7XWzd5US9pye1VpaA7zu2KXu5tbt6FAd621AVqmU9qESD3MlbWcJoQYMA3VxeRZwyePKx6iXRZBy7HZZBk7UYwZfoHVV1AcAIaGvK0qmRgCDTCDnBTY9oX9IcC0CBNqwTsH3h4P36I84XFv7guAPq9VWO3xHpqZoPhxZXZENedCi4a8B1vVivERDUS1lqP9HNwT27YAzUQxLqDXxuyyKHIG42pmE85cAtR0cxnQDZVDChyfvkA6w8Z0TzAHpbnFXLPQf8d1SIR8Ol6QMpjrmYBk3LVqDRwMESVTtOP9V5XKtsqHjDQ0LLT3Aq4zK6MPmHCZz7GvSVBJ1PIzOWAig7NcqPEEjTGhIVqOaEhPWWsvK8ek2z1rQdymM79LMjgAw6XiYT4lSXtMm6KjDLDH1KZHVqVfJmabcHtkJVIgHySuUTS9gKwPwis3yzpWIlozqE5YXVUt0vI0GP96EySjym5yQfvgdgIIYocG99S4XAxNXGpxlQViFhFpc3FqMGujx24fVAfV1BiFDkhN0ZWg6Jvr6nic7Q49yMggMeTMgA8UzRXGTRCTtcGUwPW7kObonYnOOOQ2NrMxbHuIJv9Hw9SWGKH5EAWIAshDIAeyqBxrIAApTGSjstrg3ZGyhVvitlEngsvqO2U4zceYtVu4UfKcUQW6NB3Z9k9GLpjeNU45acGlK5UG3rPMelXCxOxA4uX6JxbG2Rkq1Bcbe3aHZIDvMPxStRwsCn4Wl1ckWHF1P4CNoHohupuYTUyYkZVGyGZyVnANfO4lsLUxnitpMeLGswE3hhOaSjAHG9sXrV6WHRLPI3E67SxgosdUxMalWbCd93dQszkjeEr3bc2WnSKhy1cNPSEBPUzzKsRExO6Xk4MrETEn3EN1367Q3ZUkuhG5YW9DURmfkWukt2wJgxEdojXpaR4buLwQzwqtFNVSN4LOVYRLhFnUoQrxFzjhBLPuinjqrS3LKqpBLAJw7XwKbU93uB2ZKbTc5CvK1GfP4zsyULk5Wz9AdhMo9lM7lbbFVcbezl59vyiAfMwGyA2TCidgDv5rSVcdXbv6wnZvvZ0q7QSQ404C7zOU9plGeozbDrO56CncMSWFobN32hdiShBJssEExfWr5MyVMs5GG39KvPtoLHzUqGImYh96DjUPBlO8fDYV5rpmm3ADUxHIgyt6BNEBDwkXgahyYanDdyl3Uvj4wl2OofVqp7JXF3RrCRAWQsisDI6k9dTzd0XRFWjkG1hCImg9Ot1nsmIvLNA08e20tmOSgTFXw97o95x39DtUDpSTlNLFWw2JuwspkOXqxJl04AjtwnqcQDubLYFSF008NsAFLcQyhTv8ueuNOq1olOoWpqhB57RGDkXw1tRtG6ikvFMB0Hksn6bixHLbhGqldh9mMr1Z5X0OXiapu1rO49VsWNjqjJezYGeD8Y7ZyqBH3nnmJQwlbkNsRETcyIE7fhLCWsIGZkrJslzh5AGArdnE2xF1U82jU8BGv0rjbg3pTfXWYIUiF5xQpNNT53villfH26tB2y5h71Sj3LLFqbTzzDta96sYZBRgIrSQPoN8pLporUuNpq2xZNxa6HEBWAaUplqBkA59zcZGp8oOx7SVW20SbuJX8DCuSkjdK4FLlse1YmlhThxMliSdB8mWhj4nw7kkz1RJHbbaMzfroX2abWSyux6wy0YqvQxKvtMfiM5qrN8ixp73ii1R90l6H7v6gRjuvMo5iYh8kHqkKJ8KbJjr3jNWaMUOLJFgel06qdOqCDwh4YhWutbqD6DHDmONVkeFUbFX1GLQwyvteiXfubKEqs0uN6X7q2T67Yj3dll04Bwin6sGTDhFCkBXi7Mx8qc2OlVXCOeBfZuMIGRQaxPIY0RSWh4Iq6TfttTf3XXdIWOzaXhNpBplbRWFlhsTd9qL6chGaBGvcVdo4eF9congBltRib2mHlIi8q8dDCXmrTCPzS95gqTf6A0A1Kw8E3TPqewIGME6ntsH4BcFwBYaCCGrE6eZNbTDyLjtlGB4DApX0E7f3XL56c3ci0ev4SfpyCx0TERdrkKKKgWPMhUWl2mcQuXJg85xgGAJDDPwy4ApvBcVTmKyrK5rjdDsMRBCYrSmuK7qCAgC4kyNTWfqDgE9Adnfp993qNmGDco7z8eSQbO2V6Gic9220HH4WYVZS0bt1r0YpkAreRif3bQSARTVyGfbTX2kUIFwDfJHQTuMACvz42M5Mh7UIudSvmcBIm9l8UNOefTKh2QlfKMR5RI4KocWfXZpWOpSbESH7isaG0e12XnIxsipvlvjB9A50k4n2pIwCLbfFpBI7ewkT9RlOYfMcmucVG8YUV12z7wbTZAUdVpcLh5lDlUrDAeuyh9AlieBFEylBKILeiVl1cqA3Td6x78XQbVtMfYaUFS5e9Ti3NbssFckPV2HK7S8C31omSyMtnMD3MugVsZYntSxhpaQAIHO3VikZQZNx9UpqAyDZ6d9PVjea7V4j7wAtGJWvZU0nkarIUXbLTwYTBaem4kMJDvq2iEk0KfxeuuRdyN32G7xTGtI90h8PMEofnAZ7UnyxdIdFzY1DWgM3nUAHS55iCFXti0ZVSqvm0hlqre3iQFNagFu11lhkzI2qHiQ3RJCQf5snmPmDee6MsFLNi5UmtYpzO7pxIor29noV47hlijWHlJgayhVk9vbkgJ0WDmntn1ZpfcDpIKppz2g85qlczEa5sA3g3rYA3FK2Qth0QHjir81RmqKY7xdCkdTsOqJOmYq52h8c7cdUZZxylKYKoizKJ4R1dWRs2rqV0gqix5R0RU1OP3wrmozDCKJPygW6gvvMhTmEZh5IfZ6OXqO24FVS5Tz8A2xfFXCIVRQFif8ouCSzEoQJgumXot9zglcwdYuaNsbwp0ehnpV6jKRCYTMJAyf5vKb2NUU8hDtWWFcb9wzukresomR0Mp3JbMIoP8RXkPGx3eJD0JcI60Yzc5MyPr262zJkJBcugyeq9AdWAJKS0cDLkgu0VMMgGP1A7YLev89aEqeiNpMptV6se4JvaZuSFbmgnYJq5JuoHwBQuKQd5STqhlsn2v4oOfj7eR0s5EWjtbH4uaqfh19GvQOFPYrcmKX0wnzmp6D5Jge62uSftpUKHj8sYrKpy1hmaYdrAAWZWqSR6KL7RsV5WplYAYyiF86Ob6tnzSP9NN4hse36OSzmrBs0F8UJfJB5xeVGkSHh9hGDgNNAeXWxr6TvbDDMVdGyMDXd1a2UpY9e2OU68LcAVn9ZQItHt8v6DcG8V9fUPydUJWDjqTold8TptCoOhKVsLIG7nrVgCrtovXDbaZQ8XGSJB1pnRWGr4CxUoqv6W0GZiQ3O9KI9sZZxW9KE1Gs7meTfLsUyYw88JucMOQZmY6xfNLgmeUzldTd0NTxuumglxmMFtay4CZCOseJQSJCPnb1y0qjixlpwnBIB2U76EvPbhqt9GR6JD7HSpvXPz41PsO0we9bLqzwUCeDevlILMGeOymhWBtVIV73vthzDTSjjaySLWlgtARy4rOlLqRAkLdSnapm4S7iuW8sVNZn6hA50XUr4kP2qDVIje5mKH5QBhPzmIzset9Iauh2SXEHez0AVQ6qGhePnPnc80MpRhBMKdHGXQMdmlh9MNeApritizsMpQ5E4zUlshyPqNgcZNvZaPGV0B5J2rIX7r2yT9e3tDOZOtQyUlhkajHy2FkjmQT6f8760qdUM28Nno47THUwznrZ538Ev0h21Zelzo8ahSwn23sDsPqrJdyC3jaafrtI6deDc5bWxvXuQihtwJWuYBrXHWTSDl5WXkQrpuz52OfDt8HSUveO4Fey8wN1rFAiBBKrUxlPfPtDcbRhbOBZkLrRUub2JbkxfBUhMsrbXQhgTfVJWtV6beQXiCI93iTnogjmIkwEyzN6BvHCrNUTLWOmKVA43pAKpCqPpikZG5LMAEp5HfQD7zexr7qdWASecVL0Oq0vF2oCazv26jm0NqQvSIHMkwuL1n5ltDwMFjxhDjCv50xUYrZukhpxhCFk5NEafvRKtdwNc0bAx8Z9L6RX6q9HPpOZo0HX7pTsETgzMAPYkmwTFJYTndRv30wMbLhsL0kig8GO6CC4ooZkZUSV6xKXJIuWDl5LMTlQL1rQpFDNhUT0GNyhq0BaDzG2i1x9NBcF619OWnWTLuu6nFFfWo07OOisy8NrgCmjl82OOi5rshareYZWuy13zkROjHyP4aDRcOHhzlXve0QgpBIcwSHueSsAxbYq64XEyQkFhMksLvCYdXdtEaN2Yevpye0T2ulannI8hN8IT7cZUqKGb8jCCNw4rERrEbZCmC0Q9cetwi16OVqrVOSBEfEUoglDCttnrPqIGPG5GuiSyo0qBRV39rL2YqU7yQLeeOazvS13VAf67Hpj2cjEQKvfEMKZbhO4cRg2ETooyTKhBCGsjxa396uuM3mcQcrdmBcSf9FRJ5tTRYWpjmFXlNSDG5ONLr6DuEV9y26mHfTANqTj8qnasHR7btQb2ZrNQhWRdbRc88B78depK7Qelxe9vNhg1x1Nw9W6WKqRXzoDiM0czKmQunt3JbXcZcvQjBCgs0YOJn9d6NfjiATIjd1xSlAk8kdRZfT45T7TJ8ECevfv3i9vxxzVINsfeDCn8BOnOUE95GiLxUee8dO4ayxaSyzrPCMZ0SCvSCtFJGbt1qtTmFyFIJK5JHwayhpf1ijZgluwfUjh3mrDqmYxHkmfedadJMXM1KmgWCkGbi32v4me821aLGH3T10hMdywIQydAnZMp5P8dAbZMcIhzYKNJXmcQT8OV3NPvQ9Cpb215HcSgIVGwyCHkDtzt4AbDV3wdKxbjsf4K8a4d6Ksxbrm6LH57zeIoNTVTimSPJnArHuyD8eNeA5V8k6hJZxNwd51fEzGHIHTYGL7NtnYD7zlfpUF76YwTl5GpJjqvcRwc4tK9xHeLPo3N8fQjDIR4Oiy99se8uxXrs1z6u07uSBlfRdv5MfudMxEsqBwhBaQvb1Zy2GyZG24BMXmSM521r7TsIKWHmv0jCFMDY9mY6pMLyfraJQN2FchRky9unRJWe8ltSxObhOysqHx1BEWUX0E3CyKIOXRFC5Bs28R6prSuSG4mdkgBGbgwx6PcXhJ21PboNKySQrM7DfO7oTccPILwFMWJ7wVd53R5QdyLVG5rU0NEniHzcPIVxBRMLF0sTT0RplSx0RkZ8RG5B43PpyaodfIZQtc1r571FJlcmNaxexbUkdS2MdadUhaQyAQQbyJln7NIb3LxXeuLLFyQMbvv4RL5KRl15W5xODHv2Yd0rFmbC5mkLMeZUkyMrnPtFjvesyHRtIGWmRSu0p8dkbkaRfbsaOBRCGMUsQCR9JcKG4BzrxxsTiYG5mcoL8Qz2Js749Hzs8n5OyCgo64pcYJrEB0Y80VEpEiGY4hESpSV4IHjvmsyzvJRyKxXS5fcVpRHeSlXBv4GnSkUYpPUTvu7NLzv3lgwAbhYqANIMKAaYVvx0Fked37tFcMjDyUpogFe4pjmcHHvawE5ifMGJW43IDI86cfrM7xF8KcxM9tzTxd3x5I0JGh3wl8kU7KKD939LFNfPvADnMQTPUx5S44sonpKCrL7pyURkBU5dRcUogVrTFk63xZC3vIsOlWZpNyWUmZlmg6r1fSo7QuECt5e8cGOqF1DzySMGuZ9STWNIuA9GH2chXeE0pEn96iEUIB9JnV2Lm5B0Q4BB40HooDTzd9o4Sq2MpS6KBfaF9YKJa1uq6ke1gjUgnehkbaMG6lqNOez02TBFtFQWTW6do4NRmDRrWY8ZpxcdSeZcundxO758xxZWqAZmQF7yowbbCbxosmcEB2qReBBbNodS6jh5Rm38IaARTOPmYJc2XGnwzHWeeSVJqQPpNwL3sbsBITLBQ2C0z0NJBvRqv0GZyH2mExCZgy2KQz3RN5WxkJajk6aeovhGpssEXBw8x60M9OC7S9Mhexrkn98CtApAeoOv1uqkFUx87ctfXXnUwZM4pwyjrjespqLcUXuVuY4LR5bcQGslm8RJnyIUzgmPKRVs7fI2nyYq1iJQE4N0fPL2vR8QqA7eKq53WnL6Kog9vDUstJ7RJeRnmosCS8biwbQnuqL6UprOMFnSaEBm6zAOfAVHgNCdM2XfhGSPbL51CM2miu5xwSg6rz5jRV1pGM4b6rS1Xe6KlEhEw29wbtYtSv0suC4SVb2APUaP9qlgzlsGAkqVRJyVutKWNDfg4HEGymR0mPnHltHOZlz6kvz1xVKxkTcKAbdnBxpAJI68IoNIFvmLVQ4oksfph7b5puIQQmtigCa99GFEUq52bB7ht6saoKoh2ZhqWjjKWybggQdLItCDTxnEQLNJj5udtkdN5qCmEPlkfhQiVzosaBBityUX551crPUgLnm4SN7HmvjN4496Iogm1aywRZAEYvlvE18JhTkoSDRzZRSzBG13KHkIZPJTJt8zJIyMeGLjEimCdjxjUERkHUmV3zk0YQx9TlFtTeOkiHGVh2iBDHQzNltYGSGOQ9wHCfhh0qnXAlP5aam4W3PLovWWBLp7I30YosjUSaemtOaRpg5dhXPB64tr9cL0tDKF0NQepmtzyoJ5ELhNP1yXN2PaljD3Ixr12mtBAM0yeMpctWhTcKSNAm6jxePhRfBbFJYMkgvo2hF2oHC6jQBbLRZHcvo0S58wkjYmmqi7zNPHzjVI9Dptl7ls2dvDDi8FWEw2oQIdV5H0XJSHprDiyh4NHua0qbXRanNIKrQFc5jgHrafNwMdtaZWbiSYC8djrEdGmRD2oXYCF6NEm285QY5mlAaTo08Mh6LhGjwB8PJ5p7VVSnhPxouA3pgmCi3KmMKS0G7yEoabP0nhMCrnaSrAnko4SMmkM3yqKQtI71hToiu0ZA7OBS8VKAbjES87885uF2Qwgyi1gW9AawsIFBgkpUIKhFj9sGaOGSQD2bO7Z0QEE89C4lvNOD9v0XkaPKUnvxjQGUeuEl4p7TGTKnl93Jm9wzXXCEkaK4WUSCx2cNcBMDumVZ9hHqN5uDlILchDQZOBvbcsnXwh580aVYvKogUALU4OpAGLvHOgyFfSnbXniHuKZbyEegLPCdkM1ncQwxTMy9044LYg54xgmaEvXWraktYBqVJWNBCPdAbJA2t6IdUCiIyUMs0fJffE3WEU4XyfCdXgHl7ssLnWGJ1LdmXBg2eqkdQPwzGyfc0WhvPaL4Uupa1rN9WKU4c0iIguVUznATAs6LGLNNYp5HtH58mALopUFIZso5DSAkmDIBkKet7eP8OfxlBu1f9wfjRSJ2KZtCFQopL9hZqSn7C83rbx4om7DbWejAACuKJ4p5CzItA5mk4hbkJ1Sr1IqHwq2V3MlYhYZsexJ1PUazIRoN4oI8JosYoF9NvlXaojrN8F3byg03BjRZnGNdB7ynxEcabpI7LHZRUZKaOIRQhOCAL4kxAjfCanqCnxE6gML8C1HpfKMIEnui6wCDfJhfyo1FZCtdR2cKJVmvv3gZMrlIAHj8dCFM0RfHZ8raU9RFacLj2DbjPbY059vio3bAliazeclap5JRadjcsuaw8XicnxYFYpTYCuqaB0mFZA7zbcU2C8lN1eAzCCJYpqL3gwqnd0LdXtGwyqndbl4HaaKwrsG8G9eEwpQy1G17NncqcFjJbZ1elu6I5weVlkkbpj1KqdHpIyWfvMjf7CPAADMuQH1gZ60R3poCr4Lc3G0jX72mH6FdKLRJM5a0AsbJP7XjZmJOZRPNyCVifJOmpLea2aHHFSov7ytkvEt6U5kjpqG5qhYSmTii44yqE2KyXrAUYy34AGPLRmpLgVz77Tt9vwkgjvTYK1GLjbXlYG0ec5fd6RSwef2so6R9OH3l1JZVyn7DHi26h17MM6XZpDEOhalTQZYyXVU57ADvaNnbCuMW6DEnW6CMF0EbWKKsafoBY0NHcIjRnXaJyEmzT82p6JBKdcWnrJ37bUwulZ6ivgiNgoxVTBplKbJKl4yCNzSIqPeVTVoKTXPLfLe8l7EMIad5jVIRZAk0SXV7BuwF0E0j34kzJr20lp98IMK4MOCkCeJHYJRFiJBc9iSKAdEoIbZMvlpKuqktFXrih8A40U2hLk7eA1ShQwMM0YEfp0K5ovBOpJEDrZJY9FXoG0Sf8DxzEXqLxA6FqeYpTuGUhvWM1fn03RbW8HxZ3bDYE9EFc2VSbTNkVWlfmOfCcGJD6jQPDdsg7Mkh0AZCWve4RJsqCLOj5jqTenVsgpnPpOMR9N3X8WiK95Q035IJ09LNRhRB9RM8eKSmpR1ggmHgQkZJskSynbz0AII5wpkXUXDdfKlvrsHapIFlVLllZcvdZwCKc1ir72TIdHbshnwAf0IbYG6JSLivdr4HLDDkmVrAIKaxCLb5KQ0NTfla0HtqrhyEi5BsBWBowFAdU5m82uK5HMWZhmHPnkwR0qZFbjNb804K0cmhgdWXORxamJP7zTDqjxhEOkxmrlv3O4BnzinSd8AmsgZOVXyOxjfm7MLiLhB00UyjDuktL0A92aZVsgLJV8C390IuyRfdLnSd6Dv4UlgSRnONYhHBN1UWajbmKy8K8fzkwAt3kP1I0F3agxGHPhnF58xn1PvCIzQzSbNRsd5WzxSTxenZHb57QTSC3438yYaApTGtLkINSTa1NfpISXOsIneMKiyjAXP8HuB8nymvnqhcdBcbwWf9jytsX6wjvz26mw15SwF6dKZ5GiL5kvzfbWEW14vZ0zO5idmgbQVeJT42HjQ5uP0EuonxadLvFTJGqoXkQ1DiOyO1pqUWud0rnWknvEUOzXEtjlHvBwG0U9vBnJUdqf6fol6NNM85x4LB7WWOtcRDKykFOnTzpcDMeSPwwOyctFgwR5tpdvbhlqqIyjugsVcLf0uWcw5lRibXaYrxtgMXCc6ZMK9IdUwHZ5t6NYrR7X68N0d2k8j2ksd3BnIs2JrgiHdjGtOR1ePkifKTzfbejDznpX0eR3XdHW4l392I74ViqyPuCgNSxsDVKLXnSVH7WEStB1Gmg5PGhatPp6886N5aZZXwFUgOy8Lj2P4V86dw7yhAJWlhJZUXMryPkWBlleXLk4oqXfUCt49cHrFTT3aXsAHhKoKLYg9tpxE1Vtto4WHVAZjo3kcNXBnYWopOECvN0XemWNzVIfQYiCY17Ei2Vk57MmGTBWvZbV93bBSUEWJ0gKybZAdCJKlqBndMngtwvyUD2fcy2ZwAE36lBg8WWRXefmbALu0la0QGl5O2zDc9xyZuqJtjgXu0u3B5A2mTSPWWL6UOnFuePhzEOur8Nqt2ehTSJ4NpQAS70FuX9HX588u8jtvtBmtPofTjKshwCM6BtaO2LulemzcDDBUw5AlyglW91h8cx0q6Cp7Xt79OqCMO7fcebDh6C054Kg6Tox2FMWkxneGf1y30WuZxBWhavynaBstlxXzWm3pXvtwAWdMG2KhCx6vASAuoQ0MNnv8JMTLOEkRIlRjA2iLW9V1CYRdLPhbBZNujx34dfkWOKeX34QW5Nhg3QSF7DhbWoxpYV2ub6eopptGUALVsgZEAitgLoq2xedaLhDPI6pO4Y6rTleCuc6Vcn9UpByA5NtHChMUeyfHuX9b4jUwHvOtwuX5uOhlEeVosPi28YWxr2PYs9T2pyrrTVFpLcOqU9GkyoUnpi47dUH8LXeN1W4f5lia3jkkDJxbpfmod2ek3tjsZWH39kNuEdMgB9H0ZTjR2SSpqwz65McllQRoSbfcSvKkWeYSt8m9hnbR6XJKnxiNPTItr5WzG5c1FHBvKmPFU4T5IvpI8HfT09hWeR1lFprxK9urUiRH99WYBhuuPmn85uDxr5t65D21QTJVaX6d1RbFdeYpXZ0Dgude7I1TCjFfDlf5AJaxm1hc1cOlLJxRDAzV193vNkOwR0gogNvUzsIsjpe9v1hRP9MkiHiiqVZH6LYMT1NdxPJp3stbcP9EAIZCA6OAh43ZGnKSnQdW7kGPghoz0WPRvtkZDfDabxr6TqKH1vEK8252xTX0uiJ7dAk4O8AwZmfrH1QbV2zzSlckP6t47IhEuALfNd2Ya68WtvgiPF11mWbEijRR781IMzLJiHuqw81fQH7X6YDR6imdKpXc7hRDpXlFxrHA8W2lwWwiUBrdJuWud0P15CFOzwtGiLUlApF2sJZAZpdhGOPAzF4FDpEl9FjCs2j0Ml5KNEGHm84XNZgdOE0w5efc77uNFFT26CbOdrEZOgI0CwcZ5orTxpWLM59LWhOJWWVUxFAgvUVXyIRa0KPdoGYyte5DIZr47euZu4bfcEpcR9XE81cXhJn5wIVuLib3U7X0K5NxCA4C8k9GxXMDWGGmB5Y9OxYPlWEVJWXwyZcqd8ZppTDYyuzOZHXGLCsRyhKhgFT8kSduXIoXxm9JNYysYUuOtfgQVbWgH3vY6lrOtETCokphgJp82yNj3soUMFSAZPwXVdglKfD1GRjPr4TgKNkEP1LrGGHA5XB314K8DXTo9KUPTodXm4zY66scvyNQ5lNNDJw3bXJk497IVczkvq4swIfK3bNr1y6ekJ1TSX569vyEKyXCWdIxAVYWZS4aM6LeUlAZQYk6gCQEqPKyCb64X9S7b7CIBNmSJ4b9pQv0GDm46JMDJmBuOS4f2DxTBkNcMJJaXZVjjQJVOG0pSL0yN6vXRHLxgpKIMM3r1maMirI1zcjXXcd7kLujvnhLMD3SbeDOSS9fWjATkMVGQF0M5PEze7xI5XPSAzqBJx7eQyWvAcHL8acuF5ErG2Pq88qqrYMfR1qgGC9h74hpWS34OHaDO2RfiKatvOLD3no08FyCq06zoCJUDOFEDXjFtOO9ttnQaRJIlj3qwWNTA1xUMZwADkaz8XOUZ0NCPkzKSKy74EY20jihkv0KYkUtTGb8sV9BARGlaULlOuQqqiAttrptWwrwXPdJCzaOtNZxnGZJIZGcflzk0cUKBOJ9QreWbEUqGydhTl0xTxpvWtVie8W8moUGTbvy1R63xaGM7AVyKBBThM6JlMOdnU0rMrXEG70lJTAfsD2JmMMYqB3HMrYji9uX0dZ1tJ4qRfplnVjLPksrZ91sMwNiUiBmcXwtFw9gf4QsIAd2lKbwzZOuHizWDyKZMXTAemQhcsqf9sCJ9MjWK1Utdgiz2nyAFKbKTrli98nrrcRqVjjLRInGgleWY2FNwuiYZbF3yTGfd7rNS3DHinDXkUlqx14xizvo0jVeiV6Nl9uRg0OOWwD6528PTV47igFVIGcYLmB27YEkEAkiRfKpeWIQmEThnYGH8TVZ1Yg4FMUmVwErtKY8lghxdTvjlA45fXvwj7PEkHD0FAgciZwXp9GDX87ZB59JhChK41fiK5b30UcvjbttCsUrDGZxysjo8IxPcyz4npQSt1LUlLGvn3irDrNFDMm4ztrMJhBRubF2F4rXssIFDR3LWrlKrt7ExVTUcbTWfb76kL5mauzGcVrcJ4gXEtFrwbDUO0ZfyfpIrtvK8kH4C7M42EHZ0i3vR2v6WtsYSatZJV4bmLbO3rFjlRp8n1NRItfdFYIXthxvkZ2LBsn3j7TDv0gR5L9Q5Tihz06NtjQjgGrZEvuspYyYCem6c4AIMll9H66vsOzvEZlC5ZTqyTWeD2W1gnulABRo1ay8TqrmcaDRDW6rOpWJbBbo7G5eWa3YXiWCSNSGY9fj1Wjkoq1ymBgN7uE3xuOA2Ib7TVd2s75c1euRa18Uo7llcLygTggu3tj1UasPKZ98fUmSitvHZTp6xxYKPyDFPzNMPsbySEUGEGxa558t4T2gfPWnYAKf2nE6PaH82gGgmsM8g6EiuKOrhcxRGKhcb1FsduwwUstXoHNwZl7Gc9yjqHRnDXIfM9VBEq7uZD7WzQBa4Sb0VEZbl7d1H5eCDXRLNf1pH8J0nn7RjHqYhb4jiUKRjcuOA5vPMfntOWhFz9EqKbD6Nt5qJYqieo9WSH6SXS951MQqxTyPobKIZtWaS7mCDqnAiQ6fxGSCku78rgCUnymvqDJKfJeRNZwI7FLfgFnnkiXqOXDIKLZM5ZuNezxXPt2WBDWvx0Gcu7QlypcvUnmjpr1JtEIpkdKQI8tS6q0BSMLn9evOoBHsZa3QCQoOaqAmYyDW5xBQVTmHDXPHbPF7IE6D9hyYcSMF40SMhMiktfzH29FC3GXKeJSL6hXpjS4w7luk02ZNoA3kW0zvy0AaSQxxhcyQvf2Jva3FMY5lKAD0imTgfZZckpBazmWhbzyrAhgZ6kJ5rfCDhaFSP75gAn7VX0zLv5D9OBonHnfCWdNPWtakkAnFZBohWa0vJMejsFVmKP0PTF1p7sGFxzjutb4JSVu8IbDcyKI7iCtJ8TP3JyQ089XUTz5Tkm13zxAAsYbU6ev1y9kOY4ejuJg7w1gYHeUhz8U4dVW6ey0rCibn3SEvXlFdSRkJkBRE5EhLCCjRurNvHO4onYCgOXQW4I9zHtzqzGUcopDHVmXGOD3RpRDwUfVnl3oLPqeVO81ZDxrgLDdYaligO5trTSinj6GeLgQ9Rez57haCUAIXTugXKpQNkaywvuOHJ2U7VAqxip6Yc6hZMKl80L4ma3x8xw68xgqbRfpU9nu2Ycb8FQWR8RJFEXXs4Xr1Kg9A8Yhiceytn11wtuQ5exULSgiBYnAu0I4JdamC1cq1YbwpeYyrNgyd60QaSrTxQqJ2FyYN9OAeyJJC1EehORx9x1LOW2vGhxzuXCeIzOClDuP5enFdFn8TwJLbF2nsmBwwBhW2Yiq3R3hEVvbHSxC7xApcwx17d03sCquGdy3EU63mwzk69Jm3tl0Ao0WebAV34YSTPwu3LESKHAIrPY1Dkya1E7CAze4lyvYN11lfvFgZogRfI18cC0ee6UWhKNFmCOwKK9u96MoxDW1yOtF7gQ7neLvKdf51QXC0Xecu1CKJS6DGy82ZOGzFjkpQwc9Ae11Aebf6JKNkS0P4XxHCpgKH5ZIVk4vFMeZV8siWrr1Nn8xod3uUGS6BOKKtq4FICV4mlVGZmR6EJc0ttUZap8Mv1ZZY19pvx8eQZrCkklot39rtIr720ZJFxH2w7lrHfQ4fEf8KnGzqgCYgnc2boKQVMSRgHZpOFTOFH2v9ww4hbuuMHIIRBWQl2yejhUO3AJBHdFrH3mv1K3HzqaHAHiaucCQ7wKZxPNvIIJjT0IrvNR5jyuxOzAAIVQWLCWYbmXkvfJ7PAsBHXMvUGmrrHZbar98qowgMAkYRUtHL0dQahHfaefxFvG8jH1SwGKPHoGYIJroKhkwh3GNs0Xe8aprJTeOjhSd49RtoQ57kGJJQtnIQR5LNQjehWnTj7AVlnjBgpfEcniP4dHr2xmnyiAxEOSeX2x3McEmhPQQ44BE01T3r5wDEBM2p7qTbssIBAia0RZGaitSWlX4jlJKeemvtYXyVlsnacSTyWuYcKsmrF7O6OD8vu2DWFUXunzCi5qWILhPKKHkvgn3NM5bUovUM1Lv7QYvgz5gt3Ff5WvjVbFXvD3A7XpLc7EinHyUwnlHWxKYeBobgQRFyCLNH04WfqKn9vXSRMt3m4cZ9I3VM9JQ5kADzxEKoC44y76oOD2OjXeCuzcoMzmeKC9ol22GTIOyIOyEqYU6DXKO0kDp5rNvIiKu8GjcXdiDHwQHiKSlXbcnRJ9tvPKuwdiCqyvI29Tq2neXxPFR9TGM7GMBVJZGebZkzv0w45fV0jWCHUFeYw10GjepwECtkpJ6suZkf8zM4kUR3WJkBXQ6MmQMl55QieWT0QvJGuCvxUChv3gR4TXEFXuqv7wUfpXbxhGgninjdIvDT95KipyhRleIc0dymFgztdvQaLb1ROzmmcD2bpaAwPufzo3UlzSSZbKiixIjsc71yBAuU5N1Klf1TApItX7MDtIlN9DGVLWJQPZPwCg39cQawWuAGEKrWBxiJjOdpZMJUoOzCI1bf3MIlLvQkd6A9OURuXRQ73dsEAjLs37uf8wNHPZ5pm2lPXWKU1GDisBrVsGjpN2QrGZWRdpA73JL87zjxgiHGryLyNL1HxWdzJYtZz6OY7moG6u1gyMpH2D2dlLBmyJmpNzZ5AiVpxUehuECClToHw6dtZX1JihpdM8HSIzuEQmEaf6GGRKQMc88FTT0ujyaQdfSGH7ofg7MVHX0GGNxPKFEWTFNk2G0Wtp4PCGD8hnl3aX49Ct948mNIqMAkLuiQLu7EQAAQWRgS8mu7VvcfylYAEdCtwkB9WlQy55HgobZYYjOO2SuvFN7X2PCQQTX05FRpsJ6wLccUWQUbmroiIpwAecHgDW3VF0DJoNdg4grwUVbzYwquFOXoO17buJnZOi36i7HUIycppik4jRRZa8gbCVRQxm6LwWH2HTIWHKlRVAB3HuMNvhggRtYJK19ih2XQCPtOrVAPtOIesOZ0s0sWfTa2QW6RIRzDpqakAnfNYTP4NgfCkU2ip9HAfeyAIv5W2vBasdFAnowG3YcxP9HnlQDJrwFzUtHYVpWcH6y8d3BMqXQ46kfqcZM8YNfjD7vFeUlI1lAhE2caRxzTUTEtmT0m1NhcgLEMHDMX69KpGgalLpmaiXAvUmMMmUOZiC5gRqSC29xEeoLBCmmSpM8FCD21HNKvp8r16vioBbrQruwjIopVqEFv7wIuWWmgXOLk5o3hP6Q1CYdJENUUrKFNTukOYS4qoSILnld6J8wJVXguDAGDiEsJZ5mgtuXk4Rikug2IXtCDhykbL0sR695k5A1ehQira7cQU2FeGyzmwHUvb7D60rV6vgSR9M6Db2xbfyP52W7U0dfRmnHt4geTZVB3lZOofagHP0Bebqn7zzQlhPpiU0oMDFv7ZZFGs7lYuj1QQQnW93q0tBIactVgF11jBpgiDWsDXGHPToLlW7bIQs4wTco5zPqzewj7iTEzO1sXb0tzxAEiE5yrwip3Vs3SQHptznnti8aGiz6k4zXCjYEIQAHy4CthaAw5rHY8ZwABoiy0EDPkaPLg1dg4p90j8efYF5uv49NMvcZbNtvyXIVmFl6CHunq3pdGM9uWE5WlXUZek3ZcFsLbxioCxktZ5hEVBHWYKjXQ0mmnfTtpXC7LCa5hhF0He3yC5ifVfrDsOOoaU5UtMqrkMDOInmhwVeAtuoUSRKJ1jLITwz7KqvN3rwYfcyXwU3DZVlF3yS7KblSL2YydeRLpa07YeEVCzR16MPtZoZ54Q8E9gN9CsptIxnl4WC3mUbM9NC3Dxw2hPdWaXKRl0aUitysWmotnqHG1TYTja7txXiDLxYxS3LwCTCEKUCkgFVtI2Tw2rLHVlU51P8lqc4hSHnzxfG9wKW8fdniqLf8WRtwDRmOSaLjxlnJCFBYxekcF2rG5B2vstVDwsDHrsRdUeyM8GDmL80PAILTfJJPzZ9NfIN7p39O3uozDfMgLmPoXkW4j9H8gtZG1Vuv2YyadIkIPht8tMM02GSGsA1tVrrrl5HG2TGf4xXbCqrTZb3661id3fmZoyB8Hjmuj5tyYlg2OHjvmuMxhv8hnhiOnqV3sHishj47uS8UuYdrqYpFuqGDZ68xh7wo5CrtGbYoApkiZfO9x9iV26WeaYCon6IFZNQhkGscRHpzqNKPiRw7gbvc849ZeakF6x3fIzLNER7AHIEUUZ9TYkQFLTcSmcqg2qQcYYfB4VCOhOxIXcAQGuQQXdPDOIGv9dlGLIrSn4hmU7ZPXtKe4Th16s0vDZSvqBAUZmEguaoe3O96YrjAkNs6MlshIK2DBj7ljRMbASrVbND2yXonakWatP7bqRgPo6hGLjia6DeA5zfnq046Vhcrv4Km0OFbBNCRJiLhJb0wBZPuJoA4MBMcfEQBqX2y7NB9Folsj1GvwjepTrLSoujnF1ed6ILgGQDOrdxsDVTtWI7ZCY3YVgHfPKrQawc7uSqqdq4DIC2yj7KyukA8KUJ7RNEg9iX2lzs95TZkwLTXv0EFPyhPjCdY2Mp9a0AfObxwJxJrc87ohxtUQbqGKdlHiAeRyKPzbrD98Byvnoq1DnMHOemlJUJ0e9RTmRb514aveYWpkzvqqm0izSP1eXDlqc4ZgpbopkVxWuiY5YHsqFkkHUk1gv8KL58ma7d4URxAwngub05sM5bnXnZD119jsK2sMowgAdWDh5lMnv8bnNGS9Wzoa7WHuzaFlCaHDvd2YUgeJRtCpPXwOUhUTokhu4xettqBV2axp7nzE8E4AaXY7EUyL04uQynfGgY2q6GzdruwJ0ZkDakajNVaivXuKhRjr4PZU7XDaoS6Z5sENnz3aCK6dMJCqDr6ZeglnC3SZJHRWI2WvtQ6rDppgjel723sYSBuTmTjCBHQpvDqWpLWhMIvSaqgKf9PXm6ecZw3491H2EwPpozScLBVof641ZF8IGRH7y8NwZwIEhR3TutxjsuRrgWGLJpylNZJkDnvFXIx31DJdLNpoFYf2eYKFB4qaD4428Dp1ltVBPaSWh35FLZfbFhjiT9ZwdBYlbPPd3blkTb3aFHjWWtgqmXrqUmZvQ4fnQTIa0zJdn35o39r859Ow5GPGxGdGHKRIi390Dtiao3y3PNkpgZtgJ6NChf4YqFRXnXa1WHEe4LKE6LMJRrXYVURDy6iwCDQOnT3f0PO4ep2cZ1Dq4EyGDfOWyPZauLJTUuRkf5PU5oR8eifMJGOi3VXqZ9D5OLsjRH1GeRJzV9j0MMetTtdS0WE3OJj01LK1fiJLgCj93cGz1Nqed65bv4iPtJ1p9Ozcdhd8cBXiHYa2XzWtDpeKj3wZgp5qSeor3mvcD078CrcKVAv01a76M5E8gb2dhfJI9D6YIfQJN2gJWWJfleCTbu0nmYTpWXo3PGjyiAxoRy0u0IDVMHGXfSbDLEhmagvjGWYeVFqUgmqYWe2rbApRj38kaPsOYpdZ7MyqiEa7QOUbugYFhgosWIWoW2etYIw4KMRSVFTAe5XZFTWGIjRrDbE0eVeqPO6kSxS9jRMEcQk79c0x3X1yD7smfeEc89hoLhQ0PyNTCpSxY8M7EHKEEgvwDR9jIsAErmkTZXf0anQhbOQMjM2niGxbDq5R0sDlu1IHVpY5DteVEbPAojFq5PpiEMYAnaGVT2f4aWxN2Qaoe7jXggWcsQligOx6bO940pHBDUtrw3jpFq5C9xNHC5RpUTWlpX24hc8cSZkVxjQhO9Y441JO2heXEFmuiMJIu31sNSqXCJk2cjyyg9A6KerDTuePYMsVV524cZCVBU0I7P7zPQkMvYOiDa9NdR37D98K8GPyf33AH3ESkgIUUsKjnr1L9Z9FLFTJNALxZE4Dwc6lK2UUI4hWXMQdFdZU7DgFEEEnNyfW48Y13l1kSD9PqkvFdgw2LNQTm7F1mSdkraW5T1AhKOKvvsakwZpHc3SJsaBdHqyuM1nilonfkusSW8A6TPXvPRBBdspuCBA8fQmSKT2Gk2yQag34zY0MudPBHUZIEvRHOSTDA9ljjF8YqhPw10bms1crUbVHPr4rDFEQ7bheOMRXwvqbmMZ7jqeAiMKkI5m42uHOuu5sR16YZq1bQzxPMBaXdIexA2Lqh7ZNj0jCseoUcQHawIhwrGuXJSjTXBKW2qbql0paVtMiVjkzHYPEXmH4Xn6BmlY94mfaityHYochAn6HglP4EcHgiLdkgeicD5QtFWZXBJC8rg8I6XWaybZBaibKEdEpphWJUuBmlHHmCweCqsIyn0sznZTrVFVLmRgLzLEp29gguyXFTWgY7UjxUK2NZlI4PMrP7okyE1tpXPzK2aZZJLv1jhYy4MS8GPQRfweTRWdFymHXE4MQWlB3gltw4wgXmTQGhilSWN49WBkmBnyC9JCt9qvhDQzfY0PMWb6aj8nyjFSwzSZYvGZAcjE8gfF8eOkzJgQVdYaP9UtwM2xnARFNAxgRptWIP0LZuc2y4AEO9zZpxWxYQ3s4SGJsAj1wMUvFaM2pUlzmT5QQt9WFBsauQTnPtbbEQKnDMtqZanZZ6S4ubF3UUXEjhfxXkymAYoRiL9wT0jMlamjOV0m7GDnk5uFgZpBR7sNFs1nbfVUsMvvc78OKd2jMaljcgQPrv0f1o5vWsBACpmIS1pFWe43UxFbRYoMvNpGkSjO2rP86CUgoWL08x4kb8q844l0QXKVJLXVdiP3tPvBwRkORRvJW3DdUCYHCNWhbyBtDl44srQqlij3NKShVA5Jjbg9M8OnjoTvtUGUERfX6k17tS2ebrXtSjsLvgB7LTqM5DyjaCvYrHDBxjeG2NCNxi5B3b5cFvYv10yu6FizReSVzT2hPDmn25OEpDweMufP7nJwZguv5lijOuiwyiodcWgGnjkyV2FSBWmjhxeVMVqCO4tN0hTWTNauDgkMffsN31N0sK4wRtF2pE5hChkrqRFVsiwIUHgQVSQ7S8jKMIKEjqgZ7z0erFB99OP4CL8E0LXBAEtm0i4WPt6GefKWOWffV2dn0SZStZWj4cWfECI2WsR0yZBp0VodnvHxtEG7Us4zS5ak6kz0CvgqNiAqqDZYrb0aqSMSsOzCh5AtK7lMWG36JT3scb4x4caZhX16YdbN0wyhEFbI38s5ccyeGfiJKNWvWQQShkAtpFeY2l7dAQqFB4gJ82quCh5tT0tM7CW4ZnZdp3PD2bEx2AxhhO9jyO2cOCCnQGI2c1uuuhUZo7WfdIz8PNHE9GcFf6KJnKuSJU5yU5WKpiaA0W7pHgf2oFbTHC1yFEVEA2B7GiGfRzx9YUyfJ34sOhF3XtKhPrwkoWT2RntC3HyGvI5Z4km3djklKC4KXbKisb7EwortN8uNp20pQwJjArrCDiUrXUc6OnfLtMdlKzTEEQmTmwDjDRzFKP87ZKrBVMDFwYVAeKSVVqwQR6nkWj4Wmg9BPcEtECsMsklL7pywNBus8jH7fKaHIEcDJ9pAVUzaVaqN4GYmDn7UW5zzOuMYXQxXAiXFtaFT5nLEc7gmxhxFEaB8V6qy8mS9cuUECupd6WCodr3a567t9JZCHl3EFDHEqWFqEPQibMElzlaYRG9Om9pV0CUZYRIg1WG9KBFL10tBIXIDZ51YALQCWSro35KaUYBlSOOqBI1MRiXBkWztrcGgq0Ru56pvXrJta44Tl5BPs2yYivBHJMWdc5ivLMF40HUoezUWFd8qvQV6HUEcfNc6PHByIfdcQNdOg01qW1Am0CspkPobZupdhfMWunZMvvAqIvrktj6U4QIwoEjOZZdlxKMJefCq4cUv5WG5fOGWXslakgqvneiCd82yLxTU9kAgeD4HnHfwDQne1ziYjYA7LE98upgreh3Z4WFyxUHevwhDoK0koMJsNuQizOBpoSYxUZlaPNDRVglYv3NEMnw9sCv9BZR9XjRFdBddmmE0Uq7dPDdu4zmj1gtkM39GeEuwFR6w55dbCotbw7kL60HqHE8XrCFPYf9oCFxMRvrnwlGM66xDuF3PzSGUzZiUsSTd5aMLjcGgGJPb3q3DEDayDUiHlaEitbEIJoifqZdErNtWJOI9sNVWZpvvFNMjOOuTei72OlWHxwqHQmwePLWOlDzuboaJponXbekS1dQOjcB3ulbihVW5D6sTwW6XAosLP3eAFtQeUrKzqmwVnBxP3z9DQb3oN2QhDBptWpdV18CyG6KI6NxquiBu5MI2XFbzzQu3FGiyQYypRUCz6wzssK7SWpLggg5E0FRpGXTBPRaGQjeMBVCrXMTTEPNu0uLWAQIlzo8BTsmk6QDBjJWhdi8MJvdea9JSQL3fWfisDPxj37HVzF1pxHarYCkzOJu6JAYfIAWImNrVPlnNmRwF4VxGEwEXStYPydNBG3CLn23tbIsi2AX7vVsOW3wu2jHjuRcynDHqONtvf6rMrsDmLizPuqNqmkhep6B2d1XOJc0gBEP7EYRyi67P3oi0uCmBOXn8jg8mJh3klKB0qTzZKLDXQwLXYx6rNRxm2wof3fbs9DCqfiSuD1wcCB1qHcS7yKlWbRx6uhQKMHceOeg4DpgLIbqv6zWqf9JGIxQAehoUWd8JllG7iWBZd6jfJeoyDsHIDrxdGYGDSnsQlIoZVzvjMRhG48XkTBTJbUpaunVji06uevAjUKAE8VzgGonGFfoJ4A2SdP6xo6Fp4SkdMoo0hqriQbYAFlnNhwTnlmBEiFflIsIpmR3ez3XSQT81gazNFO01hOte60jaqpRbhTy6EE5hLqsctBnYLI64G7DbFMGOtkX1Tg6FlGTaHLL8MaaF2VN5h7j5LeOdEE1XxAMDGCATFtqesMOglNnGY2ODTxr46ofNbVT9QDtJqN7bjsUSPgnml3g9ZmyuGkF7Vp56RqWHGpV53MakRjeXJouDMXpYsTSBtcUxsfK3uaEQKCNQQQ7wzdF416r0lzaijKXoEYbe8untgX5HcVA8augIpl0gN37wnKntp3mUhKpxaUihor2Y065nONvXGsweTje3jnxO4azGXUPsVusNDpeA4BdBH5aSMFfszdhDlrQ1HtiDKP36Fev7ZieeJESQpHrln8lOCn5IbBQr2Wc62mc3qOwE4w8tjn7S3Rq9DETom615uUUx0RXl2z6u1eUZPMgKloc4ufX6psznLwFXc1QtsEafiHu4aTxVbXQxCe3xmPc1gzv0KlhWec3dBNRuWOn2l3CVKA3g9Vr1tOtfv31jihtnmF9lCxWP2CmSGxAlddNSESJfPHgIvYEffJ3Pa8ANiVHNoloJVMfveY7OgfwvJT3xWArMNahBpsvDBqe3qE2qzhwksWPFV8vCAcXKM9esIbzlscqGIRMM1FM22LSkTWxm20RiuuXpGkSXtTOimcU49cYnOLhdWs11Oxh8Wxo3MLTzP0WJji9Fl0WN09jWR4hPb0KoNyaWFAZuG502Rd96LZXgtNpRq8zp5zF2ElUyrDJOprkorJfkYfLo2SpsbGA7s8Pr7CfJheKrYXgpHUAO7P9FR02sixis1dJsn7QEdX9kyU1SrBNt0l9xdAY2aXfTQwq6IlEAgZLeTlPY9TRZdt9uxjCRxuf8yHiNFEFfTInsYPiGkqoMyXfzUTIRruYateTuKCEpNuwwXGxLz7wTJp9aJ6O5j4CNr2Uxwbx7jiQkiHldlykjlMDzWO6B2sqkK9D2TDUNY2OZljaC1gOM8y62ifBFPCCind1MqZjMQuhsTPti7e7aQ9ttuG2C6WbQP0hG20EzduuJyEyZm0d1wgLQz5wOlYAWUdt4g0SAM5GD7R9W95MMdAPgKSbYTMh1xiG8BgjlCX8lxxKaNc5oWto1L61fFhoP8KP9TqgBmgOlLaAFtsV1sKQFutT2YV1zhudjAtqCyQ5Ni5Vb1b93lL79eVaVlUNFTG40mjAvjomqREUHkAis8iC0KkXc6ZKl0zyU06TkYZVCPBQIerOKgYftx1L3ZSyktYUMDCYq5rX6LJUwHsNVzFCQTdKUf02XU770c6FO99vPwetnNpkoaSBf5B3A26lWT4nnZ35PLWIBj08NKM9x5cXhvQOz3elLpvmp03rJgjm5yvM7cIJDABzKxg4DcbB056wB7PqfE9aoqioyFd5rWiakW2ttYdPHcmZGRr2IIHpGJW21nhQIV2unZsKl9VTtKpUrLQvVKYiQ9DfQjbmWWAkbvcRRnWXfamoNmux0kpZfEANQfefTt0KPUtx9iH1mkM08uF7l08SvCaL3RIQZVONU395Qv0OcfHTzhhAUbO8NC8RVaIGaSNVo9hsjPcmNjTcQZmrrC5UcXGGzbx6qCnshGGGC0ukU0aEqkwhHfNO8QXJoELhzyY43dtMcyIRpT0pn2j5vehq37Pqrxyk2FEwEHBwaK0tD6VgRTCeRJBpaDgzIUUEWpLtkOteWmgGx3az2NONmhINxiAghlvqHqiZVrTltR5tC2xk2LtzNKIQBiJ202Px9mHqcF9a2BlR7equK7PYLfAhdefsxCX6j5IZnWI4yMaQWTrPclax9w4JxEir9FkP7qRqo9TL6DZafLhota7Iyu6DeJWXDpzH5c3SfE6oDHbtuxcTleOTO5eWHWgmgqbQRKzy08N5gzRnt71NakEEzMJpAJaxQTxJnXjBq1EfAZvjI7TT7IXQvVSyxLax1wXKJFJZU0O0wJxI7I2dgHu5NCehSTy1yEsEI83wFRJwV31bSLVvD14ZveObso06YrL0IDJ5qhSq4osbBBIbAN8IUle329vINPUqKRXUbBOJcc3LAyJdwAvq1r5Dg9MmWFYYB6TiDmnYdomd6eNNvV4PJUHkd4jX37Nq1QB8WmQb47boYF4xXbQ90cfL9ksMO0lNT0cIwRMZSt0vRh90B8laaLHw9QJTWR5O38Snp3s8myty9zHmF9MUyOC2xWu8tZNSpkzUUoNyHI5WlCZ18t8u5JQjavROEOrkfNHD0aeRV5zJVP0mu2LTh38Ru7tMVKuKrndSN5J6w5Zb0dNmnLP8d0vMiaUrqTKtSoavBrFvYsv0YscqbQNiTUptPvg1GxsnxgGkxoOZSjhs3Lf3SeBMbzjbX7IQ2w1n6KsOYUZruOqbv7zJItLZ17xPXYVeNzw0BqAATblXj55Eim2rKuJ3LjKj5zSWpN2clNhXGy0Sf1IWcTfDxH7IKeFAyjXrSk5A8MSLai5yResrsQbPAeYhfC0nUyW168xIIN0k04FcyRnanIZYIv8rYZJbyIEG8Ar2Wo2rRweXK8T8YxVBwUWSGXgrRGw4xSwA58scwX3deY2OEi0KJdwqylULQ1hmxR8sMJPF86P53fVwTNVy3av1aZllHhqliDb2A9RTz1Kzv4Vp5jm0pwok4ECGUVzgkLzBXzZMSeDNKAi6vUClqscIlIkuCNjjq2aSRjvylSM06SmM5t4HNtmmE2Mt4ay2mHnUvdZEVZlquS9e5pkOZzvDKfgBKzfVLidiCr1Wi3Ihiw7oB7Q6f0y88rfTSP5myKJEWTQmr0hzd9SvnN9TEXkzQh75hxdDcYRGrR3cLCqWQw9vHf5LNcnUc4DuPyZ1K8CnrLtUTQsY0WzMRW8a8D8kCqrz7FwepoCCy5uXn8dagiHwiwD9BJtHDXDAN38rNWvy0Wup172yOif72WCRdvlMqyTbmraotRE4MjEe4R9kVYDMCs9brXJhfB6xK56Cl7xFAIPm2pZTj2pWmwsCIk3nVCzxjJUAYNHOWJPDg0zQR5DRVSMTNswlRomjk5wjQgfnN57PAorTPW7F95whCqjHWxp6vApVUQxOjp2GfMYYdWA0y3JqEfrIsovuSWfBtcc6tVGAz2Feu0Te1pvb4ngpXKos57wccifAqNdneQmjP5cAOKxNddK6btFLqHJZiVLrlHjtb2pUJZqZdNWwEdM7vpijBHKAudieZX765QuTGi1rN0zaxZjoK8MPH7ClRwVeRS2Mu9IMTGC1tXlrRSWS9FSgLHuH7HX1hovo4kCTs9hdP00zFPkDRfGr9AI6zjYNPhEehRU5EMinMwYfIqEpU5ZtfkIJOmVYPvjk1Ru8uG1T5k8IOBuiQExHTbuXnstEt8TGcRsTxFK3tBqs62aF81hRNOEChgfsHXOk4yfM7DPMciiK4QsABK0ZJ7uYZmEIrl6eAnTz2YlkdOAhZJrEQIXS95jKCSUHDnBKwfu5ZH23exqTGNh6bF0850tWi9N3RFivzgmyI6Yn6wn3SBJj4ps3Fo8z0dPUpXjkOV1IXoXn4n8BsK8uIX2NRw3jbKQlwIQAY8k3KjSXPVc4yKRbYFlrB7RBwqPYIdO4193O5FIUDk87OJpNSAQugNZcRUHvA3Egg8yCZBYlnkHPQCX45GrmNWWFUeK4RCSsbdZSAPP25ijDe6bMGX5iKr2ufjllXuZm2ngtgVxfhGbN8dOgkeR2UXwcKVpHPcK2CkkFxyFwuqER2dmWUxAxFnltXRh3JjW9LC5WPNqO9vcjzWNZAJxRQfXMl0SdLltgq6vto9obI7WtQFivAidOiryEEkhZIGYPlEYjKXt7QiwUO5pzsAOt26hBgXrtuIBVSWsrtMKsTZAHORJBBRYW43uzkDlRkKp8imrQA75CRWRobRu4lPB2bRaHOP9dw1AlxDJErDde3pqb2iXibw7WDNdeVZ2ZamJhjvIAuSBo8Y5GnfDSnMnfEIrYl7loINEgGCLa5nJBEGFN5ff2zp7hVP6PTnSQAvKJaffYmmp2Ra1Mwxm17L1BmIyfjMasv4Kc2cAKQIBtLwPt68mw9laHG3eUVs9mEihSOgtjTWAQrOsQWzc5X2G2dZwafEvthLUyIvVs9JtXxVajMynP3dDE2t8eiiHydt3SeEbP71GF8NeZrphDZBibyzspRYMZ7MLDkNek3u6zPBuVNRvCfyV0e2WRoV2fFwFRdlL0GyPc5L2cdAo3vFg9FabPB8SAYcXfCWXAYXdUosGphy6TlaBRkBGYnNTPR4boEmNkszor7gQdOS93ZnSq47BBIH91W7BDEOiqSMU1gbKzJJADGhidl95jlOzYYhMGCAkAyZzguj0niIfdXEkHF7ILAQPn9WumGi4UDV2az4yZtTjIJNB9j4pG9JxPM75hC03BPNggc0J7BUHuXxZRwGCHvW4afzcmEL0bOcD7PQ0WQpey3MdmIdVRx84BssnAbODQ2urfKUTFfcdUfnyjMRp6DnLK9uFO8OJidp4Ib9GGqUuHxfUKtzjdHUvVaxAFXMO3wTdXm2P9y8s0sUcO89az97m4dyEeYncZZD6dXxiboO9zFcJl4Rm393aTBfDMVHhyUvc19yXNLeXuY8Z766V9zT52lJJgnKUGOmUTKArylH3anbgk6yoPxvc2dxIQNETAIfnvwIoCKtlcvYuJGLkPaj4cehHKblS70kbtJUYIkmLa3T3kFnLw1fPOOeTBLeqHSFk9HqhnpmCjH4WzBFGMJR1PS09Di7DABDty5caUzBXjNUGwHhxmIU1OzsiNVcO7QJiY6UmohHgw01cgNVPiGeWteGHreyzX89HaW5HNH4y1ZKkf0nMOwtgQxCg5RNzSJU2HLTSwwiQvrsofUxdH2gPIOfbF49xMYPEMkoFs3BzkMfUEvFBHizct27suTYyTaMI7sb2RzQPnAtNH37jwfHncgPsWpMeUCMZAArDYL9if4T6XaGj1ZRKImOyjcQUlY3EKN3mHSiHkbdpIKblYxeATtcDsOFfwPLJOCboQcz7NRg8O8Fk43BNfrFzUXe6LNUURWdM6Ak7xlvCVRvcPK895oXViKNEHmiGjfO3H3mb5hLr893jFbWvoPwLeiLYLv19PaeL0q3aiNeAYXIRhKOKfOWInteWtKzJ0EAtPleSQbAk0i2ViN2Tfp3eKzNNGKTTEL2QC5Txcm23S1JPjJNe6BVI1rLgQUUj2FZgPu9lauRcFX6fOVIvPJVswNYSd6Lc0n0LCnL4O7RMZ1dGhg3sDqJyXJg5dGNPxrXCPhvWMMi1bPIpyK1EiaGQ2Uiz41xZPRtEtWW5exh6z8B3ltKMX9cCFN0FkPcBxiyZsBGJHcbzSpAJCBdLN3zsS010vpUh3K5Qearx12NzSC5odLUCqrlnAy65XOIWWfYcrinKEsBoFYJVOuKHjBmfi4bJTWscftvYNaw340uNRFwuYZ7zumK4xo0hFx1arCwggrOnIHAWhCZLM7i5oklhLNYp8oARpQ0BwrM81JQ3cSG9lKH6R4nCBUWpWXuhNfyQL8sgHQtURdDUiQoyGprNGxXu7zU1z + +-- RDB_TYPE_STREAM_LIST_PACKS_2 (19) +-- LP_ENCODING_7BIT_UINT: 0 to 127 +-- LP_ENCODING_13BIT_INT: -4096 to 4095 +-- LP_ENCODING_16BIT_INT: (-32768 to -4096) and (4095 to 32767) +-- LP_ENCODING_24BIT_INT: (βˆ’8388608 to -32768) and (32767 to 8388607) +-- LP_ENCODING_32BIT_INT: (-2147483648 to βˆ’8388608) and (8388607 to 2147483647) +-- LP_ENCODING_64BIT_INT: (-9223372036854775808 to -2147483648) and (2147483647 to 9223372036854775807) +-- LP_ENCODING_6BIT_STR: 9223372036854775808 +XADD 5-1 * f_1 0 f_1 -1 f_2 -128 f_3 127 f_4 -4096 f_5 4095 f_6 -32768 f_7 32767 f_8 -8388607 f_9 8388607 f_10 -2147483648 f_11 2147483647 f_12 -9223372036854775808 f_13 9223372036854775807 f_14 9223372036854775808 + + + +-- multi field-value pairs for the same stream id +XADD 6-1 1538561700643-0 a b c d +-- multi stream ids for the same master key +XADD 6-1 * e f g h +XADD 6-1 * i j k l + +-- empty stream +XADD 7-1 MAXLEN 0 1538561700643-0 x y + +-- XDEL +XADD 8-1 1538561700640-0 a 1 +XADD 8-1 * b 2 +XADD 8-1 * c 3 +XDEL 8-1 1538561700640-0 + + +XADD 9-1 1538561700643-0 a b c d +XADD 9-1 1538561700643-1 e f g h +XADD 9-1 1538561700643-2 i j k l +XADD 9-1 1538561700643-3 m n o p + +-- XGROUP +XGROUP CREATE 9-1 my-group-1 0 +XGROUP CREATE 9-1 my-group-3 0 + +-- XAUTOCLAIM +XAUTOCLAIM 9-1 my-group-1 my-group-2 1 0-0 COUNT 2 + +-- XCLAIM +XCLAIM 9-1 my-group-3 my-group-4 1 1538561700643-1 \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/task_config.ini b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/task_config.ini new file mode 100644 index 00000000..48e72dba --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_stream_test/task_config.ini @@ -0,0 +1,35 @@ +[extractor] +db_type=redis +extract_type=snapshot +repl_port=10008 +url= + +[filter] +do_dbs= +do_events= +ignore_dbs= +ignore_tbs= +do_tbs= + +[sinker] +db_type=redis +sink_type=write +method=rewrite +url= +batch_size=1 + +[router] +db_map= +field_map= +tb_map= + +[pipeline] +buffer_size=4 +parallel_size=2 +checkpoint_interval_secs=1 +parallel_type=redis + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/dst_ddl.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/dst_ddl.sql new file mode 100644 index 00000000..c6cb0c67 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/dst_ddl.sql @@ -0,0 +1 @@ +flushall \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_ddl.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_ddl.sql new file mode 100644 index 00000000..c6cb0c67 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_ddl.sql @@ -0,0 +1 @@ +flushall \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_dml.sql new file mode 100644 index 00000000..fadeeb11 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/src_dml.sql @@ -0,0 +1,510 @@ +-- APPEND +SET 1-1 val_0 +APPEND 1-1 append_0 + +-- BITFIELD +-- SET +BITFIELD 2-1 SET i8 #0 100 SET i8 #1 200 +-- INCRBY +BITFIELD 2-2 incrby i5 100 1 +BITFIELD 2-3 incrby i5 100 1 GET u4 0 +-- OVERFLOW +BITFIELD 2-4 incrby u2 100 1 OVERFLOW SAT incrby u2 102 1 +BITFIELD 2-4 incrby u2 100 1 OVERFLOW SAT incrby u2 102 1 +BITFIELD 2-4 incrby u2 100 1 OVERFLOW SAT incrby u2 102 1 +BITFIELD 2-4 incrby u2 100 1 OVERFLOW SAT incrby u2 102 1 +BITFIELD 2-4 OVERFLOW FAIL incrby u2 102 1 + +-- BITOP +-- AND +SET 3-1 "foobar" +SET 3-2 "abcdef" +BITOP AND 3-3 3-1 3-2 +-- OR +BITOP OR 3-4 3-1 3-2 +-- XOR +BITOP XOR 3-5 3-1 3-2 +-- NOT +BITOP NOT 3-6 3-1 + +-- BLMOVE -- version: 6.2.0 +RPUSH 4-1 a b c +RPUSH 4-2 x y z +BLMOVE 4-1 4-2 LEFT LEFT 0 + +-- BLMPOP -- version: 7.0.0 +-- BLMPOP timeout numkeys key [key ...] [COUNT count] +LPUSH 5-1 a b c d +LPUSH 5-2 1 2 3 4 +BLMPOP 0 2 5-1 5-2 LEFT COUNT 3 + +-- BLPOP +RPUSH 6-1 a b c +BLPOP 6-1 0 +-- LRANGE 6-1 0 -1 + +-- BRPOP +RPUSH 7-1 a b c +BRPOP 7-1 0 +-- LRANGE 7-1 0 -1 + +-- BRPOPLPUSH +RPUSH 8-1 a b c +BRPOPLPUSH 8-1 19 0 + +-- BZMPOP +ZADD 9-1 1 a 2 b 3 c +ZADD 9-2 1 d 2 e 3 f +BZMPOP 1 2 9-1 9-2 MIN +-- ZRANGE 9-2 0 -1 WITHSCORES + +-- BZPOPMAX +ZADD 10-1 0 a 1 b 2 c +BZPOPMAX 10-1 23 0 + +-- BZPOPMIN +ZADD 11-1 0 a 1 b 2 c +BZPOPMIN 11-1 25 0 +-- ZRANGE 11-1 0 -1 WITHSCORES + +-- COPY +SET 12-1 "sheep" +COPY 12-1 12-2 +GET 12-2 + +-- DECR +SET 13-1 "10" +DECR 13-1 + +-- DECRBY +SET 14-1 "10" +DECRBY 14-1 3 + +-- EXPIRE +SET 15-1 "Hello" +EXPIRE 15-1 1 +EXPIRE 15-1 1 XX +EXPIRE 15-1 1 NX + +-- EXPIREAT +SET 16-1 "Hello" +EXPIREAT 16-1 1 +SET 16-2 "Hello" +EXPIREAT 16-2 4102416000 + +-- GEOADD +GEOADD 17-1 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" +-- GEODIST 17-1 Palermo Catania + +-- GETDEL +SET 18-1 "Hello" +GETDEL 18-1 + +-- GETEX +SET 19-1 "Hello" +GETEX 19-1 EX 1 + +-- GETSET +SET 20-1 "Hello" +GETSET 20-1 "World" + +-- HSET +HSET 21-1 field1 "hello" field2 "world" + +-- HINCRBY +HSET 22-1 field 5 +HINCRBY 22-1 field 1 +HINCRBY 22-1 field -2 + +-- HINCRBYFLOAT +HSET 23-1 field_1 10.50 +HINCRBYFLOAT 23-1 field_1 0.1 +HINCRBYFLOAT 23-1 field_2 -5 + +-- HMSET +HMSET 24-1 field1 "Hello" field2 "World" + +-- HSET +HSET 24-1 field2 "Hi" field3 "World" + +-- HSETNX +HSETNX 25-1 field "Hello" +HSETNX 25-1 field "World" + +-- INCR +SET 26-1 "10" +INCR 26-1 + +-- INCRBY +SET 27-1 "10" +INCRBY 27-1 5 + +-- INCRBYFLOAT +SET 28-1 10.50 +INCRBYFLOAT 28-1 0.1 +INCRBYFLOAT 28-1 -5 + +-- LINSERT +RPUSH 29-1 "Hello" +RPUSH 29-1 "World" +LINSERT 29-1 BEFORE "World" "There" +-- LRANGE 29-1 0 -1 + +-- LMOVE +RPUSH 30-1 "one" +RPUSH 30-1 "two" +RPUSH 30-1 "three" +LMOVE 30-1 30-2 RIGHT LEFT +LMOVE 30-1 30-2 LEFT RIGHT +-- LRANGE 30-1 0 -1 +-- LRANGE 30-2 0 -1 + +-- LMPOP +LPUSH 31-1 "one" "two" "three" "four" "five" +LMPOP 1 31-1 LEFT +-- LRANGE 31-1 0 -1 +-- LMPOP 1 31-1 RIGHT COUNT 10 + +-- LPOP +RPUSH 32-1 "one" "two" "three" "four" "five" +LPOP 32-1 +LPOP 32-1 2 +-- LRANGE 32-1 0 -1 + +-- LPUSH +LPUSH 33-1 "world" +LPUSH 33-1 "hello" +-- LRANGE 33-1 0 -1 + +-- LPUSHX +LPUSH 34-1 "World" +LPUSHX 34-1 "Hello" +LPUSHX 34-2 "Hello" +-- LRANGE 34-1 0 -1 +-- LRANGE 34-2 0 -1 + +-- LREM +RPUSH 35-1 "hello" +RPUSH 35-1 "hello" +RPUSH 35-1 "foo" +RPUSH 35-1 "hello" +LREM 35-1 -2 "hello" +-- LRANGE 35-1 0 -1 + +-- LSET +RPUSH 36-1 "one" +RPUSH 36-1 "two" +RPUSH 36-1 "three" +LSET 36-1 0 "four" +LSET 36-1 -2 "five" +-- LRANGE 36-1 0 -1 + +-- LTRIM +RPUSH 37-1 "one" +RPUSH 37-1 "two" +RPUSH 37-1 "three" +LTRIM 37-1 1 -1 +-- LRANGE 37-1 0 -1 + +-- MOVE +SET 38-1 1 +MOVE 38-1 1 + +-- MSET +MSET 39-1 "Hello" 39-2 "World" + +-- MSETNX +MSETNX 40-1 "Hello" 40-2 "there" +MSETNX 40-2 "new" 40-3 "world" +MGET 40-1 40-2 40-3 + +-- PERSIST +SET 41-1 "Hello" +EXPIRE 41-1 10 +PERSIST 41-1 + +-- PEXPIRE +SET 42-1 "Hello" +PEXPIRE 42-1 1500000000 +SET 42-2 "Hello" +PEXPIRE 42-2 1000 XX +SET 42-3 "Hello" +PEXPIRE 42-3 1000 NX + +-- PEXPIREAT +SET 43-1 "Hello" +PEXPIREAT 43-1 1555555555005 +SET 43-2 "Hello" +PEXPIREAT 43-2 15555555550050000 +-- PEXPIRETIME 43-1 + +-- PFADD +PFADD 44-1 a b c d e f g +-- PFCOUNT 44-1 +-- GET 44-1 + +-- PFMERGE +PFADD 45-1 foo bar zap a +PFADD 45-2 a b c foo +PFMERGE 45-3 45-1 45-2 +-- PFCOUNT 45-3 +-- GET 45-3 + +-- PSETEX (deprecated) +PSETEX 46-1 1000 "Hello" +-- PTTL 46-1 +PSETEX 46-2 100000000 "Hello" +-- GET 46-2 + +-- RENAME +SET 47-1 "Hello" +RENAME 47-1 47-2 +GET 47-2 + +-- RENAMENX +SET 48-1 "Hello" +SET 48-2 "World" +RENAMENX 48-1 48-2 +-- GET 48-2 + +-- RPOP +RPUSH 49-1 "one" "two" "three" "four" "five" +RPOP 49-1 +RPOP 49-1 2 +-- LRANGE 49-1 0 -1 + +-- RPOPLPUSH (deprecated) +RPUSH 50-1 "one" +RPUSH 50-1 "two" +RPUSH 50-1 "three" +RPOPLPUSH 50-1 50-2 +-- LRANGE 50-1 0 -1 +-- LRANGE 50-2 0 -1 + +-- RPUSH +RPUSH 51-1 "hello" +RPUSH 51-1 "world" +-- LRANGE 51-1 0 -1 + +-- RPUSHX +RPUSH 52-1 "Hello" +RPUSHX 52-1 "World" +RPUSHX 52-2 "World" +-- LRANGE 52-1 0 -1 +-- LRANGE 52-2 0 -1 + +-- SADD +SADD 53-1 "Hello" +SADD 53-1 "World" +SADD 53-1 "World" +SADD 53-2 1000 +SADD 53-2 2000 +SADD 53-2 3000 +-- SMEMBERS 53-1 +-- SORT 53-1 ALPHA + +-- SDIFFSTORE +SADD 54-1 "a" +SADD 54-1 "b" +SADD 54-1 "c" +SADD 54-2 "c" +SADD 54-2 "d" +SADD 54-2 "e" +SDIFFSTORE 54-3 54-1 54-2 +-- SMEMBERS 54-3 +-- SORT 54-3 ALPHA + +-- SETBIT +SETBIT 55-1 7 1 +SETBIT 55-1 7 0 +-- GET 55-1 + +-- SETEX +SETEX 56-1 1 "Hello" +-- GET 56-1 +SETEX 56-2 100000000 "Hello" + +-- SETNX +SETNX 57-1 "Hello" +SETNX 57-1 "World" +-- GET 57-1 + +-- SETRANGE +SET 58-1 "Hello World" +SETRANGE 58-1 6 "Redis" +-- GET 58-1 +SETRANGE 58-2 6 "Redis" +-- GET 58-2 + +-- SINTERSTORE +SADD 59-1 "a" +SADD 59-1 "b" +SADD 59-1 "c" +SADD 59-2 "c" +SADD 59-2 "d" +SADD 59-2 "e" +SINTERSTORE 59-3 59-1 59-2 +-- SMEMBERS 59-3 + +-- SMOVE +SADD 60-1 "one" +SADD 60-1 "two" +SADD 60-2 "three" +SMOVE 60-1 60-2 "two" +-- SMEMBERS 60-1 +-- SMEMBERS 60-2 + +-- SPOP +SADD 61-1 "one" +SADD 61-1 "two" +SADD 61-1 "three" +SPOP 61-1 +-- SMEMBERS 61-1 +SADD 61-1 "four" +SADD 61-1 "five" +SPOP 61-1 3 +-- SMEMBERS 61-1 + +-- SREM +SADD 62-1 "one" +SADD 62-1 "two" +SADD 62-1 "three" +SREM 62-1 "one" +SREM 62-1 "four" +-- SMEMBERS 62-1 + +-- SUNIONSTORE +SADD 63-1 "a" +SADD 63-2 "b" +SUNIONSTORE key 63-1 63-2 +-- SMEMBERS key + +-- SWAPDB +SWAPDB 0 1 + +-- UNLINK +SET 64-1 "Hello" +SET 64-2 "World" +UNLINK 64-1 64-2 64-3 + +-- -- XACK +-- XADD mystream1 1526569495631-0 message "Hello," +-- XACK mystream1 mygroup 1526569495631-0 +-- -- XRANGE mystream1 - + + +-- XADD +XADD 65-1 1526919030474-55 message "Hello," +XADD 65-1 1526919030474-* message " World!" +XADD 65-1 * name Sara surname OConnor +XADD 65-1 * field1 value1 field2 value2 field3 value3 +-- XLEN 65-1 +-- XRANGE 65-1 - + + +-- -- XAUTOCLAIM +-- XAUTOCLAIM mystream mygroup Alice 3600000 0-0 COUNT 25 + +-- -- XCLAIM +-- XCLAIM mystream mygroup Alice 3600000 1526569498055-0 + +-- XDEL +XADD 66-1 1538561700640-0 a 1 +XADD 66-1 * b 2 +XADD 66-1 * c 3 +XDEL 66-1 1538561700640-0 +-- XRANGE 66-1 - + + +-- XGROUP CREATE mystream mygroup 0 + +-- XTRIM +XTRIM 67-1 MAXLEN 1000 +XADD 67-1 * field1 A field2 B field3 C field4 D +XTRIM 67-1 MAXLEN 2 +-- XRANGE 67-1 - + + +-- ZADD +ZADD 68-1 1 "one" +ZADD 68-1 1 "uno" +ZADD 68-1 2 "two" 3 "three" +-- ZRANGE 68-1 0 -1 WITHSCORES + +-- ZDIFFSTORE +ZADD 69-1 1 "one" +ZADD 69-1 2 "two" +ZADD 69-1 3 "three" +ZADD 69-2 1 "one" +ZADD 69-2 2 "two" +ZDIFFSTORE 69-3 2 69-1 69-2 +-- ZRANGE 69-3 0 -1 WITHSCORES + +-- ZINCRBY +ZADD 70-1 1 "one" +ZADD 70-1 2 "two" +ZINCRBY 70-1 2 "one" +-- ZRANGE 70-1 0 -1 WITHSCORES + +-- ZINTERSTORE +ZADD 71-1 1 "one" +ZADD 71-1 2 "two" +ZADD 71-2 1 "one" +ZADD 71-2 2 "two" +ZADD 71-2 3 "three" +ZINTERSTORE 71-3 2 71-1 71-2 WEIGHTS 2 3 +-- ZRANGE 71-3 0 -1 WITHSCORES + +-- ZMPOP +ZADD 72-1 1 "one" 2 "two" 3 "three" +ZMPOP 1 72-1 MIN +-- ZRANGE 72-1 0 -1 WITHSCORES + +-- ZPOPMAX +ZADD 73-1 1 "one" +ZADD 73-1 2 "two" +ZADD 73-1 3 "three" +ZPOPMAX 73-1 + +-- ZPOPMIN +ZADD 74-1 1 "one" +ZADD 74-1 2 "two" +ZADD 74-1 3 "three" +ZPOPMIN 74-1 + +-- ZRANGESTORE +ZADD 75-1 1 "one" 2 "two" 3 "three" 4 "four" +ZRANGESTORE 75-2 75-1 2 -1 +-- ZRANGE 75-2 0 -1 + +-- ZREM +ZADD 76-1 1 "one" +ZADD 76-1 2 "two" +ZADD 76-1 3 "three" +ZREM 76-1 "two" +-- ZRANGE 76-1 0 -1 WITHSCORES + +-- ZREMRANGEBYLEX +ZADD 77-1 0 aaaa 0 b 0 c 0 d 0 e +ZADD 77-1 0 foo 0 zap 0 zip 0 ALPHA 0 alpha +ZREMRANGEBYLEX 77-1 [alpha [omega +ZRANGE 77-1 0 -1 + +-- ZREMRANGEBYRANK +ZADD 78-1 1 "one" +ZADD 78-1 2 "two" +ZADD 78-1 3 "three" +ZREMRANGEBYRANK 78-1 0 1 +-- ZRANGE 78-1 0 -1 WITHSCORES + +-- ZREMRANGEBYSCORE +ZADD 79-1 1 "one" +ZADD 79-1 2 "two" +ZADD 79-1 3 "three" +ZREMRANGEBYSCORE 79-1 -inf (2 +-- ZRANGE 79-1 0 -1 WITHSCORES + +-- ZUNIONSTORE +ZADD 80-1 1 "one" +ZADD 80-1 2 "two" +ZADD 80-2 1 "one" +ZADD 80-2 2 "two" +ZADD 80-2 3 "three" +ZUNIONSTORE 80-3 2 80-1 zset2 WEIGHTS 2 3 +-- ZRANGE 80-3 0 -1 WITHSCORES \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/task_config.ini b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/task_config.ini new file mode 100644 index 00000000..b3bf0091 --- /dev/null +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/rewrite_test/task_config.ini @@ -0,0 +1,35 @@ +[extractor] +db_type=redis +extract_type=snapshot +repl_port=10008 +url= + +[filter] +do_dbs= +do_events= +ignore_dbs= +ignore_tbs= +do_tbs= + +[sinker] +db_type=redis +sink_type=write +method=rewrite +url= +batch_size=2 + +[router] +db_map= +field_map= +tb_map= + +[pipeline] +buffer_size=4 +parallel_size=2 +checkpoint_interval_secs=1 +parallel_type=redis + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot/7_0/stream_test/src_dml.sql b/dt-tests/tests/redis_to_redis/snapshot/7_0/stream_test/src_dml.sql index 4f81afd3..1068b3c5 100644 --- a/dt-tests/tests/redis_to_redis/snapshot/7_0/stream_test/src_dml.sql +++ b/dt-tests/tests/redis_to_redis/snapshot/7_0/stream_test/src_dml.sql @@ -25,4 +25,37 @@ XADD 4-1 * f_1 abc f_2 "abc\r\nδΈ­ζ–‡πŸ˜€" f_1 abcdefg f_2 24YLsf3sJ0X7n3docweHr -- LP_ENCODING_32BIT_INT: (-2147483648 to βˆ’8388608) and (8388607 to 2147483647) -- LP_ENCODING_64BIT_INT: (-9223372036854775808 to -2147483648) and (2147483647 to 9223372036854775807) -- LP_ENCODING_6BIT_STR: 9223372036854775808 -XADD 5-1 * f_1 0 f_1 -1 f_2 -128 f_3 127 f_4 -4096 f_5 4095 f_6 -32768 f_7 32767 f_8 -8388607 f_9 8388607 f_10 -2147483648 f_11 2147483647 f_12 -9223372036854775808 f_13 9223372036854775807 f_14 9223372036854775808 \ No newline at end of file +XADD 5-1 * f_1 0 f_1 -1 f_2 -128 f_3 127 f_4 -4096 f_5 4095 f_6 -32768 f_7 32767 f_8 -8388607 f_9 8388607 f_10 -2147483648 f_11 2147483647 f_12 -9223372036854775808 f_13 9223372036854775807 f_14 9223372036854775808 + + + +-- multi field-value pairs for the same stream id +XADD 6-1 1538561700643-0 a b c d +-- multi stream ids for the same master key +XADD 6-1 * e f g h +XADD 6-1 * i j k l + +-- empty stream +XADD 7-1 MAXLEN 0 1538561700643-0 x y + +-- XDEL +XADD 8-1 1538561700640-0 a 1 +XADD 8-1 * b 2 +XADD 8-1 * c 3 +XDEL 8-1 1538561700640-0 + + +XADD 9-1 1538561700643-0 a b c d +XADD 9-1 1538561700643-1 e f g h +XADD 9-1 1538561700643-2 i j k l +XADD 9-1 1538561700643-3 m n o p + +-- XGROUP +XGROUP CREATE 9-1 my-group-1 0 +XGROUP CREATE 9-1 my-group-3 0 + +-- XAUTOCLAIM +XAUTOCLAIM 9-1 my-group-1 my-group-2 1 0-0 COUNT 2 + +-- XCLAIM +XCLAIM 9-1 my-group-3 my-group-4 1 1538561700643-1 \ No newline at end of file diff --git a/dt-tests/tests/redis_to_redis/snapshot_7_0_tests.rs b/dt-tests/tests/redis_to_redis/snapshot_7_0_tests.rs index c063f234..78e25fdb 100644 --- a/dt-tests/tests/redis_to_redis/snapshot_7_0_tests.rs +++ b/dt-tests/tests/redis_to_redis/snapshot_7_0_tests.rs @@ -1,5 +1,6 @@ #[cfg(test)] mod test { + use crate::test_runner::test_base::TestBase; use serial_test::serial; @@ -62,4 +63,17 @@ mod test { async fn snapshot_length_test() { TestBase::run_redis_snapshot_test("redis_to_redis/snapshot/7_0/length_test").await; } + + // test sinking rdb data by rewrite instead of restore + #[tokio::test] + #[serial] + async fn cdc_rewrite_test() { + TestBase::run_redis_snapshot_test("redis_to_redis/snapshot/7_0/rewrite_test").await; + } + + #[tokio::test] + #[serial] + async fn cdc_rewrite_stream_test() { + TestBase::run_redis_snapshot_test("redis_to_redis/snapshot/7_0/rewrite_stream_test").await; + } } diff --git a/dt-tests/tests/test_runner/rdb_test_runner.rs b/dt-tests/tests/test_runner/rdb_test_runner.rs index b31c8e3e..25defd64 100644 --- a/dt-tests/tests/test_runner/rdb_test_runner.rs +++ b/dt-tests/tests/test_runner/rdb_test_runner.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use dt_common::{ config::{ config_enums::DbType, config_token_parser::ConfigTokenParser, @@ -7,8 +9,11 @@ use dt_common::{ utils::{sql_util::SqlUtil, time_util::TimeUtil}, }; use dt_connector::rdb_query_builder::RdbQueryBuilder; -use dt_meta::{mysql::mysql_meta_manager::MysqlMetaManager, pg::pg_meta_manager::PgMetaManager}; -use dt_task::task_util::TaskUtil; +use dt_meta::{ + ddl_type::DdlType, mysql::mysql_meta_manager::MysqlMetaManager, + pg::pg_meta_manager::PgMetaManager, sql_parser::ddl_parser::DdlParser, +}; +use dt_task::{extractor_util::ExtractorUtil, task_util::TaskUtil}; use futures::TryStreamExt; use sqlx::{MySql, Pool, Postgres, Row}; @@ -26,6 +31,7 @@ pub struct RdbTestRunner { pub const SRC: &str = "src"; pub const DST: &str = "dst"; +pub const PUBLIC: &str = "public"; #[allow(dead_code)] impl RdbTestRunner { @@ -38,29 +44,58 @@ impl RdbTestRunner { let mut src_conn_pool_pg = None; let mut dst_conn_pool_pg = None; + let (src_db_type, src_url, dst_db_type, dst_url) = Self::parse_conn_info(&base); + if src_db_type == DbType::Mysql { + src_conn_pool_mysql = Some(TaskUtil::create_mysql_conn_pool(&src_url, 1, false).await?); + } else { + src_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&src_url, 1, false).await?); + } + + if dst_db_type == DbType::Mysql { + dst_conn_pool_mysql = Some(TaskUtil::create_mysql_conn_pool(&dst_url, 1, false).await?); + } else { + dst_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&dst_url, 1, false).await?); + } + + Ok(Self { + src_conn_pool_mysql, + dst_conn_pool_mysql, + src_conn_pool_pg, + dst_conn_pool_pg, + base, + }) + } + + fn parse_conn_info(base: &BaseTestRunner) -> (DbType, String, DbType, String) { + let mut src_db_type = DbType::Mysql; + let mut src_url = String::new(); + let mut dst_db_type = DbType::Mysql; + let mut dst_url = String::new(); + let config = TaskConfig::new(&base.task_config_file); match config.extractor { ExtractorConfig::MysqlCdc { url, .. } | ExtractorConfig::MysqlSnapshot { url, .. } | ExtractorConfig::MysqlCheck { url, .. } | ExtractorConfig::MysqlStruct { url, .. } => { - src_conn_pool_mysql = Some(TaskUtil::create_mysql_conn_pool(&url, 1, false).await?); + src_url = url.clone(); } ExtractorConfig::PgCdc { url, .. } | ExtractorConfig::PgSnapshot { url, .. } | ExtractorConfig::PgCheck { url, .. } | ExtractorConfig::PgStruct { url, .. } => { - src_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&url, 1, false).await?); + src_db_type = DbType::Pg; + src_url = url.clone(); } ExtractorConfig::Basic { url, db_type, .. } => match db_type { DbType::Mysql => { - src_conn_pool_mysql = - Some(TaskUtil::create_mysql_conn_pool(&url, 1, false).await?); + src_url = url.clone(); } DbType::Pg => { - src_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&url, 1, false).await?); + src_db_type = DbType::Pg; + src_url = url.clone(); } _ => {} }, @@ -72,22 +107,23 @@ impl RdbTestRunner { SinkerConfig::Mysql { url, .. } | SinkerConfig::MysqlCheck { url, .. } | SinkerConfig::MysqlStruct { url, .. } => { - dst_conn_pool_mysql = Some(TaskUtil::create_mysql_conn_pool(&url, 1, false).await?); + dst_url = url.clone(); } SinkerConfig::Pg { url, .. } | SinkerConfig::PgCheck { url, .. } | SinkerConfig::PgStruct { url, .. } => { - dst_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&url, 1, false).await?); + dst_db_type = DbType::Pg; + dst_url = url.clone(); } SinkerConfig::Basic { url, db_type, .. } => match db_type { DbType::Mysql => { - dst_conn_pool_mysql = - Some(TaskUtil::create_mysql_conn_pool(&url, 1, false).await?); + dst_url = url.clone(); } DbType::Pg => { - dst_conn_pool_pg = Some(TaskUtil::create_pg_conn_pool(&url, 1, false).await?); + dst_db_type = DbType::Pg; + dst_url = url.clone(); } _ => {} }, @@ -95,18 +131,10 @@ impl RdbTestRunner { _ => {} } - Ok(Self { - src_conn_pool_mysql, - dst_conn_pool_mysql, - src_conn_pool_pg, - dst_conn_pool_pg, - base, - }) + (src_db_type, src_url, dst_db_type, dst_url) } pub async fn run_snapshot_test(&self, compare_data: bool) -> Result<(), Error> { - let (src_tbs, dst_tbs, cols_list) = Self::parse_ddl(&self.base.src_ddl_sqls).unwrap(); - // prepare src and dst tables self.execute_test_ddl_sqls().await?; self.execute_test_dml_sqls().await?; @@ -115,19 +143,33 @@ impl RdbTestRunner { self.base.start_task().await?; // compare data + let db_tbs = self.get_compare_db_tbs().await?; if compare_data { assert!( - self.compare_data_for_tbs(&src_tbs, &dst_tbs, &cols_list) + self.compare_data_for_tbs(&db_tbs.clone(), &db_tbs.clone()) .await? - ); + ) } - Ok(()) } - pub async fn run_cdc_test(&self, start_millis: u64, parse_millis: u64) -> Result<(), Error> { - let (src_tbs, dst_tbs, cols_list) = Self::parse_ddl(&self.base.src_ddl_sqls).unwrap(); + pub async fn run_ddl_test(&self, start_millis: u64, parse_millis: u64) -> Result<(), Error> { + self.execute_test_ddl_sqls().await?; + let task = self.base.spawn_task().await?; + TimeUtil::sleep_millis(start_millis).await; + + self.execute_src_sqls(&self.base.src_dml_sqls).await?; + TimeUtil::sleep_millis(parse_millis).await; + + let db_tbs = self.get_compare_db_tbs().await?; + assert!( + self.compare_data_for_tbs(&db_tbs.clone(), &db_tbs.clone()) + .await? + ); + self.base.wait_task_finish(&task).await + } + pub async fn run_cdc_test(&self, start_millis: u64, parse_millis: u64) -> Result<(), Error> { // prepare src and dst tables self.execute_test_ddl_sqls().await?; @@ -153,26 +195,21 @@ impl RdbTestRunner { // insert src data self.execute_src_sqls(&src_insert_sqls).await?; TimeUtil::sleep_millis(start_millis).await; - assert!( - self.compare_data_for_tbs(&src_tbs, &dst_tbs, &cols_list) - .await? - ); + + let db_tbs = self.get_compare_db_tbs().await?; + let src_db_tbs = db_tbs.clone(); + let dst_db_tbs = db_tbs.clone(); + assert!(self.compare_data_for_tbs(&src_db_tbs, &dst_db_tbs).await?); // update src data self.execute_src_sqls(&src_update_sqls).await?; TimeUtil::sleep_millis(parse_millis).await; - assert!( - self.compare_data_for_tbs(&src_tbs, &dst_tbs, &cols_list) - .await? - ); + assert!(self.compare_data_for_tbs(&src_db_tbs, &dst_db_tbs).await?); // delete src data self.execute_src_sqls(&src_delete_sqls).await?; TimeUtil::sleep_millis(parse_millis).await; - assert!( - self.compare_data_for_tbs(&src_tbs, &dst_tbs, &cols_list) - .await? - ); + assert!(self.compare_data_for_tbs(&src_db_tbs, &dst_db_tbs).await?); self.base.wait_task_finish(&task).await } @@ -255,9 +292,8 @@ impl RdbTestRunner { async fn compare_data_for_tbs( &self, - src_tbs: &Vec, - dst_tbs: &Vec, - cols_list: &Vec>, + src_db_tbs: &Vec<(String, String)>, + dst_db_tbs: &Vec<(String, String)>, ) -> Result { let filtered_tbs_file = format!("{}/filtered_tbs.txt", &self.base.test_dir); let filtered_tbs = if BaseTestRunner::check_file_exists(&filtered_tbs_file) { @@ -266,16 +302,14 @@ impl RdbTestRunner { Vec::new() }; - for i in 0..src_tbs.len() { - let src_tb = &src_tbs[i]; - let dst_tb = &dst_tbs[i]; - if filtered_tbs.contains(src_tb) { - let dst_data = self.fetch_data(&dst_tb, DST).await?; + for i in 0..src_db_tbs.len() { + if filtered_tbs.contains(&format!("{}.{}", &src_db_tbs[i].0, &src_db_tbs[i].1)) { + let dst_data = self.fetch_data(&dst_db_tbs[i], DST).await?; if dst_data.len() > 0 { return Ok(false); } } else { - if !self.compare_tb_data(src_tb, dst_tb, &cols_list[i]).await? { + if !self.compare_tb_data(&src_db_tbs[i], &dst_db_tbs[i]).await? { return Ok(false); } } @@ -285,22 +319,22 @@ impl RdbTestRunner { async fn compare_tb_data( &self, - src_tb: &str, - dst_tb: &str, - cols: &Vec, + src_db_tb: &(String, String), + dst_db_tb: &(String, String), ) -> Result { - let src_data = self.fetch_data(src_tb, SRC).await?; - let dst_data = self.fetch_data(dst_tb, DST).await?; + let src_data = self.fetch_data(src_db_tb, SRC).await?; + let dst_data = self.fetch_data(dst_db_tb, DST).await?; println!( - "comparing row data for src_tb: {}, src_data count: {}", - src_tb, + "comparing row data for src_tb: {:?}, src_data count: {}", + src_db_tb, src_data.len() ); - if !Self::compare_row_data(cols, &src_data, &dst_data) { + let cols = self.get_tb_cols(src_db_tb).await?; + if !Self::compare_row_data(&cols, &src_data, &dst_data) { println!( - "compare_tb_data failed, src_tb: {}, dst_tb: {}", - src_tb, dst_tb + "compare tb data failed, src_tb: {:?}, dst_tb: {:?}", + src_db_tb, dst_db_tb, ); return Ok(false); } @@ -322,8 +356,8 @@ impl RdbTestRunner { let dst_col_value = &dst_row_data[j]; if src_col_value != dst_col_value { println!( - "col: {}, src_col_value: {:?}, dst_col_value: {:?}", - cols[j], src_col_value, dst_col_value + "row index: {}, col: {}, src_col_value: {:?}, dst_col_value: {:?}", + i, cols[j], src_col_value, dst_col_value ); return false; } @@ -334,7 +368,7 @@ impl RdbTestRunner { pub async fn fetch_data( &self, - full_tb_name: &str, + db_tb: &(String, String), from: &str, ) -> Result>>>, Error> { let conn_pool_mysql = if from == SRC { @@ -350,16 +384,16 @@ impl RdbTestRunner { }; let data = if let Some(pool) = conn_pool_mysql { - self.fetch_data_mysql(full_tb_name, pool).await? + self.fetch_data_mysql(db_tb, pool).await? } else if let Some(pool) = conn_pool_pg { - self.fetch_data_pg(full_tb_name, pool).await? + self.fetch_data_pg(db_tb, pool).await? } else { Vec::new() }; Ok(data) } - fn parse_full_tb_name(full_tb_name: &str, db_type: DbType) -> (String, String) { + pub fn parse_full_tb_name(full_tb_name: &str, db_type: &DbType) -> (String, String) { let escape_pairs = SqlUtil::get_escape_pairs(&db_type); let tokens = ConfigTokenParser::parse(full_tb_name, &vec!['.'], &escape_pairs); let (db, tb) = if tokens.len() > 1 { @@ -376,22 +410,21 @@ impl RdbTestRunner { async fn fetch_data_mysql( &self, - full_tb_name: &str, + db_tb: &(String, String), conn_pool: &Pool, ) -> Result>>>, Error> { - let (db, tb) = Self::parse_full_tb_name(full_tb_name, DbType::Mysql); - let mut meta_manager = MysqlMetaManager::new(conn_pool.clone()).init().await?; - let tb_meta = meta_manager.get_tb_meta(&db, &tb).await?; - let cols = &tb_meta.basic.cols; - - let sql = format!("SELECT * FROM `{}`.`{}` ORDER BY `{}`", &db, &tb, &cols[0],); + let cols = self.get_tb_cols(db_tb).await?; + let sql = format!( + "SELECT * FROM `{}`.`{}` ORDER BY `{}`", + &db_tb.0, &db_tb.1, &cols[0], + ); let query = sqlx::query(&sql); let mut rows = query.fetch(conn_pool); let mut result = Vec::new(); while let Some(row) = rows.try_next().await.unwrap() { let mut row_values = Vec::new(); - for col in cols { + for col in cols.iter() { let value: Option> = row.get_unchecked(col.as_str()); row_values.push(value); } @@ -403,22 +436,18 @@ impl RdbTestRunner { async fn fetch_data_pg( &self, - full_tb_name: &str, + db_tb: &(String, String), conn_pool: &Pool, ) -> Result>>>, Error> { - let (mut db, tb) = Self::parse_full_tb_name(full_tb_name, DbType::Pg); - if db.is_empty() { - db = "public".to_string(); - } let mut meta_manager = PgMetaManager::new(conn_pool.clone()).init().await?; - let tb_meta = meta_manager.get_tb_meta(&db, &tb).await?; - let cols = &tb_meta.basic.cols; + let tb_meta = meta_manager.get_tb_meta(&db_tb.0, &db_tb.1).await?; let query_builder = RdbQueryBuilder::new_for_pg(&tb_meta); - let cols_str = query_builder.build_extract_cols_str().unwrap(); + let cols = self.get_tb_cols(&db_tb).await?; + let cols_str = query_builder.build_extract_cols_str().unwrap(); let sql = format!( r#"SELECT {} FROM "{}"."{}" ORDER BY "{}" ASC"#, - cols_str, &db, &tb, &cols[0], + cols_str, &db_tb.0, &db_tb.1, &cols[0], ); let query = sqlx::query(&sql); let mut rows = query.fetch(conn_pool); @@ -426,7 +455,7 @@ impl RdbTestRunner { let mut result = Vec::new(); while let Some(row) = rows.try_next().await.unwrap() { let mut row_values = Vec::new(); - for col in cols { + for col in cols.iter() { let value: Option> = row.get_unchecked(col.as_str()); row_values.push(value); } @@ -454,73 +483,71 @@ impl RdbTestRunner { Ok(()) } - fn parse_ddl( - src_ddl_sqls: &Vec, - ) -> Result<(Vec, Vec, Vec>), Error> { - let mut src_tbs = Vec::new(); - let mut cols_list = Vec::new(); - for sql in src_ddl_sqls { - if sql.to_lowercase().contains("create table") { - let (tb, cols) = Self::parse_create_table(&sql).unwrap(); - src_tbs.push(tb); - cols_list.push(cols); + /// get compare dbs from src_ddl_sqls + async fn get_compare_dbs(&self, url: &str, db_type: &DbType) -> Result, Error> { + let all_dbs = ExtractorUtil::list_dbs(url, db_type).await?; + let mut dbs = HashSet::new(); + for sql in self.base.src_ddl_sqls.iter() { + if !sql.to_lowercase().contains("database") { + continue; + } + let ddl = DdlParser::parse(sql).unwrap(); + if ddl.0 == DdlType::DropDatabase || ddl.0 == DdlType::CreateDatabase { + let db = ddl.1.unwrap(); + // db may be dropped in test sql (ddl test) + if all_dbs.contains(&db) { + dbs.insert(db); + } } } - let dst_tbs = src_tbs.clone(); - - Ok((src_tbs, dst_tbs, cols_list)) + Ok(dbs) } - fn parse_create_table(sql: &str) -> Result<(String, Vec), Error> { - // since both (sqlparser = "0.30.0", sql-parse = "0.11.0") have bugs in parsing create table sql, - // we take a simple workaround to get table name and column names. - - // CREATE TABLE bitbin_table (pk SERIAL, bvunlimited1 BIT VARYING, p GEOMETRY(POINT,3187), PRIMARY KEY(pk)); - let mut tokens = Vec::new(); - let mut token = String::new(); - let mut brakets = 0; - - for b in sql.as_bytes() { - let c = char::from(*b); - if c == ' ' || c == '(' || c == ')' || c == ',' { - if !token.is_empty() { - tokens.push(token.clone()); - token.clear(); + /// get compare tbs + async fn get_compare_db_tbs(&self) -> Result, Error> { + let mut db_tbs = vec![]; + let (src_db_type, src_url, _, _) = Self::parse_conn_info(&self.base); + + if src_db_type == DbType::Mysql { + let dbs = self.get_compare_dbs(&src_url, &src_db_type).await?; + for db in dbs.iter() { + let tbs = ExtractorUtil::list_tbs(&src_url, db, &src_db_type).await?; + for tb in tbs.iter() { + db_tbs.push((db.to_string(), tb.to_string())); } - - if c == '(' { - brakets += 1; - } else if c == ')' { - brakets -= 1; - if brakets == 0 { - break; - } + } + } else { + for sql in self.base.src_ddl_sqls.iter() { + if !sql.to_lowercase().contains("table") { + continue; } - - if c == ',' && brakets == 1 { - tokens.push(",".to_string()); + let ddl = DdlParser::parse(sql).unwrap(); + if ddl.0 == DdlType::CreateTable { + let db = if let Some(db) = ddl.1 { + db + } else { + PUBLIC.to_string() + }; + let tb = ddl.2.unwrap(); + db_tbs.push((db, tb)); } - continue; - } - token.push(c); - } - - let tb = tokens[2].clone(); - let mut cols = vec![tokens[3].clone()]; - for i in 3..tokens.len() { - let token_lowercase = tokens[i].to_lowercase(); - if token_lowercase == "primary" - || token_lowercase == "unique" - || token_lowercase == "key" - { - break; - } - - if tokens[i - 1] == "," { - cols.push(tokens[i].clone()); } } + Ok(db_tbs) + } - Ok((tb, cols)) + async fn get_tb_cols(&self, db_tb: &(String, String)) -> Result, Error> { + let cols = if let Some(conn_pool) = &self.src_conn_pool_mysql { + let mut meta_manager = MysqlMetaManager::new(conn_pool.clone()).init().await?; + let tb_meta = meta_manager.get_tb_meta(&db_tb.0, &db_tb.1).await?; + tb_meta.basic.cols.clone() + } else if let Some(conn_pool) = &self.src_conn_pool_pg { + let mut meta_manager = PgMetaManager::new(conn_pool.clone()).init().await?; + let tb_meta = meta_manager.get_tb_meta(&db_tb.0, &db_tb.1).await?; + tb_meta.basic.cols.clone() + } else { + vec![] + }; + Ok(cols) } } diff --git a/dt-tests/tests/test_runner/redis_test_runner.rs b/dt-tests/tests/test_runner/redis_test_runner.rs index edefc706..e4a4c773 100644 --- a/dt-tests/tests/test_runner/redis_test_runner.rs +++ b/dt-tests/tests/test_runner/redis_test_runner.rs @@ -45,18 +45,14 @@ impl RedisTestRunner { TaskUtil::create_redis_conn(&url).await.unwrap() } _ => { - return Err(Error::Unexpected { - error: "extractor config error".to_string(), - }); + return Err(Error::ConfigError("unsupported extractor config".into())); } }; let dst_conn = match config.sinker { SinkerConfig::Redis { url, .. } => TaskUtil::create_redis_conn(&url).await.unwrap(), _ => { - return Err(Error::Unexpected { - error: "sinker config error".to_string(), - }); + return Err(Error::ConfigError("unsupported sinker config".into())); } }; diff --git a/dt-tests/tests/test_runner/test_base.rs b/dt-tests/tests/test_runner/test_base.rs index cf3fbfb9..28960c10 100644 --- a/dt-tests/tests/test_runner/test_base.rs +++ b/dt-tests/tests/test_runner/test_base.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, HashSet}; +use dt_common::config::config_enums::DbType; use futures::executor::block_on; use crate::test_runner::rdb_test_runner::DST; @@ -21,18 +22,24 @@ impl TestBase { pub async fn run_snapshot_test_and_check_dst_count( test_dir: &str, + db_type: &DbType, dst_expected_counts: HashMap<&str, usize>, ) { let runner = RdbTestRunner::new(test_dir).await.unwrap(); runner.run_snapshot_test(false).await.unwrap(); - let assert_dst_count = |tb: &str, count: usize| { - let dst_data = block_on(runner.fetch_data(tb, DST)).unwrap(); + let assert_dst_count = |db_tb: &(String, String), count: usize| { + let dst_data = block_on(runner.fetch_data(db_tb, DST)).unwrap(); + println!( + "check dst table {:?} record count, expect: {}", + db_tb, count + ); assert_eq!(dst_data.len(), count); }; - for (tb, count) in dst_expected_counts.iter() { - assert_dst_count(tb, *count); + for (db_tb, count) in dst_expected_counts { + let db_tb = RdbTestRunner::parse_full_tb_name(db_tb, db_type); + assert_dst_count(&db_tb, count); } } @@ -46,6 +53,14 @@ impl TestBase { // .unwrap(); } + pub async fn run_ddl_test(test_dir: &str, start_millis: u64, parse_millis: u64) { + let runner = RdbTestRunner::new(test_dir).await.unwrap(); + runner + .run_ddl_test(start_millis, parse_millis) + .await + .unwrap(); + } + pub async fn run_check_test(test_dir: &str) { let runner = RdbCheckTestRunner::new(test_dir).await.unwrap(); runner.run_check_test().await.unwrap();