From fb721c15e2838e51167c4e42fbec2dd5a3133c1a Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Fri, 25 Mar 2022 17:46:34 +0800 Subject: [PATCH] webui(dm): add many fix of webui (#4946) (#5024) close pingcap/tiflow#4956, close pingcap/tiflow#4993 --- dm/_utils/terror_gen/errors_release.txt | 2 +- dm/dm/config/task_converters.go | 70 ++++--- dm/dm/config/task_converters_test.go | 185 +++++++++++++++--- dm/dm/master/openapi_controller.go | 22 ++- dm/errors.toml | 2 +- dm/openapi/gen.server.go | 68 +++---- dm/openapi/gen.types.go | 2 +- dm/openapi/spec/dm.yaml | 2 +- dm/pkg/terror/error_list.go | 2 +- dm/syncer/syncer.go | 3 +- dm/ui/locales/en.json | 8 +- dm/ui/locales/zh.json | 6 - dm/ui/package.json | 2 +- .../CreateOrUpdateTask/MigrateRule.tsx | 4 + .../components/CreateOrUpdateTask/index.tsx | 23 ++- dm/ui/src/models/source.ts | 2 + dm/ui/src/models/task.ts | 7 + dm/ui/src/pages/cluster/member.tsx | 90 +-------- .../pages/migration/replication-detail.tsx | 100 +++++++--- dm/ui/src/pages/migration/source.tsx | 2 +- dm/ui/yarn.lock | 35 ++-- tools/check/go.sum | 1 + 22 files changed, 383 insertions(+), 255 deletions(-) diff --git a/dm/_utils/terror_gen/errors_release.txt b/dm/_utils/terror_gen/errors_release.txt index 96056a0afe1..306d8fadcad 100644 --- a/dm/_utils/terror_gen/errors_release.txt +++ b/dm/_utils/terror_gen/errors_release.txt @@ -541,7 +541,7 @@ ErrSchedulerRelayWorkersWrongBound,[code=46021:class=scheduler:scope=internal:le ErrSchedulerRelayWorkersWrongRelay,[code=46022:class=scheduler:scope=internal:level=high], "Message: these workers %s have started relay for another sources %s respectively, Workaround: Please correct sources in `stop-relay`." ErrSchedulerSourceOpRelayExist,[code=46023:class=scheduler:scope=internal:level=high], "Message: source with name %s need to operate has existing relay workers %s, Workaround: Please `stop-relay` first." ErrSchedulerLatchInUse,[code=46024:class=scheduler:scope=internal:level=low], "Message: when %s, resource %s is in use by other client, Workaround: Please try again later" -ErrSchedulerSourceCfgUpdate,[code=46025:class=scheduler:scope=internal:level=low], "Message: source can only update when not enable relay and no running tasks for now" +ErrSchedulerSourceCfgUpdate,[code=46025:class=scheduler:scope=internal:level=low], "Message: source %s can only be updated when relay is disabled and no tasks are running for now" ErrSchedulerWrongWorkerInput,[code=46026:class=scheduler:scope=internal:level=medium], "Message: require DM master to modify worker [%s] with source [%s], but currently the worker is bound to source [%s]" ErrSchedulerBoundDiffWithStartedRelay,[code=46027:class=scheduler:scope=internal:level=medium], "Message: require DM worker [%s] to be bound to source [%s], but it has been started relay for source [%s], Workaround: If you intend to bind the source with worker, you can stop-relay for current source." ErrSchedulerStartRelayOnSpecified,[code=46028:class=scheduler:scope=internal:level=low], "Message: the source has `start-relay` with worker name for workers %v, so it can't `start-relay` without worker name now, Workaround: Please stop all relay workers first, or specify worker name for `start-relay`." diff --git a/dm/dm/config/task_converters.go b/dm/dm/config/task_converters.go index a0fc4292e93..cd6760bd283 100644 --- a/dm/dm/config/task_converters.go +++ b/dm/dm/config/task_converters.go @@ -149,7 +149,7 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *DBConfig, sourceCf subTaskCfg.Name = task.Name subTaskCfg.Mode = string(task.TaskMode) // set task meta - subTaskCfg.MetaFile = *task.MetaSchema + subTaskCfg.MetaSchema = *task.MetaSchema // add binlog meta if sourceCfg.BinlogGtid != nil || sourceCfg.BinlogName != nil || sourceCfg.BinlogPos != nil { meta := &Meta{} @@ -186,7 +186,6 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *DBConfig, sourceCf subTaskCfg.From = sourceCfgMap[sourceCfg.SourceName].From // set target db config subTaskCfg.To = *toDBCfg.Clone() - // TODO set meet error policy // TODO ExprFilter // set full unit config subTaskCfg.MydumperConfig = DefaultMydumperConfig() @@ -218,13 +217,11 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *DBConfig, sourceCf } subTaskCfg.ValidatorCfg = defaultValidatorConfig() // set route,blockAllowList,filter config - doCnt := len(tableMigrateRuleMap[sourceCfg.SourceName]) - doDBs := make([]string, doCnt) - doTables := make([]*filter.Table, doCnt) - + doDBs := []string{} + doTables := []*filter.Table{} routeRules := []*router.TableRule{} filterRules := []*bf.BinlogEventRule{} - for j, rule := range tableMigrateRuleMap[sourceCfg.SourceName] { + for _, rule := range tableMigrateRuleMap[sourceCfg.SourceName] { // route if rule.Target != nil && (rule.Target.Schema != nil || rule.Target.Table != nil) { tableRule := &router.TableRule{SchemaPattern: rule.Source.Schema, TablePattern: rule.Source.Table} @@ -244,17 +241,31 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *DBConfig, sourceCf return nil, terror.ErrOpenAPICommonError.Generatef("filter rule name %s not found.", name) } filterRule.SchemaPattern = rule.Source.Schema - filterRule.TablePattern = rule.Source.Table + if rule.Source.Table != "" { + filterRule.TablePattern = rule.Source.Table + } filterRules = append(filterRules, &filterRule) } } // BlockAllowList - doDBs[j] = rule.Source.Schema - doTables[j] = &filter.Table{Schema: rule.Source.Schema, Name: rule.Source.Table} + if rule.Source.Table != "" { + doTables = append(doTables, &filter.Table{Schema: rule.Source.Schema, Name: rule.Source.Table}) + } else { + doDBs = append(doDBs, rule.Source.Schema) + } } subTaskCfg.RouteRules = routeRules subTaskCfg.FilterRules = filterRules - subTaskCfg.BAList = &filter.Rules{DoDBs: removeDuplication(doDBs), DoTables: doTables} + if len(doDBs) > 0 || len(doTables) > 0 { + bAList := &filter.Rules{} + if len(doDBs) > 0 { + bAList.DoDBs = removeDuplication(doDBs) + } + if len(doTables) > 0 { + bAList.DoTables = doTables + } + subTaskCfg.BAList = bAList + } // adjust sub task config if err := subTaskCfg.Adjust(true); err != nil { return nil, terror.Annotatef(err, "source name %s", sourceCfg.SourceName) @@ -513,6 +524,8 @@ func SubTaskConfigsToOpenAPITask(subTaskConfigList []*SubTaskConfig) *openapi.Ta } // set table migrate rules tableMigrateRuleList := []openapi.TaskTableMigrateRule{} + // used to remove repeated rules + ruleMap := map[string]struct{}{} appendOneRule := func(sourceName, schemaPattern, tablePattern, targetSchema, targetTable string) { tableMigrateRule := openapi.TaskTableMigrateRule{ Source: struct { @@ -525,13 +538,15 @@ func SubTaskConfigsToOpenAPITask(subTaskConfigList []*SubTaskConfig) *openapi.Ta Table: tablePattern, }, } - if targetSchema != "" || targetTable != "" { + if targetSchema != "" { tableMigrateRule.Target = &struct { Schema *string `json:"schema,omitempty"` Table *string `json:"table,omitempty"` }{ Schema: &targetSchema, - Table: &targetTable, + } + if targetTable != "" { + tableMigrateRule.Target.Table = &targetTable } } if filterRuleList, ok := filterMap[sourceName]; ok { @@ -541,26 +556,35 @@ func SubTaskConfigsToOpenAPITask(subTaskConfigList []*SubTaskConfig) *openapi.Ta } tableMigrateRule.BinlogFilterRule = &ruleNameList } + ruleKey := strings.Join([]string{sourceName, schemaPattern, tablePattern}, "-") + if _, ok := ruleMap[ruleKey]; ok { + return + } + ruleMap[ruleKey] = struct{}{} tableMigrateRuleList = append(tableMigrateRuleList, tableMigrateRule) } - + // gen migrate rules by route for sourceName, ruleList := range routeMap { for _, rule := range ruleList { appendOneRule(sourceName, rule.SchemaPattern, rule.TablePattern, rule.TargetSchema, rule.TargetTable) } } - // for user only set BlockAllowList without route rules, this means keep same with upstream db and table - if len(tableMigrateRuleList) == 0 { - for _, cfg := range subTaskConfigList { - if cfg.BAList != nil { - for idx := range cfg.BAList.DoTables { - schemaPattern := cfg.BAList.DoTables[idx].Schema - tablePattern := cfg.BAList.DoTables[idx].Name - appendOneRule(cfg.SourceID, schemaPattern, tablePattern, "", "") - } + + // gen migrate rules by BAList + for _, cfg := range subTaskConfigList { + if cfg.BAList != nil { + for idx := range cfg.BAList.DoDBs { + schemaPattern := cfg.BAList.DoDBs[idx] + appendOneRule(cfg.SourceID, schemaPattern, "", "", "") + } + for idx := range cfg.BAList.DoTables { + schemaPattern := cfg.BAList.DoTables[idx].Schema + tablePattern := cfg.BAList.DoTables[idx].Name + appendOneRule(cfg.SourceID, schemaPattern, tablePattern, "", "") } } } + // set basic global config task := openapi.Task{ Name: oneSubtaskConfig.Name, diff --git a/dm/dm/config/task_converters_test.go b/dm/dm/config/task_converters_test.go index c69b8553d0d..d7feb6fdebe 100644 --- a/dm/dm/config/task_converters_test.go +++ b/dm/dm/config/task_converters_test.go @@ -65,6 +65,9 @@ func testNoShardTaskToSubTaskConfigs(c *check.C) { User: task.TargetConfig.User, Password: task.TargetConfig.Password, } + // change meta + newMeta := "new_dm_meta" + task.MetaSchema = &newMeta subTaskConfigList, err := OpenAPITaskToSubTaskConfigs(&task, toDBCfg, sourceCfgMap) c.Assert(err, check.IsNil) c.Assert(subTaskConfigList, check.HasLen, 1) @@ -109,7 +112,6 @@ func testNoShardTaskToSubTaskConfigs(c *check.C) { // check balist c.Assert(subTaskConfig.BAList, check.NotNil) bAListFromOpenAPITask := &filter.Rules{ - DoDBs: []string{sourceSchema}, DoTables: []*filter.Table{{Schema: sourceSchema, Name: sourceTable}}, } c.Assert(subTaskConfig.BAList, check.DeepEquals, bAListFromOpenAPITask) @@ -197,7 +199,6 @@ func testShardAndFilterTaskToSubTaskConfigs(c *check.C) { // check balist c.Assert(subTask1Config.BAList, check.NotNil) bAListFromOpenAPITask := &filter.Rules{ - DoDBs: []string{source1Schema}, DoTables: []*filter.Table{{Schema: source1Schema, Name: source1Table}}, } c.Assert(subTask1Config.BAList, check.DeepEquals, bAListFromOpenAPITask) @@ -245,7 +246,6 @@ func testShardAndFilterTaskToSubTaskConfigs(c *check.C) { // check balist c.Assert(subTask2Config.BAList, check.NotNil) bAListFromOpenAPITask = &filter.Rules{ - DoDBs: []string{source2Schema}, DoTables: []*filter.Table{{Schema: source2Schema, Name: source2Table}}, } c.Assert(subTask2Config.BAList, check.DeepEquals, bAListFromOpenAPITask) @@ -350,39 +350,50 @@ func TestConvertBetweenOpenAPITaskAndTaskConfig(t *testing.T) { source1Name := task.SourceConfig.SourceConf[0].SourceName sourceCfg1.SourceID = source1Name sourceCfgMap := map[string]*SourceConfig{source1Name: sourceCfg1} + taskCfg, err := OpenAPITaskToTaskConfig(&task, sourceCfgMap) require.NoError(t, err) require.NotNil(t, taskCfg) - task1, err := TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) require.NoError(t, err) require.NotNil(t, task1) require.EqualValues(t, task1, &task) // test update some fields in task - batch := 1000 - task.SourceConfig.IncrMigrateConf.ReplBatch = &batch - taskCfg1, err := OpenAPITaskToTaskConfig(&task, sourceCfgMap) - require.NoError(t, err) - require.Equal(t, batch, taskCfg1.MySQLInstances[0].Syncer.Batch) - for _, cfg := range taskCfg1.Syncers { - require.Equal(t, batch, cfg.Batch) - } + { + batch := 1000 + task.SourceConfig.IncrMigrateConf.ReplBatch = &batch + taskCfg2, err2 := OpenAPITaskToTaskConfig(&task, sourceCfgMap) + require.NoError(t, err2) + require.Equal(t, batch, taskCfg2.MySQLInstances[0].Syncer.Batch) + for _, cfg := range taskCfg2.Syncers { + require.Equal(t, batch, cfg.Batch) + } - // test update some fields in taskConfig - batch = 1 - for _, cfg := range taskCfg1.Syncers { - cfg.Batch = batch + // test update some fields in taskConfig + batch = 1 + for _, cfg := range taskCfg2.Syncers { + cfg.Batch = batch + } + task2, err3 := TaskConfigToOpenAPITask(taskCfg2, sourceCfgMap) + require.NoError(t, err3) + require.Equal(t, batch, *task2.SourceConfig.IncrMigrateConf.ReplBatch) } - task2, err := TaskConfigToOpenAPITask(taskCfg1, sourceCfgMap) - require.NoError(t, err) - require.Equal(t, batch, *task2.SourceConfig.IncrMigrateConf.ReplBatch) - // test more complex case + // test update route { require.Len(t, task.TableMigrateRule, 1) + sourceSchema := task.TableMigrateRule[0].Source.Schema + targetSchema := *task.TableMigrateRule[0].Target.Schema // only route schema - targetSchema := "db1" + task.TableMigrateRule[0].Source = struct { + Schema string `json:"schema"` + SourceName string `json:"source_name"` + Table string `json:"table"` + }{ + SourceName: source1Name, + Schema: sourceSchema, + } task.TableMigrateRule[0].Target = &struct { Schema *string `json:"schema,omitempty"` Table *string `json:"table,omitempty"` @@ -396,10 +407,39 @@ func TestConvertBetweenOpenAPITaskAndTaskConfig(t *testing.T) { for k := range taskCfg.Routes { routeKey = k } + + // validate route rule in taskCfg + require.Equal(t, sourceSchema, taskCfg.Routes[routeKey].SchemaPattern) + require.Equal(t, "", taskCfg.Routes[routeKey].TablePattern) require.Equal(t, targetSchema, taskCfg.Routes[routeKey].TargetSchema) + require.Equal(t, "", taskCfg.Routes[routeKey].TargetTable) + + // validate baList in taskCfg, just do DBs because there is no table route rule + require.Len(t, taskCfg.BAList, 1) + require.Len(t, taskCfg.MySQLInstances, 1) + baListName := taskCfg.MySQLInstances[0].BAListName + require.Len(t, taskCfg.BAList[baListName].DoTables, 0) + require.Len(t, taskCfg.BAList[baListName].DoDBs, 1) + require.Equal(t, sourceSchema, taskCfg.BAList[baListName].DoDBs[0]) + + // convert back to openapi.Task + taskAfterConvert, err2 := TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) + require.NoError(t, err2) + require.NotNil(t, taskAfterConvert) + require.EqualValues(t, taskAfterConvert, &task) // only route table will meet error + sourceTable := "tb" targetTable := "tb1" + task.TableMigrateRule[0].Source = struct { + Schema string `json:"schema"` + SourceName string `json:"source_name"` + Table string `json:"table"` + }{ + SourceName: source1Name, + Schema: sourceSchema, + Table: sourceTable, + } task.TableMigrateRule[0].Target = &struct { Schema *string `json:"schema,omitempty"` Table *string `json:"table,omitempty"` @@ -409,19 +449,100 @@ func TestConvertBetweenOpenAPITaskAndTaskConfig(t *testing.T) { _, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) require.True(t, terror.ErrConfigGenTableRouter.Equal(err)) - // remove route target (meanus only use block-allow-list to sync) + // route both + task.TableMigrateRule[0].Source = struct { + Schema string `json:"schema"` + SourceName string `json:"source_name"` + Table string `json:"table"` + }{ + SourceName: source1Name, + Schema: sourceSchema, + Table: sourceTable, + } + task.TableMigrateRule[0].Target = &struct { + Schema *string `json:"schema,omitempty"` + Table *string `json:"table,omitempty"` + }{ + Schema: &targetSchema, + Table: &targetTable, + } + taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) + require.NoError(t, err) + + // validate route rule in taskCfg + require.Equal(t, sourceSchema, taskCfg.Routes[routeKey].SchemaPattern) + require.Equal(t, sourceTable, taskCfg.Routes[routeKey].TablePattern) + require.Equal(t, targetSchema, taskCfg.Routes[routeKey].TargetSchema) + require.Equal(t, targetTable, taskCfg.Routes[routeKey].TargetTable) + + // validate baList in taskCfg, just do Tables because there is a table route rule + require.Len(t, taskCfg.BAList[baListName].DoDBs, 0) + require.Len(t, taskCfg.BAList[baListName].DoTables, 1) + require.Equal(t, sourceSchema, taskCfg.BAList[baListName].DoTables[0].Schema) + require.Equal(t, sourceTable, taskCfg.BAList[baListName].DoTables[0].Name) + + // convert back to openapi.Task + taskAfterConvert, err = TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) + require.NoError(t, err) + require.NotNil(t, taskAfterConvert) + require.EqualValues(t, taskAfterConvert, &task) + + // no route and only sync one table task.TableMigrateRule[0].Target = nil - taskCfg, err := OpenAPITaskToTaskConfig(&task, sourceCfgMap) + taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) require.NoError(t, err) require.Len(t, taskCfg.Routes, 0) - taskAfterConvert, err := TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) + // validate baList in taskCfg, just do Tables because there is a table route rule + require.Len(t, taskCfg.BAList[baListName].DoDBs, 0) + require.Len(t, taskCfg.BAList[baListName].DoTables, 1) + require.Equal(t, sourceSchema, taskCfg.BAList[baListName].DoTables[0].Schema) + require.Equal(t, sourceTable, taskCfg.BAList[baListName].DoTables[0].Name) + + taskAfterConvert, err = TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) + require.NoError(t, err) + require.NotNil(t, taskAfterConvert) + require.EqualValues(t, taskAfterConvert, &task) + + // no route and sync one schema + task.TableMigrateRule[0].Source = struct { + Schema string `json:"schema"` + SourceName string `json:"source_name"` + Table string `json:"table"` + }{ + SourceName: source1Name, + Schema: sourceSchema, + Table: "", + } + taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) + require.NoError(t, err) + require.Len(t, taskCfg.Routes, 0) + + // validate baList in taskCfg, just do DBs because there is no table route rule + require.Len(t, taskCfg.BAList[baListName].DoTables, 0) + require.Len(t, taskCfg.BAList[baListName].DoDBs, 1) + require.Equal(t, sourceSchema, taskCfg.BAList[baListName].DoDBs[0]) + + taskAfterConvert, err = TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) require.NoError(t, err) require.NotNil(t, taskAfterConvert) require.EqualValues(t, taskAfterConvert, &task) + } - // add filter + // test update filter + { + // no filter no change require.Nil(t, task.BinlogFilterRule) + taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) + require.NoError(t, err) + taskAfterConvert, err := TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) + require.NoError(t, err) + require.NotNil(t, taskAfterConvert) + require.EqualValues(t, taskAfterConvert, &task) + + // filter both + sourceSchema := task.TableMigrateRule[0].Source.Schema + sourceTable := task.TableMigrateRule[0].Source.Table ignoreEvent := []string{"drop database"} ignoreSQL := []string{"^Drop"} ruleName := genFilterRuleName(source1Name, 0) @@ -434,6 +555,18 @@ func TestConvertBetweenOpenAPITaskAndTaskConfig(t *testing.T) { taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) require.NoError(t, err) + require.Len(t, taskCfg.Filters, 1) + require.Len(t, taskCfg.MySQLInstances[0].FilterRules, 1) + filterName := taskCfg.MySQLInstances[0].FilterRules[0] + require.Equal(t, bf.Ignore, taskCfg.Filters[filterName].Action) + require.Equal(t, sourceSchema, taskCfg.Filters[filterName].SchemaPattern) + require.Equal(t, sourceTable, taskCfg.Filters[filterName].TablePattern) + require.Len(t, taskCfg.Filters[filterName].SQLPattern, 1) + require.Equal(t, ignoreSQL[0], taskCfg.Filters[filterName].SQLPattern[0]) + require.Len(t, taskCfg.Filters[filterName].Events, 1) + require.Equal(t, ignoreEvent[0], string(taskCfg.Filters[filterName].Events[0])) + + // convert back to openapi.Task taskAfterConvert, err = TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) require.NoError(t, err) require.NotNil(t, taskAfterConvert) @@ -444,6 +577,8 @@ func TestConvertBetweenOpenAPITaskAndTaskConfig(t *testing.T) { ruleM.Set(ruleName, rule) taskCfg, err = OpenAPITaskToTaskConfig(&task, sourceCfgMap) require.NoError(t, err) + require.Len(t, taskCfg.Filters[filterName].SQLPattern, 0) + taskAfterConvert, err = TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) ruleMAfterConvert := taskAfterConvert.BinlogFilterRule ruleAfterConvert, ok := ruleMAfterConvert.Get(ruleName) diff --git a/dm/dm/master/openapi_controller.go b/dm/dm/master/openapi_controller.go index 4435ce84ae7..9a52cc40b21 100644 --- a/dm/dm/master/openapi_controller.go +++ b/dm/dm/master/openapi_controller.go @@ -86,6 +86,7 @@ func (s *Server) createSource(ctx context.Context, req openapi.CreateSourceReque if err != nil { return nil, err } + // TODO: refine relay logic https://github.com/pingcap/tiflow/issues/4985 if cfg.EnableRelay { return &req.Source, s.enableRelay(ctx, req.Source.SourceName, openapi.EnableRelayRequest{}) } @@ -653,9 +654,22 @@ func (s *Server) convertTaskConfig(ctx context.Context, req openapi.ConverterTas if err := taskCfg.RawDecode(*req.TaskConfigFile); err != nil { return nil, nil, err } + // clear extra config in MySQLInstance, use cfg.xxConfigName instead otherwise adjust will fail + for _, cfg := range taskCfg.MySQLInstances { + cfg.Mydumper = nil + cfg.Loader = nil + cfg.Syncer = nil + } + if adjustErr := taskCfg.Adjust(); adjustErr != nil { + return nil, nil, adjustErr + } sourceCfgMap := make(map[string]*config.SourceConfig, len(taskCfg.MySQLInstances)) for _, source := range taskCfg.MySQLInstances { - sourceCfgMap[source.SourceID] = s.scheduler.GetSourceCfgByID(source.SourceID) + sourceCfg := s.scheduler.GetSourceCfgByID(source.SourceID) + if sourceCfg == nil { + return nil, nil, terror.ErrConfigSourceIDNotFound.Generate(source.SourceID) + } + sourceCfgMap[source.SourceID] = sourceCfg } task, err := config.TaskConfigToOpenAPITask(taskCfg, sourceCfgMap) if err != nil { @@ -669,7 +683,11 @@ func (s *Server) convertTaskConfig(ctx context.Context, req openapi.ConverterTas } sourceCfgMap := make(map[string]*config.SourceConfig, len(task.SourceConfig.SourceConf)) for _, cfg := range task.SourceConfig.SourceConf { - sourceCfgMap[cfg.SourceName] = s.scheduler.GetSourceCfgByID(cfg.SourceName) + sourceCfg := s.scheduler.GetSourceCfgByID(cfg.SourceName) + if sourceCfg == nil { + return nil, nil, terror.ErrConfigSourceIDNotFound.Generate(cfg.SourceName) + } + sourceCfgMap[sourceCfg.SourceID] = sourceCfg } taskCfg, err := config.OpenAPITaskToTaskConfig(task, sourceCfgMap) if err != nil { diff --git a/dm/errors.toml b/dm/errors.toml index 3ae8c9f19ea..0b37755031f 100644 --- a/dm/errors.toml +++ b/dm/errors.toml @@ -3245,7 +3245,7 @@ workaround = "Please try again later" tags = ["internal", "low"] [error.DM-scheduler-46025] -message = "source can only update when not enable relay and no running tasks for now" +message = "source %s can only be updated when relay is disabled and no tasks are running for now" description = "" workaround = "" tags = ["internal", "low"] diff --git a/dm/openapi/gen.server.go b/dm/openapi/gen.server.go index 58877a1c21f..dfb22a53ec7 100644 --- a/dm/openapi/gen.server.go +++ b/dm/openapi/gen.server.go @@ -1298,40 +1298,40 @@ var swaggerSpec = []string{ "kqyb7upBWRvmezOlVQTtOz30zcSSW6K7vsOvWJkeUEYMxVDqjvbaUyk6C6MlMGfcZcnVteWNtsjWnqcu", "c6sWtxfJ9R31NmstSu4KENhweI8hD1Pm1um5oGmqHPiyiOZXU0Dv9C+ctra/7ksb9YLaIXbbxOc93e6a", "9lQP1QQ5/TpEhnzcL3fEW9PlPenS9pE9IYwhuMZxGEAW5r551fmdj365Y2C6kTvzBaxFmfFvOmE9YBVO", - "WFvzPpa68OkJ4dHDJfXc52GEFHFd5WsCJfmOee1YprfEYM8FfBq5hp7+vRwcvmsLSsswTTtOt7HIYb0a", - "h9uUHmwoL9+eifceOkpSyR7eZCK9QuyaYYHWypsWb2ljW5hVij+6r6eU63aD7rtqGEEcq8YQ/LIZnmpJ", - "6Ds41xRiOJ46Opfk8quc1Cm66jolCwLEuQfc9SrDmnMNm9hwAaXvLN1rG5r+Ukgv/sAdZWr9GtqSlS3e", - "hr+yoXnQ5YreWzTmugwHufIS1FRb8Lb2NV2p1ltUYrTXXtyoAImQMis+ooEjrHX0AXxKEXnz+QQcfXon", - "JROLB4eDrgZMI6liRtrww5SYfkzaCo+oogQs1E4aC1whxvXaB1KZqBh+ighM8eBwsKd+koJRLBW0Y5ji", - "8dV0bO4Jj/PpjVVR9OE4CdVabz6fVHtZqGyaFkBqvt3JxMTF8gpbmOqAqtzGP7kuLi2tjTZC9nTNUFiv", - "aQ/N7+r0eJYkkK0Gh3IPoOiaQSIKeBYsAeSg0kpDwAW3OmQMvshJ6mjRCTneFzNlE42HwY+jaUcbloaD", - "/XsEo9HNxbG0Fr4t52M16cvFzDoHM/6u/1Dm+41mwxhpre04qU9RFGOCNNo+6tRAChlMkD7lPxq5Cgu8", - "3IGSv0s+GuSptoEFw8AWIzpV6ApG+RsofmkQzr7DanpkJ0o1XmstF3sdZC7ee3JY2eHlYTjM0VFmyzjM", - "ahW5FoeZgxl/NzpzLQ4zur4Hh9ng+TnMguFpc1i18WfrQYbJTg6ck7PeI3FEg/85//TRw0pVsORcxT2f", - "JrmFNABquRKqkAY1iIyp1ALO3y8+nPYCRw7sAGcpdDbTB442ybtFT9mXqYuYJX/l9z3UzcGijlrR9LcM", - "sZVF1FgsZ8UIBxG7S01uho6uwSvAkMiY7pCgy1pG5hZ3Xo/sAqFyeXkdGL5sVvo6WmE5OMW+YBfnNbw1", - "OqgPKekh98hUTIT7zt9uUGpED+LiLQ1X97ZfVw9Ux27N0mAu175p4H96b/AUHuyj13O67w+AJMxLuSAg", - "6No+ddeBN2XA+LsVBu7WckfqYUEUrTJhEdO5aqeREfwtq94K9Su8alS6l8Lz3tJpCoyI6kseNM0hgTE3", - "rSvye8nK/Ta5b5foUHPcUWZsgeLVdABgF00N++iQbaSVh9Fpm9QnLfKsaJC476RFg3kqQKQaqzf1SxtB", - "pJmPIOxw4xbQxJfN6D1X0PWmGoiT4N78GNJ4ZHIoU8jqlkNdum0c6lbiKsbsN3tMw/HtItEun+HR6RaN", - "5Hs41PJiesuZ6r7ez0e6ySMtzNC7nqhyydZj1rO8P9XTVCeubyTcGH2yrZKhbBAUZUS3mMsvqdwPga0h", - "OJ44eTm+irCt1GWE1MaJq2iF0UJbZW/Fp0tazf6S/c3gx01pigIqbfHWpyXr2209XGzdRKxPsHYDpOPv", - "w7FZB7faOG1LElR5qxFdaegLzvYlj/F3/UcZwetBLKpA9/HRyrClGtOzfLn3nss7izU3SqXVG8zbRaS6", - "WPX2NFp0YegjwYreO49HG7becniQXFDtKxFbQj729yjtj3re3cISDBIemY9k+s2rCzPsqccam8WHfxUT", - "KyeEQlRRAHU/dF0r0EFdOsXTJZnyj+R0EpCkecgvHzL7bS65zFd65bwtjmvN/FlfhVW0IWpb1cEf9WXr", - "namGa4WnLZ25YVHb+BaSgwgVkmPTbO/xCNoCqpLcde1zn/T+he5rsrnkvl3c/SNT+6ZGfGsS+8X3VqpH", - "Wpdf4yD/0C7v0Ifmi7ybPHDX144d+8aRJlrMASZpJnQzVSM8dWPpfFe6t2D5OV3dlJgycIUDBK4Q43Cj", - "VOP+jPEWkNGFqohSWCamPaPpH00jAOtNuRtI3elBefnVnn46NL+88wAFrFsuy4u7U3cS6hflxatN8LpP", - "nD7Lc7c8r5zsOsw11i1AOoT7iRr0QOdev0K4Phnsbgie7ZHPprHL7cniu2pgtk7RXo061nKH7R5qDj+4", - "gKWnF+xrvrbVhXL+i691Ad5bWW7PMU2enGBv6uu2I/dWxJVXYJ8PfWtq0fqee0N+305qP1aKaKuuVjCg", - "K0QAjlQbb8Czee72saKTzHN9tc/T76EmtoYuHiA4+iOkU82J3Pf1LWupovafflcN9WMmgI2WTd8tovhk", - "dVbviKKlozwZuLwlWt7usE/8p/rV/q2RXA9e/uDMouimvaZZ7MBX1vBL/xl1X9/2CdWYXx4+692klq3L", - "fatsnF0/AUlomoPm33anmTC3zXDl6vDtubJ3tVhRJ/Z2JXH9hoS3y5E/EaZ8rl9ro293EdudqXjNorai", - "nO2ZpJ/L7LaWl5y1dvfMSvK9eYzWjEHMY3QuWBaIjD3z1GPjqaG/wagP5TkF9Ma5++s52x+vr3Aet0h8", - "3WjMM4c8c8j0xzhLVeLbfmeplQ39YbFP6qdnZXWbxZ8KI95/TLKguiYf/rWqrTXHrak2261WATsLW4ov", - "/T6xUHfjC8fbeuNWHfItg8/97g5Z35XbQmFftJje9ur5Lb2mZC5OaOpZjzpp2im89Ie7n5zsqn6vfHtF", - "F039kkt9DIJd5Sda7XK+otlOSBOIiepxPpCoNhO4ZcGgq616SIPevdRN8/TxtwwHlyMlgUe6DnVU9v2q", - "yJiByzJT294sVNdYLEdhYsGjlm1Ck/d5LcblP9x8ufl3AAAA//+44huJ86wAAA==", + "WFvzPpa68OkJ4dHDJfXc52GEFHFd5WsCJfmOee1YprfEYM8FfBq5hp7+vRwcvmsLSsswTTtOn0CRw21q", + "DzaUmG9PxXtPHSWp5A9vNpFeIXbNsEBrJU6Lt7S1LcwqxR/d91PKdbtB9901jCCOVWcIftmMT7Vk9B2s", + "ayoxHE8drUtyAVZO6pRddaWSBQHi3APueqVhzbmGTWy4gNKXlu61D01/MaQXf+CWMrWGDW3ZyhZ3w1/a", + "0DzockXvNRpzX4aDXHsJasoteFv/mq5c6y1KMdqLL25UhERImRUf0cAR1zr6AD6liLz5fAKOPr2TkonF", + "g8NBVwemkdQxI235YUpMQyZthkdUUQIWaieNBa4Q43rtA6lNVBA/RQSmeHA42FM/ScEolgraMUzx+Go6", + "NheFx/n0xqwoGnGchGqtN59Pqs0sVDpNCyA13+5kYgJjeYktTHVEVW7jn1xXl5bmRhshe9pmKKzXtIfm", + "d3V6PEsSyFaDQ7kHULTNIBEFPAuWAHJQ6aUh4IJbLTIGX+QkdbTojBzvi5myi8bD4MfRtaMNS8PB/j2C", + "0Wjn4lhaC9+W87G69OViZp2DGX/Xfyj7/UazYYy01nac1KcoijFBGm0fdW4ghQwmSJ/yH41khQVe7kHJ", + "3yUfDfJc28CCYWCLEZ0rdEWj/B0UvzQIZ99hNT2yE6Uar7Wei70OMhfvPTmsbPHyMBzmaCmzZRxm9Ypc", + "i8PMwYy/G525FocZXd+Dw2zw/BxmwfC0Oaza+bP1IMNkJwfOyVnvkTiiwf+cf/roYaUqWHKu4qJPk9xC", + "GgC1XAlVSIMaRMZUagHn7xcfTnuBIwd2gLMUOp3pA0eb5N2ip2zM1EXMkr/yCx/q6mBRSK1o+luG2Moi", + "aiyWs2KEg4jdtSY3Q0fb4BVgSGRMt0jQdS0jc407L0h2gVC5vbwODF82K30dvbAcnGLfsIvzIt4aHdSH", + "lPSQe2QqKMJ95293KDWiB3Hxloare9uvqwmqY7dmaTCXa9808D+9N3gKD/bR6znd+AdAEua1XBAQdG2f", + "uuvAmzJg/N2KA3druSP1sCCKVpmwiOlc9dPICP6WVa+F+hVeNSzdS+F5r+k0BUZE9S0PmuaQwJib3hX5", + "xWTlfpvkt0t0qDnuKDO2QPFqOgCwi6aGfXTINtLKw+i0TeqTFnlWdEjcd9KiwTwVIFKd1Zv6pY0g0sxH", + "EHa4cQto4stm9J4r6HpTDcRJcG9+DGk8MjmUKWR1y6Eu3TYOdS9xFWP2mz2m4/h2kWiXz/DodItG8j0c", + "ankzveVMdWPv5yPd5JEWZuhdT1S5ZOsx61neoOppqhPXRxJujD7ZVslQdgiKMqJ7zOW3VO6HwNYQHE+c", + "vByfRdhW6jJCauPEVfTCaKGtsrni0yWtZoPJ/mbw46Y0RQGVvnjr05L18bYeLrbuItYnWLsB0vE34tis", + "g1vtnLYlCaq814guNfQFZ/uSx/i7/qOM4PUgFlWh+/hoZdhSjulZvtx7z+Wd1ZobpdLqFebtIlJdrXp7", + "Gi3aMPSRYEXzncejDVuvOTxILqj2mYgtIR/7g5T2Vz3vbmEJBgmPzFcy/ebVhRn21GONzeLDv4qJlRNC", + "IaoogLohuq4V6KAuneLpkkz5V3I6CUjSPOSXD5n9Nrdc5iu9ct4Xx7Vm/qyvwir6ELWt6uCP+rL11lTD", + "tcLTls7csKhtfAzJQYQKybHptvd4BG0BVUnuuva5T3r/Qjc22Vxy3y7u/pGpfVMjvjWJ/eKDK9Ujrcuv", + "cZB/aZd36EPzSd5NHrjrc8eOfeNIEy3mAJM0E7qbqhGeurN0vivdXLD8nq7uSkwZuMIBAleIcbhRqnF/", + "x3gLyOhCVUQpLBPTn9E0kKYRgPWu3A2k7vSgvPxqTz8dml/eeYAC1i2X5cXdqTsJ9Yvy4tUmeN0nTp/l", + "uVueV052HeYa6x4gHcL9RA16oHOvXyFcnwx2NwTP9shn09nl9mTxXXUwW6dor0Yda7nDdhM1hx9cwNLT", + "C/Z1X9vqQjn/xde6AO+tLLfnmCZPTrA39XXbkXsr4sorsM+HvjW1aH3PvSG/bye1HytFtFVXKxjQFSIA", + "R6qPN+DZPHf7WNFK5rm+2ufp91ATW0MXDxAc/RHSqeZE7vsal7VUUftPv6uG+jETwEbLpu8WUXyyOqt3", + "RNHSUZ4MXN4TLe932Cf+U/1s/9ZIrgcvf3BmUXTXXtMtduAra/il/4y6sW/7hGrMLw+f9W5Sy9blvlU2", + "zq6fgCQ03UHzj7vTTJjbZrhydfj2XNm7WqyoE3u7krh+Q8Lb5cifCFM+16+10be7iO3OVLxmUVtRzvZM", + "0s9ldlvLS85au3tmJfnePEZrxiDmMToXLAtExp556rHx1NDfYdSH8pwCeuPc/fmc7Y/XVziPWyS+bjTm", + "mUOeOWT6Y5ylKvFtv7PUyob+sNgn9dOzsrrN4k+FEe8/JllQXZMP/1rV1prj1lSb7VargJ2FLcWnfp9Y", + "qLvxieNtvXGrDvmWwed+d4esD8ttobAvWkxve/X8ll5TMhcnNPWsR5007RRe+svdT052VT9Yvr2ii6Z+", + "yaU+BsGu8hOtdjlf0WwnpAnERPU4H0hUmwncsmDQ1VY9pEHvXuqmefr4W4aDy5GSwCNdhzoq+35VZMzA", + "ZZmpbW8WqmsslqMwseBRyzahyfu8FuPyH26+3Pw7AAD//2uF9Jr0rAAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/dm/openapi/gen.types.go b/dm/openapi/gen.types.go index 4f95919c555..33e7c102994 100644 --- a/dm/openapi/gen.types.go +++ b/dm/openapi/gen.types.go @@ -573,7 +573,7 @@ type TaskTargetDataBase struct { // source password Password string `json:"password"` - // ource port + // source port Port int `json:"port"` // data source ssl configuration, the field will be hidden when getting the data source configuration from the interface diff --git a/dm/openapi/spec/dm.yaml b/dm/openapi/spec/dm.yaml index 8324ffc7a74..a1c77a34f1f 100644 --- a/dm/openapi/spec/dm.yaml +++ b/dm/openapi/spec/dm.yaml @@ -1570,7 +1570,7 @@ components: port: type: integer example: 3306 - description: "ource port" + description: "source port" user: type: string example: "root" diff --git a/dm/pkg/terror/error_list.go b/dm/pkg/terror/error_list.go index 3e3e9745b58..072754497c4 100644 --- a/dm/pkg/terror/error_list.go +++ b/dm/pkg/terror/error_list.go @@ -1327,7 +1327,7 @@ var ( ErrSchedulerRelayWorkersWrongRelay = New(codeSchedulerRelayWorkersWrongRelay, ClassScheduler, ScopeInternal, LevelHigh, "these workers %s have started relay for another sources %s respectively", "Please correct sources in `stop-relay`.") ErrSchedulerSourceOpRelayExist = New(codeSchedulerSourceOpRelayExist, ClassScheduler, ScopeInternal, LevelHigh, "source with name %s need to operate has existing relay workers %s", "Please `stop-relay` first.") ErrSchedulerLatchInUse = New(codeSchedulerLatchInUse, ClassScheduler, ScopeInternal, LevelLow, "when %s, resource %s is in use by other client", "Please try again later") - ErrSchedulerSourceCfgUpdate = New(codeSchedulerSourceCfgUpdate, ClassScheduler, ScopeInternal, LevelLow, "source can only update when not enable relay and no running tasks for now", "") + ErrSchedulerSourceCfgUpdate = New(codeSchedulerSourceCfgUpdate, ClassScheduler, ScopeInternal, LevelLow, "source %s can only be updated when relay is disabled and no tasks are running for now", "") ErrSchedulerWrongWorkerInput = New(codeSchedulerWrongWorkerInput, ClassScheduler, ScopeInternal, LevelMedium, "require DM master to modify worker [%s] with source [%s], but currently the worker is bound to source [%s]", "") ErrSchedulerBoundDiffWithStartedRelay = New(codeSchedulerCantTransferToRelayWorker, ClassScheduler, ScopeInternal, LevelMedium, "require DM worker [%s] to be bound to source [%s], but it has been started relay for source [%s]", "If you intend to bind the source with worker, you can stop-relay for current source.") ErrSchedulerStartRelayOnSpecified = New(codeSchedulerStartRelayOnSpecified, ClassScheduler, ScopeInternal, LevelLow, "the source has `start-relay` with worker name for workers %v, so it can't `start-relay` without worker name now", "Please stop all relay workers first, or specify worker name for `start-relay`.") diff --git a/dm/syncer/syncer.go b/dm/syncer/syncer.go index 4c506650eb6..94f6fa8777a 100644 --- a/dm/syncer/syncer.go +++ b/dm/syncer/syncer.go @@ -3581,7 +3581,7 @@ func (s *Syncer) CheckCanUpdateCfg(newCfg *config.SubTaskConfig) error { oldCfg.RouteRules = newCfg.RouteRules oldCfg.FilterRules = newCfg.FilterRules oldCfg.SyncerConfig = newCfg.SyncerConfig - newCfg.To.Session = oldCfg.To.Session // session is adjusted in `createDBs` + oldCfg.To.Session = newCfg.To.Session // session is adjusted in `createDBs` // support fields that changed in func `copyConfigFromSource` oldCfg.From = newCfg.From @@ -3594,6 +3594,7 @@ func (s *Syncer) CheckCanUpdateCfg(newCfg *config.SubTaskConfig) error { oldCfg.CaseSensitive = newCfg.CaseSensitive if oldCfg.String() != newCfg.String() { + s.tctx.L().Warn("can not update cfg", zap.Stringer("old cfg", oldCfg), zap.Stringer("new cfg", newCfg)) return terror.ErrWorkerUpdateSubTaskConfig.Generatef("can't update subtask config for syncer because new config contains some fields that should not be changed, task: %s", s.cfg.Name) } return nil diff --git a/dm/ui/locales/en.json b/dm/ui/locales/en.json index 74d41f3e083..d718833d9d0 100644 --- a/dm/ui/locales/en.json +++ b/dm/ui/locales/en.json @@ -5,11 +5,9 @@ "add new source": "Add new source", "add source config": "Add source config", "address": "Address", - "are you sure to offline this master": "Are you sure to offline this master", - "are you sure to offline this worker": "Are you sure to offline this worker", "basic info": "Basics", "binlog": "binlog", - "bound stage": "bound stage", + "bound stage": "Bound stage", "cancel": "Cancel", "check": "Check", "cluster management": "Cluster", @@ -97,9 +95,6 @@ "note": "Note", "note-dashboard-config": "Please fill in the host and port of the deployed Grafana component.", "off": "Off", - "offline": "Offline", - "offline master confirm description": "Offline masters may cause the cluster to work abnormally.", - "offline worker confirm description": "When a worker is offline, its bound source and subtask will be automatically migrated to other workers, which may cause a short outage and binlog to be pulled again", "on": "On", "open task by config": "Open task as plaintext config", "open task by config desc": "Create tasks directly by writing configuration files manually, suitable to copy from existing configuration files or for skilled users who want to define more detailed configurations", @@ -140,7 +135,6 @@ "status": "Status", "stop": "Stop", "stop_task list": "Stop", - "stopped": "Stopped", "submit": "Submit", "subtask": "Subtask", "sync config": "Replication config", diff --git a/dm/ui/locales/zh.json b/dm/ui/locales/zh.json index b8d4b9cdd13..37bb64da3d4 100644 --- a/dm/ui/locales/zh.json +++ b/dm/ui/locales/zh.json @@ -5,8 +5,6 @@ "add new source": "添加上游", "add source config": "添加上游配置", "address": "地址", - "are you sure to offline this master": "确定要下线此主节点吗？", - "are you sure to offline this worker": "确定要下线此从节点吗？", "basic info": "基本信息", "binlog": "binlog", "bound stage": "bound stage", @@ -97,9 +95,6 @@ "note": "注意", "note-dashboard-config": "请填写部署时 Grafana 组件的地址和端口", "off": "Off", - "offline": "下线", - "offline master confirm description": "下线 master 可能导致集群工作不正常，请确认后操作", - "offline worker confirm description": "下线 worker 后，其绑定的 source 和 subtask 将自动迁移到其他 worker，可能导致短暂中断，以及 binlog 需要重新拉取", "on": "On", "open task by config": "以配置文件方式打开", "open task by config desc": "通过手动编写配置文件直接创建任务，适合从已有配置文件复制或定义更多配置的熟练用户", @@ -140,7 +135,6 @@ "status": "状态", "stop": "停用", "stop_task list": "停止", - "stopped": "已停用", "submit": "提交", "subtask": "子任务", "sync config": "同步配置", diff --git a/dm/ui/package.json b/dm/ui/package.json index 584586efc67..fbb541df892 100644 --- a/dm/ui/package.json +++ b/dm/ui/package.json @@ -29,7 +29,7 @@ "react-i18next": "^11.15.1", "react-redux": "^7.2.6", "react-router-dom": "^6.1.1", - "react-syntax-highlighter": "^15.4.5" + "react-syntax-highlighter": "^15.5.0" }, "devDependencies": { "@faker-js/faker": "^6.0.0-alpha.7", diff --git a/dm/ui/src/components/CreateOrUpdateTask/MigrateRule.tsx b/dm/ui/src/components/CreateOrUpdateTask/MigrateRule.tsx index 98b14c76cb1..ebaa04f453a 100644 --- a/dm/ui/src/components/CreateOrUpdateTask/MigrateRule.tsx +++ b/dm/ui/src/components/CreateOrUpdateTask/MigrateRule.tsx @@ -10,6 +10,7 @@ import { Cascader, message, Space, + Checkbox, } from '~/uikit' import { FileAddOutlined, @@ -284,6 +285,9 @@ const MigrateRule: StepCompnent = ({ prev, initialValues }) => { )} + diff --git a/dm/ui/src/components/CreateOrUpdateTask/index.tsx b/dm/ui/src/components/CreateOrUpdateTask/index.tsx index f5ea5a8d3ed..80ada564222 100644 --- a/dm/ui/src/components/CreateOrUpdateTask/index.tsx +++ b/dm/ui/src/components/CreateOrUpdateTask/index.tsx @@ -1,6 +1,6 @@ import React, { useState, useCallback, useEffect } from 'react' import { useTranslation } from 'react-i18next' -import { merge, omit } from 'lodash-es' +import { omit } from 'lodash-es' import { useNavigate, useLocation } from 'react-router-dom' import { Card, Form, Steps, message } from '~/uikit' @@ -83,8 +83,6 @@ const CreateTaskConfig: React.FC<{ const handleSubmit = async (taskData: TaskFormData) => { const isEditing = Boolean(data) - const key = 'createTask-' + Date.now() - message.loading({ content: t('requesting'), key }) const payload = { ...taskData } const startAfterSaved = payload.start_after_saved if (payload.shard_mode === TaskShardMode.NONE) { @@ -105,12 +103,14 @@ const CreateTaskConfig: React.FC<{ ) } const handler = isEditing ? updateTask : createTask + const key = 'createTask-' + Date.now() + message.loading({ content: t('requesting'), key }) try { await handler({ task: payload as Task }).unwrap() - message.success({ content: t('request success'), key }) if (startAfterSaved) { await startTask({ taskName: payload.name }).unwrap() } + message.success({ content: t('request success'), key }) navigate('/migration/task') } catch (e) { message.destroy(key) @@ -130,13 +130,12 @@ const CreateTaskConfig: React.FC<{ useEffect(() => { if (data) { - setTaskData( - merge({}, defaultValue, omit(data, 'status_list'), { - binlog_filter_rule_array: Object.entries( - data?.binlog_filter_rule ?? {} - ).map(([name, value]) => ({ name, ...value })), - }) - ) + setTaskData({ + ...omit(data, 'status_list'), + binlog_filter_rule_array: Object.entries( + data?.binlog_filter_rule ?? {} + ).map(([name, value]) => ({ name, ...value })), + }) } setLoading(false) }, [data]) @@ -168,7 +167,7 @@ const CreateTaskConfig: React.FC<{
{ - const nextTaskData = merge({}, taskData, values) + const nextTaskData = { ...taskData, ...values } setTaskData(nextTaskData) if (currentStep === stepComponents.length - 1) { diff --git a/dm/ui/src/models/source.ts b/dm/ui/src/models/source.ts index c912206e6b9..fcc4f8d8769 100644 --- a/dm/ui/src/models/source.ts +++ b/dm/ui/src/models/source.ts @@ -119,6 +119,7 @@ const injectedRtkApi = api.injectEndpoints({ method: 'POST', body: queryArg.payload, }), + invalidatesTags: ['Source'], }), dmapiEnableRelay: build.mutation< void, @@ -137,6 +138,7 @@ const injectedRtkApi = api.injectEndpoints({ method: 'POST', body: queryArg.payload, }), + invalidatesTags: ['Source'], }), dmapiPurgeRelay: build.mutation< void, diff --git a/dm/ui/src/models/task.ts b/dm/ui/src/models/task.ts index cffd52e20f4..e4672cce84a 100644 --- a/dm/ui/src/models/task.ts +++ b/dm/ui/src/models/task.ts @@ -383,6 +383,13 @@ export const calculateTaskStatus = (subtasks?: SubTaskStatus[]): TaskStage => { } // TODO Error status + if (subtasks.some(subtask => subtask.stage === TaskStage.Resuming)) { + return TaskStage.Resuming + } + + if (subtasks.some(subtask => subtask.stage === TaskStage.Pausing)) { + return TaskStage.Pausing + } if (subtasks.some(subtask => subtask.stage === TaskStage.Paused)) { return TaskStage.Paused diff --git a/dm/ui/src/pages/cluster/member.tsx b/dm/ui/src/pages/cluster/member.tsx index 1ebf198af88..1e54994ff46 100644 --- a/dm/ui/src/pages/cluster/member.tsx +++ b/dm/ui/src/pages/cluster/member.tsx @@ -1,28 +1,12 @@ import React from 'react' import { useTranslation } from 'react-i18next' -import { - Space, - Button, - Tabs, - Card, - Table, - Badge, - message, - Modal, - Breadcrumb, -} from '~/uikit' -import { - RedoOutlined, - CheckCircleOutlined, - ExclamationCircleOutlined, -} from '~/uikit/icons' +import { Space, Button, Tabs, Card, Table, Badge, Breadcrumb } from '~/uikit' +import { RedoOutlined, CheckCircleOutlined } from '~/uikit/icons' import i18n from '~/i18n' import { useDmapiGetClusterMasterListQuery, useDmapiGetClusterWorkerListQuery, - useDmapiOfflineMasterNodeMutation, - useDmapiOfflineWorkerNodeMutation, } from '~/models/cluster' const { TabPane } = Tabs @@ -30,24 +14,6 @@ const { TabPane } = Tabs const MasterTable: React.FC = () => { const [t] = useTranslation() const { data, isFetching, refetch } = useDmapiGetClusterMasterListQuery() - const [offlineMasterNode] = useDmapiOfflineMasterNodeMutation() - const handleConfirmOffline = (name: string) => { - Modal.confirm({ - title: t('are you sure to offline this master'), - content: t('offline master confirm description'), - icon: , - async onOk() { - const key = 'offlineMasterNode-' + Date.now() - message.loading({ content: t('requesting'), key }) - try { - await offlineMasterNode(name).unwrap() - message.success({ content: t('request success'), key }) - } catch (e) { - message.destroy(key) - } - }, - }) - } const dataSource = data?.data const columns = [ { @@ -74,23 +40,6 @@ const MasterTable: React.FC = () => { ) : null }, }, - { - title: t('operations'), - dataIndex: 'name', - render(name: string) { - return ( - - - - ) - }, - }, ] return ( @@ -120,24 +69,6 @@ const MasterTable: React.FC = () => { const WorkerTable: React.FC = () => { const [t] = useTranslation() const { data, isFetching, refetch } = useDmapiGetClusterWorkerListQuery() - const [offlineWorkerNode] = useDmapiOfflineWorkerNodeMutation() - const handleConfirmOffline = (name: string) => { - Modal.confirm({ - title: t('are you sure to offline this worker'), - content: t('offline worker confirm description'), - icon: , - async onOk() { - const key = 'offlineWorkerNode-' + Date.now() - message.loading({ content: t('requesting'), key }) - try { - await offlineWorkerNode(name).unwrap() - message.success({ content: t('request success'), key }) - } catch (e) { - message.destroy(key) - } - }, - }) - } const dataSource = data?.data const columns = [ { @@ -164,23 +95,6 @@ const WorkerTable: React.FC = () => { title: t('source'), dataIndex: 'bound_source_name', }, - { - title: t('operations'), - dataIndex: 'name', - render(name: string) { - return ( - - - - ) - }, - }, ] return ( diff --git a/dm/ui/src/pages/migration/replication-detail.tsx b/dm/ui/src/pages/migration/replication-detail.tsx index f61228bcdd0..fbdb5812114 100644 --- a/dm/ui/src/pages/migration/replication-detail.tsx +++ b/dm/ui/src/pages/migration/replication-detail.tsx @@ -1,6 +1,6 @@ -import React, { useEffect, useState } from 'react' +import React, { useState } from 'react' import { useTranslation } from 'react-i18next' -import { useDebounce } from 'ahooks' +import { isEqual } from 'lodash-es' import i18n from '~/i18n' import { @@ -22,16 +22,33 @@ import { } from '~/models/task' import { useDmapiGetSourceQuery } from '~/models/source' +interface ReplicationDetailSearchOptions { + currentTask?: Task + currentSourceName: string + dbPattern: string + tablePattern: string +} + const ReplicationDetail: React.FC = () => { const [t] = useTranslation() - const [currentTask, setCurrentTask] = useState() - const [currentSourceName, setCurrentSourceName] = useState() - const [dbPattern, setDbPattern] = useState('') - const [tablePattern, setTablePattern] = useState('') - const debouncedDbPattern = useDebounce(dbPattern, { wait: 500 }) - const debouncedTablePattern = useDebounce(tablePattern, { wait: 500 }) - - const currentTaskName = currentTask?.name ?? '' + + const [ + { currentTask, currentSourceName, dbPattern, tablePattern }, + setSearchOptions, + ] = useState({ + currentTask: undefined, + currentSourceName: '', + dbPattern: '', + tablePattern: '', + }) + + const [payload, setPayload] = useState({ + taskName: currentTask?.name ?? '', + sourceName: currentSourceName ?? '', + schemaPattern: dbPattern, + tablePattern: tablePattern, + }) + const { data: taskList, isFetching: isFetchingTaskList } = useDmapiGetTaskListQuery({ withStatus: true, @@ -40,15 +57,9 @@ const ReplicationDetail: React.FC = () => { data: migrateTagetData, refetch, isFetching: isFetchingMigrateTarget, - } = useDmapiGetTaskMigrateTargetsQuery( - { - taskName: currentTaskName, - sourceName: currentSourceName ?? '', - schemaPattern: debouncedDbPattern, - tablePattern: debouncedTablePattern, - }, - { skip: !currentTask || !currentSourceName } - ) + } = useDmapiGetTaskMigrateTargetsQuery(payload, { + skip: !payload.taskName || !payload.sourceName, + }) const { data: sourceData } = useDmapiGetSourceQuery( { sourceName: currentSourceName ?? '' }, { skip: !currentSourceName } @@ -113,45 +124,74 @@ const ReplicationDetail: React.FC = () => { ({ label: i.source_name, value: i.source_name, }))} onSelect={(value: string) => { - setCurrentSourceName(value) + setSearchOptions(prev => ({ + ...prev, + currentSourceName: value, + })) }} /> setDbPattern(e.target.value)} + onChange={e => { + setSearchOptions(prev => ({ + ...prev, + dbPattern: e.target.value, + })) + }} /> setTablePattern(e.target.value)} + onChange={e => { + setSearchOptions(prev => ({ + ...prev, + tablePattern: e.target.value, + })) + }} /> - + diff --git a/dm/ui/src/pages/migration/source.tsx b/dm/ui/src/pages/migration/source.tsx index aabb3b19419..12bcbe963f8 100644 --- a/dm/ui/src/pages/migration/source.tsx +++ b/dm/ui/src/pages/migration/source.tsx @@ -206,7 +206,7 @@ const SourceList: React.FC = () => { return ( ) }, diff --git a/dm/ui/yarn.lock b/dm/ui/yarn.lock index 1006561876f..eb8c60f7dcd 100644 --- a/dm/ui/yarn.lock +++ b/dm/ui/yarn.lock @@ -3578,10 +3578,10 @@ prettier@^2.2.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.1.tgz#671e11c89c14a4cfc876ce564106c4a6726c9f5c" integrity sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA== -prismjs@^1.25.0, prismjs@~1.25.0: - version "1.25.0" - resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.25.0.tgz#6f822df1bdad965734b310b315a23315cf999756" - integrity sha512-WCjJHl1KEWbnkQom1+SzftbtXMKQoezOCYs5rECqMN+jP+apI7ftoflyqigqzopSO3hMhTEb0mFClA8lkolgEg== +prismjs@^1.27.0, prismjs@~1.27.0: + version "1.27.0" + resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057" + integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== process-nextick-args@^2.0.0, process-nextick-args@~2.0.0: version "2.0.1" @@ -4152,16 +4152,16 @@ react-router@6.1.1: dependencies: history "^5.1.0" -react-syntax-highlighter@^15.4.5: - version "15.4.5" - resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-15.4.5.tgz#db900d411d32a65c8e90c39cd64555bf463e712e" - integrity sha512-RC90KQTxZ/b7+9iE6s9nmiFLFjWswUcfULi4GwVzdFVKVMQySkJWBuOmJFfjwjMVCo0IUUuJrWebNKyviKpwLQ== +react-syntax-highlighter@^15.5.0: + version "15.5.0" + resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-15.5.0.tgz#4b3eccc2325fa2ec8eff1e2d6c18fa4a9e07ab20" + integrity sha512-+zq2myprEnQmH5yw6Gqc8lD55QHnpKaU8TOcFeC/Lg/MQSs8UknEA0JC4nTZGFAXC2J2Hyj/ijJ7NlabyPi2gg== dependencies: "@babel/runtime" "^7.3.1" highlight.js "^10.4.1" lowlight "^1.17.0" - prismjs "^1.25.0" - refractor "^3.2.0" + prismjs "^1.27.0" + refractor "^3.6.0" react@^17.0.2: version "17.0.2" @@ -4229,14 +4229,14 @@ redux@^4.0.0, redux@^4.1.2: dependencies: "@babel/runtime" "^7.9.2" -refractor@^3.2.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/refractor/-/refractor-3.5.0.tgz#334586f352dda4beaf354099b48c2d18e0819aec" - integrity sha512-QwPJd3ferTZ4cSPPjdP5bsYHMytwWYnAN5EEnLtGvkqp/FCCnGsBgxrm9EuIDnjUC3Uc/kETtvVi7fSIVC74Dg== +refractor@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/refractor/-/refractor-3.6.0.tgz#ac318f5a0715ead790fcfb0c71f4dd83d977935a" + integrity sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA== dependencies: hastscript "^6.0.0" parse-entities "^2.0.0" - prismjs "~1.25.0" + prismjs "~1.27.0" reftools@^1.1.9: version "1.1.9" @@ -4298,11 +4298,6 @@ reselect@^4.1.5: resolved "https://registry.yarnpkg.com/reselect/-/reselect-4.1.5.tgz#852c361247198da6756d07d9296c2b51eddb79f6" integrity sha512-uVdlz8J7OO+ASpBYoz1Zypgx0KasCY20H+N8JD13oUMtPvSHQuscrHop4KbXrbsBcdB9Ds7lVK7eRkBIfO43vQ== -reset-css@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/reset-css/-/reset-css-5.0.1.tgz#1ced0604fdb6836567e40d82de3537afaf3bb4c8" - integrity sha512-VyuJdNFfp5x/W6e5wauJM59C02Vs0P22sxzZGhQMPaqu/NGTeFxlBFOOw3eq9vQd19gIDdZp7zi89ylyKOJ33Q== - resize-observer-polyfill@^1.5.0, resize-observer-polyfill@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464" diff --git a/tools/check/go.sum b/tools/check/go.sum index e115da9d410..5a69401aaca 100644 --- a/tools/check/go.sum +++ b/tools/check/go.sum @@ -1120,6 +1120,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=