*: Remove and add partitioning | tidb-test=pr/2125

ddl/column.go

@@ -1069,7 +1069,12 @@
 				return dbterror.ErrCancelledDDLJob.GenWithStack("Can not find partition id %d for table %d", reorgInfo.PhysicalTableID, t.Meta().ID)
 			}
 			workType := typeReorgPartitionWorker
-			if reorgInfo.Job.Type != model.ActionReorganizePartition {
+			switch reorgInfo.Job.Type {
+			case model.ActionReorganizePartition,
+				model.ActionRemovePartitioning,
+				model.ActionAlterTablePartitioning:
+				// Expected
+			default:
 				// workType = typeUpdateColumnWorker
 				// TODO: Support Modify Column on partitioned table
 				// https://github.com/pingcap/tidb/issues/38297

ddl/ddl.go

@@ -983,7 +983,9 @@ func getIntervalFromPolicy(policy []time.Duration, i int) (time.Duration, bool)
 func getJobCheckInterval(job *model.Job, i int) (time.Duration, bool) {
 	switch job.Type {
 	case model.ActionAddIndex, model.ActionAddPrimaryKey, model.ActionModifyColumn,
-		model.ActionReorganizePartition:
+		model.ActionReorganizePartition,
+		model.ActionRemovePartitioning,
+		model.ActionAlterTablePartitioning:
 		return getIntervalFromPolicy(slowDDLIntervalPolicy, i)
 	case model.ActionCreateTable, model.ActionCreateSchema:
 		return getIntervalFromPolicy(fastDDLIntervalPolicy, i)

ddl/ddl_api.go

@@ -3496,6 +3496,7 @@
 		} else {
 			validSpecs = append(validSpecs, spec)
 		}
+		// TODO: Only allow REMOVE PARTITIONING as a single ALTER TABLE statement?
 	}
 
 	// Verify whether the algorithm is supported.
@@ -3588,7 +3589,7 @@
 		case ast.AlterTableOptimizePartition:
 			err = errors.Trace(dbterror.ErrUnsupportedOptimizePartition)
 		case ast.AlterTableRemovePartitioning:
-			err = errors.Trace(dbterror.ErrUnsupportedRemovePartition)
+			err = d.RemovePartitioning(sctx, ident, spec)
 		case ast.AlterTableRepairPartition:
 			err = errors.Trace(dbterror.ErrUnsupportedRepairPartition)
 		case ast.AlterTableDropColumn:
@@ -3665,8 +3666,7 @@
 			isAlterTable := true
 			err = d.renameTable(sctx, ident, newIdent, isAlterTable)
 		case ast.AlterTablePartition:
-			// Prevent silent succeed if user executes ALTER TABLE x PARTITION BY ...
-			err = errors.New("alter table partition is unsupported")
+			err = d.AlterTablePartitioning(sctx, ident, spec)
 		case ast.AlterTableOption:
 			var placementPolicyRef *model.PolicyRefInfo
 			for i, opt := range spec.Options {
@@ -4211,11 +4211,12 @@
 	return tmpDefs
 }
 
-func getReplacedPartitionIDs(names []model.CIStr, pi *model.PartitionInfo) (firstPartIdx int, lastPartIdx int, idMap map[int]struct{}, err error) {
+func getReplacedPartitionIDs(names []string, pi *model.PartitionInfo) (firstPartIdx int, lastPartIdx int, idMap map[int]struct{}, err error) {
 	idMap = make(map[int]struct{})
 	firstPartIdx, lastPartIdx = -1, -1
 	for _, name := range names {
-		partIdx := pi.FindPartitionDefinitionByName(name.L)
+		nameL := strings.ToLower(name)
+		partIdx := pi.FindPartitionDefinitionByName(nameL)
 		if partIdx == -1 {
 			return 0, 0, nil, errors.Trace(dbterror.ErrWrongPartitionName)
 		}
@@ -4250,6 +4251,86 @@
 	return firstPartIdx, lastPartIdx, idMap, nil
 }
 
+func getPartitionInfoTypeNone() *model.PartitionInfo {
+	return &model.PartitionInfo{
+		Type:   model.PartitionTypeNone,
+		Enable: true,
+		Definitions: []model.PartitionDefinition{{
+			Name:    model.NewCIStr("pFullTable"),
+			Comment: "Intermediate partition during ALTER TABLE ... PARTITION BY ...",
+		}},
+		Num: 1,
+	}
+}
+
+// AlterTablePartitioning reorganize one set of partitions to a new set of partitions.
+func (d *ddl) AlterTablePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+	schema, t, err := d.getSchemaAndTableByIdent(ctx, ident)
+	if err != nil {
+		return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+	}
+
+	meta := t.Meta().Clone()
+	piOld := meta.GetPartitionInfo()
+	var partNames []string
+	if piOld != nil {
+		partNames = make([]string, 0, len(piOld.Definitions))
+		for i := range piOld.Definitions {
+			partNames = append(partNames, piOld.Definitions[i].Name.L)
+		}
+	} else {
+		piOld = getPartitionInfoTypeNone()
+		meta.Partition = piOld
+		partNames = append(partNames, piOld.Definitions[0].Name.L)
+	}
+	newMeta := meta.Clone()
+	err = buildTablePartitionInfo(ctx, spec.Partition, newMeta)
+	if err != nil {
+		return err
+	}
+	newPartInfo := newMeta.Partition
+
+	if err = d.assignPartitionIDs(newPartInfo.Definitions); err != nil {
+		return errors.Trace(err)
+	}
+	// A new table ID would be needed for
+	// the global index, which cannot be the same as the current table id,
+	// since this table id will be removed in the final state when removing
+	// all the data with this table id.
+	var newID []int64
+	newID, err = d.genGlobalIDs(1)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	newPartInfo.NewTableID = newID[0]
+	newPartInfo.DDLType = piOld.Type
+
+	tzName, tzOffset := ddlutil.GetTimeZone(ctx)
+	job := &model.Job{
+		SchemaID:   schema.ID,
+		TableID:    meta.ID,
+		SchemaName: schema.Name.L,
+		TableName:  t.Meta().Name.L,
+		Type:       model.ActionAlterTablePartitioning,
+		BinlogInfo: &model.HistoryInfo{},
+		Args:       []interface{}{partNames, newPartInfo},
+		ReorgMeta: &model.DDLReorgMeta{
+			SQLMode:       ctx.GetSessionVars().SQLMode,
+			Warnings:      make(map[errors.ErrorID]*terror.Error),
+			WarningsCount: make(map[errors.ErrorID]int64),
+			Location:      &model.TimeZoneLocation{Name: tzName, Offset: tzOffset},
+		},
+	}
+
+	// No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
+	err = d.DoDDLJob(ctx, job)
+	err = d.callHookOnChanged(job, err)
+	if err == nil {
+		ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("The statistics of new partitions will be outdated after reorganizing partitions. Please use 'ANALYZE TABLE' statement if you want to update it now"))
+	}
+	return errors.Trace(err)
+}
+
 // ReorganizePartitions reorganize one set of partitions to a new set of partitions.
 func (d *ddl) ReorganizePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
 	schema, t, err := d.getSchemaAndTableByIdent(ctx, ident)
@@ -4272,7 +4353,11 @@
 	default:
 		return errors.Trace(dbterror.ErrUnsupportedReorganizePartition)
 	}
-	firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(spec.PartitionNames, pi)
+	partNames := make([]string, 0, len(spec.PartitionNames))
+	for _, name := range spec.PartitionNames {
+		partNames = append(partNames, name.L)
+	}
+	firstPartIdx, lastPartIdx, idMap, err := getReplacedPartitionIDs(partNames, pi)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -4283,7 +4368,7 @@
 	if err = d.assignPartitionIDs(partInfo.Definitions); err != nil {
 		return errors.Trace(err)
 	}
-	if err = checkReorgPartitionDefs(ctx, meta, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
+	if err = checkReorgPartitionDefs(ctx, model.ActionReorganizePartition, meta, partInfo, firstPartIdx, lastPartIdx, idMap); err != nil {
 		return errors.Trace(err)
 	}
 	if err = handlePartitionPlacement(ctx, partInfo); err != nil {
@@ -4298,7 +4383,7 @@
 		TableName:  t.Meta().Name.L,
 		Type:       model.ActionReorganizePartition,
 		BinlogInfo: &model.HistoryInfo{},
-		Args:       []interface{}{spec.PartitionNames, partInfo},
+		Args:       []interface{}{partNames, partInfo},
 		ReorgMeta: &model.DDLReorgMeta{
 			SQLMode:       ctx.GetSessionVars().SQLMode,
 			Warnings:      make(map[errors.ErrorID]*terror.Error),
@@ -4316,55 +4401,137 @@
 	return errors.Trace(err)
 }
 
-func checkReorgPartitionDefs(ctx sessionctx.Context, tblInfo *model.TableInfo, partInfo *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) error {
+// RemovePartitioning removes partitioning from a table.
+func (d *ddl) RemovePartitioning(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error {
+	schema, t, err := d.getSchemaAndTableByIdent(ctx, ident)
+	if err != nil {
+		return errors.Trace(infoschema.ErrTableNotExists.FastGenByArgs(ident.Schema, ident.Name))
+	}
+
+	meta := t.Meta().Clone()
+	pi := meta.GetPartitionInfo()
+	if pi == nil {
+		return dbterror.ErrPartitionMgmtOnNonpartitioned
+	}
+	// TODO: Optimize for remove partitioning with a single partition
+	// TODO: Add the support for this in onReorganizePartition
+	// skip if only one partition
+	// If there are only one partition, then we can do:
+	// change the table id to the partition id
+	// and keep the statistics for the partition id (which should be similar to the global statistics)
+	// and it let the GC clean up the old table metadata including possible global index.
+
+	newSpec := &ast.AlterTableSpec{}
+	newSpec.Tp = spec.Tp
+	defs := make([]*ast.PartitionDefinition, 1)
+	defs[0] = &ast.PartitionDefinition{}
+	defs[0].Name = model.NewCIStr("CollapsedPartitions")
+	newSpec.PartDefinitions = defs
+	partNames := make([]string, len(pi.Definitions))
+	for i := range pi.Definitions {
+		partNames[i] = pi.Definitions[i].Name.L
+	}
+	meta.Partition.Type = model.PartitionTypeNone
+	partInfo, err := BuildAddedPartitionInfo(ctx, meta, newSpec)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	if err = d.assignPartitionIDs(partInfo.Definitions); err != nil {
+		return errors.Trace(err)
+	}
+	// TODO: check where the default placement comes from (i.e. table level)
+	if err = handlePartitionPlacement(ctx, partInfo); err != nil {
+		return errors.Trace(err)
+	}
+	partInfo.NewTableID = partInfo.Definitions[0].ID
+
+	tzName, tzOffset := ddlutil.GetTimeZone(ctx)
+	job := &model.Job{
+		SchemaID:   schema.ID,
+		TableID:    meta.ID,
+		SchemaName: schema.Name.L,
+		TableName:  meta.Name.L,
+		Type:       model.ActionRemovePartitioning,
+		BinlogInfo: &model.HistoryInfo{},
+		Args:       []interface{}{partNames, partInfo},
+		ReorgMeta: &model.DDLReorgMeta{
+			SQLMode:       ctx.GetSessionVars().SQLMode,
+			Warnings:      make(map[errors.ErrorID]*terror.Error),
+			WarningsCount: make(map[errors.ErrorID]int64),
+			Location:      &model.TimeZoneLocation{Name: tzName, Offset: tzOffset},
+		},
+	}
+
+	// No preSplitAndScatter here, it will be done by the worker in onReorganizePartition instead.
+	err = d.DoDDLJob(ctx, job)
+	err = d.callHookOnChanged(job, err)
+	return errors.Trace(err)
+}
+
+func checkReorgPartitionDefs(ctx sessionctx.Context, action model.ActionType, tblInfo *model.TableInfo, partInfo *model.PartitionInfo, firstPartIdx, lastPartIdx int, idMap map[int]struct{}) error {
 	// partInfo contains only the new added partition, we have to combine it with the
 	// old partitions to check all partitions is strictly increasing.
 	pi := tblInfo.Partition
 	clonedMeta := tblInfo.Clone()
-	clonedMeta.Partition.AddingDefinitions = partInfo.Definitions
-	clonedMeta.Partition.Definitions = getReorganizedDefinitions(clonedMeta.Partition, firstPartIdx, lastPartIdx, idMap)
+	switch action {
+	case model.ActionRemovePartitioning, model.ActionAlterTablePartitioning:
+		clonedMeta.Partition = partInfo
+		clonedMeta.ID = partInfo.NewTableID
+	case model.ActionReorganizePartition:
+		clonedMeta.Partition.AddingDefinitions = partInfo.Definitions
+		clonedMeta.Partition.Definitions = getReorganizedDefinitions(clonedMeta.Partition, firstPartIdx, lastPartIdx, idMap)
+	default:
+		return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("partition type")
+	}
 	if err := checkPartitionDefinitionConstraints(ctx, clonedMeta); err != nil {
 		return errors.Trace(err)
 	}
-	if pi.Type == model.PartitionTypeRange {
-		if lastPartIdx == len(pi.Definitions)-1 {
-			// Last partition dropped, OK to change the end range
-			// Also includes MAXVALUE
-			return nil
-		}
-		// Check if the replaced end range is the same as before
-		lastAddingPartition := partInfo.Definitions[len(partInfo.Definitions)-1]
-		lastOldPartition := pi.Definitions[lastPartIdx]
-		if len(pi.Columns) > 0 {
-			newGtOld, err := checkTwoRangeColumns(ctx, &lastAddingPartition, &lastOldPartition, pi, tblInfo)
+	if action == model.ActionReorganizePartition {
+		if pi.Type == model.PartitionTypeRange {
+			if lastPartIdx == len(pi.Definitions)-1 {
+				// Last partition dropped, OK to change the end range
+				// Also includes MAXVALUE
+				return nil
+			}
+			// Check if the replaced end range is the same as before
+			lastAddingPartition := partInfo.Definitions[len(partInfo.Definitions)-1]
+			lastOldPartition := pi.Definitions[lastPartIdx]
+			if len(pi.Columns) > 0 {
+				newGtOld, err := checkTwoRangeColumns(ctx, &lastAddingPartition, &lastOldPartition, pi, tblInfo)
+				if err != nil {
+					return errors.Trace(err)
+				}
+				if newGtOld {
+					return errors.Trace(dbterror.ErrRangeNotIncreasing)
+				}
+				oldGtNew, err := checkTwoRangeColumns(ctx, &lastOldPartition, &lastAddingPartition, pi, tblInfo)
+				if err != nil {
+					return errors.Trace(err)
+				}
+				if oldGtNew {
+					return errors.Trace(dbterror.ErrRangeNotIncreasing)
+				}
+				return nil
+			}
+
+			isUnsigned := isPartExprUnsigned(tblInfo)
+			currentRangeValue, _, err := getRangeValue(ctx, pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
 			if err != nil {
 				return errors.Trace(err)
 			}
-			if newGtOld {
-				return errors.Trace(dbterror.ErrRangeNotIncreasing)
-			}
-			oldGtNew, err := checkTwoRangeColumns(ctx, &lastOldPartition, &lastAddingPartition, pi, tblInfo)
+			newRangeValue, _, err := getRangeValue(ctx, partInfo.Definitions[len(partInfo.Definitions)-1].LessThan[0], isUnsigned)
 			if err != nil {
 				return errors.Trace(err)
 			}
-			if oldGtNew {
+
+			if currentRangeValue != newRangeValue {
 				return errors.Trace(dbterror.ErrRangeNotIncreasing)
 			}
-			return nil
-		}
-
-		isUnsigned := isPartExprUnsigned(tblInfo)
-		currentRangeValue, _, err := getRangeValue(ctx, pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
-		if err != nil {
-			return errors.Trace(err)
 		}
-		newRangeValue, _, err := getRangeValue(ctx, partInfo.Definitions[len(partInfo.Definitions)-1].LessThan[0], isUnsigned)
-		if err != nil {
-			return errors.Trace(err)
-		}
-
-		if currentRangeValue != newRangeValue {
-			return errors.Trace(dbterror.ErrRangeNotIncreasing)
+	} else {
+		if len(pi.Definitions) != (lastPartIdx - firstPartIdx + 1) {
+			// if not ActionReorganizePartition, require all partitions to be changed.
+			return errors.Trace(dbterror.ErrAlterOperationNotSupported)
 		}
 	}
 	return nil
@@ -7482,6 +7649,8 @@
 func BuildAddedPartitionInfo(ctx sessionctx.Context, meta *model.TableInfo, spec *ast.AlterTableSpec) (*model.PartitionInfo, error) {
 	numParts := uint64(0)
 	switch meta.Partition.Type {
+	case model.PartitionTypeNone:
+		// OK
 	case model.PartitionTypeList:
 		if len(spec.PartDefinitions) == 0 {
 			return nil, ast.ErrPartitionsMustBeDefined.GenWithStackByArgs(meta.Partition.Type)
@@ -7505,6 +7674,10 @@
 		}
 	case model.PartitionTypeHash, model.PartitionTypeKey:
 		switch spec.Tp {
+		case ast.AlterTableRemovePartitioning:
+			numParts = 1
+		default:
+			return nil, errors.Trace(dbterror.ErrUnsupportedAddPartition)
 		case ast.AlterTableCoalescePartitions:
 			if int(spec.Num) >= len(meta.Partition.Definitions) {
 				return nil, dbterror.ErrDropLastPartition
@@ -7535,6 +7708,7 @@
 	}
 
 	part.Definitions = defs
+	part.Num = uint64(len(defs))
 	return part, nil
 }