ddl: support create/drop expression index

ddl/column.go

@@ -77,6 +77,9 @@ func adjustColumnInfoInDropColumn(tblInfo *model.TableInfo, offset int) {
 		oldCols[i].Offset = i - 1
 	}
 	oldCols[offset].Offset = len(oldCols) - 1
+	// For expression index, we drop hidden columns and index simultaneously.
+	// So we need to change the offset of expression index.
+	offsetChanged[offset] = len(oldCols) - 1
 	// Update index column offset info.
 	// TODO: There may be some corner cases for index column offsets, we may check this later.
 	for _, idx := range tblInfo.Indices {

ddl/db_change_test.go

@@ -773,6 +773,16 @@ func (s *testStateChangeSuite) TestParallelAlterAddIndex(c *C) {
 	s.testControlParallelExecSQL(c, sql1, sql2, f)
 }
 
+func (s *testStateChangeSuite) TestParallelAlterAddExpressionIndex(c *C) {
+	sql1 := "ALTER TABLE t add index expr_index_b((b+1));"
+	sql2 := "CREATE INDEX expr_index_b ON t ((c+1));"
+	f := func(c *C, err1, err2 error) {
+		c.Assert(err1, IsNil)
+		c.Assert(err2.Error(), Equals, "[ddl:1061]index already exist expr_index_b")
+	}
+	s.testControlParallelExecSQL(c, sql1, sql2, f)
+}
+
 func (s *testStateChangeSuite) TestParallelAddPrimaryKey(c *C) {
 	sql1 := "ALTER TABLE t add primary key index_b(b);"
 	sql2 := "ALTER TABLE t add primary key index_b(c);"

ddl/db_integration_test.go

@@ -51,6 +51,7 @@ var _ = Suite(&testIntegrationSuite2{&testIntegrationSuite{}})
 var _ = Suite(&testIntegrationSuite3{&testIntegrationSuite{}})
 var _ = Suite(&testIntegrationSuite4{&testIntegrationSuite{}})
 var _ = Suite(&testIntegrationSuite5{&testIntegrationSuite{}})
+var _ = Suite(&testIntegrationSuite6{&testIntegrationSuite{}})
 
 type testIntegrationSuite struct {
 	lease     time.Duration
@@ -121,6 +122,7 @@ func (s *testIntegrationSuite2) TearDownTest(c *C) {
 type testIntegrationSuite3 struct{ *testIntegrationSuite }
 type testIntegrationSuite4 struct{ *testIntegrationSuite }
 type testIntegrationSuite5 struct{ *testIntegrationSuite }
+type testIntegrationSuite6 struct{ *testIntegrationSuite }
 
 func (s *testIntegrationSuite5) TestNoZeroDateMode(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
@@ -1010,22 +1012,22 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) {
 
 	unique := false
 	indexName := model.NewCIStr("idx_b")
-	idxColName := &ast.IndexPartSpecification{
+	indexPartSpecification := &ast.IndexPartSpecification{
 		Column: &ast.ColumnName{
 			Schema: schemaName,
 			Table:  tableName,
 			Name:   model.NewCIStr("b"),
 		},
 		Length: types.UnspecifiedLength,
 	}
-	idxColNames := []*ast.IndexPartSpecification{idxColName}
+	indexPartSpecifications := []*ast.IndexPartSpecification{indexPartSpecification}
 	var indexOption *ast.IndexOption
 	job := &model.Job{
 		SchemaID:   schema.ID,
 		TableID:    tbl.Meta().ID,
 		Type:       model.ActionAddIndex,
 		BinlogInfo: &model.HistoryInfo{},
-		Args:       []interface{}{unique, indexName, idxColNames, indexOption},
+		Args:       []interface{}{unique, indexName, indexPartSpecifications, indexOption},
 	}
 	txn, err := s.store.Begin()
 	c.Assert(err, IsNil)
@@ -1946,3 +1948,55 @@ func (s *testIntegrationSuite3) TestParserIssue284(c *C) {
 	tk.MustExec("drop table test.t_parser_issue_284")
 	tk.MustExec("drop table test.t_parser_issue_284_2")
 }
+
+func (s *testIntegrationSuite6) TestAddExpressionIndex(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t;")
+	tk.MustExec("create table t (a int, b real);")
+	tk.MustExec("insert into t values (1, 2.1);")
+	tk.MustExec("alter table t add index idx((a+b));")
+
+	tblInfo, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	c.Assert(err, IsNil)
+	columns := tblInfo.Meta().Columns
+	c.Assert(len(columns), Equals, 3)
+	c.Assert(columns[2].Hidden, IsTrue)
+
+	tk.MustQuery("select * from t;").Check(testkit.Rows("1 2.1"))
+
+	tk.MustExec("alter table t add index idx_multi((a+b),(a+1), b);")
+	tblInfo, err = s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	c.Assert(err, IsNil)
+	columns = tblInfo.Meta().Columns
+	c.Assert(len(columns), Equals, 5)
+	c.Assert(columns[3].Hidden, IsTrue)
+	c.Assert(columns[4].Hidden, IsTrue)
+
+	tk.MustQuery("select * from t;").Check(testkit.Rows("1 2.1"))
+
+	tk.MustExec("alter table t drop index idx;")
+	tblInfo, err = s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	c.Assert(err, IsNil)
+	columns = tblInfo.Meta().Columns
+	c.Assert(len(columns), Equals, 4)
+
+	tk.MustQuery("select * from t;").Check(testkit.Rows("1 2.1"))
+
+	tk.MustExec("alter table t drop index idx_multi;")
+	tblInfo, err = s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	c.Assert(err, IsNil)
+	columns = tblInfo.Meta().Columns
+	c.Assert(len(columns), Equals, 2)
+
+	tk.MustQuery("select * from t;").Check(testkit.Rows("1 2.1"))
+}
+
+func (s *testIntegrationSuite6) TestCreateExpressionIndexError(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t;")
+	tk.MustExec("create table t (a int, b real);")
+	tk.MustGetErrCode("alter table t add primary key ((a+b));", mysql.ErrFunctionalIndexPrimaryKey)
+
+}

ddl/db_test.go

@@ -520,8 +520,8 @@ func testCancelDropIndex(c *C, store kv.Storage, d ddl.DDL, idxName, addIdxSQL,
 	}{
 		// model.JobStateNone means the jobs is canceled before the first run.
 		{true, model.JobStateNone, model.StateNone, true},
-		{false, model.JobStateRunning, model.StateWriteOnly, true},
-		{false, model.JobStateRunning, model.StateDeleteOnly, false},
+		{false, model.JobStateRunning, model.StateWriteOnly, false},
+		{true, model.JobStateRunning, model.StateDeleteOnly, false},
 		{true, model.JobStateRunning, model.StateDeleteReorganization, false},
 	}
 	var checkErr error

ddl/ddl.go

@@ -226,6 +226,10 @@ var (
 	ErrFieldNotFoundPart = terror.ClassDDL.New(mysql.ErrFieldNotFoundPart, mysql.MySQLErrName[mysql.ErrFieldNotFoundPart])
 	// ErrWrongTypeColumnValue returns 'Partition column values of incorrect type'
 	ErrWrongTypeColumnValue = terror.ClassDDL.New(mysql.ErrWrongTypeColumnValue, mysql.MySQLErrName[mysql.ErrWrongTypeColumnValue])
+	// ErrFunctionalIndexPrimaryKey returns 'The primary key cannot be a functional index'
+	ErrFunctionalIndexPrimaryKey = terror.ClassDDL.New(mysql.ErrFunctionalIndexPrimaryKey, mysql.MySQLErrName[mysql.ErrFunctionalIndexPrimaryKey])
+	// ErrFunctionalIndexOnField returns 'Functional index on a column is not supported. Consider using a regular index instead'
+	ErrFunctionalIndexOnField = terror.ClassDDL.New(mysql.ErrFunctionalIndexOnField, mysql.MySQLErrName[mysql.ErrFunctionalIndexOnField])
 	// ErrInvalidAutoRandom returns when auto_random is used incorrectly.
 	ErrInvalidAutoRandom = terror.ClassDDL.New(mysql.ErrInvalidAutoRandom, mysql.MySQLErrName[mysql.ErrInvalidAutoRandom])
 )
@@ -669,6 +673,7 @@ func init() {
 		mysql.ErrFieldNotFoundPart:                    mysql.ErrFieldNotFoundPart,
 		mysql.ErrFieldTypeNotAllowedAsPartitionField:  mysql.ErrFieldTypeNotAllowedAsPartitionField,
 		mysql.ErrFileNotFound:                         mysql.ErrFileNotFound,
+		mysql.ErrFunctionalIndexPrimaryKey:            mysql.ErrFunctionalIndexPrimaryKey,
 		mysql.ErrGeneratedColumnFunctionIsNotAllowed:  mysql.ErrGeneratedColumnFunctionIsNotAllowed,
 		mysql.ErrGeneratedColumnNonPrior:              mysql.ErrGeneratedColumnNonPrior,
 		mysql.ErrGeneratedColumnRefAutoInc:            mysql.ErrGeneratedColumnRefAutoInc,
@@ -733,6 +738,7 @@ func init() {
 		mysql.ErrWrongTableName:                       mysql.ErrWrongTableName,
 		mysql.ErrWrongTypeColumnValue:                 mysql.ErrWrongTypeColumnValue,
 		mysql.WarnDataTruncated:                       mysql.WarnDataTruncated,
+		mysql.ErrFunctionalIndexOnField:               mysql.ErrFunctionalIndexOnField,
 		mysql.ErrFkColumnCannotDrop:                   mysql.ErrFkColumnCannotDrop,
 		mysql.ErrFKIncompatibleColumns:                mysql.ErrFKIncompatibleColumns,
 	}

ddl/ddl_api.go

@@ -3426,7 +3426,7 @@ func getAnonymousIndex(t table.Table, colName model.CIStr) model.CIStr {
 }
 
 func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName model.CIStr,
-	idxColNames []*ast.IndexPartSpecification, indexOption *ast.IndexOption) error {
+	indexPartSpecifications []*ast.IndexPartSpecification, indexOption *ast.IndexOption) error {
 	if !config.GetGlobalConfig().AlterPrimaryKey {
 		return ErrUnsupportedModifyPrimaryKey.GenWithStack("Unsupported add primary key, alter-primary-key is false")
 	}
@@ -3445,22 +3445,30 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
 		return infoschema.ErrMultiplePriKey
 	}
 
+	// Primary keys cannot include expression index parts. A primary key requires the generated column to be stored,
+	// but expression index parts are implemented as virtual generated columns, not stored generated columns.
+	for _, idxPart := range indexPartSpecifications {
+		if idxPart.Expr != nil {
+			return ErrFunctionalIndexPrimaryKey
+		}
+	}
+
 	tblInfo := t.Meta()
 	// Check before the job is put to the queue.
 	// This check is redundant, but useful. If DDL check fail before the job is put
 	// to job queue, the fail path logic is super fast.
 	// After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic.
 	// The recover step causes DDL wait a few seconds, makes the unit test painfully slow.
-	_, err = buildIndexColumns(tblInfo.Columns, idxColNames)
+	_, err = buildIndexColumns(tblInfo.Columns, indexPartSpecifications)
 	if err != nil {
 		return errors.Trace(err)
 	}
-	if _, err = checkPKOnGeneratedColumn(tblInfo, idxColNames); err != nil {
+	if _, err = checkPKOnGeneratedColumn(tblInfo, indexPartSpecifications); err != nil {
 		return err
 	}
 
 	if tblInfo.GetPartitionInfo() != nil {
-		if err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), idxColNames, tblInfo, true); err != nil {
+		if err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexPartSpecifications, tblInfo, true); err != nil {
 			return err
 		}
 	}
@@ -3478,7 +3486,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
 		SchemaName: schema.Name.L,
 		Type:       model.ActionAddPrimaryKey,
 		BinlogInfo: &model.HistoryInfo{},
-		Args:       []interface{}{unique, indexName, idxColNames, indexOption, sqlMode},
+		Args:       []interface{}{unique, indexName, indexPartSpecifications, indexOption, sqlMode},
 		Priority:   ctx.GetSessionVars().DDLReorgPriority,
 	}
 
@@ -3488,8 +3496,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m
 }
 
 func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.IndexKeyType, indexName model.CIStr,
-	idxColNames []*ast.IndexPartSpecification, indexOption *ast.IndexOption, ifNotExists bool) error {
-
+	indexPartSpecifications []*ast.IndexPartSpecification, indexOption *ast.IndexOption, ifNotExists bool) error {
 	// not support Spatial and FullText index
 	if keyType == ast.IndexKeyTypeFullText || keyType == ast.IndexKeyTypeSpatial {
 		return errUnsupportedIndexType.GenWithStack("FULLTEXT and SPATIAL index is not supported")
@@ -3502,7 +3509,7 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
 
 	// Deal with anonymous index.
 	if len(indexName.L) == 0 {
-		indexName = getAnonymousIndex(t, idxColNames[0].Column.Name)
+		indexName = getAnonymousIndex(t, indexPartSpecifications[0].Column.Name)
 	}
 
 	if indexInfo := t.Meta().FindIndexByName(indexName.L); indexInfo != nil {
@@ -3519,32 +3526,111 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
 	}
 
 	tblInfo := t.Meta()
+	hiddenCols := make([]*model.ColumnInfo, 0, len(indexPartSpecifications))
+	// build hidden columns if necessary
+	for i, idxPart := range indexPartSpecifications {
+		if idxPart.Expr != nil {
+			idxPart.Column = &ast.ColumnName{Name: model.NewCIStr(fmt.Sprintf("_v$_%s_%d", indexName, i))}
+			idxPart.Length = types.UnspecifiedLength
+			// The index part is an expression, prepare a hidden column for it.
+			if len(idxPart.Column.Name.L) > mysql.MaxColumnNameLength {
+				// TODO: refine the error message
+				return ErrTooLongIdent.GenWithStackByArgs("hidden column")
+			}
+			// // TODO: refine the error message
+			if err := checkIllegalFn4GeneratedColumn("expression index", idxPart.Expr); err != nil {
+				return errors.Trace(err)
+			}
+
+			var sb strings.Builder
+			restoreFlags := format.RestoreStringSingleQuotes | format.RestoreKeyWordLowercase | format.RestoreNameBackQuotes |
+				format.RestoreSpacesAroundBinaryOperation
+			restoreCtx := format.NewRestoreCtx(restoreFlags, &sb)
+			sb.Reset()
+			err = idxPart.Expr.Restore(restoreCtx)
+			if err != nil {
+				return errors.Trace(err)
+			}
+			expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr)
+			if err != nil {
+				// TODO: refine the error message
+				return err
+			}
+			if _, ok := expr.(*expression.Column); ok {
+				return ErrFunctionalIndexOnField
+			}
+
+			colInfo := &model.ColumnInfo{
+				Name:                idxPart.Column.Name,
+				GeneratedExprString: sb.String(),
+				GeneratedStored:     false,
+				Version:             model.CurrLatestColumnInfoVersion,
+				Dependences:         make(map[string]struct{}),
+				Hidden:              true,
+				FieldType:           *expr.GetType(),
+			}
+			for _, colName := range findColumnNamesInExpr(idxPart.Expr) {
+				colInfo.Dependences[colName.Name.O] = struct{}{}
+			}
+			if err = checkDependedColExist(colInfo.Dependences, t.Cols()); err != nil {
+				return errors.Trace(err)
+			}
+			idxPart.Expr = nil
+			hiddenCols = append(hiddenCols, colInfo)
+		}
+	}
+
 	// Check before the job is put to the queue.
 	// This check is redundant, but useful. If DDL check fail before the job is put
 	// to job queue, the fail path logic is super fast.
 	// After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic.
 	// The recover step causes DDL wait a few seconds, makes the unit test painfully slow.
-	_, err = buildIndexColumns(tblInfo.Columns, idxColNames)
+	_, err = buildIndexColumns(append(tblInfo.Columns, hiddenCols...), indexPartSpecifications)
 	if err != nil {
 		return errors.Trace(err)
 	}
 	if unique && tblInfo.GetPartitionInfo() != nil {
-		if err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), idxColNames, tblInfo, false); err != nil {
+		if err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexPartSpecifications, tblInfo, false); err != nil {
 			return err
 		}
 	}
 	// May be truncate comment here, when index comment too long and sql_mode is't strict.
 	if _, err = validateCommentLength(ctx.GetSessionVars(), indexName.String(), indexOption); err != nil {
 		return errors.Trace(err)
 	}
-
+	if len(hiddenCols) > 0 {
+		// Check hidden column
+		if err = checkAddColumnTooManyColumns(len(t.Cols()) + len(hiddenCols)); err != nil {
+			return errors.Trace(err)
+		}
+		// Check whether the hidden columns have existed.
+		for _, colInfo := range hiddenCols {
+			// Check whether the hidden columns have existed.
+			col := table.FindCol(t.Cols(), colInfo.Name.String())
+			if col != nil {
+				// TODO: use expression index related error
+				return infoschema.ErrColumnExists.GenWithStackByArgs(colInfo.Name.String())
+			}
+			if err = checkAutoIncrementRef("", colInfo.Dependences, tblInfo); err != nil {
+				return errors.Trace(err)
+			}
+		}
+		for _, indexPart := range indexPartSpecifications {
+			if indexPart.Expr != nil {
+				// TODO: refine the error message
+				if err := checkIllegalFn4GeneratedColumn("", indexPart.Expr); err != nil {
+					return errors.Trace(err)
+				}
+			}
+		}
+	}
 	job := &model.Job{
 		SchemaID:   schema.ID,
 		TableID:    t.Meta().ID,
 		SchemaName: schema.Name.L,
 		Type:       model.ActionAddIndex,
 		BinlogInfo: &model.HistoryInfo{},
-		Args:       []interface{}{unique, indexName, idxColNames, indexOption},
+		Args:       []interface{}{unique, indexName, indexPartSpecifications, indexOption, hiddenCols},
 		Priority:   ctx.GetSessionVars().DDLReorgPriority,
 	}
 

ddl/generated_column.go

@@ -69,11 +69,13 @@ func verifyColumnGenerationSingle(dependColNames map[string]struct{}, cols []*ta
 	return nil
 }
 
-// checkDependedColExist ensure all depended columns exist.
+// checkDependedColExist ensure all depended columns exist and not hidden.
 // NOTE: this will MODIFY parameter `dependCols`.
 func checkDependedColExist(dependCols map[string]struct{}, cols []*table.Column) error {
 	for _, col := range cols {
-		delete(dependCols, col.Name.L)
+		if !col.Hidden {
+			delete(dependCols, col.Name.L)
+		}
 	}
 	if len(dependCols) != 0 {
 		for arbitraryCol := range dependCols {