NEOS-1730: mysql index statements broken when using non column (#3198)

backend/pkg/dbschemas/sql/mysql/queries/system.sql

@@ -52,7 +52,7 @@ LEFT JOIN information_schema.check_constraints as cc
 	AND tc.constraint_name = cc.constraint_name
 WHERE
     tc.table_schema IN (sqlc.slice('schemas'))
-GROUP BY 
+GROUP BY
     tc.table_schema,
     tc.table_name,
     tc.constraint_name,
@@ -97,7 +97,7 @@ LEFT JOIN information_schema.check_constraints as cc
 WHERE
     tc.table_schema = sqlc.arg('schema')
     AND tc.table_name IN (sqlc.slice('tables'))
-GROUP BY 
+GROUP BY
     tc.table_schema,
     tc.table_name,
     tc.constraint_name,
@@ -178,7 +178,7 @@ SELECT
     ACTION_TIMING AS timing
 FROM
     information_schema.TRIGGERS
-WHERE 
+WHERE
     EVENT_OBJECT_SCHEMA = sqlc.arg('schema') AND EVENT_OBJECT_TABLE IN (sqlc.slice('tables'));
 
 
@@ -191,9 +191,9 @@ SELECT
    IFNULL(REPLACE(REPLACE(REPLACE(REPLACE(c.COLUMN_DEFAULT, '_utf8mb4\\\'', '_utf8mb4\''), '_utf8mb3\\\'', '_utf8mb3\''), '\\\'', '\''), '\\\'', '\''), '') AS column_default, -- hack to fix this bug https://bugs.mysql.com/bug.php?
    CASE WHEN c.IS_NULLABLE = 'YES' THEN 1 ELSE 0 END AS is_nullable,
    CAST(IF(c.DATA_TYPE IN ('varchar', 'char'), c.CHARACTER_MAXIMUM_LENGTH, -1) AS SIGNED) AS character_maximum_length,
-   CAST(IF(c.DATA_TYPE IN ('decimal', 'numeric'), c.NUMERIC_PRECISION, 
-     IF(c.DATA_TYPE = 'smallint', 16, 
-        IF(c.DATA_TYPE = 'int', 32, 
+   CAST(IF(c.DATA_TYPE IN ('decimal', 'numeric'), c.NUMERIC_PRECISION,
+     IF(c.DATA_TYPE = 'smallint', 16,
+        IF(c.DATA_TYPE = 'int', 32,
            IF(c.DATA_TYPE = 'bigint', 64, -1))))AS SIGNED) AS numeric_precision,
    CAST(IF(c.DATA_TYPE IN ('decimal', 'numeric'), c.NUMERIC_SCALE, 0)AS SIGNED) AS numeric_scale,
    c.ORDINAL_POSITION AS ordinal_position,
@@ -211,41 +211,41 @@ ORDER BY
 
 
 -- name: GetIndicesBySchemasAndTables :many
-SELECT 
+SELECT
     s.TABLE_SCHEMA as schema_name,
     s.TABLE_NAME as table_name,
     s.COLUMN_NAME as column_name,
+    s.EXPRESSION as expression,
     s.INDEX_NAME as index_name,
     s.INDEX_TYPE as index_type,
     s.SEQ_IN_INDEX as seq_in_index,
     s.NULLABLE as nullable
-FROM 
+FROM
     INFORMATION_SCHEMA.STATISTICS s
-LEFT JOIN 
+LEFT JOIN
     INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
     ON s.TABLE_SCHEMA = kcu.CONSTRAINT_SCHEMA
     AND s.TABLE_NAME = kcu.TABLE_NAME
     AND s.COLUMN_NAME = kcu.COLUMN_NAME
-WHERE 
-    -- CONCAT(s.TABLE_SCHEMA, '.', s.TABLE_NAME) IN (sqlc.slice('schematables')) broken
-		s.TABLE_SCHEMA = sqlc.arg('schema') AND s.TABLE_NAME in (sqlc.slice('tables'))
+WHERE
+    s.TABLE_SCHEMA = sqlc.arg('schema') AND s.TABLE_NAME in (sqlc.slice('tables'))
     AND s.INDEX_NAME != 'PRIMARY'
     AND kcu.CONSTRAINT_NAME IS NULL
-ORDER BY 
+ORDER BY
     s.TABLE_NAME,
     s.INDEX_NAME,
     s.SEQ_IN_INDEX;
 
 
 -- name: GetCustomFunctionsBySchemas :many
-SELECT 
-    ROUTINE_NAME as function_name, 
+SELECT
+    ROUTINE_NAME as function_name,
     ROUTINE_SCHEMA as schema_name,
     DTD_IDENTIFIER as return_data_type,
     ROUTINE_DEFINITION as definition,
     CASE WHEN IS_DETERMINISTIC = 'YES' THEN 1 ELSE 0 END as is_deterministic
-FROM 
-    INFORMATION_SCHEMA.ROUTINES 
-WHERE 
+FROM
+    INFORMATION_SCHEMA.ROUTINES
+WHERE
     ROUTINE_TYPE = 'FUNCTION'
     AND ROUTINE_SCHEMA in (sqlc.slice('schemas'));

backend/pkg/dbschemas/sql/mysql/schema/system.sql

@@ -60,7 +60,8 @@ create table information_schema.table_privileges (
 create table information_schema.statistics (
   table_schema text not null,
   table_name text not null,
-  column_name text not null,
+  column_name text null,
+  expression text null,
   index_name text not null,
   index_type text not null,
   seq_in_index bigint,

backend/pkg/sqlmanager/mysql/mysql-manager.go

@@ -316,7 +316,7 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 				Tables: tables,
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to build mysql database table schemas by schemas and tables: %w", err)
 			}
 			colDefMapMu.Lock()
 			defer colDefMapMu.Unlock()
@@ -337,7 +337,7 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 				Tables: tables,
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to build mysql table constraints: %w", err)
 			}
 			constraintMapMu.Lock()
 			defer constraintMapMu.Unlock()
@@ -358,7 +358,7 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 				Tables: tables,
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to build mysql indices by schemas and tables: %w", err)
 			}
 
 			indexMapMu.Lock()
@@ -368,11 +368,19 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 				if _, exists := indexmap[key.String()]; !exists {
 					indexmap[key.String()] = make(map[string][]string)
 				}
-				// Group columns by index name
-				indexmap[key.String()][record.IndexName] = append(
-					indexmap[key.String()][record.IndexName],
-					record.ColumnName,
-				)
+				// Group columns/expressions by index name
+				if record.ColumnName.Valid {
+					indexmap[key.String()][record.IndexName] = append(
+						indexmap[key.String()][record.IndexName],
+						record.ColumnName.String,
+					)
+				} else if record.Expression.Valid {
+					indexmap[key.String()][record.IndexName] = append(
+						indexmap[key.String()][record.IndexName],
+						// expressions must be wrapped in parentheses on creation, but don't come out of the DB in that format /shrug
+						fmt.Sprintf("(%s)", record.Expression.String),
+					)
+				}
 			}
 			return nil
 		})
@@ -399,7 +407,7 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 
 			columnDefaultStr, err := convertUInt8ToString(record.ColumnDefault)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to convert column default to string: %w", err)
 			}
 			var columnDefaultType *string
 			if identityType != nil && columnDefaultStr != "" && *identityType == "" {
@@ -411,12 +419,12 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 			}
 			columnDefaultStr, err = EscapeMysqlDefaultColumn(columnDefaultStr, columnDefaultType)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to escape column default: %w", err)
 			}
 
 			genExp, err := convertUInt8ToString(record.GenerationExp)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to convert generation expression to string: %w", err)
 			}
 			columns = append(columns, buildTableCol(&buildTableColRequest{
 				ColumnName:          record.ColumnName,
@@ -436,7 +444,7 @@ func (m *MysqlManager) GetTableInitStatements(ctx context.Context, tables []*sql
 		for _, constraint := range constraintmap[key] {
 			stmt, err := buildAlterStatementByConstraint(constraint)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to build alter table statement by constraint: %w", err)
 			}
 			info.AlterTableStatements = append(info.AlterTableStatements, stmt)
 		}
@@ -793,7 +801,11 @@ func wrapIdempotentIndex(
 
 	columnInput := []string{}
 	for _, col := range cols {
-		columnInput = append(columnInput, EscapeMysqlColumn(col))
+		if strings.HasPrefix(col, "(") {
+			columnInput = append(columnInput, col)
+		} else {
+			columnInput = append(columnInput, EscapeMysqlColumn(col))
+		}
 	}
 	procedureName := fmt.Sprintf("NeosyncAddIndex_%s", hashInput(hashParams...))[:64]
 	stmt := fmt.Sprintf(`

backend/pkg/sqlmanager/mysql/mysql-manager_integration_test.go

@@ -295,6 +295,7 @@ func Test_MysqlManager(t *testing.T) {
 				{Schema: schema, Table: "parent1"},
 				{Schema: schema, Table: "child1"},
 				{Schema: schema, Table: "order"},
+				{Schema: schema, Table: "test_mixed_index"},
 			},
 		)
 

backend/pkg/sqlmanager/mysql/testdata/setup.sql

@@ -196,3 +196,15 @@ CREATE TABLE `order` (
 
 -- Creates an index that uses reserved MySQL words
 CREATE INDEX `order_index_on_reserved_words` ON `order` (`select`, `from`, `where`);
+
+-- Create a table with some columns
+CREATE TABLE test_mixed_index (
+    id INT PRIMARY KEY,
+    first_name VARCHAR(50),
+    last_name VARCHAR(50),
+    birth_date DATE
+);
+
+-- Create a composite index that uses both regular columns and expressions
+CREATE INDEX idx_mixed ON test_mixed_index
+    (first_name, (UPPER(last_name)), birth_date, (YEAR(birth_date)));