NEOS-1787: add auto map support to mysql

backend/pkg/dbschemas/sql/mysql/queries/system.sql

@@ -7,6 +7,7 @@ SELECT
     IFNULL(REPLACE(REPLACE(REPLACE(REPLACE(c.COLUMN_DEFAULT, '_utf8mb4\\\'', '_utf8mb4\''), '_utf8mb3\\\'', '_utf8mb3\''), '\\\'', '\''), '\\\'', '\''), '') AS column_default, -- hack to fix this bug https://bugs.mysql.com/bug.php?
 	c.is_nullable,
 	c.data_type,
+    c.column_type, -- same as data_type but includes things like the length, or set/enum information
 	c.character_maximum_length,
     c.numeric_precision,
     c.numeric_scale,
@@ -214,6 +215,7 @@ SELECT
    c.TABLE_NAME AS table_name,
    c.COLUMN_NAME AS column_name,
    c.COLUMN_TYPE AS data_type,
+   c.COLUMN_TYPE AS column_type, -- same as data_type but includes things like the length, or set/enum information
    IFNULL(REPLACE(REPLACE(REPLACE(REPLACE(c.COLUMN_DEFAULT, '_utf8mb4\\\'', '_utf8mb4\''), '_utf8mb3\\\'', '_utf8mb3\''), '\\\'', '\''), '\\\'', '\''), '') AS column_default, -- hack to fix this bug https://bugs.mysql.com/bug.php?
    CASE WHEN c.IS_NULLABLE = 'YES' THEN 1 ELSE 0 END AS is_nullable,
    CAST(IF(c.DATA_TYPE IN ('varchar', 'char'), c.CHARACTER_MAXIMUM_LENGTH, -1) AS SIGNED) AS character_maximum_length,

backend/pkg/sqlmanager/mysql/mysql-manager.go

@@ -96,6 +96,7 @@ func (m *MysqlManager) GetDatabaseSchema(ctx context.Context) ([]*sqlmanager_sha
 			TableName:              row.TableName,
 			ColumnName:             row.ColumnName,
 			DataType:               row.DataType,
+			MysqlColumnType:        row.ColumnType,
 			ColumnDefault:          columnDefaultStr,
 			ColumnDefaultType:      columnDefaultType,
 			IsNullable:             row.IsNullable != "NO",
@@ -197,6 +198,7 @@ func (m *MysqlManager) GetDatabaseTableSchemasBySchemasAndTables(ctx context.Con
 				TableName:              row.TableName,
 				ColumnName:             row.ColumnName,
 				DataType:               row.DataType,
+				MysqlColumnType:        row.ColumnType,
 				ColumnDefault:          columnDefaultStr,
 				ColumnDefaultType:      columnDefaultType,
 				IsNullable:             row.IsNullable == 1,

backend/pkg/sqlmanager/shared/types.go

@@ -13,6 +13,7 @@ type DatabaseSchemaRow struct {
 	TableName              string
 	ColumnName             string
 	DataType               string
+	MysqlColumnType        string // will only be populated for mysql. Same as the DataType but includes length, etc. varchar(255), enum('a', 'b'), etc.
 	ColumnDefault          string
 	ColumnDefaultType      *string
 	IsNullable             bool

backend/protos/mgmt/v1alpha1/job.proto

@@ -207,7 +207,8 @@ message PostgresSourceTableOption {
 
 message MysqlSourceConnectionOptions {
   // Whether to halt the job if a new column is added
-  bool halt_on_new_column_addition = 1;
+  // Deprecated: Use new_column_addition_strategy instead
+  bool halt_on_new_column_addition = 1 [deprecated = true];
   // The list of schemas (and their tables) along with any configuration options that will be used.
   repeated MysqlSourceSchemaOption schemas = 2;
   // The unique connection id to a mysql connection configuration
@@ -218,6 +219,9 @@ message MysqlSourceConnectionOptions {
   // Provide a strategy of what to do in the event Neosync encounters a column that is removed from the source table.
   ColumnRemovalStrategy column_removal_strategy = 5;
 
+  // Provide a strategy of what to do in the event Neosync encounters an unmapped column for the job's mapped tables.
+  NewColumnAdditionStrategy new_column_addition_strategy = 6;
+
   message ColumnRemovalStrategy {
     oneof strategy {
       // halt job if a column is removed
@@ -232,6 +236,21 @@ message MysqlSourceConnectionOptions {
     // Configuration for the ContinueJob strategy
     message ContinueJob {}
   }
+
+  message NewColumnAdditionStrategy {
+    oneof strategy {
+      // halt job if a new column is detected.
+      HaltJob halt_job = 1;
+      // automatically handle unmapped columns. It handles this by using the DBs default/nullable values.
+      // If this doesn't exist, will fall back to configuring generators for supported datatypes.
+      // If none of the criteria above can be met, the job run will fail to prevent leaking of PII.
+      AutoMap auto_map = 2;
+    }
+    // Configuration for the HaltJob strategy
+    message HaltJob {}
+    // Configuration for the AutoMap strategy
+    message AutoMap {}
+  }
 }
 
 message MysqlSourceSchemaOption {

backend/sql/postgresql/models/models.go

@@ -1209,13 +1209,23 @@ func (s *MongoDbSourceOptions) FromDto(dto *mgmtv1alpha1.MongoDBSourceConnection
 }
 
 type MysqlSourceOptions struct {
-	HaltOnNewColumnAddition       bool                        `json:"haltOnNewColumnAddition"`
-	SubsetByForeignKeyConstraints bool                        `json:"subsetByForeignKeyConstraints"`
-	Schemas                       []*MysqlSourceSchemaOption  `json:"schemas"`
-	ConnectionId                  string                      `json:"connectionId"`
-	ColumnRemovalStrategy         *MysqlColumnRemovalStrategy `json:"columnRemovalStrategy,omitempty"`
+	// @deprecated
+	HaltOnNewColumnAddition       bool                            `json:"haltOnNewColumnAddition"`
+	SubsetByForeignKeyConstraints bool                            `json:"subsetByForeignKeyConstraints"`
+	Schemas                       []*MysqlSourceSchemaOption      `json:"schemas"`
+	ConnectionId                  string                          `json:"connectionId"`
+	ColumnRemovalStrategy         *MysqlColumnRemovalStrategy     `json:"columnRemovalStrategy,omitempty"`
+	NewColumnAdditionStrategy     *MysqlNewColumnAdditionStrategy `json:"newColumnAdditionStrategy,omitempty"`
 }
 
+type MysqlNewColumnAdditionStrategy struct {
+	HaltJob *MysqlHaltJobNewColumnAdditionStrategy `json:"haltJob,omitempty"`
+	AutoMap *MysqlAutoMapNewColumnAdditionStrategy `json:"autoMap,omitempty"`
+}
+
+type MysqlHaltJobNewColumnAdditionStrategy struct{}
+type MysqlAutoMapNewColumnAdditionStrategy struct{}
+
 type MysqlColumnRemovalStrategy struct {
 	HaltJob     *MysqlHaltJobColumnRemovalStrategy     `json:"haltJob,omitempty"`
 	ContinueJob *MysqlContinueJobColumnRemovalStrategy `json:"continueJob,omitempty"`
@@ -1466,11 +1476,60 @@ func (s *MysqlSourceOptions) ToDto() *mgmtv1alpha1.MysqlSourceConnectionOptions
 	if s.ColumnRemovalStrategy != nil {
 		dto.ColumnRemovalStrategy = s.ColumnRemovalStrategy.ToDto()
 	}
+	if s.NewColumnAdditionStrategy != nil {
+		dto.NewColumnAdditionStrategy = s.NewColumnAdditionStrategy.ToDto()
+	} else if s.HaltOnNewColumnAddition {
+		dto.NewColumnAdditionStrategy = &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy{
+			Strategy: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_HaltJob_{
+				HaltJob: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_HaltJob{},
+			},
+		}
+		dto.HaltOnNewColumnAddition = true //nolint:staticcheck
+	}
 
 	return dto
 }
+
+func (s *MysqlNewColumnAdditionStrategy) ToDto() *mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy {
+	if s.HaltJob != nil {
+		return &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy{
+			Strategy: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_HaltJob_{
+				HaltJob: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_HaltJob{},
+			},
+		}
+	}
+	if s.AutoMap != nil {
+		return &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy{
+			Strategy: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_AutoMap_{
+				AutoMap: &mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_AutoMap{},
+			},
+		}
+	}
+	return nil
+}
+
+func (s *MysqlNewColumnAdditionStrategy) FromDto(dto *mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy) {
+	if dto.GetStrategy() != nil {
+		switch dto.GetStrategy().(type) {
+		case *mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_HaltJob_:
+			s.HaltJob = &MysqlHaltJobNewColumnAdditionStrategy{}
+		case *mgmtv1alpha1.MysqlSourceConnectionOptions_NewColumnAdditionStrategy_AutoMap_:
+			s.AutoMap = &MysqlAutoMapNewColumnAdditionStrategy{}
+		}
+	}
+}
+
 func (s *MysqlSourceOptions) FromDto(dto *mgmtv1alpha1.MysqlSourceConnectionOptions) {
-	s.HaltOnNewColumnAddition = dto.HaltOnNewColumnAddition
+	s.HaltOnNewColumnAddition = dto.HaltOnNewColumnAddition //nolint:staticcheck
+	if dto.GetNewColumnAdditionStrategy().GetStrategy() != nil {
+		s.NewColumnAdditionStrategy = &MysqlNewColumnAdditionStrategy{}
+		s.NewColumnAdditionStrategy.FromDto(dto.GetNewColumnAdditionStrategy())
+	} else if dto.HaltOnNewColumnAddition { //nolint:staticcheck
+		s.NewColumnAdditionStrategy = &MysqlNewColumnAdditionStrategy{
+			HaltJob: &MysqlHaltJobNewColumnAdditionStrategy{},
+		}
+		s.HaltOnNewColumnAddition = true
+	}
 	s.SubsetByForeignKeyConstraints = dto.SubsetByForeignKeyConstraints
 	s.Schemas = FromDtoMysqlSourceSchemaOptions(dto.Schemas)
 	s.ConnectionId = dto.ConnectionId

docs/docs/guides/new-column-addition-strategies.md

@@ -19,7 +19,7 @@ This is a common occurrence for any company that is adding new columns to a data
 | Strategy | Description                                                                                     | PostgreSQL | MySQL | MS SQL Server |
 | -------- | ----------------------------------------------------------------------------------------------- | ---------- | ----- | ------------- |
 | Halt     | Stops the job run if a new column is detected that is not found in the configured job mappings. | ✅         | ✅    | ✅            |
-| AutoMap  | Automatically generates a fake value. See more below.                                           | ✅         | ❌    | ❌            |
+| AutoMap  | Automatically generates a fake value. See more below.                                           | ✅         | ✅    | ❌            |
 | Continue | Ignores new columns; may fail if column doesn't have default. See more below.                   | ✅         | ✅    | ✅            |
 
 ## Halt Strategy
@@ -111,3 +111,47 @@ Postgres has many data types and not all of them are currently supported in the
 | text[]           | ❌      |                 |
 
 <!-- cspell:enable  -->
+
+### MySQL
+
+MySQL has many data types and not all of them are currently supported in the auto map mode. Support will continue to increase over time.
+
+<!-- cspell:disable  -->
+
+| Data Type        | Support | Generator           |
+| ---------------- | ------- | ------------------- |
+| tinyint          | ✅      | GenerateInt64       |
+| smallint         | ✅      | GenerateInt64       |
+| mediumint        | ✅      | GenerateInt64       |
+| integer          | ✅      | GenerateInt64       |
+| int              | ✅      | GenerateInt64       |
+| bigint           | ✅      | GenerateInt64       |
+| float            | ✅      | GenerateFloat64     |
+| decimal          | ✅      | GenerateFloat64     |
+| dec              | ✅      | GenerateFloat64     |
+| double           | ✅      | GenerateFloat64     |
+| double precision | ✅      | GenerateFloat64     |
+| char             | ✅      | GenerateString      |
+| varchar          | ✅      | GenerateString      |
+| tinytext         | ✅      | GenerateString      |
+| text             | ✅      | GenerateString      |
+| mediumtext       | ✅      | GenerateString      |
+| longtext         | ✅      | GenerateString      |
+| boolean          | ✅      | GenerateBool        |
+| bool             | ✅      | GenerateBool        |
+| enum             | ✅      | GenerateCategorical |
+| set              | ✅      | GenerateCategorical |
+| date             | ✅      | GenerateJavaScript  |
+| datetime         | ✅      | GenerateJavaScript  |
+| timestamp        | ✅      | GenerateJavaScript  |
+| time             | ✅      | GenerateJavaScript  |
+| year             | ✅      | GenerateJavaScript  |
+| bit              | ❌      |                     |
+| binary           | ❌      |                     |
+| varbinary        | ❌      |                     |
+| tinyblob         | ❌      |                     |
+| blob             | ❌      |                     |
+| mediumblob       | ❌      |                     |
+| longblob         | ❌      |                     |
+
+<!-- cspell:enable  -->

docs/openapi/mgmt/v1alpha1/job.openapi.yaml

@@ -3504,7 +3504,9 @@ components:
         haltOnNewColumnAddition:
           type: boolean
           title: halt_on_new_column_addition
-          description: Whether to halt the job if a new column is added
+          description: |-
+            Whether to halt the job if a new column is added
+             Deprecated: Use new_column_addition_strategy instead
         schemas:
           type: array
           items:
@@ -3525,6 +3527,11 @@ components:
             - $ref: '#/components/schemas/mgmt.v1alpha1.MysqlSourceConnectionOptions.ColumnRemovalStrategy'
           title: column_removal_strategy
           description: Provide a strategy of what to do in the event Neosync encounters a column that is removed from the source table.
+        newColumnAdditionStrategy:
+          allOf:
+            - $ref: '#/components/schemas/mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy'
+          title: new_column_addition_strategy
+          description: Provide a strategy of what to do in the event Neosync encounters an unmapped column for the job's mapped tables.
       title: MysqlSourceConnectionOptions
       additionalProperties: false
     mgmt.v1alpha1.MysqlSourceConnectionOptions.ColumnRemovalStrategy:
@@ -3575,6 +3582,57 @@ components:
       title: HaltJob
       additionalProperties: false
       description: Configuration for the HaltJob strategy
+    mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy:
+      type: object
+      allOf:
+        - anyOf:
+            - required:
+                - autoMap
+            - required:
+                - haltJob
+            - not:
+                anyOf:
+                  - required:
+                      - autoMap
+                  - required:
+                      - haltJob
+      anyOf:
+        - required:
+            - autoMap
+        - required:
+            - haltJob
+        - not:
+            anyOf:
+              - required:
+                  - autoMap
+              - required:
+                  - haltJob
+      properties:
+        haltJob:
+          allOf:
+            - $ref: '#/components/schemas/mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy.HaltJob'
+          title: halt_job
+          description: halt job if a new column is detected.
+        autoMap:
+          allOf:
+            - $ref: '#/components/schemas/mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy.AutoMap'
+          title: auto_map
+          description: |-
+            automatically handle unmapped columns. It handles this by using the DBs default/nullable values.
+             If this doesn't exist, will fall back to configuring generators for supported datatypes.
+             If none of the criteria above can be met, the job run will fail to prevent leaking of PII.
+      title: NewColumnAdditionStrategy
+      additionalProperties: false
+    mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy.AutoMap:
+      type: object
+      title: AutoMap
+      additionalProperties: false
+      description: Configuration for the AutoMap strategy
+    mgmt.v1alpha1.MysqlSourceConnectionOptions.NewColumnAdditionStrategy.HaltJob:
+      type: object
+      title: HaltJob
+      additionalProperties: false
+      description: Configuration for the HaltJob strategy
     mgmt.v1alpha1.MysqlSourceSchemaOption:
       type: object
       properties: