Fix collation and charset after migrations (every run) (#4002)

* fix collation and charset after migrations (every run)

server/services/store/sqlstore/data_migrations.go

@@ -3,9 +3,11 @@ package sqlstore
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 
 	sq "github.com/Masterminds/squirrel"
+	"github.com/wiggin77/merror"
 
 	"github.com/mattermost/focalboard/server/model"
 	"github.com/mattermost/focalboard/server/utils"
@@ -645,3 +647,125 @@ func (s *SQLStore) getDeletedMembershipBoards(tx sq.BaseRunner) ([]*model.Board,
 
 	return boards, err
 }
+
+func (s *SQLStore) RunFixCollationsAndCharsetsMigration() error {
+	// This is for MySQL only
+	if s.dbType != model.MysqlDBType {
+		return nil
+	}
+
+	// get collation and charSet setting that Channels is using.
+	// when unit testing, no channels tables exist so just set to a default.
+	var collation string
+	var charSet string
+	var err error
+	if os.Getenv("FOCALBOARD_UNIT_TESTING") == "1" {
+		collation = "utf8mb4_general_ci"
+		charSet = "utf8mb4"
+	} else {
+		collation, charSet, err = s.getCollationAndCharset()
+		if err != nil {
+			return err
+		}
+	}
+
+	// get all FocalBoard tables
+	tableNames, err := s.getFocalBoardTableNames()
+	if err != nil {
+		return err
+	}
+
+	merr := merror.New()
+
+	// alter each table; this is idempotent
+	for _, name := range tableNames {
+		sql := fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET '%s' COLLATE '%s'", name, charSet, collation)
+		result, err := s.db.Exec(sql)
+		if err != nil {
+			merr.Append(err)
+			continue
+		}
+		num, err := result.RowsAffected()
+		if err != nil {
+			merr.Append(err)
+		}
+		if num > 0 {
+			s.logger.Debug("table collation and/or charSet fixed",
+				mlog.String("table_name", name),
+			)
+		}
+	}
+	return merr.ErrorOrNil()
+}
+
+func (s *SQLStore) getFocalBoardTableNames() ([]string, error) {
+	if s.dbType != model.MysqlDBType {
+		return nil, newErrInvalidDBType("getFocalBoardTableNames requires MySQL")
+	}
+
+	query := s.getQueryBuilder(s.db).
+		Select("table_name").
+		From("information_schema.tables").
+		Where(sq.Like{"table_name": s.tablePrefix + "%"}).
+		Where("table_schema=(SELECT DATABASE())")
+
+	rows, err := query.Query()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching FocalBoard table names: %w", err)
+	}
+	defer rows.Close()
+
+	names := make([]string, 0)
+
+	for rows.Next() {
+		var tableName string
+
+		err := rows.Scan(&tableName)
+		if err != nil {
+			return nil, fmt.Errorf("cannot scan result while fetching table names: %w", err)
+		}
+
+		names = append(names, tableName)
+	}
+
+	return names, nil
+}
+
+func (s *SQLStore) getCollationAndCharset() (string, string, error) {
+	if s.dbType != model.MysqlDBType {
+		return "", "", newErrInvalidDBType("getCollationAndCharset requires MySQL")
+	}
+
+	query := s.getQueryBuilder(s.db).
+		Select("table_collation").
+		From("information_schema.tables").
+		Where(sq.Eq{"table_name": "Channels"}).
+		Where("table_schema=(SELECT DATABASE())")
+
+	row := query.QueryRow()
+
+	var collation string
+	err := row.Scan(&collation)
+	if err != nil {
+		return "", "", fmt.Errorf("error fetching collation: %w", err)
+	}
+
+	query = s.getQueryBuilder(s.db).
+		Select("CHARACTER_SET_NAME").
+		From("information_schema.columns").
+		Where(sq.Eq{
+			"table_name":  "Channels",
+			"COLUMN_NAME": "Name",
+		}).
+		Where("table_schema=(SELECT DATABASE())")
+
+	row = query.QueryRow()
+
+	var charSet string
+	err = row.Scan(&charSet)
+	if err != nil {
+		return "", "", fmt.Errorf("error fetching charSet: %w", err)
+	}
+
+	return collation, charSet, nil
+}

server/services/store/sqlstore/data_migrations_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/mattermost/focalboard/server/model"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -234,3 +235,39 @@ func TestRunUniqueIDsMigration(t *testing.T) {
 	require.NotEqual(t, block4.ID, newBlock4.BoardID)
 	require.NotEqual(t, block5.ID, newBlock5.ParentID)
 }
+
+func TestCheckForMismatchedCollation(t *testing.T) {
+	store, tearDown := SetupTests(t)
+	sqlStore := store.(*SQLStore)
+	defer tearDown()
+
+	if sqlStore.dbType != model.MysqlDBType {
+		return
+	}
+
+	// make sure all collations are consistent.
+	tableNames, err := sqlStore.getFocalBoardTableNames()
+	require.NoError(t, err)
+
+	sqlCollation := "SELECT table_collation FROM information_schema.tables WHERE table_name=? and table_schema=(SELECT DATABASE())"
+	stmtCollation, err := sqlStore.db.Prepare(sqlCollation)
+	require.NoError(t, err)
+	defer stmtCollation.Close()
+
+	var collation string
+
+	// make sure the correct charset is applied to each table.
+	for i, name := range tableNames {
+		row := stmtCollation.QueryRow(name)
+
+		var actualCollation string
+		err = row.Scan(&actualCollation)
+		require.NoError(t, err)
+
+		if collation == "" {
+			collation = actualCollation
+		}
+
+		assert.Equalf(t, collation, actualCollation, "for table_name='%s', index=%d", name, i)
+	}
+}

server/services/store/sqlstore/helpers_test.go

@@ -12,6 +12,9 @@ import (
 )
 
 func SetupTests(t *testing.T) (store.Store, func()) {
+	origUnitTesting := os.Getenv("FOCALBOARD_UNIT_TESTING")
+	os.Setenv("FOCALBOARD_UNIT_TESTING", "1")
+
 	dbType, connectionString, err := PrepareNewTestDatabase()
 	require.NoError(t, err)
 
@@ -40,6 +43,7 @@ func SetupTests(t *testing.T) (store.Store, func()) {
 		if err = os.Remove(connectionString); err == nil {
 			logger.Debug("Removed test database", mlog.String("file", connectionString))
 		}
+		os.Setenv("FOCALBOARD_UNIT_TESTING", origUnitTesting)
 	}
 
 	return store, tearDown

server/services/store/sqlstore/migrate.go

@@ -242,9 +242,18 @@ func (s *SQLStore) runMigrationSequence(engine *morph.Morph, driver drivers.Driv
 	}
 
 	s.logger.Debug("== Applying all remaining migrations ====================",
-		mlog.Int("current_version", len(appliedMigrations)))
+		mlog.Int("current_version", len(appliedMigrations)),
+	)
 
-	return engine.ApplyAll()
+	if err := engine.ApplyAll(); err != nil {
+		return err
+	}
+
+	// always run the collations & charset fix-ups
+	if mErr := s.RunFixCollationsAndCharsetsMigration(); mErr != nil {
+		return fmt.Errorf("error running fix collations and charsets migration: %w", mErr)
+	}
+	return nil
 }
 
 func (s *SQLStore) ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error {