Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integrate morph into the plugin #619

Merged
merged 5 commits into from
May 29, 2024
Merged

Integrate morph into the plugin #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c3e6f8b
Integrate morph into the plugin
mgdelacroix Apr 20, 2024
49aa79c
Fix system settings functions and move last migration to morph
mgdelacroix May 17, 2024
3225998
Change down migrations to be empty files
mgdelacroix May 17, 2024
0a606e3
Merge branch 'main' into integrate-morph
mattermost-build May 20, 2024
6cb7dd1
Merge remote-tracking branch 'origin/main' into integrate-morph
mgdelacroix May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/lib/pq v1.10.9
github.com/mattermost/mattermost/server/public v0.1.3
github.com/mattermost/mattermost/server/v8 v8.0.0-20240520151130-4bdd8bb18e47
github.com/mattermost/morph v1.1.0
github.com/microsoft/kiota-abstractions-go v1.6.0
github.com/microsoft/kiota-http-go v1.4.1
github.com/microsoftgraph/msgraph-sdk-go v1.43.0
Expand Down Expand Up @@ -141,7 +142,6 @@ require (
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 // indirect
github.com/mattermost/logr/v2 v2.0.21 // indirect
github.com/mattermost/morph v1.1.0 // indirect
github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0 // indirect
github.com/mattermost/squirrel v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down
205 changes: 100 additions & 105 deletions server/store/sqlstore/migrations.go → server/store/sqlstore/data_migrations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,101 @@ package sqlstore
import (
"database/sql"
"fmt"
"strconv"
"time"

sq "github.com/Masterminds/squirrel"
)

const (
RemoteIDMigrationKey = "RemoteIDMigrationComplete"
SetEmailVerifiedToTrueForRemoteUsersMigrationKey = "SetEmailVerifiedToTrueForRemoteUsersMigrationComplete"
MSTeamUserIDDedupMigrationKey = "MSTeamUserIDDedupMigrationComplete"
WhitelistedUsersMigrationKey = "WhitelistedUsersMigrationComplete"

DedupScoreDefault byte = 0
DedupScoreNotSynthetic byte = 1
)

func (s *SQLStore) runMigrationRemoteID(remoteID string) error {
_, err := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{
setting, err := s.getSystemSetting(s.db, RemoteIDMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Remote ID migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running Remote ID migration")
start := time.Now()

_, qErr := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{
sq.NotEq{"RemoteID": nil},
sq.NotEq{"RemoteID": ""},
sq.Expr("RemoteID NOT IN (SELECT remoteid FROM remoteclusters)"),
sq.Like{"Username": "msteams_%"},
}).Exec()
return err

if qErr != nil {
return qErr
}

if err := s.setSystemSetting(s.db, RemoteIDMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Remote ID migration as completed: %w", err)
}

s.api.LogDebug("Remote ID migration run successfully", "elapsed", time.Since(start))

return nil
}

func (s *SQLStore) runSetEmailVerifiedToTrueForRemoteUsers(remoteID string) error {
_, err := s.getQueryBuilder(s.db).
setting, err := s.getSystemSetting(s.db, SetEmailVerifiedToTrueForRemoteUsersMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Set Email Verified to True for Remote Users migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running Set Email Verified to True for Remote Users migration")
start := time.Now()

_, qErr := s.getQueryBuilder(s.db).
Update("Users").
Set("EmailVerified", true).
Where(sq.And{
sq.Eq{"RemoteID": remoteID},
sq.Eq{"EmailVerified": false},
}).Exec()

return err
}
if qErr != nil {
return qErr
}

const (
DedupScoreDefault byte = 0
DedupScoreNotSynthetic byte = 1
)
if err := s.setSystemSetting(s.db, SetEmailVerifiedToTrueForRemoteUsersMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Set Email Verified to True for Remote Users migration as completed: %w", err)
}

s.api.LogDebug("Set Email Verified to True for Remote Users migration run successfully", "elapsed", time.Since(start))

return nil
}

func (s *SQLStore) runMSTeamUserIDDedup() error {
setting, err := s.getSystemSetting(s.db, MSTeamUserIDDedupMigrationKey)
if err != nil {
return fmt.Errorf("cannot get MSTeam User ID Dedup migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running MSTeam User ID Dedup migration")

// get all users with duplicate msteamsuserid
rows, err := s.getQueryBuilder(s.replica).Select(
"mmuserid",
Expand Down Expand Up @@ -106,22 +168,44 @@ func (s *SQLStore) runMSTeamUserIDDedup() error {
_, err = s.getQueryBuilder(s.db).Delete(usersTableName).
Where(orCond).
Exec()
if err != nil {
return err
}

if err := s.setSystemSetting(s.db, MSTeamUserIDDedupMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark MSTeam User ID Dedup migration as completed: %w", err)
}

s.api.LogDebug("MSTeam User ID Dedup migration run successfully")

return err
return nil
}

func (s *SQLStore) ensureMigrationWhitelistedUsers() error {
oldWhitelistToProcess, err := s.tableExist(whitelistedUsersLegacyTableName)
func (s *SQLStore) runWhitelistedUsersMigration() error {
setting, err := s.getSystemSetting(s.db, WhitelistedUsersMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Whitelisted Users migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

oldWhitelistToProcess, err := tableExist(s, whitelistedUsersLegacyTableName)
if err != nil {
return err
}

if !oldWhitelistToProcess {
// migration already done, no rows to process
if sErr := s.setSystemSetting(s.db, WhitelistedUsersMigrationKey, strconv.FormatBool(true)); sErr != nil {
return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", sErr)
}

return nil
}

s.api.LogInfo("Migrating old whitelist rows")
s.api.LogDebug("Running Whitelisted Users migration")

now := time.Now()

Expand Down Expand Up @@ -154,100 +238,11 @@ func (s *SQLStore) ensureMigrationWhitelistedUsers() error {
return err
}

err = s.deleteTable(whitelistedUsersLegacyTableName)

if err != nil {
return err
}

return nil
}

func (s *SQLStore) createTable(tableName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", tableName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) deleteTable(tableName string) error {
if _, err := s.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil {
return err
if err := s.setSystemSetting(s.db, WhitelistedUsersMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", err)
}

return nil
}

func (s *SQLStore) createIndex(tableName, indexName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil {
return err
}
s.api.LogDebug("Whitelisted Users migration run successfully", "elapsed", time.Since(now))

return nil
}

func (s *SQLStore) createUniqueIndex(tableName, indexName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) addColumn(tableName, columnName, columnDefinition string) error {
if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, columnName, columnDefinition)); err != nil {
return err
}

return nil
}

func (s *SQLStore) indexExist(tableName, indexName string) (bool, error) {
rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_indexes WHERE tablename = '%s' AND indexname = '%s'", tableName, indexName))
if err != nil {
return false, err
}

defer rows.Close()
return rows.Next(), nil
}

func (s *SQLStore) tableExist(tableName string) (bool, error) {
rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_tables WHERE schemaname = current_schema() AND tablename = '%s'", tableName))
if err != nil {
return false, err
}

defer rows.Close()
return rows.Next(), nil
}

func (s *SQLStore) addPrimaryKey(tableName, columnList string) error {
rows, err := s.db.Query(fmt.Sprintf("SELECT constraint_name from information_schema.table_constraints where table_name = '%s' and constraint_type='PRIMARY KEY'", tableName))
if err != nil {
return err
}
defer rows.Close()

var constraintName string
if rows.Next() {
if scanErr := rows.Scan(&constraintName); scanErr != nil {
return scanErr
}
}

if constraintName == "" {
if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD PRIMARY KEY(%s)", tableName, columnList)); err != nil {
return err
}
} else if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s, ADD PRIMARY KEY(%s)", tableName, constraintName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) createMSTeamsUserIDUniqueIndex() error {
return s.createUniqueIndex(usersTableName, "idx_msteamssync_users_msteamsuserid_unq", "msteamsuserid")
}
18 changes: 14 additions & 4 deletions server/store/sqlstore/helper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/jmoiron/sqlx"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/plugin/plugintest"
"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/testcontainers/testcontainers-go"
Expand Down Expand Up @@ -78,19 +79,28 @@ func createTestDB() (*sql.DB, func(), error) {

func setupTestStore(t *testing.T) (*SQLStore, *plugintest.API) {
api := &plugintest.API{}
api.On("LogDebug", mock.AnythingOfType("string")).Return()
api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
api.On("LogInfo", mock.AnythingOfType("string")).Return()
api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
api.On("LogError", mock.AnythingOfType("string")).Return()
api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()

store := &SQLStore{}
store.api = api
store.db = db
store.replica = db

err := store.createTable("Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)")
err := createTable(store, "Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)")
require.NoError(t, err)
err = store.createTable("Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)")
err = createTable(store, "Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)")
require.NoError(t, err)
err = store.createTable("Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT")
err = createTable(store, "Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT")
require.NoError(t, err)
err = store.createTable("Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL")
err = createTable(store, "Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL")
require.NoError(t, err)
err = store.Init("")
require.NoError(t, err)
Expand Down
Loading
Loading