Skip to content

Commit

Permalink
Integrate morph into the plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@mgdelacroix
mgdelacroix committed May 17, 2024
1 parent 1292481 commit fab6d4c
Show file tree
Hide file tree
Showing 36 changed files with 412 additions and 198 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/lib/pq v1.10.9
github.com/mattermost/mattermost/server/public v0.0.18-0.20240404202637-65d589935ff9
github.com/mattermost/mattermost/server/v8 v8.0.0-20240404204026-0a3667bf58c5
github.com/mattermost/morph v1.1.0
github.com/microsoft/kiota-abstractions-go v1.5.6
github.com/microsoft/kiota-http-go v1.3.1
github.com/microsoftgraph/msgraph-sdk-go v1.36.0
Expand Down Expand Up @@ -141,7 +142,6 @@ require (
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 // indirect
github.com/mattermost/logr/v2 v2.0.21 // indirect
github.com/mattermost/morph v1.1.0 // indirect
github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0 // indirect
github.com/mattermost/squirrel v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down
205 changes: 100 additions & 105 deletions server/store/sqlstore/migrations.go → server/store/sqlstore/data_migrations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,101 @@ package sqlstore
import (
"database/sql"
"fmt"
"strconv"
"time"

sq "github.com/Masterminds/squirrel"
)

const (
RemoteIDMigrationKey = "RemoteIDMigrationComplete"
SetEmailVerifiedToTrueForRemoteUsersMigrationKey = "SetEmailVerifiedToTrueForRemoteUsersMigrationComplete"
MSTeamUserIDDedupMigrationKey = "MSTeamUserIDDedupMigrationComplete"
WhitelistedUsersMigrationKey = "WhitelistedUsersMigrationComplete"

DedupScoreDefault byte = 0
DedupScoreNotSynthetic byte = 1
)

func (s *SQLStore) runMigrationRemoteID(remoteID string) error {
_, err := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{
setting, err := s.getSystemSetting(RemoteIDMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Remote ID migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running Remote ID migration")
start := time.Now()

_, qErr := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{
sq.NotEq{"RemoteID": nil},
sq.NotEq{"RemoteID": ""},
sq.Expr("RemoteID NOT IN (SELECT remoteid FROM remoteclusters)"),
sq.Like{"Username": "msteams_%"},
}).Exec()
return err

if qErr != nil {
return qErr
}

if err := s.setSystemSetting(RemoteIDMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Remote ID migration as completed: %w", err)
}

s.api.LogDebug("Remote ID migration run successfully", "elapsed", time.Since(start))

return nil
}

func (s *SQLStore) runSetEmailVerifiedToTrueForRemoteUsers(remoteID string) error {
_, err := s.getQueryBuilder(s.db).
setting, err := s.getSystemSetting(SetEmailVerifiedToTrueForRemoteUsersMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Set Email Verified to True for Remote Users migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running Set Email Verified to True for Remote Users migration")
start := time.Now()

_, qErr := s.getQueryBuilder(s.db).
Update("Users").
Set("EmailVerified", true).
Where(sq.And{
sq.Eq{"RemoteID": remoteID},
sq.Eq{"EmailVerified": false},
}).Exec()

return err
}
if qErr != nil {
return qErr
}

const (
DedupScoreDefault byte = 0
DedupScoreNotSynthetic byte = 1
)
if err := s.setSystemSetting(SetEmailVerifiedToTrueForRemoteUsersMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Set Email Verified to True for Remote Users migration as completed: %w", err)
}

s.api.LogDebug("Set Email Verified to True for Remote Users migration run successfully", "elapsed", time.Since(start))

return nil
}

func (s *SQLStore) runMSTeamUserIDDedup() error {
setting, err := s.getSystemSetting(MSTeamUserIDDedupMigrationKey)
if err != nil {
return fmt.Errorf("cannot get MSTeam User ID Dedup migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

s.api.LogDebug("Running MSTeam User ID Dedup migration")

// get all users with duplicate msteamsuserid
rows, err := s.getQueryBuilder(s.replica).Select(
"mmuserid",
Expand Down Expand Up @@ -106,22 +168,44 @@ func (s *SQLStore) runMSTeamUserIDDedup() error {
_, err = s.getQueryBuilder(s.db).Delete(usersTableName).
Where(orCond).
Exec()
if err != nil {
return err
}

if err := s.setSystemSetting(MSTeamUserIDDedupMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark MSTeam User ID Dedup migration as completed: %w", err)
}

s.api.LogDebug("MSTeam User ID Dedup migration run successfully")

return err
return nil
}

func (s *SQLStore) ensureMigrationWhitelistedUsers() error {
oldWhitelistToProcess, err := s.tableExist(whitelistedUsersLegacyTableName)
func (s *SQLStore) runWhitelistedUsersMigration() error {
setting, err := s.getSystemSetting(WhitelistedUsersMigrationKey)
if err != nil {
return fmt.Errorf("cannot get Whitelisted Users migration state: %w", err)
}

if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun {
return nil
}

oldWhitelistToProcess, err := tableExist(s, whitelistedUsersLegacyTableName)
if err != nil {
return err
}

if !oldWhitelistToProcess {
// migration already done, no rows to process
if sErr := s.setSystemSetting(WhitelistedUsersMigrationKey, strconv.FormatBool(true)); sErr != nil {
return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", sErr)
}

return nil
}

s.api.LogInfo("Migrating old whitelist rows")
s.api.LogDebug("Running Whitelisted Users migration")

now := time.Now()

Expand Down Expand Up @@ -154,100 +238,11 @@ func (s *SQLStore) ensureMigrationWhitelistedUsers() error {
return err
}

err = s.deleteTable(whitelistedUsersLegacyTableName)

if err != nil {
return err
}

return nil
}

func (s *SQLStore) createTable(tableName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", tableName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) deleteTable(tableName string) error {
if _, err := s.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil {
return err
if err := s.setSystemSetting(WhitelistedUsersMigrationKey, strconv.FormatBool(true)); err != nil {
return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", err)
}

return nil
}

func (s *SQLStore) createIndex(tableName, indexName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil {
return err
}
s.api.LogDebug("Whitelisted Users migration run successfully", "elapsed", time.Since(now))

return nil
}

func (s *SQLStore) createUniqueIndex(tableName, indexName, columnList string) error {
if _, err := s.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) addColumn(tableName, columnName, columnDefinition string) error {
if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, columnName, columnDefinition)); err != nil {
return err
}

return nil
}

func (s *SQLStore) indexExist(tableName, indexName string) (bool, error) {
rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_indexes WHERE tablename = '%s' AND indexname = '%s'", tableName, indexName))
if err != nil {
return false, err
}

defer rows.Close()
return rows.Next(), nil
}

func (s *SQLStore) tableExist(tableName string) (bool, error) {
rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_tables WHERE schemaname = current_schema() AND tablename = '%s'", tableName))
if err != nil {
return false, err
}

defer rows.Close()
return rows.Next(), nil
}

func (s *SQLStore) addPrimaryKey(tableName, columnList string) error {
rows, err := s.db.Query(fmt.Sprintf("SELECT constraint_name from information_schema.table_constraints where table_name = '%s' and constraint_type='PRIMARY KEY'", tableName))
if err != nil {
return err
}
defer rows.Close()

var constraintName string
if rows.Next() {
if scanErr := rows.Scan(&constraintName); scanErr != nil {
return scanErr
}
}

if constraintName == "" {
if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD PRIMARY KEY(%s)", tableName, columnList)); err != nil {
return err
}
} else if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s, ADD PRIMARY KEY(%s)", tableName, constraintName, columnList)); err != nil {
return err
}

return nil
}

func (s *SQLStore) createMSTeamsUserIDUniqueIndex() error {
return s.createUniqueIndex(usersTableName, "idx_msteamssync_users_msteamsuserid_unq", "msteamsuserid")
}
18 changes: 14 additions & 4 deletions server/store/sqlstore/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/jmoiron/sqlx"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/plugin/plugintest"
"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/testcontainers/testcontainers-go"
Expand Down Expand Up @@ -78,19 +79,28 @@ func createTestDB() (*sql.DB, func(), error) {

func setupTestStore(t *testing.T) (*SQLStore, *plugintest.API) {
api := &plugintest.API{}
api.On("LogDebug", mock.AnythingOfType("string")).Return()
api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
api.On("LogInfo", mock.AnythingOfType("string")).Return()
api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
api.On("LogError", mock.AnythingOfType("string")).Return()
api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return()
api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()

store := &SQLStore{}
store.api = api
store.db = db
store.replica = db

err := store.createTable("Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)")
err := createTable(store, "Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)")
require.NoError(t, err)
err = store.createTable("Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)")
err = createTable(store, "Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)")
require.NoError(t, err)
err = store.createTable("Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT")
err = createTable(store, "Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT")
require.NoError(t, err)
err = store.createTable("Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL")
err = createTable(store, "Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL")
require.NoError(t, err)
err = store.Init("")
require.NoError(t, err)
Expand Down
Loading

0 comments on commit fab6d4c

Please sign in to comment.