diff --git a/go.mod b/go.mod index 9b8b6815d..bed353f23 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/lib/pq v1.10.9 github.com/mattermost/mattermost/server/public v0.0.18-0.20240404202637-65d589935ff9 github.com/mattermost/mattermost/server/v8 v8.0.0-20240404204026-0a3667bf58c5 + github.com/mattermost/morph v1.1.0 github.com/microsoft/kiota-abstractions-go v1.5.6 github.com/microsoft/kiota-http-go v1.3.1 github.com/microsoftgraph/msgraph-sdk-go v1.36.0 @@ -141,7 +142,6 @@ require ( github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect github.com/mattermost/ldap v0.0.0-20231116144001-0f480c025956 // indirect github.com/mattermost/logr/v2 v2.0.21 // indirect - github.com/mattermost/morph v1.1.0 // indirect github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0 // indirect github.com/mattermost/squirrel v0.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/server/store/sqlstore/migrations.go b/server/store/sqlstore/data_migrations.go similarity index 53% rename from server/store/sqlstore/migrations.go rename to server/store/sqlstore/data_migrations.go index 110eac87f..ed92db465 100644 --- a/server/store/sqlstore/migrations.go +++ b/server/store/sqlstore/data_migrations.go @@ -3,23 +3,69 @@ package sqlstore import ( "database/sql" "fmt" + "strconv" "time" sq "github.com/Masterminds/squirrel" ) +const ( + RemoteIDMigrationKey = "RemoteIDMigrationComplete" + SetEmailVerifiedToTrueForRemoteUsersMigrationKey = "SetEmailVerifiedToTrueForRemoteUsersMigrationComplete" + MSTeamUserIDDedupMigrationKey = "MSTeamUserIDDedupMigrationComplete" + WhitelistedUsersMigrationKey = "WhitelistedUsersMigrationComplete" + + DedupScoreDefault byte = 0 + DedupScoreNotSynthetic byte = 1 +) + func (s *SQLStore) runMigrationRemoteID(remoteID string) error { - _, err := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{ + setting, err := s.getSystemSetting(RemoteIDMigrationKey) + if err != nil { + return fmt.Errorf("cannot get Remote ID migration state: %w", err) + } + + if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun { + return nil + } + + s.api.LogDebug("Running Remote ID migration") + start := time.Now() + + _, qErr := s.getQueryBuilder(s.db).Update("Users").Set("RemoteID", remoteID).Where(sq.And{ sq.NotEq{"RemoteID": nil}, sq.NotEq{"RemoteID": ""}, sq.Expr("RemoteID NOT IN (SELECT remoteid FROM remoteclusters)"), sq.Like{"Username": "msteams_%"}, }).Exec() - return err + + if qErr != nil { + return qErr + } + + if err := s.setSystemSetting(RemoteIDMigrationKey, strconv.FormatBool(true)); err != nil { + return fmt.Errorf("cannot mark Remote ID migration as completed: %w", err) + } + + s.api.LogDebug("Remote ID migration run successfully", "elapsed", time.Since(start)) + + return nil } func (s *SQLStore) runSetEmailVerifiedToTrueForRemoteUsers(remoteID string) error { - _, err := s.getQueryBuilder(s.db). + setting, err := s.getSystemSetting(SetEmailVerifiedToTrueForRemoteUsersMigrationKey) + if err != nil { + return fmt.Errorf("cannot get Set Email Verified to True for Remote Users migration state: %w", err) + } + + if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun { + return nil + } + + s.api.LogDebug("Running Set Email Verified to True for Remote Users migration") + start := time.Now() + + _, qErr := s.getQueryBuilder(s.db). Update("Users"). Set("EmailVerified", true). Where(sq.And{ @@ -27,15 +73,31 @@ func (s *SQLStore) runSetEmailVerifiedToTrueForRemoteUsers(remoteID string) erro sq.Eq{"EmailVerified": false}, }).Exec() - return err -} + if qErr != nil { + return qErr + } -const ( - DedupScoreDefault byte = 0 - DedupScoreNotSynthetic byte = 1 -) + if err := s.setSystemSetting(SetEmailVerifiedToTrueForRemoteUsersMigrationKey, strconv.FormatBool(true)); err != nil { + return fmt.Errorf("cannot mark Set Email Verified to True for Remote Users migration as completed: %w", err) + } + + s.api.LogDebug("Set Email Verified to True for Remote Users migration run successfully", "elapsed", time.Since(start)) + + return nil +} func (s *SQLStore) runMSTeamUserIDDedup() error { + setting, err := s.getSystemSetting(MSTeamUserIDDedupMigrationKey) + if err != nil { + return fmt.Errorf("cannot get MSTeam User ID Dedup migration state: %w", err) + } + + if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun { + return nil + } + + s.api.LogDebug("Running MSTeam User ID Dedup migration") + // get all users with duplicate msteamsuserid rows, err := s.getQueryBuilder(s.replica).Select( "mmuserid", @@ -106,22 +168,44 @@ func (s *SQLStore) runMSTeamUserIDDedup() error { _, err = s.getQueryBuilder(s.db).Delete(usersTableName). Where(orCond). Exec() + if err != nil { + return err + } + + if err := s.setSystemSetting(MSTeamUserIDDedupMigrationKey, strconv.FormatBool(true)); err != nil { + return fmt.Errorf("cannot mark MSTeam User ID Dedup migration as completed: %w", err) + } + + s.api.LogDebug("MSTeam User ID Dedup migration run successfully") - return err + return nil } -func (s *SQLStore) ensureMigrationWhitelistedUsers() error { - oldWhitelistToProcess, err := s.tableExist(whitelistedUsersLegacyTableName) +func (s *SQLStore) runWhitelistedUsersMigration() error { + setting, err := s.getSystemSetting(WhitelistedUsersMigrationKey) + if err != nil { + return fmt.Errorf("cannot get Whitelisted Users migration state: %w", err) + } + + if hasAlreadyRun, _ := strconv.ParseBool(setting); hasAlreadyRun { + return nil + } + + oldWhitelistToProcess, err := tableExist(s, whitelistedUsersLegacyTableName) if err != nil { return err } if !oldWhitelistToProcess { // migration already done, no rows to process + if sErr := s.setSystemSetting(WhitelistedUsersMigrationKey, strconv.FormatBool(true)); sErr != nil { + return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", sErr) + } + return nil } - s.api.LogInfo("Migrating old whitelist rows") + s.api.LogDebug("Running Whitelisted Users migration") now := time.Now() @@ -154,100 +238,11 @@ func (s *SQLStore) ensureMigrationWhitelistedUsers() error { return err } - err = s.deleteTable(whitelistedUsersLegacyTableName) - - if err != nil { - return err - } - - return nil -} - -func (s *SQLStore) createTable(tableName, columnList string) error { - if _, err := s.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", tableName, columnList)); err != nil { - return err - } - - return nil -} - -func (s *SQLStore) deleteTable(tableName string) error { - if _, err := s.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { - return err + if err := s.setSystemSetting(WhitelistedUsersMigrationKey, strconv.FormatBool(true)); err != nil { + return fmt.Errorf("cannot mark Whitelisted Users migration as completed: %w", err) } - return nil -} - -func (s *SQLStore) createIndex(tableName, indexName, columnList string) error { - if _, err := s.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil { - return err - } + s.api.LogDebug("Whitelisted Users migration run successfully", "elapsed", time.Since(now)) return nil } - -func (s *SQLStore) createUniqueIndex(tableName, indexName, columnList string) error { - if _, err := s.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil { - return err - } - - return nil -} - -func (s *SQLStore) addColumn(tableName, columnName, columnDefinition string) error { - if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, columnName, columnDefinition)); err != nil { - return err - } - - return nil -} - -func (s *SQLStore) indexExist(tableName, indexName string) (bool, error) { - rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_indexes WHERE tablename = '%s' AND indexname = '%s'", tableName, indexName)) - if err != nil { - return false, err - } - - defer rows.Close() - return rows.Next(), nil -} - -func (s *SQLStore) tableExist(tableName string) (bool, error) { - rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_tables WHERE schemaname = current_schema() AND tablename = '%s'", tableName)) - if err != nil { - return false, err - } - - defer rows.Close() - return rows.Next(), nil -} - -func (s *SQLStore) addPrimaryKey(tableName, columnList string) error { - rows, err := s.db.Query(fmt.Sprintf("SELECT constraint_name from information_schema.table_constraints where table_name = '%s' and constraint_type='PRIMARY KEY'", tableName)) - if err != nil { - return err - } - defer rows.Close() - - var constraintName string - if rows.Next() { - if scanErr := rows.Scan(&constraintName); scanErr != nil { - return scanErr - } - } - - if constraintName == "" { - if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD PRIMARY KEY(%s)", tableName, columnList)); err != nil { - return err - } - } else if _, err := s.db.Exec(fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s, ADD PRIMARY KEY(%s)", tableName, constraintName, columnList)); err != nil { - return err - } - - return nil -} - -func (s *SQLStore) createMSTeamsUserIDUniqueIndex() error { - return s.createUniqueIndex(usersTableName, "idx_msteamssync_users_msteamsuserid_unq", "msteamsuserid") -} diff --git a/server/store/sqlstore/helper_test.go b/server/store/sqlstore/helper_test.go index faa6999bc..098b2fdc3 100644 --- a/server/store/sqlstore/helper_test.go +++ b/server/store/sqlstore/helper_test.go @@ -11,6 +11,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/plugin/plugintest" + "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock" "github.com/pkg/errors" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -78,19 +79,28 @@ func createTestDB() (*sql.DB, func(), error) { func setupTestStore(t *testing.T) (*SQLStore, *plugintest.API) { api := &plugintest.API{} + api.On("LogDebug", mock.AnythingOfType("string")).Return() + api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + api.On("LogInfo", mock.AnythingOfType("string")).Return() + api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + api.On("LogError", mock.AnythingOfType("string")).Return() + api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() store := &SQLStore{} store.api = api store.db = db store.replica = db - err := store.createTable("Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)") + err := createTable(store, "Teams", "Id VARCHAR(255), DisplayName VARCHAR(255)") require.NoError(t, err) - err = store.createTable("Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)") + err = createTable(store, "Channels", "Id VARCHAR(255), DisplayName VARCHAR(255)") require.NoError(t, err) - err = store.createTable("Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT") + err = createTable(store, "Users", "Id VARCHAR(255), FirstName VARCHAR(255), LastName VARCHAR(255), Email VARCHAR(255), remoteid VARCHAR(26), createat BIGINT, deleteat BIGINT") require.NoError(t, err) - err = store.createTable("Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL") + err = createTable(store, "Preferences", "userid VARCHAR(26) NOT NULL, category VARCHAR(32) NOT NULL, name VARCHAR(32) NOT NULL, value VARCHAR(2000) NULL") require.NoError(t, err) err = store.Init("") require.NoError(t, err) diff --git a/server/store/sqlstore/migrate.go b/server/store/sqlstore/migrate.go new file mode 100644 index 000000000..1ed9573ae --- /dev/null +++ b/server/store/sqlstore/migrate.go @@ -0,0 +1,155 @@ +package sqlstore + +import ( + "context" + "embed" + "fmt" + + "github.com/mattermost/morph" + "github.com/mattermost/morph/drivers" + "github.com/mattermost/morph/drivers/postgres" + "github.com/mattermost/morph/sources/embedded" +) + +const ( + migrationRemoteIDRequiredVersion = 12 + migrationWhitelistedUsersRequiredVersion = 13 +) + +//go:embed migrations/*.sql +var Assets embed.FS + +func (s *SQLStore) Migrate(remoteID string) error { + // ToDo: do we need to handle mutex? Most probably not, but still + + driver, err := postgres.WithInstance(s.db) + if err != nil { + return fmt.Errorf("cannot create postgres driver: %w", err) + } + + assetsList, err := Assets.ReadDir("migrations") + if err != nil { + return fmt.Errorf("cannot read assets dir: %w", err) + } + + assetsNames := make([]string, len(assetsList)) + for i, entry := range assetsList { + assetsNames[i] = entry.Name() + } + + migrationAssets := &embedded.AssetSource{ + Names: assetsNames, + AssetFunc: func(name string) ([]byte, error) { + return Assets.ReadFile("migrations/" + name) + }, + } + + src, err := embedded.WithInstance(migrationAssets) + if err != nil { + return err + } + + opts := []morph.EngineOption{ + morph.WithLock("msteams-lock-key"), + morph.SetMigrationTableName("msteamssync_schema_migrations"), + morph.SetStatementTimeoutInSeconds(1000000), + } + + s.api.LogDebug("Creating migration engine") + engine, err := morph.New(context.Background(), driver, src, opts...) + if err != nil { + return err + } + defer func() { + s.api.LogDebug("Closing migration engine") + engine.Close() + }() + + return s.runMigrationSequence(engine, driver, remoteID) +} + +// runMigrationSequence executes all the migrations in order, both +// plain SQL and data migrations. +func (s *SQLStore) runMigrationSequence(engine *morph.Morph, driver drivers.Driver, remoteID string) error { + // ToDo: we may wanna add a condition here if we want to skip + // running these for existing installs. Something like: if system + // settings table is not present and the + // idx_msteamssync_users_msteamsuserid_unq index is present, then + // we know that the version is 1.12.X and we can simply execute + // migration 1, then dump the keys for all migrations into both + // schema_migrations and system_settings and exit + // + // Next boot will find all keys and run nothing but the new stuff + + if mErr := s.ensureMigrationsAppliedUpToVersion(engine, driver, migrationRemoteIDRequiredVersion); mErr != nil { + return mErr + } + + // ToDo: check that the system settings table is working as + // expected for data migrations + // ToDo: remove this? tests + if remoteID != "" { + if mErr := s.runMigrationRemoteID(remoteID); mErr != nil { + return fmt.Errorf("error running remote ID migration: %w", mErr) + } + + if mErr := s.runSetEmailVerifiedToTrueForRemoteUsers(remoteID); mErr != nil { + return fmt.Errorf("error running set email verified to true for remote users migration: %w", mErr) + } + } + + if err := s.runMSTeamUserIDDedup(); err != nil { + return err + } + + if mErr := s.ensureMigrationsAppliedUpToVersion(engine, driver, migrationWhitelistedUsersRequiredVersion); mErr != nil { + return mErr + } + + // ToDo: test permutations of this, as the table might not exists + // by the time that this runs + // ToDo: where was the table created? maybe I need to readd a + // migration that creates the table + if err := s.runWhitelistedUsersMigration(); err != nil { + return err + } + + appliedMigrations, err := driver.AppliedMigrations() + if err != nil { + return err + } + + s.api.LogDebug("== Applying all remaining migrations ====================", + "current_version", len(appliedMigrations), + ) + + return engine.ApplyAll() +} + +func (s *SQLStore) ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error { + applied, err := driver.AppliedMigrations() + if err != nil { + return err + } + currentVersion := len(applied) + + s.api.LogDebug("== Ensuring migrations applied up to version ====================", "version", version, "current_version", currentVersion) + + // if the target version is below or equal to the current one, do + // not migrate either because is not needed (both are equal) or + // because it would downgrade the database (is below) + if version <= currentVersion { + s.api.LogDebug("-- There is no need of applying any migration --------------------") + return nil + } + + for _, migration := range applied { + s.api.LogDebug("-- Found applied migration --------------------", "version", migration.Version, "name", migration.Name) + } + + if _, err = engine.Apply(version - currentVersion); err != nil { + return err + } + + return nil +} diff --git a/server/store/sqlstore/migrations/000001_system_settings_table.down.sql b/server/store/sqlstore/migrations/000001_system_settings_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000001_system_settings_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000001_system_settings_table.up.sql b/server/store/sqlstore/migrations/000001_system_settings_table.up.sql new file mode 100644 index 000000000..eecc7520b --- /dev/null +++ b/server/store/sqlstore/migrations/000001_system_settings_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS msteamssync_system_settings ( + id VARCHAR(100), + value TEXT, + PRIMARY KEY(id) +); diff --git a/server/store/sqlstore/migrations/000002_subscriptions_table.down.sql b/server/store/sqlstore/migrations/000002_subscriptions_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000002_subscriptions_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000002_subscriptions_table.up.sql b/server/store/sqlstore/migrations/000002_subscriptions_table.up.sql new file mode 100644 index 000000000..72f4066b9 --- /dev/null +++ b/server/store/sqlstore/migrations/000002_subscriptions_table.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS msteamssync_subscriptions ( + subscriptionID VARCHAR(255) PRIMARY KEY, + type VARCHAR(255), + msTeamsTeamID VARCHAR(255), + msTeamsChannelID VARCHAR(255), + msTeamsUserID VARCHAR(255), + secret VARCHAR(255), + expiresOn BIGINT +); diff --git a/server/store/sqlstore/migrations/000003_links_table.down.sql b/server/store/sqlstore/migrations/000003_links_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000003_links_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000003_links_table.up.sql b/server/store/sqlstore/migrations/000003_links_table.up.sql new file mode 100644 index 000000000..7f2920860 --- /dev/null +++ b/server/store/sqlstore/migrations/000003_links_table.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS msteamssync_links ( + mmChannelID VARCHAR(255) PRIMARY KEY, + mmTeamID VARCHAR(255), + msTeamsChannelID VARCHAR(255), + msTeamsTeamID VARCHAR(255), + creator VARCHAR(255) +); + +ALTER TABLE msteamssync_links ADD COLUMN IF NOT EXISTS creator VARCHAR(255); diff --git a/server/store/sqlstore/migrations/000004_users_table.down.sql b/server/store/sqlstore/migrations/000004_users_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000004_users_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000004_users_table.up.sql b/server/store/sqlstore/migrations/000004_users_table.up.sql new file mode 100644 index 000000000..da1de765c --- /dev/null +++ b/server/store/sqlstore/migrations/000004_users_table.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS msteamssync_users ( + mmUserID VARCHAR(255), + msTeamsUserID VARCHAR(255), + token TEXT, + + PRIMARY KEY(mmUserID, msTeamsUserID) +); diff --git a/server/store/sqlstore/migrations/000005_posts_table.down.sql b/server/store/sqlstore/migrations/000005_posts_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000005_posts_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000005_posts_table.up.sql b/server/store/sqlstore/migrations/000005_posts_table.up.sql new file mode 100644 index 000000000..5aa413933 --- /dev/null +++ b/server/store/sqlstore/migrations/000005_posts_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS msteamssync_posts ( + mmPostID VARCHAR(255) PRIMARY KEY, + msTeamsPostID VARCHAR(255), + msTeamsChannelID VARCHAR(255), + msTeamsLastUpdateAt BIGINT +); diff --git a/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.down.sql b/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.up.sql b/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.up.sql new file mode 100644 index 000000000..39a0383e7 --- /dev/null +++ b/server/store/sqlstore/migrations/000006_links_teamid_channelid_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS idx_msteamssync_links_msteamsteamid_msteamschannelid ON msteamssync_links (msTeamsTeamID, msTeamsChannelID); diff --git a/server/store/sqlstore/migrations/000007_users_userid_index.down.sql b/server/store/sqlstore/migrations/000007_users_userid_index.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000007_users_userid_index.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000007_users_userid_index.up.sql b/server/store/sqlstore/migrations/000007_users_userid_index.up.sql new file mode 100644 index 000000000..e5fe85732 --- /dev/null +++ b/server/store/sqlstore/migrations/000007_users_userid_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS idx_msteamssync_users_msteamsuserid ON msteamssync_users (msTeamsUserID); diff --git a/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.down.sql b/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.up.sql b/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.up.sql new file mode 100644 index 000000000..14aac2b64 --- /dev/null +++ b/server/store/sqlstore/migrations/000008_posts_channelid_postid_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS idx_msteamssync_posts_msteamschannelid_msteamspostid ON msteamssync_posts (msTeamsChannelID, msTeamsPostID); diff --git a/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.down.sql b/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.up.sql b/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.up.sql new file mode 100644 index 000000000..b79b98fbc --- /dev/null +++ b/server/store/sqlstore/migrations/000009_subscriptions_add_certificate_and_lastactivityat_columns.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE msteamssync_subscriptions ADD COLUMN IF NOT EXISTS certificate TEXT; +ALTER TABLE msteamssync_subscriptions ADD COLUMN IF NOT EXISTS lastActivityAt BIGINT; diff --git a/server/store/sqlstore/migrations/000010_invited_users_table.down.sql b/server/store/sqlstore/migrations/000010_invited_users_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000010_invited_users_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000010_invited_users_table.up.sql b/server/store/sqlstore/migrations/000010_invited_users_table.up.sql new file mode 100644 index 000000000..8ce73ae69 --- /dev/null +++ b/server/store/sqlstore/migrations/000010_invited_users_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS msteamssync_invited_users ( + mmUserID VARCHAR(255) PRIMARY KEY +); + +ALTER TABLE msteamssync_invited_users ADD COLUMN IF NOT EXISTS invitePendingSince BIGINT; +ALTER TABLE msteamssync_invited_users ADD COLUMN IF NOT EXISTS inviteLastSentAt BIGINT; diff --git a/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.down.sql b/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.up.sql b/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.up.sql new file mode 100644 index 000000000..d8b565853 --- /dev/null +++ b/server/store/sqlstore/migrations/000011_users_add_lastconnectat_lastdisconnectat_columns.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE msteamssync_users ADD COLUMN IF NOT EXISTS lastConnectAt BIGINT NOT NULL DEFAULT 0; +ALTER TABLE msteamssync_users ADD COLUMN IF NOT EXISTS lastDisconnectAt BIGINT NOT NULL DEFAULT 0; diff --git a/server/store/sqlstore/migrations/000012_whitelist_table.down.sql b/server/store/sqlstore/migrations/000012_whitelist_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000012_whitelist_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000012_whitelist_table.up.sql b/server/store/sqlstore/migrations/000012_whitelist_table.up.sql new file mode 100644 index 000000000..f5f442e85 --- /dev/null +++ b/server/store/sqlstore/migrations/000012_whitelist_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE IF NOT EXISTS msteamssync_whitelist ( + mmUserID VARCHAR(255) PRIMARY KEY +); diff --git a/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.down.sql b/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.up.sql b/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.up.sql new file mode 100644 index 000000000..ebefa5bf2 --- /dev/null +++ b/server/store/sqlstore/migrations/000013_users_teamsuserid_unique_index.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX IF NOT EXISTS idx_msteamssync_users_msteamsuserid_unq ON msteamssync_users (msTeamsUserID); diff --git a/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.down.sql b/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.down.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.down.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.up.sql b/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.up.sql new file mode 100644 index 000000000..3a97298af --- /dev/null +++ b/server/store/sqlstore/migrations/000014_delete_whitelisted_users_table.up.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS msteamssync_whitelisted_users; diff --git a/server/store/sqlstore/migrations_test.go b/server/store/sqlstore/migrations_test.go index d078d9284..4b45f391e 100644 --- a/server/store/sqlstore/migrations_test.go +++ b/server/store/sqlstore/migrations_test.go @@ -46,7 +46,7 @@ func TestRunMSTeamUserIDDedup(t *testing.T) { _, err := store.db.Exec("DROP INDEX IF EXISTS idx_msteamssync_users_msteamsuserid_unq") assert.NoError(err) - defer func() { _ = store.createMSTeamsUserIDUniqueIndex() }() + defer func() { _ = createMSTeamsUserIDUniqueIndex(store) }() res, err := store.getQueryBuilder(store.db).Insert("users"). Columns("id", "createat", "remoteid"). diff --git a/server/store/sqlstore/store.go b/server/store/sqlstore/store.go index cf4679d24..2f474df19 100644 --- a/server/store/sqlstore/store.go +++ b/server/store/sqlstore/store.go @@ -26,6 +26,7 @@ const ( oAuth2StateTimeToLive = 300 // seconds oAuth2KeyPrefix = "oauth2_" backgroundJobPrefix = "background_job" + systemSettingsTableName = "msteamssync_system_settings" usersTableName = "msteamssync_users" linksTableName = "msteamssync_links" postsTableName = "msteamssync_posts" @@ -56,102 +57,39 @@ func New(db, replica *sql.DB, api plugin.API, enabledTeams func() []string, encr } func (s *SQLStore) Init(remoteID string) error { - if err := s.createTable(subscriptionsTableName, "subscriptionID VARCHAR(255) PRIMARY KEY, type VARCHAR(255), msTeamsTeamID VARCHAR(255), msTeamsChannelID VARCHAR(255), msTeamsUserID VARCHAR(255), secret VARCHAR(255), expiresOn BIGINT"); err != nil { - return err - } - - if err := s.createTable(linksTableName, "mmChannelID VARCHAR(255) PRIMARY KEY, mmTeamID VARCHAR(255), msTeamsChannelID VARCHAR(255), msTeamsTeamID VARCHAR(255), creator VARCHAR(255)"); err != nil { - return err - } - - if err := s.addColumn(linksTableName, "creator", "VARCHAR(255)"); err != nil { - return err - } - - if err := s.createTable(usersTableName, "mmUserID VARCHAR(255) PRIMARY KEY, msTeamsUserID VARCHAR(255), token TEXT"); err != nil { - return err - } - - if err := s.addPrimaryKey(usersTableName, "mmUserID, msTeamsUserID"); err != nil { - return err - } - - if err := s.createTable(postsTableName, "mmPostID VARCHAR(255) PRIMARY KEY, msTeamsPostID VARCHAR(255), msTeamsChannelID VARCHAR(255), msTeamsLastUpdateAt BIGINT"); err != nil { - return err - } - - if err := s.createIndex(linksTableName, "idx_msteamssync_links_msteamsteamid_msteamschannelid", "msTeamsTeamID, msTeamsChannelID"); err != nil { - return err - } - - if err := s.createIndex(usersTableName, "idx_msteamssync_users_msteamsuserid", "msTeamsUserID"); err != nil { - return err - } - - if err := s.createIndex(postsTableName, "idx_msteamssync_posts_msteamschannelid_msteamspostid", "msTeamsChannelID, msTeamsPostID"); err != nil { - return err - } - - if err := s.addColumn(subscriptionsTableName, "certificate", "TEXT"); err != nil { - return err - } - - if err := s.addColumn(subscriptionsTableName, "lastActivityAt", "BIGINT"); err != nil { - return err - } - - if err := s.createTable(invitedUsersTableName, "mmUserID VARCHAR(255) PRIMARY KEY"); err != nil { - return err + if err := s.Migrate(remoteID); err != nil { + return fmt.Errorf("error running database migrations: %w", err) } - if err := s.addColumn(invitedUsersTableName, "invitePendingSince", "BIGINT"); err != nil { - return err - } - - if err := s.addColumn(invitedUsersTableName, "inviteLastSentAt", "BIGINT"); err != nil { - return err - } - - if err := s.addColumn(usersTableName, "lastConnectAt", "BIGINT NOT NULL DEFAULT 0"); err != nil { - return err - } + return nil +} - if err := s.addColumn(usersTableName, "lastDisconnectAt", "BIGINT NOT NULL DEFAULT 0"); err != nil { - return err - } +func (s *SQLStore) getSystemSetting(key string) (string, error) { + scanner := s.getQueryBuilder(). + Select("value"). + From(systemSettingsTableName). + Where(sq.Eq{"id": key}). + QueryRow() - if err := s.createTable(whitelistTableName, "mmUserID VARCHAR(255) PRIMARY KEY"); err != nil { - return err + var result string + err := scanner.Scan(&result) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return "", err } - if remoteID != "" { - if err := s.runMigrationRemoteID(remoteID); err != nil { - return err - } - - if err := s.runSetEmailVerifiedToTrueForRemoteUsers(remoteID); err != nil { - return err - } - } + return result, nil +} - exist, err := s.indexExist(usersTableName, "idx_msteamssync_users_msteamsuserid_unq") +func (s *SQLStore) setSystemSetting(id, value string) error { + _, err := s.getQueryBuilder(). + Insert(systemSettingsTableName). + Columns("id", "value"). + Values(id, value). + Suffix("ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value"). + Exec() if err != nil { return err } - if !exist { - // dedup entries with multiples ms teams id - if err := s.runMSTeamUserIDDedup(); err != nil { - return err - } - - if err := s.createMSTeamsUserIDUniqueIndex(); err != nil { - return err - } - } - - if err := s.ensureMigrationWhitelistedUsers(); err != nil { - return err - } if err := s.addColumn(usersTableName, "LastChatSentAt", "BIGINT NOT NULL DEFAULT 0"); err != nil { return err diff --git a/server/store/sqlstore/utils.go b/server/store/sqlstore/utils.go new file mode 100644 index 000000000..43ea0b327 --- /dev/null +++ b/server/store/sqlstore/utils.go @@ -0,0 +1,35 @@ +package sqlstore + +import ( + "fmt" +) + +func createTable(store *SQLStore, tableName, columnList string) error { + if _, err := store.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", tableName, columnList)); err != nil { + return err + } + + return nil +} + +func tableExist(store *SQLStore, tableName string) (bool, error) { + rows, err := store.db.Query(fmt.Sprintf("SELECT 1 FROM pg_tables WHERE schemaname = current_schema() AND tablename = '%s'", tableName)) + if err != nil { + return false, err + } + + defer rows.Close() + return rows.Next(), nil +} + +func createUniqueIndex(store *SQLStore, tableName, indexName, columnList string) error { + if _, err := store.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil { + return err + } + + return nil +} + +func createMSTeamsUserIDUniqueIndex(store *SQLStore) error { + return createUniqueIndex(store, usersTableName, "idx_msteamssync_users_msteamsuserid_unq", "msteamsuserid") +} diff --git a/server/testutils/containere2e/containere2e.go b/server/testutils/containere2e/containere2e.go index 12b9ec309..11244a17b 100644 --- a/server/testutils/containere2e/containere2e.go +++ b/server/testutils/containere2e/containere2e.go @@ -12,6 +12,8 @@ import ( "github.com/mattermost/mattermost-plugin-msteams/server/store/sqlstore" "github.com/mattermost/mattermost-plugin-msteams/server/testutils/mmcontainer" + "github.com/mattermost/mattermost/server/public/plugin/plugintest" + "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/mockserver" @@ -143,7 +145,18 @@ func NewE2ETestPlugin(t *testing.T, extraOptions ...mmcontainer.MattermostCustom } require.NoError(t, err) - store := sqlstore.New(conn, conn, nil, func() []string { return []string{""} }, func() []byte { return []byte("eyPBz0mBhwfGGwce9hp4TWaYzgY7MdIB") }) + api := &plugintest.API{} + api.On("LogDebug", mock.AnythingOfType("string")).Return() + api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogDebug", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + api.On("LogInfo", mock.AnythingOfType("string")).Return() + api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogInfo", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + api.On("LogError", mock.AnythingOfType("string")).Return() + api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything).Return() + api.On("LogError", mock.AnythingOfType("string"), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + + store := sqlstore.New(conn, conn, api, func() []string { return []string{""} }, func() []byte { return []byte("eyPBz0mBhwfGGwce9hp4TWaYzgY7MdIB") }) if err2 := store.Init(""); err2 != nil { _ = mockserverContainer.Terminate(ctx) _ = mattermost.Terminate(ctx)