mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-13799] Avoid queries timing out during upgrade (#17444)

Просмотр исходного кода 
* Avoid timing out during upgrade

* Update gorp dep

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Claudio Costa
2021-04-19 20:34:21 +02:00
коммит произвёл GitHub
родитель 93c8c6e9d8
Коммит 3984381011
8 изменённых файлов: 70 добавлений и 50 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

19
store/sqlstore/store_test.go
Просмотреть файл

@@ -655,3 +655,22 @@ func makeSqlSettings(driver string) *model.SqlSettings {
return nil
}
func TestExecNoTimeout(t *testing.T) {
StoreTest(t, func(t *testing.T, ss store.Store) {
sqlStore := ss.(*SqlStore)
var query string
timeout := sqlStore.master.QueryTimeout
sqlStore.master.QueryTimeout = 1
defer func() {
sqlStore.master.QueryTimeout = timeout
}()
if sqlStore.DriverName() == model.DATABASE_DRIVER_MYSQL {
query = `SELECT SLEEP(2);`
} else if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = `SELECT pg_sleep(2);`
}
_, err := sqlStore.GetMaster().ExecNoTimeout(query)
require.NoError(t, err)
})
}

52
store/sqlstore/upgrade.go
Просмотреть файл

@@ -265,19 +265,19 @@ func upgradeDatabaseToVersion33(sqlStore *SqlStore) {
// increase size of Value column of Preferences table to match the size of the ThemeProps column
if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
if _, err := transaction.Exec("ALTER TABLE Preferences ALTER COLUMN Value TYPE varchar(2000)"); err != nil {
if _, err := transaction.ExecNoTimeout("ALTER TABLE Preferences ALTER COLUMN Value TYPE varchar(2000)"); err != nil {
themeMigrationFailed(err)
return
}
} else if sqlStore.DriverName() == model.DATABASE_DRIVER_MYSQL {
if _, err := transaction.Exec("ALTER TABLE Preferences MODIFY Value text"); err != nil {
if _, err := transaction.ExecNoTimeout("ALTER TABLE Preferences MODIFY Value text"); err != nil {
themeMigrationFailed(err)
return
}
}
// copy data across
if _, err := transaction.Exec(
if _, err := transaction.ExecNoTimeout(
`INSERT INTO
Preferences(UserId, Category, Name, Value)
SELECT
@@ -291,7 +291,7 @@ func upgradeDatabaseToVersion33(sqlStore *SqlStore) {
}
// delete old data
if _, err := transaction.Exec("ALTER TABLE Users DROP COLUMN ThemeProps"); err != nil {
if _, err := transaction.ExecNoTimeout("ALTER TABLE Users DROP COLUMN ThemeProps"); err != nil {
themeMigrationFailed(err)
return
}
@@ -343,12 +343,12 @@ func upgradeDatabaseToVersion34(sqlStore *SqlStore) {
func upgradeDatabaseToVersion35(sqlStore *SqlStore) {
if shouldPerformUpgrade(sqlStore, Version340, Version350) {
sqlStore.GetMaster().Exec("UPDATE Users SET Roles = 'system_user' WHERE Roles = ''")
sqlStore.GetMaster().Exec("UPDATE Users SET Roles = 'system_user system_admin' WHERE Roles = 'system_admin'")
sqlStore.GetMaster().Exec("UPDATE TeamMembers SET Roles = 'team_user' WHERE Roles = ''")
sqlStore.GetMaster().Exec("UPDATE TeamMembers SET Roles = 'team_user team_admin' WHERE Roles = 'admin'")
sqlStore.GetMaster().Exec("UPDATE ChannelMembers SET Roles = 'channel_user' WHERE Roles = ''")
sqlStore.GetMaster().Exec("UPDATE ChannelMembers SET Roles = 'channel_user channel_admin' WHERE Roles = 'admin'")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Users SET Roles = 'system_user' WHERE Roles = ''")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Users SET Roles = 'system_user system_admin' WHERE Roles = 'system_admin'")
sqlStore.GetMaster().ExecNoTimeout("UPDATE TeamMembers SET Roles = 'team_user' WHERE Roles = ''")
sqlStore.GetMaster().ExecNoTimeout("UPDATE TeamMembers SET Roles = 'team_user team_admin' WHERE Roles = 'admin'")
sqlStore.GetMaster().ExecNoTimeout("UPDATE ChannelMembers SET Roles = 'channel_user' WHERE Roles = ''")
sqlStore.GetMaster().ExecNoTimeout("UPDATE ChannelMembers SET Roles = 'channel_user channel_admin' WHERE Roles = 'admin'")
// The rest of the migration from Filenames -> FileIds is done lazily in api.GetFileInfosForPost
sqlStore.CreateColumnIfNotExists("Posts", "FileIds", "varchar(150)", "varchar(150)", "[]")
@@ -530,7 +530,7 @@ func upgradeDatabaseToVersion410(sqlStore *SqlStore) {
sqlStore.RemoveIndexIfExists("ClientId_2", "OAuthAccessData")
saveSchemaVersion(sqlStore, Version4100)
sqlStore.GetMaster().Exec("UPDATE Users SET AuthData=LOWER(AuthData) WHERE AuthService = 'saml'")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Users SET AuthData=LOWER(AuthData) WHERE AuthService = 'saml'")
}
}
@@ -559,8 +559,8 @@ func upgradeDatabaseToVersion50(sqlStore *SqlStore) {
sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeAdmin", "boolean", "boolean")
sqlStore.CreateColumnIfNotExists("Roles", "BuiltIn", "boolean", "boolean", "0")
sqlStore.GetMaster().Exec("UPDATE Roles SET BuiltIn=true")
sqlStore.GetMaster().Exec("UPDATE Roles SET SchemeManaged=false WHERE Name NOT IN ('system_user', 'system_admin', 'team_user', 'team_admin', 'channel_user', 'channel_admin')")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Roles SET BuiltIn=true")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Roles SET SchemeManaged=false WHERE Name NOT IN ('system_user', 'system_admin', 'team_user', 'team_admin', 'channel_user', 'channel_admin')")
sqlStore.CreateColumnIfNotExists("IncomingWebhooks", "ChannelLocked", "boolean", "boolean", "0")
sqlStore.RemoveIndexIfExists("idx_channels_txt", "Channels")
@@ -613,7 +613,7 @@ func upgradeDatabaseToVersion56(sqlStore *SqlStore) {
sqlStore.CreateColumnIfNotExists("PluginKeyValueStore", "ExpireAt", "bigint(20)", "bigint", "0")
// migrating user's accepted terms of service data into the new table
sqlStore.GetMaster().Exec("INSERT INTO UserTermsOfService SELECT Id, AcceptedTermsOfServiceId as TermsOfServiceId, :CreateAt FROM Users WHERE AcceptedTermsOfServiceId != \"\" AND AcceptedTermsOfServiceId IS NOT NULL", map[string]interface{}{"CreateAt": model.GetMillis()})
sqlStore.GetMaster().ExecNoTimeout("INSERT INTO UserTermsOfService SELECT Id, AcceptedTermsOfServiceId as TermsOfServiceId, :CreateAt FROM Users WHERE AcceptedTermsOfServiceId != \"\" AND AcceptedTermsOfServiceId IS NOT NULL", map[string]interface{}{"CreateAt": model.GetMillis()})
if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
sqlStore.RemoveIndexIfExists("idx_users_email_lower", "lower(Email)")
@@ -696,10 +696,10 @@ func upgradeDatabaseToVersion512(sqlStore *SqlStore) {
sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultTeamGuestRole", "text", "VARCHAR(64)")
sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultChannelGuestRole", "text", "VARCHAR(64)")
sqlStore.GetMaster().Exec("UPDATE Schemes SET DefaultTeamGuestRole = '', DefaultChannelGuestRole = ''")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Schemes SET DefaultTeamGuestRole = '', DefaultChannelGuestRole = ''")
// Saturday, January 24, 2065 5:20:00 AM GMT. To remove all personal access token sessions.
sqlStore.GetMaster().Exec("DELETE FROM Sessions WHERE ExpiresAt > 3000000000000")
sqlStore.GetMaster().ExecNoTimeout("DELETE FROM Sessions WHERE ExpiresAt > 3000000000000")
saveSchemaVersion(sqlStore, Version5120)
}
@@ -708,7 +708,7 @@ func upgradeDatabaseToVersion512(sqlStore *SqlStore) {
func upgradeDatabaseToVersion513(sqlStore *SqlStore) {
if shouldPerformUpgrade(sqlStore, Version5120, Version5130) {
// The previous jobs ran once per minute, cluttering the Jobs table with somewhat useless entries. Clean that up.
sqlStore.GetMaster().Exec("DELETE FROM Jobs WHERE Type = 'plugins'")
sqlStore.GetMaster().ExecNoTimeout("DELETE FROM Jobs WHERE Type = 'plugins'")
saveSchemaVersion(sqlStore, Version5130)
}
@@ -729,9 +729,9 @@ func upgradeDatabaseToVersion515(sqlStore *SqlStore) {
func upgradeDatabaseToVersion516(sqlStore *SqlStore) {
if shouldPerformUpgrade(sqlStore, Version5150, Version5160) {
if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
sqlStore.GetMaster().Exec("ALTER TABLE Tokens ALTER COLUMN Extra TYPE varchar(2048)")
sqlStore.GetMaster().ExecNoTimeout("ALTER TABLE Tokens ALTER COLUMN Extra TYPE varchar(2048)")
} else if sqlStore.DriverName() == model.DATABASE_DRIVER_MYSQL {
sqlStore.GetMaster().Exec("ALTER TABLE Tokens MODIFY Extra text")
sqlStore.GetMaster().ExecNoTimeout("ALTER TABLE Tokens MODIFY Extra text")
}
saveSchemaVersion(sqlStore, Version5160)
@@ -812,7 +812,7 @@ func upgradeDatabaseToVersion523(sqlStore *SqlStore) {
func upgradeDatabaseToVersion524(sqlStore *SqlStore) {
if shouldPerformUpgrade(sqlStore, Version5230, Version5240) {
sqlStore.CreateColumnIfNotExists("UserGroups", "AllowReference", "boolean", "boolean", "0")
sqlStore.GetMaster().Exec("UPDATE UserGroups SET Name = null, AllowReference = false")
sqlStore.GetMaster().ExecNoTimeout("UPDATE UserGroups SET Name = null, AllowReference = false")
sqlStore.AlterPrimaryKey("Reactions", []string{"PostId", "UserId", "EmojiName"})
saveSchemaVersion(sqlStore, Version5240)
@@ -847,7 +847,7 @@ func upgradeDatabaseToVersion528(sqlStore *SqlStore) {
}
sqlStore.CreateColumnIfNotExistsNoDefault("Commands", "PluginId", "VARCHAR(190)", "VARCHAR(190)")
sqlStore.GetMaster().Exec("UPDATE Commands SET PluginId = '' WHERE PluginId IS NULL")
sqlStore.GetMaster().ExecNoTimeout("UPDATE Commands SET PluginId = '' WHERE PluginId IS NULL")
sqlStore.AlterColumnTypeIfExists("Teams", "Type", "VARCHAR(255)", "VARCHAR(255)")
sqlStore.AlterColumnTypeIfExists("Teams", "SchemeId", "VARCHAR(26)", "VARCHAR(26)")
@@ -932,7 +932,7 @@ func upgradeDatabaseToVersion529(sqlStore *SqlStore) {
if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
updateThreadChannelsQuery = "UPDATE Threads SET ChannelId=Posts.ChannelId FROM Posts WHERE Posts.Id=Threads.PostId AND Threads.ChannelId IS NULL"
}
if _, err := sqlStore.GetMaster().Exec(updateThreadChannelsQuery); err != nil {
if _, err := sqlStore.GetMaster().ExecNoTimeout(updateThreadChannelsQuery); err != nil {
mlog.Error("Error updating ChannelId in Threads table", mlog.Err(err))
}
@@ -1045,7 +1045,7 @@ func upgradeDatabaseToVersion535(sqlStore *SqlStore) {
sqlStore.CreateColumnIfNotExistsNoDefault("FileInfo", "RemoteId", "VARCHAR(26)", "VARCHAR(26)")
sqlStore.CreateColumnIfNotExists("UploadSessions", "RemoteId", "VARCHAR(26)", "VARCHAR(26)", "")
sqlStore.CreateColumnIfNotExists("UploadSessions", "ReqFileId", "VARCHAR(26)", "VARCHAR(26)", "")
if _, err := sqlStore.GetMaster().Exec("UPDATE UploadSessions SET RemoteId='', ReqFileId='' WHERE RemoteId IS NULL"); err != nil {
if _, err := sqlStore.GetMaster().ExecNoTimeout("UPDATE UploadSessions SET RemoteId='', ReqFileId='' WHERE RemoteId IS NULL"); err != nil {
mlog.Error("Error updating RemoteId,ReqFileId in UploadsSession table", mlog.Err(err))
}
uniquenessColumns := []string{"SiteUrl", "RemoteTeamId"}
@@ -1093,7 +1093,7 @@ func rootCountMigration(sqlStore *SqlStore) {
ChannelMembers.MentionCount > 0
`
}
if _, err := sqlStore.GetMaster().Exec(updateMentionCountRootQuery); err != nil {
if _, err := sqlStore.GetMaster().ExecNoTimeout(updateMentionCountRootQuery); err != nil {
mlog.Error("Error updating ChannelId in Threads table", mlog.Err(err))
}
sqlStore.CreateColumnIfNotExists("Channels", "TotalMsgCountRoot", "bigint", "bigint", "0")
@@ -1141,12 +1141,12 @@ func rootCountMigration(sqlStore *SqlStore) {
}
if !totalMsgCountRootExists {
if _, err := sqlStore.GetMaster().Exec(updateChannels); err != nil {
if _, err := sqlStore.GetMaster().ExecNoTimeout(updateChannels); err != nil {
mlog.Error("Error updating Channels table", mlog.Err(err))
}
}
if !msgCountRootExists {
if _, err := sqlStore.GetMaster().Exec(updateChannelMembers); err != nil {
if _, err := sqlStore.GetMaster().ExecNoTimeout(updateChannelMembers); err != nil {
mlog.Error("Error updating ChannelMembers table", mlog.Err(err))
}
}