From 70318462431eb41bd7322cf1e8f95e580e18fb23 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Mon, 22 Apr 2019 11:30:58 -0400 Subject: [PATCH] MM-15064: allow running with future minor db versions (#10636) * MM-15064: allow running with future minor db versions This is to support running another community cluster that stays on the most recent stable version while we test upcoming versions. During the past few releases, we have committed to not making backwards incompatible schema changes across minor versions. This means that a 5.x server can run against any 5.y schema, y >= x. (It will upgrade to x if y < x). * check errors on incoming versions vs. Must/panic * fix TestSaveSchemaVersion --- store/sqlstore/supplier.go | 7 ++- store/sqlstore/upgrade.go | 84 +++++++++++++++++++---------- store/sqlstore/upgrade_test.go | 98 ++++++++++++++++++++++++++++------ 3 files changed, 145 insertions(+), 44 deletions(-) diff --git a/store/sqlstore/supplier.go b/store/sqlstore/supplier.go index 9f93a5e807..50fef8bd01 100644 --- a/store/sqlstore/supplier.go +++ b/store/sqlstore/supplier.go @@ -164,7 +164,12 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter os.Exit(EXIT_CREATE_TABLE) } - UpgradeDatabase(supplier) + err = UpgradeDatabase(supplier, model.CurrentVersion) + if err != nil { + mlog.Critical("Failed to upgrade database", mlog.Err(err)) + time.Sleep(time.Second) + os.Exit(EXIT_GENERIC_FAILURE) + } supplier.oldStores.team.(*SqlTeamStore).CreateIndexesIfNotExists() supplier.oldStores.channel.(*SqlChannelStore).CreateIndexesIfNotExists() diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index 7104684a7c..af2653edc0 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/blang/semver" "github.com/pkg/errors" "github.com/mattermost/mattermost-server/mlog" @@ -60,15 +61,62 @@ const ( ) const ( - EXIT_VERSION_SAVE_MISSING = 1001 - EXIT_TOO_OLD = 1002 - EXIT_VERSION_SAVE = 1003 - EXIT_THEME_MIGRATION = 1004 - EXIT_ROLE_MIGRATION_FAILED = 1005 + EXIT_VERSION_SAVE = 1003 + EXIT_THEME_MIGRATION = 1004 ) -func UpgradeDatabase(sqlStore SqlStore) { +// UpgradeDatabase attempts to migrate the schema to the latest supported version. +// The value of model.CurrentVersion is accepted as a parameter for unit testing, but it is not +// used to stop migrations at that version. +func UpgradeDatabase(sqlStore SqlStore, currentModelVersionString string) error { + currentModelVersion, err := semver.Parse(currentModelVersionString) + if err != nil { + return errors.Wrapf(err, "failed to parse current model version %s", currentModelVersionString) + } + nextUnsupportedMajorVersion := semver.Version{ + Major: currentModelVersion.Major + 1, + } + + oldestSupportedVersion, err := semver.Parse(OLDEST_SUPPORTED_VERSION) + if err != nil { + return errors.Wrapf(err, "failed to parse oldest supported version %s", OLDEST_SUPPORTED_VERSION) + } + + var currentSchemaVersion *semver.Version + currentSchemaVersionString := sqlStore.GetCurrentSchemaVersion() + if currentSchemaVersionString != "" { + currentSchemaVersion, err = semver.New(currentSchemaVersionString) + if err != nil { + return errors.Wrapf(err, "failed to parse database schema version %s", currentSchemaVersionString) + } + } + + // Assume a fresh database if no schema version has been recorded. + if currentSchemaVersion == nil { + if result := <-sqlStore.System().SaveOrUpdate(&model.System{Name: "Version", Value: currentModelVersion.String()}); result.Err != nil { + return errors.Wrap(err, "failed to initialize schema version for fresh database") + } + + currentSchemaVersion = ¤tModelVersion + mlog.Info(fmt.Sprintf("The database schema has been set to version %s", *currentSchemaVersion)) + return nil + } + + // Upgrades prior to the oldest supported version are not supported. + if currentSchemaVersion.LT(oldestSupportedVersion) { + return errors.Errorf("Database schema version %s is no longer supported. This Mattermost server supports automatic upgrades from schema version %s through schema version %s. Please manually upgrade to at least version %s before continuing.", *currentSchemaVersion, oldestSupportedVersion, currentModelVersion, oldestSupportedVersion) + } + + // Allow forwards compatibility only within the same major version. + if currentSchemaVersion.GTE(nextUnsupportedMajorVersion) { + return errors.Errorf("Database schema version %s is not supported. This Mattermost server supports only >=%s, <%s. Please upgrade to at least version %s before continuing.", *currentSchemaVersion, currentModelVersion, nextUnsupportedMajorVersion, nextUnsupportedMajorVersion) + } else if currentSchemaVersion.GT(currentModelVersion) { + mlog.Warn(fmt.Sprintf("The database schema with version %s is newer than Mattermost version %s.", currentSchemaVersion, currentModelVersion)) + } + + // Otherwise, apply any necessary migrations. Note that these methods currently invoke + // os.Exit instead of returning an error. UpgradeDatabaseToVersion31(sqlStore) UpgradeDatabaseToVersion32(sqlStore) UpgradeDatabaseToVersion33(sqlStore) @@ -106,28 +154,11 @@ func UpgradeDatabase(sqlStore SqlStore) { UpgradeDatabaseToVersion510(sqlStore) UpgradeDatabaseToVersion511(sqlStore) - // If the SchemaVersion is empty this this is the first time it has ran - // so lets set it to the current version. - if sqlStore.GetCurrentSchemaVersion() == "" { - if result := <-sqlStore.System().SaveOrUpdate(&model.System{Name: "Version", Value: model.CurrentVersion}); result.Err != nil { - mlog.Critical(result.Err.Error()) - time.Sleep(time.Second) - os.Exit(EXIT_VERSION_SAVE_MISSING) - } - - mlog.Info(fmt.Sprintf("The database schema has been set to version %v", model.CurrentVersion)) - } - - // If we're not on the current version then it's too old to be upgraded - if sqlStore.GetCurrentSchemaVersion() != model.CurrentVersion { - mlog.Critical(fmt.Sprintf("Database schema version %v is no longer supported. This Mattermost server supports automatic upgrades from schema version %v through schema version %v. Downgrades are not supported. Please manually upgrade to at least version %v before continuing", sqlStore.GetCurrentSchemaVersion(), OLDEST_SUPPORTED_VERSION, model.CurrentVersion, OLDEST_SUPPORTED_VERSION)) - time.Sleep(time.Second) - os.Exit(EXIT_TOO_OLD) - } + return nil } func saveSchemaVersion(sqlStore SqlStore, version string) { - if result := <-sqlStore.System().Update(&model.System{Name: "Version", Value: version}); result.Err != nil { + if result := <-sqlStore.System().SaveOrUpdate(&model.System{Name: "Version", Value: version}); result.Err != nil { mlog.Critical(result.Err.Error()) time.Sleep(time.Second) os.Exit(EXIT_VERSION_SAVE) @@ -138,8 +169,7 @@ func saveSchemaVersion(sqlStore SqlStore, version string) { func shouldPerformUpgrade(sqlStore SqlStore, currentSchemaVersion string, expectedSchemaVersion string) bool { if sqlStore.GetCurrentSchemaVersion() == currentSchemaVersion { - mlog.Warn(fmt.Sprintf("The database schema version of %v appears to be out of date", currentSchemaVersion)) - mlog.Warn(fmt.Sprintf("Attempting to upgrade the database schema version to %v", expectedSchemaVersion)) + mlog.Warn(fmt.Sprintf("Attempting to upgrade the database schema version from %s to %v", currentSchemaVersion, expectedSchemaVersion)) return true } diff --git a/store/sqlstore/upgrade_test.go b/store/sqlstore/upgrade_test.go index 7c3b2f1246..ab6847ec24 100644 --- a/store/sqlstore/upgrade_test.go +++ b/store/sqlstore/upgrade_test.go @@ -8,34 +8,100 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" + "github.com/stretchr/testify/require" ) func TestStoreUpgrade(t *testing.T) { StoreTest(t, func(t *testing.T, ss store.Store) { - saveSchemaVersion(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore), VERSION_3_0_0) - UpgradeDatabase(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore)) + sqlStore := ss.(*store.LayeredStore).DatabaseLayer.(SqlStore) - saveSchemaVersion(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore), "") - UpgradeDatabase(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore)) + t.Run("invalid currentModelVersion", func(t *testing.T) { + err := UpgradeDatabase(sqlStore, "notaversion") + require.EqualError(t, err, "failed to parse current model version notaversion: No Major.Minor.Patch elements found") + }) + + t.Run("upgrade from invalid version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "invalid") + err := UpgradeDatabase(sqlStore, "5.8.0") + require.EqualError(t, err, "failed to parse database schema version invalid: No Major.Minor.Patch elements found") + require.Equal(t, "invalid", sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade from unsupported version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "2.0.0") + err := UpgradeDatabase(sqlStore, "5.8.0") + require.EqualError(t, err, "Database schema version 2.0.0 is no longer supported. This Mattermost server supports automatic upgrades from schema version 3.0.0 through schema version 5.8.0. Please manually upgrade to at least version 3.0.0 before continuing.") + require.Equal(t, "2.0.0", sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade from earliest supported version", func(t *testing.T) { + saveSchemaVersion(sqlStore, VERSION_3_0_0) + err := UpgradeDatabase(sqlStore, model.CurrentVersion) + require.NoError(t, err) + require.Equal(t, model.CurrentVersion, sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade from no existing version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "") + err := UpgradeDatabase(sqlStore, model.CurrentVersion) + require.NoError(t, err) + require.Equal(t, model.CurrentVersion, sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade schema running earlier minor version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "5.1.0") + err := UpgradeDatabase(sqlStore, "5.8.0") + require.NoError(t, err) + // Assert model.CurrentVersion, not 5.8.0, since the migrations will move + // past 5.8.0 regardless of the input parameter. + require.Equal(t, model.CurrentVersion, sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade schema running later minor version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "5.29.0") + err := UpgradeDatabase(sqlStore, "5.8.0") + require.NoError(t, err) + require.Equal(t, "5.29.0", sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade schema running earlier major version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "4.1.0") + err := UpgradeDatabase(sqlStore, model.CurrentVersion) + require.NoError(t, err) + require.Equal(t, model.CurrentVersion, sqlStore.GetCurrentSchemaVersion()) + }) + + t.Run("upgrade schema running later major version", func(t *testing.T) { + saveSchemaVersion(sqlStore, "6.0.0") + err := UpgradeDatabase(sqlStore, "5.8.0") + require.EqualError(t, err, "Database schema version 6.0.0 is not supported. This Mattermost server supports only >=5.8.0, <6.0.0. Please upgrade to at least version 6.0.0 before continuing.") + require.Equal(t, "6.0.0", sqlStore.GetCurrentSchemaVersion()) + }) }) } func TestSaveSchemaVersion(t *testing.T) { StoreTest(t, func(t *testing.T, ss store.Store) { - saveSchemaVersion(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore), VERSION_3_0_0) - if result := <-ss.System().Get(); result.Err != nil { - t.Fatal(result.Err) - } else { + sqlStore := ss.(*store.LayeredStore).DatabaseLayer.(SqlStore) + + t.Run("set earliest version", func(t *testing.T) { + saveSchemaVersion(sqlStore, VERSION_3_0_0) + result := <-ss.System().Get() + require.Nil(t, result.Err) + props := result.Data.(model.StringMap) - if props["Version"] != VERSION_3_0_0 { - t.Fatal("version not updated") - } - } + require.Equal(t, VERSION_3_0_0, props["Version"]) + require.Equal(t, VERSION_3_0_0, sqlStore.GetCurrentSchemaVersion()) + }) - if ss.(*store.LayeredStore).DatabaseLayer.(SqlStore).GetCurrentSchemaVersion() != VERSION_3_0_0 { - t.Fatal("version not updated") - } + t.Run("set current version", func(t *testing.T) { + saveSchemaVersion(sqlStore, model.CurrentVersion) + result := <-ss.System().Get() + require.Nil(t, result.Err) - saveSchemaVersion(ss.(*store.LayeredStore).DatabaseLayer.(SqlStore), model.CurrentVersion) + props := result.Data.(model.StringMap) + require.Equal(t, model.CurrentVersion, props["Version"]) + require.Equal(t, model.CurrentVersion, sqlStore.GetCurrentSchemaVersion()) + }) }) }