Create basic make commands for configuring golang-migrate (#16938)

* Create basic make commands for configuring golang-migrate * Showcase full flow with new migrations * Apply PR suggestions * Migrate over team members * Update mocks * Fix specs * Move columns that added after table creation onto separate stmts * Put back gorp table definitions * Fix issues with golang-migrate that not tracks underlying db driver * Help prompt after new migration and consistent checksum for bindata * Put gorp mapping back * Apply PR suggestiong * Close migrations after they run * Add migration file to bindata check * Updates needed * Reset store_test * Add copyright * Apply PR suggestions * Fix new circleci check * Put back upgrade step for backwards comp * Add store test to test migration directions * Apply PR suggestions * Add go-bindata to tools * Apply PR suggestios Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2021-02-23 16:22:31 +02:00
--- a/store/sqlstore/store.go
+++ b/store/sqlstore/store.go
@@ -8,7 +8,9 @@ import (
 	dbsql "database/sql"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -18,12 +20,18 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/dyatlov/go-opengraph/opengraph"
 	"github.com/go-sql-driver/mysql"
-	_ "github.com/go-sql-driver/mysql"
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	mysqlmigrate "github.com/golang-migrate/migrate/v4/database/mysql"
+	"github.com/golang-migrate/migrate/v4/database/postgres"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+	bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
 	"github.com/mattermost/gorp"
 	"github.com/pkg/errors"

+	"github.com/mattermost/mattermost-server/v5/db/migrations"
 	"github.com/mattermost/mattermost-server/v5/einterfaces"
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
@@ -31,6 +39,8 @@ import (
 	"github.com/mattermost/mattermost-server/v5/utils"
 )

+type migrationDirection string
+
 const (
 	IndexTypeFullText      = "full_text"
 	IndexTypeDefault       = "default"
@@ -44,6 +54,9 @@ const (
 	// 10.1 would be 100001.
 	// 9.6.3 would be 90603.
 	MinimumRequiredPostgresVersion = 100000
+
+	migrationsDirectionUp   migrationDirection = "up"
+	migrationsDirectionDown migrationDirection = "down"
 )

 const (
@@ -148,6 +161,12 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS

 	store.initConnection()

+	err := store.migrate(migrationsDirectionUp)
+	if err != nil {
+		mlog.Critical("Failed to apply database migrations.", mlog.Err(err))
+		os.Exit(ExitGenericFailure)
+	}
+
 	store.stores.team = newSqlTeamStore(store)
 	store.stores.channel = newSqlChannelStore(store, metrics)
 	store.stores.post = newSqlPostStore(store, metrics)
@@ -182,7 +201,8 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS
 	store.stores.scheme = newSqlSchemeStore(store)
 	store.stores.group = newSqlGroupStore(store)
 	store.stores.productNotices = newSqlProductNoticesStore(store)
-	err := store.GetMaster().CreateTablesIfNotExists()
+	err = store.GetMaster().CreateTablesIfNotExists()
+
 	if err != nil {
 		if IsDuplicate(err) {
 			mlog.Warn("Duplicate key error occurred; assuming table already created and proceeding.", mlog.Err(err))
@@ -199,7 +219,6 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS
 		os.Exit(ExitGenericFailure)
 	}

-	store.stores.team.(*SqlTeamStore).createIndexesIfNotExists()
 	store.stores.channel.(*SqlChannelStore).createIndexesIfNotExists()
 	store.stores.post.(*SqlPostStore).createIndexesIfNotExists()
 	store.stores.thread.(*SqlThreadStore).createIndexesIfNotExists()
@@ -234,7 +253,7 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS
 	return store
 }

-func setupConnection(con_type string, dataSource string, settings *model.SqlSettings) *gorp.DbMap {
+func setupConnection(connType string, dataSource string, settings *model.SqlSettings) *gorp.DbMap {
 	db, err := dbsql.Open(*settings.DriverName, dataSource)
 	if err != nil {
 		mlog.Critical("Failed to open SQL connection to err.", mlog.Err(err))
@@ -243,7 +262,7 @@ func setupConnection(con_type string, dataSource string, settings *model.SqlSett
 	}

 	for i := 0; i < DBPingAttempts; i++ {
-		mlog.Info("Pinging SQL", mlog.String("database", con_type))
+		mlog.Info("Pinging SQL", mlog.String("database", connType))
 		ctx, cancel := context.WithTimeout(context.Background(), DBPingTimeoutSecs*time.Second)
 		defer cancel()
 		err = db.PingContext(ctx)
@@ -1191,6 +1210,88 @@ func (ss *SqlStore) UpdateLicense(license *model.License) {
 	ss.license = license
 }

+func (ss *SqlStore) migrate(direction migrationDirection) error {
+	var driver database.Driver
+	var err error
+
+	// When WithInstance is used in golang-migrate, the underlying driver connections are not tracked.
+	// So we will have to open a fresh connection for migrations and explicitly close it when all is done.
+	dataSource := ss.appendMultipleStatementsFlag(*ss.settings.DataSource)
+	conn := setupConnection("migrations", dataSource, ss.settings)
+	defer conn.Db.Close()
+
+	if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
+		driver, err = mysqlmigrate.WithInstance(conn.Db, &mysqlmigrate.Config{})
+		if err != nil {
+			return err
+		}
+	} else {
+		driver, err = postgres.WithInstance(conn.Db, &postgres.Config{})
+		if err != nil {
+			return err
+		}
+	}
+
+	var assetNamesForDriver []string
+	for _, assetName := range migrations.AssetNames() {
+		if strings.HasPrefix(assetName, ss.DriverName()) {
+			assetNamesForDriver = append(assetNamesForDriver, filepath.Base(assetName))
+		}
+	}
+
+	source := bindata.Resource(assetNamesForDriver, func(name string) ([]byte, error) {
+		return migrations.Asset(filepath.Join(ss.DriverName(), name))
+	})
+
+	sourceDriver, err := bindata.WithInstance(source)
+	if err != nil {
+		return err
+	}
+
+	migrations, err := migrate.NewWithInstance("go-bindata",
+		sourceDriver,
+		ss.DriverName(),
+		driver)
+
+	if err != nil {
+		return err
+	}
+	defer migrations.Close()
+
+	switch direction {
+	case migrationsDirectionUp:
+		err = migrations.Up()
+	case migrationsDirectionDown:
+		err = migrations.Down()
+	default:
+		return errors.New(fmt.Sprintf("unsupported migration direction %s", direction))
+	}
+
+	if err != nil && err != migrate.ErrNoChange && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	return nil
+}
+
+func (ss *SqlStore) appendMultipleStatementsFlag(dataSource string) string {
+	// We need to tell the MySQL driver that we want to use multiStatements
+	// in order to make migrations work.
+	if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
+		u, err := url.Parse(dataSource)
+		if err != nil {
+			mlog.Critical("Invalid database url found", mlog.Err(err))
+			os.Exit(ExitGenericFailure)
+		}
+		q := u.Query()
+		q.Set("multiStatements", "true")
+		u.RawQuery = q.Encode()
+		return u.String()
+	}
+
+	return dataSource
+}
+
 type mattermConverter struct{}

 func (me mattermConverter) ToDb(val interface{}) (interface{}, error) {
--- a/store/sqlstore/store_test.go
+++ b/store/sqlstore/store_test.go
@@ -405,6 +405,25 @@ func TestGetDbVersion(t *testing.T) {
 	}
 }

+func TestUpAndDownMigrations(t *testing.T) {
+	testDrivers := []string{
+		model.DATABASE_DRIVER_POSTGRES,
+		model.DATABASE_DRIVER_MYSQL,
+	}
+
+	for _, driver := range testDrivers {
+		t.Run("Should be reversible for "+driver, func(t *testing.T) {
+			t.Parallel()
+			settings := makeSqlSettings(driver)
+			store := New(*settings, nil)
+			defer store.Close()
+
+			err := store.migrate(migrationsDirectionDown)
+			assert.NoError(t, err, "downing migrations should not error")
+		})
+	}
+}
+
 func TestGetAllConns(t *testing.T) {
 	t.Parallel()
 	testCases := []struct {
--- a/store/sqlstore/team_store.go
+++ b/store/sqlstore/team_store.go
@@ -233,20 +233,6 @@ func newSqlTeamStore(sqlStore *SqlStore) store.TeamStore {
 	return s
 }

-func (s SqlTeamStore) createIndexesIfNotExists() {
-	s.CreateIndexIfNotExists("idx_teams_name", "Teams", "Name")
-	s.RemoveIndexIfExists("idx_teams_description", "Teams")
-	s.CreateIndexIfNotExists("idx_teams_invite_id", "Teams", "InviteId")
-	s.CreateIndexIfNotExists("idx_teams_update_at", "Teams", "UpdateAt")
-	s.CreateIndexIfNotExists("idx_teams_create_at", "Teams", "CreateAt")
-	s.CreateIndexIfNotExists("idx_teams_delete_at", "Teams", "DeleteAt")
-	s.CreateIndexIfNotExists("idx_teams_scheme_id", "Teams", "SchemeId")
-
-	s.CreateIndexIfNotExists("idx_teammembers_team_id", "TeamMembers", "TeamId")
-	s.CreateIndexIfNotExists("idx_teammembers_user_id", "TeamMembers", "UserId")
-	s.CreateIndexIfNotExists("idx_teammembers_delete_at", "TeamMembers", "DeleteAt")
-}
-
 // Save adds the team to the database if a team with the same name does not already
 // exist in the database. It returns the team added if the operation is successful.
 func (s SqlTeamStore) Save(team *model.Team) (*model.Team, error) {
--- a/store/sqlstore/upgrade.go
+++ b/store/sqlstore/upgrade.go
@@ -231,8 +231,6 @@ func upgradeDatabaseToVersion31(sqlStore *SqlStore) {

 func upgradeDatabaseToVersion32(sqlStore *SqlStore) {
 	if shouldPerformUpgrade(sqlStore, Version310, Version320) {
-		sqlStore.CreateColumnIfNotExists("TeamMembers", "DeleteAt", "bigint(20)", "bigint", "0")
-
 		saveSchemaVersion(sqlStore, Version320)
 	}
 }
@@ -362,9 +360,6 @@ func upgradeDatabaseToVersion36(sqlStore *SqlStore) {
 	if shouldPerformUpgrade(sqlStore, Version350, Version360) {
 		sqlStore.CreateColumnIfNotExists("Posts", "HasReactions", "tinyint", "boolean", "0")

-		// Create Team Description column
-		sqlStore.CreateColumnIfNotExists("Teams", "Description", "varchar(255)", "varchar(255)", "")
-
 		// Add a Position column to users.
 		sqlStore.CreateColumnIfNotExists("Users", "Position", "varchar(64)", "varchar(64)", "")

@@ -508,7 +503,6 @@ func upgradeDatabaseToVersion49(sqlStore *SqlStore) {
 	// in the file `app/app.go` in the function `DoAdvancedPermissionsMigration()`.

 	if shouldPerformUpgrade(sqlStore, Version481, Version490) {
-		sqlStore.CreateColumnIfNotExists("Teams", "LastTeamIconUpdate", "bigint", "bigint", "0")
 		defaultTimezone := timezones.DefaultUserTimezone()
 		defaultTimezoneValue, err := json.Marshal(defaultTimezone)
 		if err != nil {
@@ -551,12 +545,8 @@ func upgradeDatabaseToVersion50(sqlStore *SqlStore) {
 	//    DELETE from Systems WHERE Name = 'migration_advanced_permissions_phase_2';

 	if shouldPerformUpgrade(sqlStore, Version4100, Version500) {
-
-		sqlStore.CreateColumnIfNotExistsNoDefault("Teams", "SchemeId", "varchar(26)", "varchar(26)")
 		sqlStore.CreateColumnIfNotExistsNoDefault("Channels", "SchemeId", "varchar(26)", "varchar(26)")

-		sqlStore.CreateColumnIfNotExistsNoDefault("TeamMembers", "SchemeUser", "boolean", "boolean")
-		sqlStore.CreateColumnIfNotExistsNoDefault("TeamMembers", "SchemeAdmin", "boolean", "boolean")
 		sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeUser", "boolean", "boolean")
 		sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeAdmin", "boolean", "boolean")

@@ -665,7 +655,6 @@ func upgradeDatabaseToVersion59(sqlStore *SqlStore) {
 func upgradeDatabaseToVersion510(sqlStore *SqlStore) {
 	if shouldPerformUpgrade(sqlStore, Version590, Version5100) {
 		sqlStore.CreateColumnIfNotExistsNoDefault("Channels", "GroupConstrained", "tinyint(4)", "boolean")
-		sqlStore.CreateColumnIfNotExistsNoDefault("Teams", "GroupConstrained", "tinyint(4)", "boolean")

 		sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId")
 		sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId")
@@ -695,7 +684,6 @@ func upgradeDatabaseToVersion511(sqlStore *SqlStore) {

 func upgradeDatabaseToVersion512(sqlStore *SqlStore) {
 	if shouldPerformUpgrade(sqlStore, Version5110, Version5120) {
-		sqlStore.CreateColumnIfNotExistsNoDefault("TeamMembers", "SchemeGuest", "boolean", "boolean")
 		sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeGuest", "boolean", "boolean")
 		sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultTeamGuestRole", "text", "VARCHAR(64)")
 		sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultChannelGuestRole", "text", "VARCHAR(64)")
@@ -740,7 +728,6 @@ func upgradeDatabaseToVersion516(sqlStore *SqlStore) {
 		saveSchemaVersion(sqlStore, Version5160)

 		// Fix mismatches between the canonical and migrated schemas.
-		sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean")
 		sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)")
 		sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)")
 		sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)")