MM-16888 - Fix canonical vs. migration mismatches (#11825)

* fixed migration errors, added check to Jenkins

app/import_types.go

@@ -167,9 +167,9 @@ type SchemeImportData struct {
 	Scope                   *string         `json:"scope"`
 	DefaultTeamAdminRole    *RoleImportData `json:"default_team_admin_role"`
 	DefaultTeamUserRole     *RoleImportData `json:"default_team_user_role"`
-	DefaultTeamGuestRole    *RoleImportData `json:"default_team_guest_role"`
 	DefaultChannelAdminRole *RoleImportData `json:"default_channel_admin_role"`
 	DefaultChannelUserRole  *RoleImportData `json:"default_channel_user_role"`
+	DefaultTeamGuestRole    *RoleImportData `json:"default_team_guest_role"`
 	DefaultChannelGuestRole *RoleImportData `json:"default_channel_guest_role"`
 }
 

build/Jenkinsfile.pr

@@ -141,6 +141,114 @@ pipeline {
                     }
                 }
 
+                dir('src/github.com/mattermost/mattermost-server') {
+                    ansiColor('xterm') {
+                        sh """
+                        echo "Creating databases"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
+                        echo "Importing postgres dump from version 5.0"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T postgres psql -U mmuser -d migrated < \$(pwd)/scripts/mattermost-postgresql-5.0.sql
+                        """
+                    }
+                }
+
+                withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') {
+                    ansiColor('xterm') {
+                        sh """
+                            cd /go/src/github.com/mattermost/mattermost-server
+                            echo "Setting up config for db migration"
+                            export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10\"
+                            export MM_SQLSETTINGS_DRIVERNAME=\"postgres\"
+                            make ARGS="config get SqlSettings.DataSource" run-cli
+                            echo "Running the migration"
+                            make ARGS="version" run-cli
+
+                            echo "Setting up config for fresh db setup"
+                            export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10\"
+                            make ARGS="config get SqlSettings.DataSource" run-cli
+
+                            echo "Setting up fresh db"
+                            make ARGS="version" run-cli
+                        """
+                    }
+                }
+
+
+                dir('src/github.com/mattermost/mattermost-server') {
+                    ansiColor('xterm') {
+                        sh """
+                        echo "Generating dump"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
+
+                        echo "Removing databases created for db comparison"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T postgres sh -c \'exec echo \"DROP DATABASE migrated; DROP DATABASE latest;\" | exec psql -U mmuser mattermost_test\'
+
+                        echo "Generating diff"
+                        diff migrated.sql latest.sql > diff.txt
+                        export diffErrorCode=\$?
+
+                        if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi
+
+                        exit \$diffErrorCode
+                        """
+                    }
+                }
+
+                dir('src/github.com/mattermost/mattermost-server') {
+                    ansiColor('xterm') {
+                        sh """
+                        echo "Creating databases"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T mysql mysql -uroot -pmostest -e \"CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser\"
+                        echo "Importing mysql dump from version 5.0"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T mysql mysql -D migrated -uroot -pmostest < \$(pwd)/scripts/mattermost-mysql-5.0.sql
+                        """
+                    }
+                }
+
+                withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') {
+                    ansiColor('xterm') {
+                        sh """
+                            cd /go/src/github.com/mattermost/mattermost-server
+                            echo "Setting up config for db migration"
+                            export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\"
+                            export MM_SQLSETTINGS_DRIVERNAME=\"mysql\"
+                            make ARGS="config get SqlSettings.DataSource" run-cli
+                            echo "Running the migration"
+                            make ARGS="version" run-cli
+
+                            echo "Setting up config for fresh db setup"
+                            export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\"
+                            make ARGS="config get SqlSettings.DataSource" run-cli
+
+                            echo "Setting up fresh db"
+                            make ARGS="version" run-cli
+                        """
+                    }
+                }
+
+
+                dir('src/github.com/mattermost/mattermost-server') {
+                    ansiColor('xterm') {
+                        sh """
+                        echo "Generating dump"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
+
+                        echo "Removing databases created for db comparison"
+                        /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml  exec -T mysql mysql -uroot -pmostest -e \"DROP DATABASE migrated; DROP DATABASE latest\"
+
+                        echo "Generating diff"
+                        diff migrated.sql latest.sql > diff.txt
+                        export diffErrorCode=\$?
+
+                        if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi
+
+                        exit \$diffErrorCode
+                        """
+                    }
+                }
+
                 withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') {
                     ansiColor('xterm') {
                         sh """
@@ -150,7 +258,6 @@ pipeline {
                             cat config/config.json
 
                             make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE=
-
                         """
                     }
                     withCredentials([string(credentialsId: 'CODECOV_TOKEN', variable: 'CODECOV')]) {

model/scheme.go

@@ -29,9 +29,9 @@ type Scheme struct {
 	Scope                   string `json:"scope"`
 	DefaultTeamAdminRole    string `json:"default_team_admin_role"`
 	DefaultTeamUserRole     string `json:"default_team_user_role"`
-	DefaultTeamGuestRole    string `json:"default_team_guest_role"`
 	DefaultChannelAdminRole string `json:"default_channel_admin_role"`
 	DefaultChannelUserRole  string `json:"default_channel_user_role"`
+	DefaultTeamGuestRole    string `json:"default_team_guest_role"`
 	DefaultChannelGuestRole string `json:"default_channel_guest_role"`
 }
 

store/sqlstore/channel_store.go

@@ -52,9 +52,9 @@ type channelMember struct {
 	MentionCount int64
 	NotifyProps  model.StringMap
 	LastUpdateAt int64
-	SchemeGuest  sql.NullBool
 	SchemeUser   sql.NullBool
 	SchemeAdmin  sql.NullBool
+	SchemeGuest  sql.NullBool
 }
 
 func NewChannelMemberFromModel(cm *model.ChannelMember) *channelMember {

store/sqlstore/group_store.go

@@ -82,6 +82,10 @@ func (s *SqlGroupStore) CreateIndexesIfNotExists() {
 	s.CreateIndexIfNotExists("idx_groupmembers_create_at", "GroupMembers", "CreateAt")
 	s.CreateIndexIfNotExists("idx_usergroups_remote_id", "UserGroups", "RemoteId")
 	s.CreateIndexIfNotExists("idx_usergroups_delete_at", "UserGroups", "DeleteAt")
+	s.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId")
+	s.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId")
+	s.CreateColumnIfNotExistsNoDefault("Channels", "GroupConstrained", "tinyint(1)", "boolean")
+	s.CreateColumnIfNotExistsNoDefault("Teams", "GroupConstrained", "tinyint(1)", "boolean")
 }
 
 func (s *SqlGroupStore) Create(group *model.Group) (*model.Group, *model.AppError) {

store/sqlstore/team_store.go

@@ -33,9 +33,9 @@ type teamMember struct {
 	UserId      string
 	Roles       string
 	DeleteAt    int64
-	SchemeGuest sql.NullBool
 	SchemeUser  sql.NullBool
 	SchemeAdmin sql.NullBool
+	SchemeGuest sql.NullBool
 }
 
 func NewTeamMemberFromModel(tm *model.TeamMember) *teamMember {

store/sqlstore/upgrade.go

@@ -692,6 +692,7 @@ func UpgradeDatabaseToVersion512(sqlStore SqlStore) {
 		sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeGuest", "boolean", "boolean")
 		sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultTeamGuestRole", "text", "VARCHAR(64)")
 		sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultChannelGuestRole", "text", "VARCHAR(64)")
+
 		sqlStore.GetMaster().Exec("UPDATE Schemes SET DefaultTeamGuestRole = '', DefaultChannelGuestRole = ''")
 
 		// Saturday, January 24, 2065 5:20:00 AM GMT. To remove all personal access token sessions.
@@ -712,6 +713,17 @@ func UpgradeDatabaseToVersion513(sqlStore SqlStore) {
 
 func UpgradeDatabaseToVersion514(sqlStore SqlStore) {
 	if shouldPerformUpgrade(sqlStore, VERSION_5_13_0, VERSION_5_14_0) {
+		sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean")
+		sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean")
+		sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)")
+		sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)")
+		sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)")
+		sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean")
+		sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean")
+
+		sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId")
+		sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId")
+
 		saveSchemaVersion(sqlStore, VERSION_5_14_0)
 	}
 }