From da768d677e9d0c76262be91b58bf2ab0a2f0407a Mon Sep 17 00:00:00 2001 From: Eli Yukelzon Date: Wed, 14 Aug 2019 14:58:01 +0300 Subject: [PATCH] MM-16888 - Fix canonical vs. migration mismatches (#11825) * fixed migration errors, added check to Jenkins --- app/import_types.go | 2 +- build/Jenkinsfile.pr | 109 +++++++++++++++++++++++++++++++- model/scheme.go | 2 +- store/sqlstore/channel_store.go | 2 +- store/sqlstore/group_store.go | 4 ++ store/sqlstore/team_store.go | 2 +- store/sqlstore/upgrade.go | 12 ++++ 7 files changed, 128 insertions(+), 5 deletions(-) diff --git a/app/import_types.go b/app/import_types.go index 26c0e69d22..2df205847e 100644 --- a/app/import_types.go +++ b/app/import_types.go @@ -167,9 +167,9 @@ type SchemeImportData struct { Scope *string `json:"scope"` DefaultTeamAdminRole *RoleImportData `json:"default_team_admin_role"` DefaultTeamUserRole *RoleImportData `json:"default_team_user_role"` - DefaultTeamGuestRole *RoleImportData `json:"default_team_guest_role"` DefaultChannelAdminRole *RoleImportData `json:"default_channel_admin_role"` DefaultChannelUserRole *RoleImportData `json:"default_channel_user_role"` + DefaultTeamGuestRole *RoleImportData `json:"default_team_guest_role"` DefaultChannelGuestRole *RoleImportData `json:"default_channel_guest_role"` } diff --git a/build/Jenkinsfile.pr b/build/Jenkinsfile.pr index ceedc35e72..2fc80b6789 100644 --- a/build/Jenkinsfile.pr +++ b/build/Jenkinsfile.pr @@ -141,6 +141,114 @@ pipeline { } } + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Creating databases" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' + echo "Importing postgres dump from version 5.0" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres psql -U mmuser -d migrated < \$(pwd)/scripts/mattermost-postgresql-5.0.sql + """ + } + } + + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + echo "Setting up config for db migration" + export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10\" + export MM_SQLSETTINGS_DRIVERNAME=\"postgres\" + make ARGS="config get SqlSettings.DataSource" run-cli + echo "Running the migration" + make ARGS="version" run-cli + + echo "Setting up config for fresh db setup" + export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10\" + make ARGS="config get SqlSettings.DataSource" run-cli + + echo "Setting up fresh db" + make ARGS="version" run-cli + """ + } + } + + + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Generating dump" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql + + echo "Removing databases created for db comparison" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c \'exec echo \"DROP DATABASE migrated; DROP DATABASE latest;\" | exec psql -U mmuser mattermost_test\' + + echo "Generating diff" + diff migrated.sql latest.sql > diff.txt + export diffErrorCode=\$? + + if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi + + exit \$diffErrorCode + """ + } + } + + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Creating databases" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser\" + echo "Importing mysql dump from version 5.0" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest < \$(pwd)/scripts/mattermost-mysql-5.0.sql + """ + } + } + + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + echo "Setting up config for db migration" + export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" + export MM_SQLSETTINGS_DRIVERNAME=\"mysql\" + make ARGS="config get SqlSettings.DataSource" run-cli + echo "Running the migration" + make ARGS="version" run-cli + + echo "Setting up config for fresh db setup" + export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" + make ARGS="config get SqlSettings.DataSource" run-cli + + echo "Setting up fresh db" + make ARGS="version" run-cli + """ + } + } + + + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Generating dump" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql + + echo "Removing databases created for db comparison" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"DROP DATABASE migrated; DROP DATABASE latest\" + + echo "Generating diff" + diff migrated.sql latest.sql > diff.txt + export diffErrorCode=\$? + + if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi + + exit \$diffErrorCode + """ + } + } + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { ansiColor('xterm') { sh """ @@ -150,7 +258,6 @@ pipeline { cat config/config.json make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= - """ } withCredentials([string(credentialsId: 'CODECOV_TOKEN', variable: 'CODECOV')]) { diff --git a/model/scheme.go b/model/scheme.go index f40a9599c3..2a6fccc01b 100644 --- a/model/scheme.go +++ b/model/scheme.go @@ -29,9 +29,9 @@ type Scheme struct { Scope string `json:"scope"` DefaultTeamAdminRole string `json:"default_team_admin_role"` DefaultTeamUserRole string `json:"default_team_user_role"` - DefaultTeamGuestRole string `json:"default_team_guest_role"` DefaultChannelAdminRole string `json:"default_channel_admin_role"` DefaultChannelUserRole string `json:"default_channel_user_role"` + DefaultTeamGuestRole string `json:"default_team_guest_role"` DefaultChannelGuestRole string `json:"default_channel_guest_role"` } diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index 78d37f3470..5ff8af8513 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -52,9 +52,9 @@ type channelMember struct { MentionCount int64 NotifyProps model.StringMap LastUpdateAt int64 - SchemeGuest sql.NullBool SchemeUser sql.NullBool SchemeAdmin sql.NullBool + SchemeGuest sql.NullBool } func NewChannelMemberFromModel(cm *model.ChannelMember) *channelMember { diff --git a/store/sqlstore/group_store.go b/store/sqlstore/group_store.go index fbb6708765..b9f6c6ac5e 100644 --- a/store/sqlstore/group_store.go +++ b/store/sqlstore/group_store.go @@ -82,6 +82,10 @@ func (s *SqlGroupStore) CreateIndexesIfNotExists() { s.CreateIndexIfNotExists("idx_groupmembers_create_at", "GroupMembers", "CreateAt") s.CreateIndexIfNotExists("idx_usergroups_remote_id", "UserGroups", "RemoteId") s.CreateIndexIfNotExists("idx_usergroups_delete_at", "UserGroups", "DeleteAt") + s.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId") + s.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId") + s.CreateColumnIfNotExistsNoDefault("Channels", "GroupConstrained", "tinyint(1)", "boolean") + s.CreateColumnIfNotExistsNoDefault("Teams", "GroupConstrained", "tinyint(1)", "boolean") } func (s *SqlGroupStore) Create(group *model.Group) (*model.Group, *model.AppError) { diff --git a/store/sqlstore/team_store.go b/store/sqlstore/team_store.go index cd15ca0f51..8b4c84ad52 100644 --- a/store/sqlstore/team_store.go +++ b/store/sqlstore/team_store.go @@ -33,9 +33,9 @@ type teamMember struct { UserId string Roles string DeleteAt int64 - SchemeGuest sql.NullBool SchemeUser sql.NullBool SchemeAdmin sql.NullBool + SchemeGuest sql.NullBool } func NewTeamMemberFromModel(tm *model.TeamMember) *teamMember { diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index f7635b0a1c..603b57ecfb 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -692,6 +692,7 @@ func UpgradeDatabaseToVersion512(sqlStore SqlStore) { sqlStore.CreateColumnIfNotExistsNoDefault("ChannelMembers", "SchemeGuest", "boolean", "boolean") sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultTeamGuestRole", "text", "VARCHAR(64)") sqlStore.CreateColumnIfNotExistsNoDefault("Schemes", "DefaultChannelGuestRole", "text", "VARCHAR(64)") + sqlStore.GetMaster().Exec("UPDATE Schemes SET DefaultTeamGuestRole = '', DefaultChannelGuestRole = ''") // Saturday, January 24, 2065 5:20:00 AM GMT. To remove all personal access token sessions. @@ -712,6 +713,17 @@ func UpgradeDatabaseToVersion513(sqlStore SqlStore) { func UpgradeDatabaseToVersion514(sqlStore SqlStore) { if shouldPerformUpgrade(sqlStore, VERSION_5_13_0, VERSION_5_14_0) { + sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean") + sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean") + sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)") + sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)") + sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)") + sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean") + sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean") + + sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId") + sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId") + saveSchemaVersion(sqlStore, VERSION_5_14_0) } }