Fix migration scripts (#18756)

* Fix migration scripts Set the DB schema version in DB prior to running migration https://community-daily.mattermost.com/boards/workspace/zyoahc9uapdn3xdptac6jb69ic/285b80a3-257d-41f6-8cf4-ed80ca9d92e5/495cdb4d-c13a-4992-8eb9-80cfee2819a4?c=c2525dbb-23eb-4fe7-8fb4-e9a5f503dc8c ```release-note NONE ``` * Revert schema to 6.0.0 and update CI scripts * Fix typo * Add missing default * Fix extra comma Co-authored-by: Claudio Costa <cstcld91@gmail.com>
2021-10-19 19:56:09 +05:30
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -376,8 +376,9 @@ jobs:
            mkdir -p mattermost-server/client/plugins
            echo "Creating databases"
            docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
-            echo "Importing postgres dump from version 6.0"
-            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.sql
+            echo "Importing postgres dump from version 6.0.0"
+            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.0.sql
+            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
            docker run -it --net circleci_mm-test \
              --env-file="dotenv/test-schema-validation.env" \
              --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
@@ -404,8 +405,9 @@ jobs:
            mkdir -p mattermost-server/client/plugins
            echo "Creating databases"
            docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
-            echo "Importing mysql dump from version 6.0"
-            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.sql
+            echo "Importing mysql dump from version 6.0.0"
+            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
+            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"

            docker run -it --net circleci_mm-test \
              --env-file="dotenv/test-schema-validation.env" \
--- a/.gitlab-ci/scripts/test-schema/mysql.sh
+++ b/.gitlab-ci/scripts/test-schema/mysql.sh
@@ -24,8 +24,9 @@ docker run --net $DOCKER_NETWORK "$CI_REGISTRY"/mattermost/ci/images/curl:7.59.0

 echo "Creating databases"
 docker exec $CONTAINER_DB mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser; "
-echo "Importing mysql dump from version 6.0"
-docker exec $CONTAINER_DB mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.sql
+echo "Importing mysql dump from version 6.0.0"
+docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.0.sql
+docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
 docker run -d -it --rm --name "$CONTAINER_SERVER" --net $DOCKER_NETWORK \
  --env-file="dotenv/test-schema-validation.env" \
  --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
--- a/.gitlab-ci/scripts/test-schema/postgres.sh
+++ b/.gitlab-ci/scripts/test-schema/postgres.sh
@@ -23,8 +23,9 @@ timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isr

 echo "Creating databases"
 docker exec $CONTAINER_DB sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test;'
-echo "Importing postgres dump from version 6.0"
-docker exec $CONTAINER_DB psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.sql
+echo "Importing postgres dump from version 6.0.0"
+docker exec -i $CONTAINER_DB psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.0.sql
+docker exec -i $CONTAINER_DB psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
 docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \
  --env-file="dotenv/test-schema-validation.env" \
  --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
--- a/8
+++ b/8
@@ -353,12 +353,12 @@ test-compile: ## Compile tests.
 	done

 test-db-migration: start-docker ## Gets diff of upgrade vs new instance schemas.
-	./scripts/mysql-migration-test.sh 6.0
-	./scripts/psql-migration-test.sh 6.0
+	./scripts/mysql-migration-test.sh 6.0.0
+	./scripts/psql-migration-test.sh 6.0.0

 test-db-migration-v5: start-docker ## Gets diff of upgrade vs new instance schemas.
-	./scripts/mysql-migration-v5-test.sh 5.0
-	./scripts/psql-migration-v5-test.sh 5.0
+	./scripts/mysql-migration-test.sh 5.0.0
+	./scripts/psql-migration-test.sh 5.0.0

 gomodtidy:
 	@cp go.mod go.mod.orig
--- a/model/channel.go
+++ b/model/channel.go
@@ -47,7 +47,6 @@ type Channel struct {
 	Header            string                 `json:"header"`
 	Purpose           string                 `json:"purpose"`
 	LastPostAt        int64                  `json:"last_post_at"`
-	LastRootPostAt    int64                  `json:"last_root_post_at"`
 	TotalMsgCount     int64                  `json:"total_msg_count"`
 	ExtraUpdateAt     int64                  `json:"extra_update_at"`
 	CreatorId         string                 `json:"creator_id"`
@@ -57,6 +56,7 @@ type Channel struct {
 	Shared            *bool                  `json:"shared"`
 	TotalMsgCountRoot int64                  `json:"total_msg_count_root"`
 	PolicyID          *string                `json:"policy_id" db:"-"`
+	LastRootPostAt    int64                  `json:"last_root_post_at"`
 }

 type ChannelWithTeamData struct {
--- a/scripts/mattermost-mysql-5.0.0.sql
+++ b/scripts/mattermost-mysql-5.0.0.sql
--- a/scripts/mattermost-mysql-6.0.0.sql
+++ b/scripts/mattermost-mysql-6.0.0.sql
@@ -71,7 +71,7 @@ CREATE TABLE `ChannelMemberHistory` (
 CREATE TABLE `ChannelMembers` (
  `ChannelId` varchar(26) NOT NULL,
  `UserId` varchar(26) NOT NULL,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
  `LastViewedAt` bigint(20) DEFAULT NULL,
  `MsgCount` bigint(20) DEFAULT NULL,
  `MentionCount` bigint(20) DEFAULT NULL,
@@ -106,7 +106,6 @@ CREATE TABLE `Channels` (
  `Header` text,
  `Purpose` varchar(250) DEFAULT NULL,
  `LastPostAt` bigint(20) DEFAULT NULL,
-  `LastRootPostAt` bigint(20) DEFAULT NULL,
  `TotalMsgCount` bigint(20) DEFAULT NULL,
  `ExtraUpdateAt` bigint(20) DEFAULT NULL,
  `CreatorId` varchar(26) DEFAULT NULL,
@@ -768,7 +767,7 @@ CREATE TABLE `Sessions` (
  `LastActivityAt` bigint(20) DEFAULT NULL,
  `UserId` varchar(26) DEFAULT NULL,
  `DeviceId` text,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
  `IsOAuth` tinyint(1) DEFAULT NULL,
  `ExpiredNotify` tinyint(1) DEFAULT NULL,
  `Props` json DEFAULT NULL,
@@ -938,7 +937,7 @@ CREATE TABLE `Systems` (
 CREATE TABLE `TeamMembers` (
  `TeamId` varchar(26) NOT NULL,
  `UserId` varchar(26) NOT NULL,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
  `DeleteAt` bigint(20) DEFAULT NULL,
  `SchemeUser` tinyint(4) DEFAULT NULL,
  `SchemeAdmin` tinyint(4) DEFAULT NULL,
--- a/scripts/mattermost-postgresql-5.0.0.sql
+++ b/scripts/mattermost-postgresql-5.0.0.sql
--- a/scripts/mattermost-postgresql-6.0.0.sql
+++ b/scripts/mattermost-postgresql-6.0.0.sql
@@ -73,7 +73,7 @@ ALTER TABLE public.channelmemberhistory OWNER TO mmuser;
 CREATE TABLE public.channelmembers (
    channelid character varying(26) NOT NULL,
    userid character varying(26) NOT NULL,
-    roles character varying(256),
+    roles character varying(64),
    lastviewedat bigint,
    msgcount bigint,
    mentioncount bigint,
@@ -105,14 +105,14 @@ CREATE TABLE public.channels (
    header character varying(1024),
    purpose character varying(250),
    lastpostat bigint,
-    lastrootpostat bigint,
    totalmsgcount bigint,
    extraupdateat bigint,
    creatorid character varying(26),
    schemeid character varying(26),
    groupconstrained boolean,
    shared boolean,
-    totalmsgcountroot bigint
+    totalmsgcountroot bigint,
+    lastrootpostat bigint DEFAULT '0'::bigint
 );


@@ -676,7 +676,7 @@ CREATE TABLE public.sessions (
    lastactivityat bigint,
    userid character varying(26),
    deviceid character varying(512),
-    roles character varying(256),
+    roles character varying(64),
    isoauth boolean,
    expirednotify boolean,
    props jsonb
@@ -826,7 +826,7 @@ ALTER TABLE public.systems OWNER TO mmuser;
 CREATE TABLE public.teammembers (
    teamid character varying(26) NOT NULL,
    userid character varying(26) NOT NULL,
-    roles character varying(256),
+    roles character varying(64),
    deleteat bigint,
    schemeuser boolean,
    schemeadmin boolean,
--- a/scripts/mysql-migration-test.sh
+++ b/scripts/mysql-migration-test.sh
@@ -10,6 +10,8 @@ docker exec mattermost-mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated
 echo "Importing mysql dump from version ${SCHEMA_VERSION}"
 docker exec -i mattermost-mysql mysql -D migrated -uroot -pmostest < $(pwd)/scripts/mattermost-mysql-$SCHEMA_VERSION.sql

+docker exec -i mattermost-mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '$SCHEMA_VERSION')"
+
 echo "Setting up config for db migration"
 cat config/config.json | \
    jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s"' | \
@@ -26,7 +28,7 @@ cat config/config.json | \
 echo "Setting up fresh db"
 make ARGS="version --config $TMPDIR/config.json" run-cli

-if [ "$SCHEMA_VERSION" == "5.0" ]; then
+if [ "$SCHEMA_VERSION" == "5.0.0" ]; then
  for i in "ChannelMembers SchemeGuest" "ChannelMembers MsgCountRoot" "ChannelMembers MentionCountRoot" "Channels TotalMsgCountRoot"; do
    a=( $i );
    echo "Ignoring known MySQL mismatch: ${a[0]}.${a[1]}"
--- a/scripts/psql-migration-test.sh
+++ b/scripts/psql-migration-test.sh
@@ -10,6 +10,8 @@ docker exec mattermost-postgres sh -c 'exec echo "CREATE DATABASE migrated; CREA
 echo "Importing postgres dump from version ${SCHEMA_VERSION}"
 docker exec -i mattermost-postgres psql -U mmuser -d migrated < $(pwd)/scripts/mattermost-postgresql-$SCHEMA_VERSION.sql

+docker exec -i mattermost-postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '$SCHEMA_VERSION')"
+
 echo "Setting up config for db migration"
 cat config/config.json | \
    jq '.SqlSettings.DataSource = "postgres://mmuser:mostest@localhost:5432/migrated?sslmode=disable&connect_timeout=10"'| \
@@ -26,7 +28,7 @@ cat config/config.json | \
 echo "Setting up fresh db"
 make ARGS="version --config $TMPDIR/config.json" run-cli

-if [ "$SCHEMA_VERSION" == "5.0" ]; then
+if [ "$SCHEMA_VERSION" == "5.0.0" ]; then
  for i in "ChannelMembers MentionCountRoot" "ChannelMembers MsgCountRoot" "Channels TotalMsgCountRoot"; do
    a=( $i );
    echo "Ignoring known Postgres mismatch: ${a[0]}.${a[1]}"
--- a/store/sqlstore/channel_store.go
+++ b/store/sqlstore/channel_store.go
@@ -386,6 +386,7 @@ func newSqlChannelStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface
 		table.ColMap("Purpose").SetMaxSize(250)
 		table.ColMap("CreatorId").SetMaxSize(26)
 		table.ColMap("SchemeId").SetMaxSize(26)
+		table.ColMap("LastRootPostAt").SetDefaultConstraint(model.NewString("0"))

 		tablem := db.AddTableWithName(channelMember{}, "ChannelMembers").SetKeys(false, "ChannelId", "UserId")
 		tablem.ColMap("ChannelId").SetMaxSize(26)