Fix migration scripts (#18756)

* Fix migration scripts

Set the DB schema version in DB prior to running migration

https://community-daily.mattermost.com/boards/workspace/zyoahc9uapdn3xdptac6jb69ic/285b80a3-257d-41f6-8cf4-ed80ca9d92e5/495cdb4d-c13a-4992-8eb9-80cfee2819a4?c=c2525dbb-23eb-4fe7-8fb4-e9a5f503dc8c

```release-note
NONE
```

* Revert schema to 6.0.0 and update CI scripts

* Fix typo

* Add missing default

* Fix extra comma

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

.circleci/config.yml

@@ -376,8 +376,9 @@ jobs:
             mkdir -p mattermost-server/client/plugins
             echo "Creating databases"
             docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
-            echo "Importing postgres dump from version 6.0"
+            echo "Importing postgres dump from version 6.0.0"
-            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.sql
+            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.0.sql
+            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
             docker run -it --net circleci_mm-test \
               --env-file="dotenv/test-schema-validation.env" \
               --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
@@ -404,8 +405,9 @@ jobs:
             mkdir -p mattermost-server/client/plugins
             echo "Creating databases"
             docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
-            echo "Importing mysql dump from version 6.0"
+            echo "Importing mysql dump from version 6.0.0"
-            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.sql
+            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
+            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
 
             docker run -it --net circleci_mm-test \
               --env-file="dotenv/test-schema-validation.env" \

.gitlab-ci/scripts/test-schema/mysql.sh

@@ -24,8 +24,9 @@ docker run --net $DOCKER_NETWORK "$CI_REGISTRY"/mattermost/ci/images/curl:7.59.0
 
 echo "Creating databases"
 docker exec $CONTAINER_DB mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser; "
-echo "Importing mysql dump from version 6.0"
+echo "Importing mysql dump from version 6.0.0"
-docker exec $CONTAINER_DB mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.sql
+docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.0.sql
+docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
 docker run -d -it --rm --name "$CONTAINER_SERVER" --net $DOCKER_NETWORK \
   --env-file="dotenv/test-schema-validation.env" \
   --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \

.gitlab-ci/scripts/test-schema/postgres.sh

@@ -23,8 +23,9 @@ timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isr
 
 echo "Creating databases"
 docker exec $CONTAINER_DB sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test;'
-echo "Importing postgres dump from version 6.0"
+echo "Importing postgres dump from version 6.0.0"
-docker exec $CONTAINER_DB psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.sql
+docker exec -i $CONTAINER_DB psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.0.sql
+docker exec -i $CONTAINER_DB psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
 docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \
   --env-file="dotenv/test-schema-validation.env" \
   --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \

Makefile

@@ -353,12 +353,12 @@ test-compile: ## Compile tests.
 	done
 
 test-db-migration: start-docker ## Gets diff of upgrade vs new instance schemas.
-	./scripts/mysql-migration-test.sh 6.0
+	./scripts/mysql-migration-test.sh 6.0.0
-	./scripts/psql-migration-test.sh 6.0
+	./scripts/psql-migration-test.sh 6.0.0
 
 test-db-migration-v5: start-docker ## Gets diff of upgrade vs new instance schemas.
-	./scripts/mysql-migration-v5-test.sh 5.0
+	./scripts/mysql-migration-test.sh 5.0.0
-	./scripts/psql-migration-v5-test.sh 5.0
+	./scripts/psql-migration-test.sh 5.0.0
 
 gomodtidy:
 	@cp go.mod go.mod.orig

model/channel.go

@@ -47,7 +47,6 @@ type Channel struct {
 	Header            string                 `json:"header"`
 	Purpose           string                 `json:"purpose"`
 	LastPostAt        int64                  `json:"last_post_at"`
-	LastRootPostAt    int64                  `json:"last_root_post_at"`
 	TotalMsgCount     int64                  `json:"total_msg_count"`
 	ExtraUpdateAt     int64                  `json:"extra_update_at"`
 	CreatorId         string                 `json:"creator_id"`
@@ -57,6 +56,7 @@ type Channel struct {
 	Shared            *bool                  `json:"shared"`
 	TotalMsgCountRoot int64                  `json:"total_msg_count_root"`
 	PolicyID          *string                `json:"policy_id" db:"-"`
+	LastRootPostAt    int64                  `json:"last_root_post_at"`
 }
 
 type ChannelWithTeamData struct {

scripts/mattermost-mysql-6.0.0.sql

@@ -71,7 +71,7 @@ CREATE TABLE `ChannelMemberHistory` (
 CREATE TABLE `ChannelMembers` (
   `ChannelId` varchar(26) NOT NULL,
   `UserId` varchar(26) NOT NULL,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
   `LastViewedAt` bigint(20) DEFAULT NULL,
   `MsgCount` bigint(20) DEFAULT NULL,
   `MentionCount` bigint(20) DEFAULT NULL,
@@ -106,7 +106,6 @@ CREATE TABLE `Channels` (
   `Header` text,
   `Purpose` varchar(250) DEFAULT NULL,
   `LastPostAt` bigint(20) DEFAULT NULL,
-  `LastRootPostAt` bigint(20) DEFAULT NULL,
   `TotalMsgCount` bigint(20) DEFAULT NULL,
   `ExtraUpdateAt` bigint(20) DEFAULT NULL,
   `CreatorId` varchar(26) DEFAULT NULL,
@@ -768,7 +767,7 @@ CREATE TABLE `Sessions` (
   `LastActivityAt` bigint(20) DEFAULT NULL,
   `UserId` varchar(26) DEFAULT NULL,
   `DeviceId` text,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
   `IsOAuth` tinyint(1) DEFAULT NULL,
   `ExpiredNotify` tinyint(1) DEFAULT NULL,
   `Props` json DEFAULT NULL,
@@ -938,7 +937,7 @@ CREATE TABLE `Systems` (
 CREATE TABLE `TeamMembers` (
   `TeamId` varchar(26) NOT NULL,
   `UserId` varchar(26) NOT NULL,
-  `Roles` text DEFAULT NULL,
+  `Roles` varchar(64) DEFAULT NULL,
   `DeleteAt` bigint(20) DEFAULT NULL,
   `SchemeUser` tinyint(4) DEFAULT NULL,
   `SchemeAdmin` tinyint(4) DEFAULT NULL,

scripts/mattermost-postgresql-6.0.0.sql

@@ -73,7 +73,7 @@ ALTER TABLE public.channelmemberhistory OWNER TO mmuser;
 CREATE TABLE public.channelmembers (
     channelid character varying(26) NOT NULL,
     userid character varying(26) NOT NULL,
-    roles character varying(256),
+    roles character varying(64),
     lastviewedat bigint,
     msgcount bigint,
     mentioncount bigint,
@@ -105,14 +105,14 @@ CREATE TABLE public.channels (
     header character varying(1024),
     purpose character varying(250),
     lastpostat bigint,
-    lastrootpostat bigint,
     totalmsgcount bigint,
     extraupdateat bigint,
     creatorid character varying(26),
     schemeid character varying(26),
     groupconstrained boolean,
     shared boolean,
-    totalmsgcountroot bigint
+    totalmsgcountroot bigint,
+    lastrootpostat bigint DEFAULT '0'::bigint
 );
 
 
@@ -676,7 +676,7 @@ CREATE TABLE public.sessions (
     lastactivityat bigint,
     userid character varying(26),
     deviceid character varying(512),
-    roles character varying(256),
+    roles character varying(64),
     isoauth boolean,
     expirednotify boolean,
     props jsonb
@@ -826,7 +826,7 @@ ALTER TABLE public.systems OWNER TO mmuser;
 CREATE TABLE public.teammembers (
     teamid character varying(26) NOT NULL,
     userid character varying(26) NOT NULL,
-    roles character varying(256),
+    roles character varying(64),
     deleteat bigint,
     schemeuser boolean,
     schemeadmin boolean,

scripts/mysql-migration-test.sh

@@ -10,6 +10,8 @@ docker exec mattermost-mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated
 echo "Importing mysql dump from version ${SCHEMA_VERSION}"
 docker exec -i mattermost-mysql mysql -D migrated -uroot -pmostest < $(pwd)/scripts/mattermost-mysql-$SCHEMA_VERSION.sql
 
+docker exec -i mattermost-mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '$SCHEMA_VERSION')"
+
 echo "Setting up config for db migration"
 cat config/config.json | \
     jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s"' | \
@@ -26,7 +28,7 @@ cat config/config.json | \
 echo "Setting up fresh db"
 make ARGS="version --config $TMPDIR/config.json" run-cli
 
-if [ "$SCHEMA_VERSION" == "5.0" ]; then
+if [ "$SCHEMA_VERSION" == "5.0.0" ]; then
   for i in "ChannelMembers SchemeGuest" "ChannelMembers MsgCountRoot" "ChannelMembers MentionCountRoot" "Channels TotalMsgCountRoot"; do
     a=( $i );
     echo "Ignoring known MySQL mismatch: ${a[0]}.${a[1]}"

scripts/psql-migration-test.sh

@@ -10,6 +10,8 @@ docker exec mattermost-postgres sh -c 'exec echo "CREATE DATABASE migrated; CREA
 echo "Importing postgres dump from version ${SCHEMA_VERSION}"
 docker exec -i mattermost-postgres psql -U mmuser -d migrated < $(pwd)/scripts/mattermost-postgresql-$SCHEMA_VERSION.sql
 
+docker exec -i mattermost-postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '$SCHEMA_VERSION')"
+
 echo "Setting up config for db migration"
 cat config/config.json | \
     jq '.SqlSettings.DataSource = "postgres://mmuser:mostest@localhost:5432/migrated?sslmode=disable&connect_timeout=10"'| \
@@ -26,7 +28,7 @@ cat config/config.json | \
 echo "Setting up fresh db"
 make ARGS="version --config $TMPDIR/config.json" run-cli
 
-if [ "$SCHEMA_VERSION" == "5.0" ]; then
+if [ "$SCHEMA_VERSION" == "5.0.0" ]; then
   for i in "ChannelMembers MentionCountRoot" "ChannelMembers MsgCountRoot" "Channels TotalMsgCountRoot"; do
     a=( $i );
     echo "Ignoring known Postgres mismatch: ${a[0]}.${a[1]}"

store/sqlstore/channel_store.go

@@ -386,6 +386,7 @@ func newSqlChannelStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface
 		table.ColMap("Purpose").SetMaxSize(250)
 		table.ColMap("CreatorId").SetMaxSize(26)
 		table.ColMap("SchemeId").SetMaxSize(26)
+		table.ColMap("LastRootPostAt").SetDefaultConstraint(model.NewString("0"))
 
 		tablem := db.AddTableWithName(channelMember{}, "ChannelMembers").SetKeys(false, "ChannelId", "UserId")
 		tablem.ColMap("ChannelId").SetMaxSize(26)