[MM-53840] Add migration workflow (#24211)

server/Makefile

@@ -540,7 +540,7 @@ test-data: run-server inject-test-data ## start a local instance and add test da
 
 inject-test-data: # add test data to the local instance.
 	@if ! ./scripts/wait-for-system-start.sh; then \
-		make stop; \
+		make stop-server; \
 	fi
 
 	@echo ServiceSettings.EnableLocalMode must be set to true.
@@ -554,6 +554,8 @@ inject-test-data: # add test data to the local instance.
 	@echo Login with a regular account username=user-1 password=SampleUs@r-1
 	@echo ========================================================================
 
+	make stop-server
+
 test-mmctl-unit: export MM_SERVER_PATH := $(MM_SERVER_PATH)
 test-mmctl-unit: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
 test-mmctl-unit: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
@@ -811,7 +813,7 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
 endif
 
 mmctl-build: ## Compiles and generates the mmctl binary
-	go build -trimpath -ldflags '$(LDFLAGS)' -o $(GOBIN) ./cmd/mmctl
+	go build -trimpath -ldflags '$(LDFLAGS)' -o bin/mmctl ./cmd/mmctl
 
 mmctl-docs: ## Generate the mmctl docs
 	rm -rf ./cmd/mmctl/docs
@@ -828,3 +830,32 @@ migrations-extract:
 	@echo Listing migration files
 	@echo "# Autogenerated file to synchronize migrations sequence in the PR workflow, please do not edit.\n#" > channels/db/migrations/migrations.list
 	find channels/db/migrations -maxdepth 2 -mindepth 2 | sort >> channels/db/migrations/migrations.list
+
+test-migration:
+	$(GO) install github.com/mattermost/morph/cmd/morph@latest
+	# apply postgres migrations to the new db
+	bin/morph apply up --driver postgres --dsn "${POSTGRES_DSN}" --path ${POSTGRES_MIGRATIONS_PATH} --number -1
+
+	# create temporary file for the config
+	@cp tests/template.load tests/temp.load
+
+	# set required values for the pgloader config, these values are read from /build/dotenv/migration.env in the CI
+	# do for psql credentials
+	@sed -i'' -e 's|{{ .pg_user }}|${PG_USERNAME}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .pg_password }}|${PG_PASSWORD}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .target_schema }}|${PG_SCHEMA}|g' tests/temp.load
+
+	# do for mysql credentials
+	@sed -i'' -e 's|{{ .mysql_user }}|${MYSQL_USERNAME}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .mysql_password }}|${MYSQL_PASSWORD}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .source_schema }}|${MYSQL_SCHEMA}|g' tests/temp.load
+
+	# run pgloader and save the logs
+	pgloader tests/temp.load > migration.log
+
+	$(GO) install github.com/mattermost/dbcmp/cmd/dbcmp@latest
+	# compare two database contents
+	# db_migrations differ due to a typo in the 92. migration name
+	# for now we exclude plugins such as playbooks and focalboard
+	# we also exlude systems table temporarily due to adding some keys while running the initial migration
+	bin/dbcmp --source "${MYSQL_DSN}" --target "${POSTGRES_DSN}" --exclude="db_migrations","ir_","focalboard","systems"

server/build/dotenv/migration.env

@@ -0,0 +1,16 @@
+GOBIN=/mattermost/bin
+PG_USERNAME=mmuser
+PG_PASSWORD=mostest
+PG_SCHEMA=mattermost_test
+MYSQL_USERNAME=root
+MYSQL_PASSWORD=mostest
+MYSQL_SCHEMA=mattermost_test
+POSTGRES_MIGRATIONS_PATH=./channels/db/migrations/postgres
+MYSQL_DSN=${MYSQL_USERNAME}:${MYSQL_PASSWORD}@tcp(mysql:3306)/${MYSQL_SCHEMA}?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true
+POSTGRES_DSN=postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres:5432/${PG_SCHEMA}?sslmode=disable
+
+MM_SERVICESETTINGS_ENABLELOCALMODE=true
+MM_SQLSETTINGS_DRIVERNAME=mysql
+MM_SQLSETTINGS_DATASOURCE=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304
+MAX_WAIT_SECONDS=500
+IS_CI=true

server/build/dotenv/test.env

@@ -1,7 +1,7 @@
 TEST_DATABASE_MYSQL_DSN=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304
 TEST_DATABASE_POSTGRESQL_DSN=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
 TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest
-GOBIN=/mattermost/mattermost-server/bin
+GOBIN=/mattermost/server/bin
 
 CI_INBUCKET_HOST=inbucket
 CI_MINIO_HOST=minio

server/scripts/wait-for-system-start.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
 total=0
-max_wait_seconds=60
+MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:=60}"
 
-echo "waiting $max_wait_seconds seconds for the server to start"
+echo "waiting $MAX_WAIT_SECONDS seconds for the server to start"
 
-while [[ "$total" -le "$max_wait_seconds" ]]; do
+while [[ "$total" -le "$MAX_WAIT_SECONDS" ]]; do
     if bin/mmctl system status --local 2> /dev/null; then
         exit 0
     else
@@ -15,6 +15,6 @@ while [[ "$total" -le "$max_wait_seconds" ]]; do
     fi
 done
 
-printf "\nserver didn't start in $max_wait_seconds seconds\n"
+printf "\nserver didn't start in $MAX_WAIT_SECONDS seconds\n"
 
 exit 1

server/tests/template.load

@@ -0,0 +1,32 @@
+LOAD DATABASE
+     FROM      mysql://{{ .mysql_user }}:{{ .mysql_password }}@mysql:3306/{{ .source_schema }}
+     INTO      pgsql://{{ .pg_user }}:{{ .pg_password }}@postgres:5432/{{ .target_schema }}
+
+WITH data only,
+     workers = 8, concurrency = 1,
+     multiple readers per thread, rows per range = 50000
+
+SET PostgreSQL PARAMETERS
+     maintenance_work_mem to '128MB',
+     work_mem to '12MB'
+
+SET MySQL PARAMETERS
+      net_read_timeout  = '120',
+      net_write_timeout = '120'
+
+CAST column Drafts.Priority to text,
+     type int when (= precision 11) to integer drop typemod,
+     type bigint when (= precision 20) to bigint drop typemod,
+     type text to varchar drop typemod,
+     type tinyint when (<= precision 4) to boolean using tinyint-to-boolean,
+     type json to jsonb drop typemod
+
+MATERIALIZE VIEWS exclude_products
+     excluding table names matching ~<IR_>, ~<focalboard>
+
+BEFORE LOAD DO
+     $$ ALTER SCHEMA public RENAME TO {{ .source_schema }}; $$
+
+AFTER LOAD DO
+     $$ UPDATE {{ .source_schema }}.db_migrations set name='add_createat_to_teamembers' where version=92; $$,
+     $$ ALTER SCHEMA {{ .source_schema }} RENAME TO public; $$;