[MM-53840] Add migration workflow (#24211)

.github/workflows/migration.yml

@@ -0,0 +1,59 @@
+name: Database Migration Test
+on:
+  workflow_call:
+jobs:
+    test:
+      name: MySQL -> Postgres Migration
+      runs-on: ubuntu-22.04
+      env:
+          COMPOSE_PROJECT_NAME: ghactions
+          BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5
+          TEST_IMAGE: migration-test-image
+      defaults:
+        run:
+          working-directory: server
+      steps:
+      - name: Checkout mattermost project
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Run docker compose
+        run: |
+          cd build
+          docker-compose --ansi never run --rm start_dependencies
+          docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test';
+          docker-compose --ansi never ps
+      - name: Build test image
+        run: |
+          echo -e "FROM $BUILD_IMAGE\n\nRUN apt update && apt install -y pgloader" >> dockerfile_tmp
+          docker build -t $TEST_IMAGE - < dockerfile_tmp
+      - name: Generate test-data
+        run: |
+          docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \
+            --ulimit nofile=8096:8096 \
+            --env-file=build/dotenv/migration.env \
+            -v $(go env GOCACHE):/go/cache \
+            -e GOCACHE=/go/cache \
+            -v $PWD:/mattermost \
+            -w /mattermost \
+            $TEST_IMAGE \
+            make test-data
+      - name: Migrate the DB and compare
+        run: |
+           docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \
+           --ulimit nofile=8096:8096 \
+           --env-file=build/dotenv/migration.env \
+           -v $(go env GOCACHE):/go/cache \
+           -e GOCACHE=/go/cache \
+           -v $PWD:/mattermost \
+           -w /mattermost \
+           $TEST_IMAGE \
+           make test-migration
+      - name: Upload artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+            name: Migration logs
+            path: server/migration.log
+            retention-days: 7
+      - name: Stop docker compose
+        run: |
+          cd build
+          docker-compose --ansi never stop

.github/workflows/server-ci-template.yml

@@ -297,3 +297,7 @@ jobs:
           name: server-build-artifact
           path: server/build/
           retention-days: 14
+  test-migration:
+    name: MySQL to PostgreSQL Migration
+    uses: ./.github/workflows/migration.yml
+    secrets: inherit

.gitignore

@@ -44,6 +44,9 @@ e2e-tests/playwright/tests/**/*-window.png
 e2e-tests/playwright/tests/accessibility/**/*-snapshots
 e2e-tests/playwright/.eslintcache
 
+# ignore temporary added configuration for pgloader
+server/tests/temp.load
+
 # Enterprise & products imports files
 imports/imports.go
 

server/Makefile

@@ -540,7 +540,7 @@ test-data: run-server inject-test-data ## start a local instance and add test da
 
 inject-test-data: # add test data to the local instance.
 	@if ! ./scripts/wait-for-system-start.sh; then \
-		make stop; \
+		make stop-server; \
 	fi
 
 	@echo ServiceSettings.EnableLocalMode must be set to true.
@@ -554,6 +554,8 @@ inject-test-data: # add test data to the local instance.
 	@echo Login with a regular account username=user-1 password=SampleUs@r-1
 	@echo ========================================================================
 
+	make stop-server
+
 test-mmctl-unit: export MM_SERVER_PATH := $(MM_SERVER_PATH)
 test-mmctl-unit: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
 test-mmctl-unit: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
@@ -811,7 +813,7 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
 endif
 
 mmctl-build: ## Compiles and generates the mmctl binary
-	go build -trimpath -ldflags '$(LDFLAGS)' -o $(GOBIN) ./cmd/mmctl
+	go build -trimpath -ldflags '$(LDFLAGS)' -o bin/mmctl ./cmd/mmctl
 
 mmctl-docs: ## Generate the mmctl docs
 	rm -rf ./cmd/mmctl/docs
@@ -828,3 +830,32 @@ migrations-extract:
 	@echo Listing migration files
 	@echo "# Autogenerated file to synchronize migrations sequence in the PR workflow, please do not edit.\n#" > channels/db/migrations/migrations.list
 	find channels/db/migrations -maxdepth 2 -mindepth 2 | sort >> channels/db/migrations/migrations.list
+
+test-migration:
+	$(GO) install github.com/mattermost/morph/cmd/morph@latest
+	# apply postgres migrations to the new db
+	bin/morph apply up --driver postgres --dsn "${POSTGRES_DSN}" --path ${POSTGRES_MIGRATIONS_PATH} --number -1
+
+	# create temporary file for the config
+	@cp tests/template.load tests/temp.load
+
+	# set required values for the pgloader config, these values are read from /build/dotenv/migration.env in the CI
+	# do for psql credentials
+	@sed -i'' -e 's|{{ .pg_user }}|${PG_USERNAME}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .pg_password }}|${PG_PASSWORD}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .target_schema }}|${PG_SCHEMA}|g' tests/temp.load
+
+	# do for mysql credentials
+	@sed -i'' -e 's|{{ .mysql_user }}|${MYSQL_USERNAME}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .mysql_password }}|${MYSQL_PASSWORD}|g' tests/temp.load
+	@sed -i'' -e 's|{{ .source_schema }}|${MYSQL_SCHEMA}|g' tests/temp.load
+
+	# run pgloader and save the logs
+	pgloader tests/temp.load > migration.log
+
+	$(GO) install github.com/mattermost/dbcmp/cmd/dbcmp@latest
+	# compare two database contents
+	# db_migrations differ due to a typo in the 92. migration name
+	# for now we exclude plugins such as playbooks and focalboard
+	# we also exlude systems table temporarily due to adding some keys while running the initial migration
+	bin/dbcmp --source "${MYSQL_DSN}" --target "${POSTGRES_DSN}" --exclude="db_migrations","ir_","focalboard","systems"

server/build/dotenv/migration.env

@@ -0,0 +1,16 @@
+GOBIN=/mattermost/bin
+PG_USERNAME=mmuser
+PG_PASSWORD=mostest
+PG_SCHEMA=mattermost_test
+MYSQL_USERNAME=root
+MYSQL_PASSWORD=mostest
+MYSQL_SCHEMA=mattermost_test
+POSTGRES_MIGRATIONS_PATH=./channels/db/migrations/postgres
+MYSQL_DSN=${MYSQL_USERNAME}:${MYSQL_PASSWORD}@tcp(mysql:3306)/${MYSQL_SCHEMA}?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true
+POSTGRES_DSN=postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres:5432/${PG_SCHEMA}?sslmode=disable
+
+MM_SERVICESETTINGS_ENABLELOCALMODE=true
+MM_SQLSETTINGS_DRIVERNAME=mysql
+MM_SQLSETTINGS_DATASOURCE=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304
+MAX_WAIT_SECONDS=500
+IS_CI=true

server/build/dotenv/test.env

@@ -1,7 +1,7 @@
 TEST_DATABASE_MYSQL_DSN=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304
 TEST_DATABASE_POSTGRESQL_DSN=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
 TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest
-GOBIN=/mattermost/mattermost-server/bin
+GOBIN=/mattermost/server/bin
 
 CI_INBUCKET_HOST=inbucket
 CI_MINIO_HOST=minio

server/scripts/wait-for-system-start.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
 total=0
-max_wait_seconds=60
+MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:=60}"
 
-echo "waiting $max_wait_seconds seconds for the server to start"
+echo "waiting $MAX_WAIT_SECONDS seconds for the server to start"
 
-while [[ "$total" -le "$max_wait_seconds" ]]; do
+while [[ "$total" -le "$MAX_WAIT_SECONDS" ]]; do
     if bin/mmctl system status --local 2> /dev/null; then
         exit 0
     else
@@ -15,6 +15,6 @@ while [[ "$total" -le "$max_wait_seconds" ]]; do
     fi
 done
 
-printf "\nserver didn't start in $max_wait_seconds seconds\n"
+printf "\nserver didn't start in $MAX_WAIT_SECONDS seconds\n"
 
 exit 1

server/tests/template.load

@@ -0,0 +1,32 @@
+LOAD DATABASE
+     FROM      mysql://{{ .mysql_user }}:{{ .mysql_password }}@mysql:3306/{{ .source_schema }}
+     INTO      pgsql://{{ .pg_user }}:{{ .pg_password }}@postgres:5432/{{ .target_schema }}
+
+WITH data only,
+     workers = 8, concurrency = 1,
+     multiple readers per thread, rows per range = 50000
+
+SET PostgreSQL PARAMETERS
+     maintenance_work_mem to '128MB',
+     work_mem to '12MB'
+
+SET MySQL PARAMETERS
+      net_read_timeout  = '120',
+      net_write_timeout = '120'
+
+CAST column Drafts.Priority to text,
+     type int when (= precision 11) to integer drop typemod,
+     type bigint when (= precision 20) to bigint drop typemod,
+     type text to varchar drop typemod,
+     type tinyint when (<= precision 4) to boolean using tinyint-to-boolean,
+     type json to jsonb drop typemod
+
+MATERIALIZE VIEWS exclude_products
+     excluding table names matching ~<IR_>, ~<focalboard>
+
+BEFORE LOAD DO
+     $$ ALTER SCHEMA public RENAME TO {{ .source_schema }}; $$
+
+AFTER LOAD DO
+     $$ UPDATE {{ .source_schema }}.db_migrations set name='add_createat_to_teamembers' where version=92; $$,
+     $$ ALTER SCHEMA {{ .source_schema }} RENAME TO public; $$;