From 83652aa3c284c7359a4b5ea6faeed64a2acc0b0e Mon Sep 17 00:00:00 2001 From: Ibrahim Serdar Acikgoz Date: Wed, 16 Aug 2023 10:09:11 +0300 Subject: [PATCH] [MM-53840] Add migration workflow (#24211) --- .github/workflows/migration.yml | 59 ++++++++++++++++++++++++ .github/workflows/server-ci-template.yml | 4 ++ .gitignore | 3 ++ server/Makefile | 35 +++++++++++++- server/build/dotenv/migration.env | 16 +++++++ server/build/dotenv/test.env | 2 +- server/scripts/wait-for-system-start.sh | 8 ++-- server/tests/template.load | 32 +++++++++++++ 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/migration.yml create mode 100644 server/build/dotenv/migration.env create mode 100644 server/tests/template.load diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml new file mode 100644 index 0000000000..35a4bffb05 --- /dev/null +++ b/.github/workflows/migration.yml @@ -0,0 +1,59 @@ +name: Database Migration Test +on: + workflow_call: +jobs: + test: + name: MySQL -> Postgres Migration + runs-on: ubuntu-22.04 + env: + COMPOSE_PROJECT_NAME: ghactions + BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5 + TEST_IMAGE: migration-test-image + defaults: + run: + working-directory: server + steps: + - name: Checkout mattermost project + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Run docker compose + run: | + cd build + docker-compose --ansi never run --rm start_dependencies + docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker-compose --ansi never ps + - name: Build test image + run: | + echo -e "FROM $BUILD_IMAGE\n\nRUN apt update && apt install -y pgloader" >> dockerfile_tmp + docker build -t $TEST_IMAGE - < dockerfile_tmp + - name: Generate test-data + run: | + docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ + --ulimit nofile=8096:8096 \ + --env-file=build/dotenv/migration.env \ + -v $(go env GOCACHE):/go/cache \ + -e GOCACHE=/go/cache \ + -v $PWD:/mattermost \ + -w /mattermost \ + $TEST_IMAGE \ + make test-data + - name: Migrate the DB and compare + run: | + docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ + --ulimit nofile=8096:8096 \ + --env-file=build/dotenv/migration.env \ + -v $(go env GOCACHE):/go/cache \ + -e GOCACHE=/go/cache \ + -v $PWD:/mattermost \ + -w /mattermost \ + $TEST_IMAGE \ + make test-migration + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: Migration logs + path: server/migration.log + retention-days: 7 + - name: Stop docker compose + run: | + cd build + docker-compose --ansi never stop diff --git a/.github/workflows/server-ci-template.yml b/.github/workflows/server-ci-template.yml index 3de3fd690e..6897e4f79d 100644 --- a/.github/workflows/server-ci-template.yml +++ b/.github/workflows/server-ci-template.yml @@ -297,3 +297,7 @@ jobs: name: server-build-artifact path: server/build/ retention-days: 14 + test-migration: + name: MySQL to PostgreSQL Migration + uses: ./.github/workflows/migration.yml + secrets: inherit diff --git a/.gitignore b/.gitignore index 821153772f..b6e49b8944 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ e2e-tests/playwright/tests/**/*-window.png e2e-tests/playwright/tests/accessibility/**/*-snapshots e2e-tests/playwright/.eslintcache +# ignore temporary added configuration for pgloader +server/tests/temp.load + # Enterprise & products imports files imports/imports.go diff --git a/server/Makefile b/server/Makefile index 705e017489..e48dc5bd55 100644 --- a/server/Makefile +++ b/server/Makefile @@ -540,7 +540,7 @@ test-data: run-server inject-test-data ## start a local instance and add test da inject-test-data: # add test data to the local instance. @if ! ./scripts/wait-for-system-start.sh; then \ - make stop; \ + make stop-server; \ fi @echo ServiceSettings.EnableLocalMode must be set to true. @@ -554,6 +554,8 @@ inject-test-data: # add test data to the local instance. @echo Login with a regular account username=user-1 password=SampleUs@r-1 @echo ======================================================================== + make stop-server + test-mmctl-unit: export MM_SERVER_PATH := $(MM_SERVER_PATH) test-mmctl-unit: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) test-mmctl-unit: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) @@ -811,7 +813,7 @@ ifeq ($(BUILD_ENTERPRISE_READY),true) endif mmctl-build: ## Compiles and generates the mmctl binary - go build -trimpath -ldflags '$(LDFLAGS)' -o $(GOBIN) ./cmd/mmctl + go build -trimpath -ldflags '$(LDFLAGS)' -o bin/mmctl ./cmd/mmctl mmctl-docs: ## Generate the mmctl docs rm -rf ./cmd/mmctl/docs @@ -828,3 +830,32 @@ migrations-extract: @echo Listing migration files @echo "# Autogenerated file to synchronize migrations sequence in the PR workflow, please do not edit.\n#" > channels/db/migrations/migrations.list find channels/db/migrations -maxdepth 2 -mindepth 2 | sort >> channels/db/migrations/migrations.list + +test-migration: + $(GO) install github.com/mattermost/morph/cmd/morph@latest + # apply postgres migrations to the new db + bin/morph apply up --driver postgres --dsn "${POSTGRES_DSN}" --path ${POSTGRES_MIGRATIONS_PATH} --number -1 + + # create temporary file for the config + @cp tests/template.load tests/temp.load + + # set required values for the pgloader config, these values are read from /build/dotenv/migration.env in the CI + # do for psql credentials + @sed -i'' -e 's|{{ .pg_user }}|${PG_USERNAME}|g' tests/temp.load + @sed -i'' -e 's|{{ .pg_password }}|${PG_PASSWORD}|g' tests/temp.load + @sed -i'' -e 's|{{ .target_schema }}|${PG_SCHEMA}|g' tests/temp.load + + # do for mysql credentials + @sed -i'' -e 's|{{ .mysql_user }}|${MYSQL_USERNAME}|g' tests/temp.load + @sed -i'' -e 's|{{ .mysql_password }}|${MYSQL_PASSWORD}|g' tests/temp.load + @sed -i'' -e 's|{{ .source_schema }}|${MYSQL_SCHEMA}|g' tests/temp.load + + # run pgloader and save the logs + pgloader tests/temp.load > migration.log + + $(GO) install github.com/mattermost/dbcmp/cmd/dbcmp@latest + # compare two database contents + # db_migrations differ due to a typo in the 92. migration name + # for now we exclude plugins such as playbooks and focalboard + # we also exlude systems table temporarily due to adding some keys while running the initial migration + bin/dbcmp --source "${MYSQL_DSN}" --target "${POSTGRES_DSN}" --exclude="db_migrations","ir_","focalboard","systems" diff --git a/server/build/dotenv/migration.env b/server/build/dotenv/migration.env new file mode 100644 index 0000000000..20339301d9 --- /dev/null +++ b/server/build/dotenv/migration.env @@ -0,0 +1,16 @@ +GOBIN=/mattermost/bin +PG_USERNAME=mmuser +PG_PASSWORD=mostest +PG_SCHEMA=mattermost_test +MYSQL_USERNAME=root +MYSQL_PASSWORD=mostest +MYSQL_SCHEMA=mattermost_test +POSTGRES_MIGRATIONS_PATH=./channels/db/migrations/postgres +MYSQL_DSN=${MYSQL_USERNAME}:${MYSQL_PASSWORD}@tcp(mysql:3306)/${MYSQL_SCHEMA}?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true +POSTGRES_DSN=postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres:5432/${PG_SCHEMA}?sslmode=disable + +MM_SERVICESETTINGS_ENABLELOCALMODE=true +MM_SQLSETTINGS_DRIVERNAME=mysql +MM_SQLSETTINGS_DATASOURCE=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 +MAX_WAIT_SECONDS=500 +IS_CI=true diff --git a/server/build/dotenv/test.env b/server/build/dotenv/test.env index e40cb65af6..929c5607de 100644 --- a/server/build/dotenv/test.env +++ b/server/build/dotenv/test.env @@ -1,7 +1,7 @@ TEST_DATABASE_MYSQL_DSN=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 TEST_DATABASE_POSTGRESQL_DSN=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest -GOBIN=/mattermost/mattermost-server/bin +GOBIN=/mattermost/server/bin CI_INBUCKET_HOST=inbucket CI_MINIO_HOST=minio diff --git a/server/scripts/wait-for-system-start.sh b/server/scripts/wait-for-system-start.sh index 109ce4c720..612be62f60 100755 --- a/server/scripts/wait-for-system-start.sh +++ b/server/scripts/wait-for-system-start.sh @@ -1,11 +1,11 @@ #!/bin/bash total=0 -max_wait_seconds=60 +MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:=60}" -echo "waiting $max_wait_seconds seconds for the server to start" +echo "waiting $MAX_WAIT_SECONDS seconds for the server to start" -while [[ "$total" -le "$max_wait_seconds" ]]; do +while [[ "$total" -le "$MAX_WAIT_SECONDS" ]]; do if bin/mmctl system status --local 2> /dev/null; then exit 0 else @@ -15,6 +15,6 @@ while [[ "$total" -le "$max_wait_seconds" ]]; do fi done -printf "\nserver didn't start in $max_wait_seconds seconds\n" +printf "\nserver didn't start in $MAX_WAIT_SECONDS seconds\n" exit 1 \ No newline at end of file diff --git a/server/tests/template.load b/server/tests/template.load new file mode 100644 index 0000000000..44f4d07409 --- /dev/null +++ b/server/tests/template.load @@ -0,0 +1,32 @@ +LOAD DATABASE + FROM mysql://{{ .mysql_user }}:{{ .mysql_password }}@mysql:3306/{{ .source_schema }} + INTO pgsql://{{ .pg_user }}:{{ .pg_password }}@postgres:5432/{{ .target_schema }} + +WITH data only, + workers = 8, concurrency = 1, + multiple readers per thread, rows per range = 50000 + +SET PostgreSQL PARAMETERS + maintenance_work_mem to '128MB', + work_mem to '12MB' + +SET MySQL PARAMETERS + net_read_timeout = '120', + net_write_timeout = '120' + +CAST column Drafts.Priority to text, + type int when (= precision 11) to integer drop typemod, + type bigint when (= precision 20) to bigint drop typemod, + type text to varchar drop typemod, + type tinyint when (<= precision 4) to boolean using tinyint-to-boolean, + type json to jsonb drop typemod + +MATERIALIZE VIEWS exclude_products + excluding table names matching ~, ~ + +BEFORE LOAD DO + $$ ALTER SCHEMA public RENAME TO {{ .source_schema }}; $$ + +AFTER LOAD DO + $$ UPDATE {{ .source_schema }}.db_migrations set name='add_createat_to_teamembers' where version=92; $$, + $$ ALTER SCHEMA {{ .source_schema }} RENAME TO public; $$;