From 0765f27483f61765543647b94ac33cc4a09f3ac0 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Wed, 23 Jun 2021 15:31:57 +0200 Subject: [PATCH] Enable e2e testing with docker-compose on GitLab (#17765) * Make testing with docker-compose work on GitLab. * Comment heavy jobs. * Make testing with docker-compose work on GitLab. * Comment heavy jobs. * Fix images * Addressing review. --- .circleci/config.yml | 7 +- .gitlab-ci.yml | 238 +++++++++++---------- .gitlab-ci/scripts/test-schema/mysql.sh | 52 +++++ .gitlab-ci/scripts/test-schema/postgres.sh | 47 ++++ .gitlab-ci/scripts/test/mysql.sh | 36 ++++ .gitlab-ci/scripts/test/postgres.sh | 35 +++ .gitlab-ci/test-schema.yml | 23 -- .gitlab-ci/test.yml | 53 ----- build/.gitignore | 3 + build/dotenv/test-schema-validation.env | 9 - build/gitlab-dc.common.yml | 57 +++++ build/gitlab-dc.mysql.yml | 102 +++++++++ build/gitlab-dc.postgres.yml | 101 +++++++++ build/gitlab-dc.schemamysql.yml | 36 ++++ build/gitlab-dc.schemapostgres.yml | 34 +++ 15 files changed, 632 insertions(+), 201 deletions(-) create mode 100755 .gitlab-ci/scripts/test-schema/mysql.sh create mode 100755 .gitlab-ci/scripts/test-schema/postgres.sh create mode 100755 .gitlab-ci/scripts/test/mysql.sh create mode 100644 .gitlab-ci/scripts/test/postgres.sh delete mode 100644 .gitlab-ci/test-schema.yml delete mode 100644 .gitlab-ci/test.yml create mode 100644 build/.gitignore create mode 100644 build/gitlab-dc.common.yml create mode 100644 build/gitlab-dc.mysql.yml create mode 100644 build/gitlab-dc.postgres.yml create mode 100644 build/gitlab-dc.schemamysql.yml create mode 100644 build/gitlab-dc.schemapostgres.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index efa1d71625..3589e93d5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -638,9 +638,4 @@ workflows: - check-store-layers - check-mocks - check-migrations - - test-schema: - requires: - - check-app-layers - - check-store-layers - - check-mocks - - check-migrations + # test-schema for master run in .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 540f1284c0..d969cab6b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ stages: -# - test + - test - build - publish - create-vars @@ -9,118 +9,136 @@ include: - project: mattermost/ci/mattermost-server ref: master file: private.yml -# - local: .gitlab-ci/test.yml -# - local: .gitlab-ci/test-schema.yml - -variables: - BUILD: "yes" - TEST: "no" empty: stage: create-vars script: - echo "empty" -#lint: -# image: $CI_REGISTRY/mattermost/ci/images/golangci-lint:v1.38.0 -# stage: .pre -# script: -# - GO111MODULE=off GOBIN=$PWD/bin go get -u github.com/mattermost/mattermost-govet -# - make config-reset -# - make check-style #todo MM_VET_OPENSPEC_PATH='$CI_PROJECT_DIR/mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' -# timeout: 60 mins -# rules: -# - if: '$TEST == "yes"' -# -#layers: -# image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 -# stage: .pre -# script: -# - make store-layers -# - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi -# - git reset --hard -# - make app-layers -# - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi -# - git reset --hard -# rules: -# - if: '$TEST == "yes"' -# -#test-postgres: -# extends: .test -# tags: -# - docker -# variables: -# MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" -# MM_SQLSETTINGS_DRIVERNAME: postgres -# rules: -# - if: '$TEST == "yes"' -# -#test-mysql: -# extends: .test -# tags: -# - docker -# variables: -# MM_SQLSETTINGS_DATASOURCE: "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true" -# MM_SQLSETTINGS_DRIVERNAME: mysql -# rules: -# - if: '$TEST == "yes"' -# -#test-schema-postgres: -# extends: .test-schema -# tags: -# - docker -# script: -# - echo "Creating databases" -# - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' -# - echo "Importing postgres dump from version 5.0" -# - docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-5.0.sql -# - > -# docker run -d -it --name server-postgres --net build_mm-test \ -# --env-file="dotenv/test-schema-validation.env" \ -# --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \ -# --env MM_SQLSETTINGS_DRIVERNAME=postgres \ -# -v $CI_PROJECT_DIR:/mattermost-server \ -# -w /mattermost-server \ -# mattermost/mattermost-build-server:20201119_golang-1.15.5 \ -# bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='version' run-cli" -# - docker logs -f server-postgres -# - echo "Generating dump" -# - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql -# - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql -# - echo "Removing databases created for db comparison" -# - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' -# - echo "Generating diff" -# - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) -# rules: -# - if: '$TEST == "yes"' -# -#test-schema-mysql: -# extends: .test-schema -# tags: -# - docker -# script: -# - echo "Creating databases" -# - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser" -# - echo "Importing mysql dump from version 5.0" -# - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-5.0.sql -# - > -# docker run -d -it --name server-mysql --net build_mm-test \ -# --env-file="dotenv/test-schema-validation.env" \ -# --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ -# --env MM_SQLSETTINGS_DRIVERNAME=mysql \ -# -v $CI_PROJECT_DIR:/mattermost-server \ -# -w /mattermost-server \ -# mattermost/mattermost-build-server:20201119_golang-1.15.5 \ -# bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='version' run-cli" -# - docker logs -f server-mysql -# - echo "Ignoring known MySQL mismatch 'ChannelMembers.SchemeGuest'" -# - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" || echo "drop failed" -# - docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" || echo "drop failed" -# - echo "Generating dump" -# - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql -# - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql -# - echo "Removing databases created for db comparison" -# - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' -# - echo "Generating diff" -# rules: -# - if: '$TEST == "yes"' +.lint: # Commenting this job, because it is run in CircleCI + image: $CI_REGISTRY/mattermost/ci/images/golangci-lint:v1.39.0-1 + stage: test + script: + - GO111MODULE=off GOBIN=$PWD/bin go get -u github.com/mattermost/mattermost-govet + - make config-reset + - make check-style #todo MM_VET_OPENSPEC_PATH='$CI_PROJECT_DIR/mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' + timeout: 60 mins + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + +.layers: # Commenting this job, because it is run in CircleCI + image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 + stage: test + script: + - make store-layers + - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi + - git reset --hard + - make app-layers + - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi + - git reset --hard + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + +.test-mysql: # Commenting this job, because it is run in CircleCI + stage: test + image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1 + services: + - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1 + alias: docker + variables: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + DOCKER_CONTENT_TRUST: 0 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" + before_script: + - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} + script: + - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}mysql" + - export RACE_MODE="" + - time cat .gitlab-ci/scripts/test/mysql.sh | /bin/bash + artifacts: + when: always + reports: + junit: report.xml + paths: + - build/logs + timeout: 2 hours + tags: + - k8s-memory + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + +.test-postgres: # Commenting this job, because it is run in CircleCI + stage: test + image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1 + services: + - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1 + alias: docker + variables: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + DOCKER_CONTENT_TRUST: 0 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" + before_script: + - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} + script: + - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}postgres" + - export RACE_MODE="" + - time cat .gitlab-ci/scripts/test/postgres.sh | /bin/bash + artifacts: + when: always + reports: + junit: report.xml + paths: + - build/logs + timeout: 2 hours + tags: + - k8s-memory + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + +test-schema-mysql: + stage: test + image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1 + services: + - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1 + alias: docker + variables: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + DOCKER_CONTENT_TRUST: 0 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" + before_script: + - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} + script: + - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemamysql" + - time cat .gitlab-ci/scripts/test-schema/mysql.sh | /bin/bash + tags: + - k8s-memory + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + +test-schema-postgres: + stage: test + image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1 + services: + - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1 + alias: docker + variables: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + DOCKER_CONTENT_TRUST: 0 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" + before_script: + - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} + script: + - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemapostgres" + - time cat .gitlab-ci/scripts/test-schema/postgres.sh | /bin/bash + tags: + - k8s-memory + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' diff --git a/.gitlab-ci/scripts/test-schema/mysql.sh b/.gitlab-ci/scripts/test-schema/mysql.sh new file mode 100755 index 0000000000..7e24ddd864 --- /dev/null +++ b/.gitlab-ci/scripts/test-schema/mysql.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -xe + +echo $DOCKER_HOST +docker ps +DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}" +DOCKER_COMPOSE_FILE="gitlab-dc.schemamysql.yml" +CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" +docker network create ${DOCKER_NETWORK} +ulimit -n 8096 +cd ${CI_PROJECT_DIR}/build +docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies +sleep 5 +docker run --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" + +echo "Creating databases" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser" +echo "Importing mysql dump from version 5.0" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest < ${CI_PROJECT_DIR}/scripts/mattermost-mysql-5.0.sql +docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \ + --env-file="dotenv/test-schema-validation.env" \ + --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ + --env MM_SQLSETTINGS_DRIVERNAME=mysql \ + -v $CI_PROJECT_DIR:/mattermost-server \ + -w /mattermost-server \ + ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \ + bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='version' run-cli" +docker logs -f "${CONTAINER_SERVER}" + +echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" +echo "Ignoring known MySQL mismatch: ChannelMembers.MentionCountRoot and MsgCountRoot" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" +echo "Ignoring known MySQL mismatch: Channels.TotalMsgCountRoot" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" + +echo "Generating dump" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql +echo "Removing databases created for db comparison" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' + +echo "Generating diff" +diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) + +docker-compose -f $DOCKER_COMPOSE_FILE down +docker network remove ${DOCKER_NETWORK} diff --git a/.gitlab-ci/scripts/test-schema/postgres.sh b/.gitlab-ci/scripts/test-schema/postgres.sh new file mode 100755 index 0000000000..4d5d48e78b --- /dev/null +++ b/.gitlab-ci/scripts/test-schema/postgres.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -xe + +echo $DOCKER_HOST +docker ps +DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}" +DOCKER_COMPOSE_FILE="gitlab-dc.schemapostgres.yml" +CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" +docker network create ${DOCKER_NETWORK} +ulimit -n 8096 +cd ${CI_PROJECT_DIR}/build +docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies +timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done" + +echo "Creating databases" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' +echo "Importing postgres dump from version 5.0" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres psql -U mmuser -d migrated < ${CI_PROJECT_DIR}/scripts/mattermost-postgresql-5.0.sql +docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \ + --env-file="dotenv/test-schema-validation.env" \ + --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \ + --env MM_SQLSETTINGS_DRIVERNAME=postgres \ + -v $CI_PROJECT_DIR:/mattermost-server \ + -w /mattermost-server \ + ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \ + bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='version' run-cli" +docker logs -f "${CONTAINER_SERVER}" + +echo "Ignoring known mismatch: ChannelMembers.MentionCountRoot" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" | exec psql -U mmuser -d migrated' +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" | exec psql -U mmuser -d latest' +echo "Ignoring known mismatch: ChannelMembers.MsgCountRoot and Channels.TotalMsgCountRoot" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" | exec psql -U mmuser -d migrated' +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" | exec psql -U mmuser -d latest' +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" | exec psql -U mmuser -d migrated' +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" | exec psql -U mmuser -d latest' + +echo "Generating dump" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql +echo "Removing databases created for db comparison" +docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' +echo "Generating diff" +diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) + +docker-compose -f $DOCKER_COMPOSE_FILE down +docker network remove ${DOCKER_NETWORK} diff --git a/.gitlab-ci/scripts/test/mysql.sh b/.gitlab-ci/scripts/test/mysql.sh new file mode 100755 index 0000000000..79cb98c155 --- /dev/null +++ b/.gitlab-ci/scripts/test/mysql.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -xe + +echo $DOCKER_HOST +docker ps +DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}" +DOCKER_COMPOSE_FILE="gitlab-dc.mysql.yml" +CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" +docker network create ${DOCKER_NETWORK} +ulimit -n 8096 +cd ${CI_PROJECT_DIR}/build +docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies +sleep 5 +cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; +docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test'; +docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_mysql" --net ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" +docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" + +docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \ + --env-file="dotenv/test.env" \ + --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true" \ + --env MM_SQLSETTINGS_DRIVERNAME=mysql \ + -v ${CI_PROJECT_DIR}:/mattermost-server \ + -w /mattermost-server \ + ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \ + bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \ + bash -c scripts/diff-email-templates.sh +mkdir -p logs +docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs +docker ps -a --no-trunc > logs/docker_ps +docker stats -a --no-stream > logs/docker_stats +docker logs -f "${CONTAINER_SERVER}" +tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats + +docker-compose -f $DOCKER_COMPOSE_FILE down +docker network remove ${DOCKER_NETWORK} diff --git a/.gitlab-ci/scripts/test/postgres.sh b/.gitlab-ci/scripts/test/postgres.sh new file mode 100644 index 0000000000..f51028e03c --- /dev/null +++ b/.gitlab-ci/scripts/test/postgres.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -xe + +echo $DOCKER_HOST +docker ps +DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}" +DOCKER_COMPOSE_FILE="gitlab-dc.postgres.yml" +CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" +docker network create ${DOCKER_NETWORK} +ulimit -n 8096 +cd ${CI_PROJECT_DIR}/build +docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies +sleep 5 +cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; +docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test'; +timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done" +docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" +docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \ + --env-file="dotenv/test.env" \ + --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ + --env MM_SQLSETTINGS_DRIVERNAME=postgres \ + -v $CI_PROJECT_DIR:/mattermost-server \ + -w /mattermost-server \ + ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \ + bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \ + bash -c scripts/diff-email-templates.sh +mkdir -p logs +docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs +docker ps -a --no-trunc > logs/docker_ps +docker stats -a --no-stream > logs/docker_stats +docker logs -f "${CONTAINER_SERVER}" +tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats + +docker-compose -f $DOCKER_COMPOSE_FILE down +docker network remove ${DOCKER_NETWORK} diff --git a/.gitlab-ci/test-schema.yml b/.gitlab-ci/test-schema.yml deleted file mode 100644 index 8207e8ca96..0000000000 --- a/.gitlab-ci/test-schema.yml +++ /dev/null @@ -1,23 +0,0 @@ -.test-schema: - stage: test - image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.12-1 - services: - - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.12-1 - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - before_script: - - docker ps - - - ulimit -n 8096 - - cd build - - docker-compose --no-ansi run --rm start_dependencies - - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test' - - sleep 5 - - docker run --net build_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - - docker run --net build_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" diff --git a/.gitlab-ci/test.yml b/.gitlab-ci/test.yml deleted file mode 100644 index e68e50d210..0000000000 --- a/.gitlab-ci/test.yml +++ /dev/null @@ -1,53 +0,0 @@ -.test: - stage: test - image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.12-1 - services: - - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.12-1 - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - - MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" - MM_SQLSETTINGS_DRIVERNAME: postgres - script: - - mkdir -p client/plugins - - mkdir -p build/test-results - - mkdir -p build/logs - - cd build - - docker-compose --no-ansi run --rm start_dependencies - - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test' - - sleep 5 - - docker run --net build_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - - docker run --net build_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" - - ulimit -n 8096 - # todo: remove setting GOBIN here, once circleci can be removed (see build/dotenv/test.env) - - > - time docker run -it -d --name server --net build_mm-test \ - --env-file=dotenv/test.env \ - --env MM_SQLSETTINGS_DATASOURCE=$MM_SQLSETTINGS_DATASOURCE \ - --env MM_SQLSETTINGS_DRIVERNAME=$MM_SQLSETTINGS_DRIVERNAME \ - --env GOBIN=$PWD/bin \ - -v $CI_PROJECT_DIR:/mattermost-server \ - -w /mattermost-server \ - $CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \ - bash -c "ulimit -n 8096; make test-server BUILD_NUMBER=$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=;" - - docker logs -f server - - docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs - - docker ps -a --no-trunc > logs/docker_ps - - docker stats -a --no-stream > logs/docker_stats - - tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats - - cp ../report.xml test-results - - DOCKER_EXIT_CODE=$(docker inspect server --format='{{.State.ExitCode}}') - - echo $DOCKER_EXIT_CODE - - exit $DOCKER_EXIT_CODE - artifacts: - paths: - - build/logs - reports: - junit: build/test-results/report.xml - expire_in: 7 days diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000000..0bd93a0212 --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,3 @@ +diff.txt +latest.sql +migrated.sql diff --git a/build/dotenv/test-schema-validation.env b/build/dotenv/test-schema-validation.env index 272a15daac..f5a4b3cbcf 100644 --- a/build/dotenv/test-schema-validation.env +++ b/build/dotenv/test-schema-validation.env @@ -1,10 +1 @@ -CI_INBUCKET_HOST=inbucket -CI_MINIO_HOST=minio -CI_INBUCKET_PORT=10080 -CI_MINIO_PORT=9000 -CI_INBUCKET_SMTP_PORT=10025 -CI_LDAP_HOST=openldap IS_CI=true -MM_EMAILSETTINGS_SMTPSERVER=inbucket -MM_EMAILSETTINGS_SMTPPORT=10025 -MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 diff --git a/build/gitlab-dc.common.yml b/build/gitlab-dc.common.yml new file mode 100644 index 0000000000..fe4ffc1560 --- /dev/null +++ b/build/gitlab-dc.common.yml @@ -0,0 +1,57 @@ +version: '2.4' +services: + minio: + image: ${CI_REGISTRY}/mattermost/ci/images/minio:RELEASE.2019-10-11T00-38-09Z-1 + command: "server /data" + environment: + MINIO_ACCESS_KEY: minioaccesskey + MINIO_SECRET_KEY: miniosecretkey + MINIO_SSE_MASTER_KEY: "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574" + inbucket: + image: ${CI_REGISTRY}/mattermost/ci/images/inbucket:v3.0.0-rc1-2-gc64e7a6-1 + restart: always + environment: + INBUCKET_WEB_ADDR: "0.0.0.0:10080" + INBUCKET_POP3_ADDR: "0.0.0.0:10110" + INBUCKET_SMTP_ADDR: "0.0.0.0:10025" + openldap: + image: ${CI_REGISTRY}/mattermost/ci/images/openldap:1.4.0-1 + restart: always + environment: + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_ORGANISATION: "Mattermost Test" + LDAP_DOMAIN: "mm.test.com" + LDAP_ADMIN_PASSWORD: "mostest" + elasticsearch: + image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-elasticsearch-docker:6.5.1-1 + environment: + http.host: "0.0.0.0" + http.port: 9200 + http.cors.enabled: "true" + http.cors.allow-origin: "http://localhost:1358,http://127.0.0.1:1358" + http.cors.allow-headers: "X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" + http.cors.allow-credentials: "true" + transport.host: "127.0.0.1" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + dejavu: + image: ${CI_REGISTRY}/mattermost/ci/images/dejavu:3.4.2-1 + keycloak: + image: ${CI_REGISTRY}/mattermost/ci/images/keycloak:10.0.2-1 + restart: always + environment: + KEYCLOAK_USER: mmuser + KEYCLOAK_PASSWORD: mostest + DB_VENDOR: h2 + KEYCLOAK_IMPORT: /setup/realm.json + volumes: + - "./docker/keycloak:/setup" + prometheus: + image: ${CI_REGISTRY}/mattermost/ci/images/prometheus:2.27.1-1 + volumes: + - "./docker/prometheus-linux.yml:/etc/prometheus/prometheus.yml" + grafana: + image: ${CI_REGISTRY}/mattermost/ci/images/grafana:8.0.1-1 + volumes: + - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini" + - "./docker/grafana/provisioning:/etc/grafana/provisioning" + - "./docker/grafana/dashboards:/var/lib/grafana/dashboards" diff --git a/build/gitlab-dc.mysql.yml b/build/gitlab-dc.mysql.yml new file mode 100644 index 0000000000..b82478954d --- /dev/null +++ b/build/gitlab-dc.mysql.yml @@ -0,0 +1,102 @@ +version: '2.4' +services: + mysql: + image: ${CI_REGISTRY}/mattermost/ci/images/mysql:5.6-1 + restart: always + environment: + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: mostest + MYSQL_PASSWORD: mostest + MYSQL_USER: mmuser + MYSQL_DATABASE: mattermost_test + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + interval: 5s + timeout: 10s + retries: 3 + tmpfs: /var/lib/mysql + volumes: + - "./docker/mysql.conf.d:/etc/mysql/conf.d" + networks: + default: + aliases: + - mysql + minio: + extends: + file: gitlab-dc.common.yml + service: minio + networks: + default: + aliases: + - minio + inbucket: + extends: + file: gitlab-dc.common.yml + service: inbucket + networks: + default: + aliases: + - inbucket + openldap: + extends: + file: gitlab-dc.common.yml + service: openldap + networks: + default: + aliases: + - openldap + elasticsearch: + extends: + file: gitlab-dc.common.yml + service: elasticsearch + networks: + default: + aliases: + - elasticsearch + dejavu: + extends: + file: gitlab-dc.common.yml + service: dejavu + networks: + default: + aliases: + - dejavu + keycloak: + extends: + file: gitlab-dc.common.yml + service: keycloak + prometheus: + extends: + file: gitlab-dc.common.yml + service: prometheus + networks: + default: + aliases: + - prometheus + grafana: + extends: + file: gitlab-dc.common.yml + service: grafana + networks: + default: + aliases: + - grafana + + start_dependencies: + image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest + depends_on: + - mysql + - minio + - inbucket + - openldap + - elasticsearch + - prometheus + - grafana + command: mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 + networks: + default: + +networks: + default: + external: + name: ${COMPOSE_PROJECT_NAME} diff --git a/build/gitlab-dc.postgres.yml b/build/gitlab-dc.postgres.yml new file mode 100644 index 0000000000..48b2ff124e --- /dev/null +++ b/build/gitlab-dc.postgres.yml @@ -0,0 +1,101 @@ +version: '2.4' +services: + postgres: + image: ${CI_REGISTRY}/mattermost/ci/images/postgres:10-1 + restart: always + environment: + POSTGRES_USER: mmuser + POSTGRES_PASSWORD: mostest + POSTGRES_DB: mattermost_test + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' + volumes: + - "./docker/postgres.conf:/etc/postgresql/postgresql.conf" + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost" ] + interval: 5s + timeout: 10s + retries: 3 + networks: + default: + aliases: + - postgres + minio: + extends: + file: gitlab-dc.common.yml + service: minio + networks: + default: + aliases: + - minio + inbucket: + extends: + file: gitlab-dc.common.yml + service: inbucket + networks: + default: + aliases: + - inbucket + openldap: + extends: + file: gitlab-dc.common.yml + service: openldap + networks: + default: + aliases: + - openldap + elasticsearch: + extends: + file: gitlab-dc.common.yml + service: elasticsearch + networks: + default: + aliases: + - elasticsearch + dejavu: + extends: + file: gitlab-dc.common.yml + service: dejavu + networks: + default: + aliases: + - dejavu + keycloak: + extends: + file: gitlab-dc.common.yml + service: keycloak + prometheus: + extends: + file: gitlab-dc.common.yml + service: prometheus + networks: + default: + aliases: + - prometheus + grafana: + extends: + file: gitlab-dc.common.yml + service: grafana + networks: + default: + aliases: + - grafana + + start_dependencies: + image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1 + depends_on: + - postgres + - minio + - inbucket + - openldap + - elasticsearch + - prometheus + - grafana + command: postgres:5432 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 + networks: + default: + + +networks: + default: + external: + name: ${COMPOSE_PROJECT_NAME} diff --git a/build/gitlab-dc.schemamysql.yml b/build/gitlab-dc.schemamysql.yml new file mode 100644 index 0000000000..c8a5e6e890 --- /dev/null +++ b/build/gitlab-dc.schemamysql.yml @@ -0,0 +1,36 @@ +version: '2.4' +services: + mysql: + image: ${CI_REGISTRY}/mattermost/ci/images/mysql:5.6-1 + restart: always + environment: + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: mostest + MYSQL_PASSWORD: mostest + MYSQL_USER: mmuser + MYSQL_DATABASE: mattermost_test + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + interval: 5s + timeout: 10s + retries: 3 + tmpfs: /var/lib/mysql + volumes: + - "./docker/mysql.conf.d:/etc/mysql/conf.d" + networks: + default: + aliases: + - mysql + + start_dependencies: + image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1 + depends_on: + - mysql + command: mysql:3306 + networks: + default: + +networks: + default: + external: + name: ${COMPOSE_PROJECT_NAME} diff --git a/build/gitlab-dc.schemapostgres.yml b/build/gitlab-dc.schemapostgres.yml new file mode 100644 index 0000000000..458566adc6 --- /dev/null +++ b/build/gitlab-dc.schemapostgres.yml @@ -0,0 +1,34 @@ +version: '2.4' +services: + postgres: + image: ${CI_REGISTRY}/mattermost/ci/images/postgres:10-1 + restart: always + environment: + POSTGRES_USER: mmuser + POSTGRES_PASSWORD: mostest + POSTGRES_DB: mattermost_test + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' + volumes: + - "./docker/postgres.conf:/etc/postgresql/postgresql.conf" + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost" ] + interval: 5s + timeout: 10s + retries: 3 + networks: + default: + aliases: + - postgres + + start_dependencies: + image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1 + depends_on: + - postgres + command: postgres:5432 + networks: + default: + +networks: + default: + external: + name: ${COMPOSE_PROJECT_NAME}