Enable e2e testing with docker-compose on GitLab (#17765)

* Make testing with docker-compose work on GitLab.

* Comment heavy jobs.

* Make testing with docker-compose work on GitLab.

* Comment heavy jobs.

* Fix images

* Addressing review.

.gitlab-ci/scripts/test-schema/mysql.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -xe
+
+echo $DOCKER_HOST
+docker ps
+DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
+DOCKER_COMPOSE_FILE="gitlab-dc.schemamysql.yml"
+CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
+docker network create ${DOCKER_NETWORK}
+ulimit -n 8096
+cd ${CI_PROJECT_DIR}/build
+docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
+sleep 5
+docker run --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
+
+echo "Creating databases"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
+echo "Importing mysql dump from version 5.0"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest < ${CI_PROJECT_DIR}/scripts/mattermost-mysql-5.0.sql
+docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
+  --env-file="dotenv/test-schema-validation.env" \
+  --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
+  --env MM_SQLSETTINGS_DRIVERNAME=mysql \
+  -v $CI_PROJECT_DIR:/mattermost-server \
+  -w /mattermost-server \
+  ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \
+  bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='version' run-cli"
+docker logs -f "${CONTAINER_SERVER}"
+
+echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
+echo "Ignoring known MySQL mismatch: ChannelMembers.MentionCountRoot and MsgCountRoot"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;"
+echo "Ignoring known MySQL mismatch: Channels.TotalMsgCountRoot"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;"
+
+echo "Generating dump"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
+echo "Removing databases created for db comparison"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
+
+echo "Generating diff"
+diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
+
+docker-compose -f $DOCKER_COMPOSE_FILE down
+docker network remove ${DOCKER_NETWORK}

.gitlab-ci/scripts/test-schema/postgres.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -xe
+
+echo $DOCKER_HOST
+docker ps
+DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
+DOCKER_COMPOSE_FILE="gitlab-dc.schemapostgres.yml"
+CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
+docker network create ${DOCKER_NETWORK}
+ulimit -n 8096
+cd ${CI_PROJECT_DIR}/build
+docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
+timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done"
+
+echo "Creating databases"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
+echo "Importing postgres dump from version 5.0"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres psql -U mmuser -d migrated < ${CI_PROJECT_DIR}/scripts/mattermost-postgresql-5.0.sql
+docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
+  --env-file="dotenv/test-schema-validation.env" \
+  --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
+  --env MM_SQLSETTINGS_DRIVERNAME=postgres \
+  -v $CI_PROJECT_DIR:/mattermost-server \
+  -w /mattermost-server \
+  ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \
+  bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='version' run-cli"
+docker logs -f "${CONTAINER_SERVER}"
+
+echo "Ignoring known mismatch: ChannelMembers.MentionCountRoot"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" | exec psql -U mmuser -d migrated'
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MentionCountRoot;" | exec psql -U mmuser -d latest'
+echo "Ignoring known mismatch: ChannelMembers.MsgCountRoot and Channels.TotalMsgCountRoot"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" | exec psql -U mmuser -d migrated'
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE ChannelMembers DROP COLUMN MsgCountRoot;" | exec psql -U mmuser -d latest'
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" | exec psql -U mmuser -d migrated'
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "ALTER TABLE Channels DROP COLUMN TotalMsgCountRoot;" | exec psql -U mmuser -d latest'
+
+echo "Generating dump"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
+echo "Removing databases created for db comparison"
+docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
+echo "Generating diff"
+diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
+
+docker-compose -f $DOCKER_COMPOSE_FILE down
+docker network remove ${DOCKER_NETWORK}

.gitlab-ci/scripts/test/mysql.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -xe
+
+echo $DOCKER_HOST
+docker ps
+DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
+DOCKER_COMPOSE_FILE="gitlab-dc.mysql.yml"
+CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
+docker network create ${DOCKER_NETWORK}
+ulimit -n 8096
+cd ${CI_PROJECT_DIR}/build
+docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
+sleep 5
+cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test';
+docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_mysql" --net ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
+docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
+
+docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
+  --env-file="dotenv/test.env" \
+  --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true" \
+  --env MM_SQLSETTINGS_DRIVERNAME=mysql \
+  -v ${CI_PROJECT_DIR}:/mattermost-server \
+  -w /mattermost-server \
+  ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \
+  bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \
+  bash -c scripts/diff-email-templates.sh
+mkdir -p logs
+docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
+docker ps -a --no-trunc > logs/docker_ps
+docker stats -a --no-stream > logs/docker_stats
+docker logs -f "${CONTAINER_SERVER}"
+tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
+
+docker-compose -f $DOCKER_COMPOSE_FILE down
+docker network remove ${DOCKER_NETWORK}

.gitlab-ci/scripts/test/postgres.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -xe
+
+echo $DOCKER_HOST
+docker ps
+DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
+DOCKER_COMPOSE_FILE="gitlab-dc.postgres.yml"
+CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
+docker network create ${DOCKER_NETWORK}
+ulimit -n 8096
+cd ${CI_PROJECT_DIR}/build
+docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
+sleep 5
+cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test';
+timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done"
+docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
+docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
+  --env-file="dotenv/test.env" \
+  --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
+  --env MM_SQLSETTINGS_DRIVERNAME=postgres \
+  -v $CI_PROJECT_DIR:/mattermost-server \
+  -w /mattermost-server \
+  ${CI_REGISTRY}/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5 \
+  bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \
+  bash -c scripts/diff-email-templates.sh
+mkdir -p logs
+docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
+docker ps -a --no-trunc > logs/docker_ps
+docker stats -a --no-stream > logs/docker_stats
+docker logs -f "${CONTAINER_SERVER}"
+tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
+
+docker-compose -f $DOCKER_COMPOSE_FILE down
+docker network remove ${DOCKER_NETWORK}