Enable e2e testing with docker-compose on GitLab (#17765)

* Make testing with docker-compose work on GitLab.

* Comment heavy jobs.

* Make testing with docker-compose work on GitLab.

* Comment heavy jobs.

* Fix images

* Addressing review.

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 stages:
-#  - test
+  - test
   - build
   - publish
   - create-vars
@@ -9,118 +9,136 @@ include:
   - project: mattermost/ci/mattermost-server
     ref: master
     file: private.yml
-#  - local: .gitlab-ci/test.yml
-#  - local: .gitlab-ci/test-schema.yml
-
-variables:
-  BUILD: "yes"
-  TEST: "no"
 
 empty:
   stage: create-vars
   script:
     - echo "empty"
 
-#lint:
-#  image: $CI_REGISTRY/mattermost/ci/images/golangci-lint:v1.38.0
-#  stage: .pre
-#  script:
-#    - GO111MODULE=off GOBIN=$PWD/bin go get -u github.com/mattermost/mattermost-govet
-#    - make config-reset
-#    - make check-style #todo MM_VET_OPENSPEC_PATH='$CI_PROJECT_DIR/mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
-#  timeout: 60 mins
-#  rules:
-#    - if: '$TEST == "yes"'
-#
-#layers:
-#  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5
-#  stage: .pre
-#  script:
-#    - make store-layers
-#    - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
-#    - git reset --hard
-#    - make app-layers
-#    - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
-#    - git reset --hard
-#  rules:
-#    - if: '$TEST == "yes"'
-#
-#test-postgres:
-#  extends: .test
-#  tags:
-#    - docker
-#  variables:
-#    MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
-#    MM_SQLSETTINGS_DRIVERNAME: postgres
-#  rules:
-#    - if: '$TEST == "yes"'
-#
-#test-mysql:
-#  extends: .test
-#  tags:
-#    - docker
-#  variables:
-#    MM_SQLSETTINGS_DATASOURCE: "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true"
-#    MM_SQLSETTINGS_DRIVERNAME: mysql
-#  rules:
-#    - if: '$TEST == "yes"'
-#
-#test-schema-postgres:
-#  extends: .test-schema
-#  tags:
-#    - docker
-#  script:
-#    - echo "Creating databases"
-#    - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
-#    - echo "Importing postgres dump from version 5.0"
-#    - docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-5.0.sql
-#    - >
-#      docker run -d -it --name server-postgres --net build_mm-test \
-#        --env-file="dotenv/test-schema-validation.env" \
-#        --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
-#        --env MM_SQLSETTINGS_DRIVERNAME=postgres \
-#        -v $CI_PROJECT_DIR:/mattermost-server \
-#        -w /mattermost-server \
-#        mattermost/mattermost-build-server:20201119_golang-1.15.5 \
-#        bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='version' run-cli"
-#    - docker logs -f server-postgres
-#    - echo "Generating dump"
-#    - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
-#    - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
-#    - echo "Removing databases created for db comparison"
-#    - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
-#    - echo "Generating diff"
-#    - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
-#  rules:
-#    - if: '$TEST == "yes"'
-#
-#test-schema-mysql:
-#  extends: .test-schema
-#  tags:
-#    - docker
-#  script:
-#    - echo "Creating databases"
-#    - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
-#    - echo "Importing mysql dump from version 5.0"
-#    - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-5.0.sql
-#    - >
-#      docker run -d -it --name server-mysql --net build_mm-test \
-#        --env-file="dotenv/test-schema-validation.env" \
-#        --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
-#        --env MM_SQLSETTINGS_DRIVERNAME=mysql \
-#        -v $CI_PROJECT_DIR:/mattermost-server \
-#        -w /mattermost-server \
-#        mattermost/mattermost-build-server:20201119_golang-1.15.5 \
-#        bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='version' run-cli"
-#    - docker logs -f server-mysql
-#    - echo "Ignoring known MySQL mismatch 'ChannelMembers.SchemeGuest'"
-#    - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" || echo "drop failed"
-#    - docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" || echo "drop failed"
-#    - echo "Generating dump"
-#    - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
-#    - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
-#    - echo "Removing databases created for db comparison"
-#    - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
-#    - echo "Generating diff"
-#  rules:
-#    - if: '$TEST == "yes"'
+.lint: # Commenting this job, because it is run in CircleCI
+  image: $CI_REGISTRY/mattermost/ci/images/golangci-lint:v1.39.0-1
+  stage: test
+  script:
+    - GO111MODULE=off GOBIN=$PWD/bin go get -u github.com/mattermost/mattermost-govet
+    - make config-reset
+    - make check-style #todo MM_VET_OPENSPEC_PATH='$CI_PROJECT_DIR/mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
+  timeout: 60 mins
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+
+.layers: # Commenting this job, because it is run in CircleCI
+  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20201119_golang-1.15.5
+  stage: test
+  script:
+    - make store-layers
+    - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
+    - git reset --hard
+    - make app-layers
+    - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
+    - git reset --hard
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+
+.test-mysql: # Commenting this job, because it is run in CircleCI
+  stage: test
+  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1
+  services:
+    - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1
+      alias: docker
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_CONTENT_TRUST: 0
+    DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ""
+  before_script:
+    - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD}
+  script:
+    - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}mysql"
+    - export RACE_MODE=""
+    - time cat .gitlab-ci/scripts/test/mysql.sh | /bin/bash
+  artifacts:
+    when: always
+    reports:
+      junit: report.xml
+    paths:
+      - build/logs
+  timeout: 2 hours
+  tags:
+    - k8s-memory
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+
+.test-postgres: # Commenting this job, because it is run in CircleCI
+  stage: test
+  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1
+  services:
+    - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1
+      alias: docker
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_CONTENT_TRUST: 0
+    DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ""
+  before_script:
+    - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD}
+  script:
+    - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}postgres"
+    - export RACE_MODE=""
+    - time cat .gitlab-ci/scripts/test/postgres.sh | /bin/bash
+  artifacts:
+    when: always
+    reports:
+      junit: report.xml
+    paths:
+      - build/logs
+  timeout: 2 hours
+  tags:
+    - k8s-memory
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+
+test-schema-mysql:
+  stage: test
+  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1
+  services:
+    - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1
+      alias: docker
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_CONTENT_TRUST: 0
+    DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ""
+  before_script:
+    - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD}
+  script:
+    - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemamysql"
+    - time cat .gitlab-ci/scripts/test-schema/mysql.sh | /bin/bash
+  tags:
+    - k8s-memory
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+
+test-schema-postgres:
+  stage: test
+  image: $CI_REGISTRY/mattermost/ci/images/mattermost-build-docker:19.03.14-1
+  services:
+    - name: $CI_REGISTRY/mattermost/ci/images/docker-dind:19.03.14-1
+      alias: docker
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_CONTENT_TRUST: 0
+    DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ""
+  before_script:
+    - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD}
+  script:
+    - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemapostgres"
+    - time cat .gitlab-ci/scripts/test-schema/postgres.sh | /bin/bash
+  tags:
+    - k8s-memory
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'