From 5025fd756121251575ce372fa413f02a9ee4a07f Mon Sep 17 00:00:00 2001 From: Ibrahim Serdar Acikgoz Date: Wed, 8 Mar 2023 18:43:31 +0300 Subject: [PATCH] remove unsused/unnecessary gitlab ci steps (#22458) --- .gitlab-ci.yml | 137 --------------------- .gitlab-ci/scripts/test-schema/mysql.sh | 53 -------- .gitlab-ci/scripts/test-schema/postgres.sh | 51 -------- .gitlab-ci/scripts/test/mysql.sh | 44 ------- .gitlab-ci/scripts/test/postgres.sh | 43 ------- 5 files changed, 328 deletions(-) delete mode 100755 .gitlab-ci/scripts/test-schema/mysql.sh delete mode 100755 .gitlab-ci/scripts/test-schema/postgres.sh delete mode 100755 .gitlab-ci/scripts/test/mysql.sh delete mode 100755 .gitlab-ci/scripts/test/postgres.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7ed1660d5..e6b27c644d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,140 +20,3 @@ empty: stage: create-vars script: - echo "empty" - -.lint: # Commenting this job, because it is run in CircleCI - image: $CI_REGISTRY/mattermost/ci/images/golangci-lint:v1.39.0-1 - stage: test - script: - - make config-reset - - make check-style #todo MM_VET_OPENSPEC_PATH='$CI_PROJECT_DIR/mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' - timeout: 60 mins - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - -.layers: # Commenting this job, because it is run in CircleCI - image: $IMAGE_BUILD_SERVER - stage: test - script: - - make store-layers - - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi - - git reset --hard - - make app-layers - - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi - - git reset --hard - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - -.test-mysql: # Commenting this job, because it is run in CircleCI - stage: test - image: $IMAGE_BUILD_DOCKER - services: - - name: $IMAGE_DIND - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - before_script: - - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} - script: - - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}mysql" - - export RACE_MODE="" - - time cat .gitlab-ci/scripts/test/mysql.sh | /bin/bash - artifacts: - when: always - reports: - junit: report.xml - paths: - - build/logs - timeout: 2 hours - tags: - - k8s-memory - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - -.test-postgres: # Commenting this job, because it is run in CircleCI - stage: test - image: $IMAGE_BUILD_DOCKER - services: - - name: $IMAGE_DIND - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - before_script: - - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} - script: - - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}postgres" - - export RACE_MODE="" - - time cat .gitlab-ci/scripts/test/postgres.sh | /bin/bash - artifacts: - when: always - reports: - junit: report.xml - paths: - - build/logs - timeout: 2 hours - tags: - - k8s-memory - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - -test-schema-mysql: - stage: test - image: $IMAGE_BUILD_DOCKER - services: - - name: $IMAGE_DIND - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - before_script: - - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} - script: - - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemamysql" - - time cat .gitlab-ci/scripts/test-schema/mysql.sh | /bin/bash - artifacts: - paths: - - build/logs - tags: - - k8s-memory - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - - if: '$CI_COMMIT_REF_NAME == "cloud"' - - if: '$CI_COMMIT_REF_NAME =~ /^release-/' - -test-schema-postgres: - stage: test - image: $IMAGE_BUILD_DOCKER - services: - - name: $IMAGE_DIND - alias: docker - variables: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_CONTENT_TRUST: 0 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: "" - before_script: - - docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} - script: - - export COMPOSE_PROJECT_NAME="${CI_PIPELINE_IID}schemapostgres" - - time cat .gitlab-ci/scripts/test-schema/postgres.sh | /bin/bash - artifacts: - paths: - - build/logs - tags: - - k8s-memory - rules: - - if: '$CI_COMMIT_REF_NAME == "master"' - - if: '$CI_COMMIT_REF_NAME == "cloud"' - - if: '$CI_COMMIT_REF_NAME =~ /^release-/' diff --git a/.gitlab-ci/scripts/test-schema/mysql.sh b/.gitlab-ci/scripts/test-schema/mysql.sh deleted file mode 100755 index a3f387d2e8..0000000000 --- a/.gitlab-ci/scripts/test-schema/mysql.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -set -xe - -if [[ "$GITLAB_CI" == "" ]]; then - export CI_PROJECT_DIR=$PWD - export CI_REGISTRY=registry.internal.mattermost.com - export COMPOSE_PROJECT_NAME="1schemamysql" - export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7 - # You need to log in to internal registry to run this script locally -fi - -echo "$DOCKER_HOST" -docker ps -DOCKER_NETWORK=$COMPOSE_PROJECT_NAME -DOCKER_COMPOSE_FILE="gitlab-dc.schemamysql.yml" -CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" -CONTAINER_DB="${COMPOSE_PROJECT_NAME}_mysql_1" -docker network create $DOCKER_NETWORK -ulimit -n 8096 -cd "$CI_PROJECT_DIR"/build -docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies -sleep 5 -docker run --net $DOCKER_NETWORK "$CI_REGISTRY"/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - -echo "Creating databases" -docker exec $CONTAINER_DB mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser; " -echo "Importing mysql dump from version 6.0.0" -docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.0.sql -docker exec -i $CONTAINER_DB mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" -docker run -d -it --rm --name "$CONTAINER_SERVER" --net $DOCKER_NETWORK \ - --env-file="dotenv/test-schema-validation.env" \ - --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ - --env MM_SQLSETTINGS_DRIVERNAME=mysql \ - -v "$CI_PROJECT_DIR":/mattermost-server \ - -w /mattermost-server \ - $IMAGE_BUILD_SERVER \ - bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli" -mkdir -p logs -docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME -docker logs -f $CONTAINER_SERVER -tar -czvf logs/docker_logs$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME - -echo "Generating dump" -docker exec $CONTAINER_DB mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql -docker exec $CONTAINER_DB mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql -echo "Removing databases created for db comparison" -docker exec $CONTAINER_DB mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' - -echo "Generating diff" -diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) - -docker-compose -f $DOCKER_COMPOSE_FILE down -docker network remove $DOCKER_NETWORK diff --git a/.gitlab-ci/scripts/test-schema/postgres.sh b/.gitlab-ci/scripts/test-schema/postgres.sh deleted file mode 100755 index 3c1a20896c..0000000000 --- a/.gitlab-ci/scripts/test-schema/postgres.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -xe - -if [[ "$GITLAB_CI" == "" ]]; then - export CI_PROJECT_DIR=$PWD - export CI_REGISTRY=registry.internal.mattermost.com - export COMPOSE_PROJECT_NAME="1schemapostgres" - export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7 - # You need to log in to internal registry to run this script locally -fi - -echo $DOCKER_HOST -docker ps -DOCKER_NETWORK=$COMPOSE_PROJECT_NAME -DOCKER_COMPOSE_FILE="gitlab-dc.schemapostgres.yml" -CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" -CONTAINER_DB="${COMPOSE_PROJECT_NAME}_postgres_1" -docker network create $DOCKER_NETWORK -ulimit -n 8096 -cd "$CI_PROJECT_DIR"/build -docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies -timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done" - -echo "Creating databases" -docker exec $CONTAINER_DB sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test;' -echo "Importing postgres dump from version 6.0.0" -docker exec -i $CONTAINER_DB psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.0.sql -docker exec -i $CONTAINER_DB psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" -docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \ - --env-file="dotenv/test-schema-validation.env" \ - --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \ - --env MM_SQLSETTINGS_DRIVERNAME=postgres \ - -v "$CI_PROJECT_DIR":/mattermost-server \ - -w /mattermost-server \ - $IMAGE_BUILD_SERVER \ - bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli" -mkdir -p logs -docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME -docker logs -f $CONTAINER_SERVER -tar -czvf logs/docker_logs$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME - -echo "Generating dump" -docker exec $CONTAINER_DB pg_dump --schema-only -d migrated -U mmuser > migrated.sql -docker exec $CONTAINER_DB pg_dump --schema-only -d latest -U mmuser > latest.sql -echo "Removing databases created for db comparison" -docker exec $CONTAINER_DB sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' -echo "Generating diff" -diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) - -docker-compose -f $DOCKER_COMPOSE_FILE down -docker network remove $DOCKER_NETWORK diff --git a/.gitlab-ci/scripts/test/mysql.sh b/.gitlab-ci/scripts/test/mysql.sh deleted file mode 100755 index f424ab2011..0000000000 --- a/.gitlab-ci/scripts/test/mysql.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -xe - -if [[ "$GITLAB_CI" == "" ]]; then - export CI_PROJECT_DIR=$PWD - export CI_REGISTRY=registry.internal.mattermost.com - export COMPOSE_PROJECT_NAME="1mysql" - export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7 - # You need to log in to internal registry to run this script locally -fi - -echo "$DOCKER_HOST" -docker ps -DOCKER_NETWORK=$COMPOSE_PROJECT_NAME -DOCKER_COMPOSE_FILE="gitlab-dc.mysql.yml" -CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" -docker network create $DOCKER_NETWORK -ulimit -n 8096 -cd "$CI_PROJECT_DIR"/build -docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies -sleep 5 -cat ../tests/test-data.ldif | docker-compose exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; -docker-compose exec -T minio sh -c 'mkdir -p /data/mattermost-test'; -docker run --name "${COMPOSE_PROJECT_NAME}_curl_mysql" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" -docker run --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" - -docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \ - --env-file="dotenv/test.env" \ - --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true" \ - --env MM_SQLSETTINGS_DRIVERNAME=mysql \ - -v "$CI_PROJECT_DIR":/mattermost-server \ - -w /mattermost-server \ - $IMAGE_BUILD_SERVER \ - bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \ - bash -c scripts/diff-email-templates.sh -mkdir -p logs -docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME -docker ps -a --no-trunc > logs/docker_ps_$COMPOSE_PROJECT_NAME -docker stats -a --no-stream > logs/docker_stats_$COMPOSE_PROJECT_NAME -docker logs -f $CONTAINER_SERVER -tar -czvf logs/docker_logs_$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME logs/docker_ps_$COMPOSE_PROJECT_NAME logs/docker_stats_$COMPOSE_PROJECT_NAME - -docker-compose -f $DOCKER_COMPOSE_FILE down -docker network remove $DOCKER_NETWORK diff --git a/.gitlab-ci/scripts/test/postgres.sh b/.gitlab-ci/scripts/test/postgres.sh deleted file mode 100755 index f51730c42c..0000000000 --- a/.gitlab-ci/scripts/test/postgres.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -xe - -if [[ "$GITLAB_CI" == "" ]]; then - export CI_PROJECT_DIR=$PWD - export CI_REGISTRY=registry.internal.mattermost.com - export COMPOSE_PROJECT_NAME="1postgres" - export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7 - # You need to log in to internal registry to run this script locally -fi - -echo "$DOCKER_HOST" -docker ps -DOCKER_NETWORK=$COMPOSE_PROJECT_NAME -DOCKER_COMPOSE_FILE="gitlab-dc.postgres.yml" -CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1" -docker network create $DOCKER_NETWORK -ulimit -n 8096 -cd "$CI_PROJECT_DIR"/build -docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies -sleep 5 -cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; -docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test'; -timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done" -docker run --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" -docker run -d -it --rm --name "${CONTAINER_SERVER}" --net $DOCKER_NETWORK \ - --env-file="dotenv/test.env" \ - --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ - --env MM_SQLSETTINGS_DRIVERNAME=postgres \ - -v "$CI_PROJECT_DIR":/mattermost-server \ - -w /mattermost-server \ - $IMAGE_BUILD_SERVER \ - bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \ - bash -c scripts/diff-email-templates.sh -mkdir -p logs -docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME -docker ps -a --no-trunc > logs/docker_ps_$COMPOSE_PROJECT_NAME -docker stats -a --no-stream > logs/docker_stats_$COMPOSE_PROJECT_NAME -docker logs -f $CONTAINER_SERVER -tar -czvf logs/docker_logs_$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME logs/docker_ps_$COMPOSE_PROJECT_NAME logs/docker_stats_$COMPOSE_PROJECT_NAME - -docker-compose -f $DOCKER_COMPOSE_FILE down -docker network remove $DOCKER_NETWORK