From 931142b64005dbabb59f4bc53ac0fffcf392aab6 Mon Sep 17 00:00:00 2001 From: Ibrahim Serdar Acikgoz Date: Thu, 2 Mar 2023 16:12:24 +0300 Subject: [PATCH] Migrate to CI pipeline to GitHub actions (#22392) * migrate to CI pipeline to github actions * simplfy workflow --------- Co-authored-by: Akis Maziotis <36594156+phoinixgrr@users.noreply.github.com> --- .circleci/config.yml | 801 ------------------------------------- .github/workflows/ci.yml | 291 ++++++++++++++ .github/workflows/test.yml | 48 +++ 3 files changed, 339 insertions(+), 801 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 60b9fd225e..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,801 +0,0 @@ -version: 2.1 - -orbs: - aws-s3: circleci/aws-s3@3.0.0 - -executors: - ubuntu: - working_directory: ~/mattermost/ - resource_class: 2xlarge - machine: - image: "ubuntu-2004:202201-02" - environment: - COMPOSE_PROJECT_NAME: "circleci" - ubuntu_xlarge: - working_directory: ~/mattermost/ - resource_class: xlarge - machine: - image: "ubuntu-2004:202201-02" - environment: - COMPOSE_PROJECT_NAME: "circleci" - -jobs: - setup-multi-product-repositories: - parameters: - target-branch: - type: string - default: '$(curl -s $(echo https://api.github.com/repos/${CIRCLE_PULL_REQUEST:19} | sed "s/\/pull\//\/pulls\//") | jq ".base.ref" | tr -d "\042" )' - working_directory: /mnt/ramdisk/mattermost-server - docker: - - image: cimg/go:1.18 - resource_class: large - steps: - - checkout - - run: | - cd .. - git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git - cd focalboard - git checkout $CIRCLE_BRANCH || git checkout <> || git checkout rolling-stable - echo $(git rev-parse HEAD) - cd ../mattermost-server - make setup-go-work - cd ../focalboard - EXCLUDE_ENTERPRISE=TRUE make templates-archive - - persist_to_workspace: - root: /mnt/ramdisk - paths: - - mattermost-server - - focalboard - setup-mattermost-webapp: - working_directory: /mnt/ramdisk/ - docker: - - image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d - resource_class: xlarge - steps: - - run: | - git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git - cd mattermost-webapp - git checkout $CIRCLE_BRANCH || git checkout master - export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) - echo "$WEBAPP_GIT_COMMIT" - FILE_DIST=dist.tar.gz - curl --version - runtime="2 minute" - endtime=$(date -ud "$runtime" +%s) - while [[ $(date -u +%s) -le $endtime ]]; do - if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then - break - fi - echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`" - sleep 5 - done - if [[ -f "$FILE_DIST" ]]; then - echo "Precompiled version of web app found" - mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1 - else - echo "Building web app from source" - make dist - fi - - persist_to_workspace: - root: /mnt/ramdisk - paths: - - mattermost-webapp - setup-focalboard-product: - working_directory: /mnt/ramdisk/ - docker: - - image: cimg/go:1.18-node - resource_class: large - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: | - cd focalboard - echo "Building Boards product for web app" - # make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile - make prebuild - cd mattermost-plugin/webapp - npm run build:product - cd ../.. - - persist_to_workspace: - root: /mnt/ramdisk - paths: - - focalboard/mattermost-plugin/webapp/dist - - check-app-layers: - docker: - - image: cimg/go:1.18 - resource_class: medium - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make app-layers - if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi - - check-generate-worktemplates: - docker: - - image: cimg/go:1.18 - resource_class: medium - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make generate-worktemplates - if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi - check-go-mod-tidy: - docker: - - image: cimg/go:1.18 - resource_class: medium+ - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make modules-tidy - if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi - check-store-layers: - docker: - - image: cimg/go:1.18 - resource_class: medium - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make store-layers - if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi - check-mocks: - docker: - - image: cimg/go:1.18 - resource_class: xlarge - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make mocks - if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi - check-email-templates: - docker: - - image: cimg/go:1.18-node - resource_class: medium+ - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - sudo npm install -g mjml@4.9.0 - make build-templates - if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi - check-gen-serialized: - docker: - - image: cimg/go:1.18 - resource_class: small - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make gen-serialized - if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi - check-migrations: - docker: - - image: cimg/go:1.18 - resource_class: small - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make migrations-extract - if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi - # Dedicated job for golangci-lint it does not need anything only the server code for that - # and to make more clear when the job fails - check-golangci-lint: - docker: - # Keep the version in sync with the command in Makefile - - image: golangci/golangci-lint:v1.50.1 - resource_class: xlarge - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - go vet ./... - make golangci-lint - # Dedicate job for mattermost-vet to make more clear when the job fails - check-mattermost-vet: - docker: - - image: mattermost/mattermost-build-server:20230118_golang-1.19.5 - resource_class: xlarge - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - export GOBIN=${PWD}/mattermost-server/bin - export PATH=$PATH:$GOBIN - cd mattermost-server - make config-reset - make plugin-checker - make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' - build-api-spec: - docker: - - image: cimg/node:lts - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git - cd mattermost-api-reference - echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server" - git checkout ${CIRCLE_BRANCH} || true - make build - build-focalboard: - docker: - - image: mattermost/mattermost-build-server:20230118_golang-1.19.5 - resource_class: medium - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd focalboard - make server-linux - build-mattermost-server: - docker: - - image: mattermost/mattermost-build-server:20230118_golang-1.19.5 - resource_class: xlarge - working_directory: /mnt/ramdisk - steps: - - attach_workspace: - at: /mnt/ramdisk - - run: - command: | - cd mattermost-server - make config-reset - make build-cmd BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' - make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' - - store_artifacts: - path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz - - store_artifacts: - path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz - - store_artifacts: - path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-arm64.tar.gz - - store_artifacts: - path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-arm64.tar.gz - - store_artifacts: - path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-windows-amd64.zip - - persist_to_workspace: - root: /mnt/ramdisk - paths: - - mattermost-server/dist - upload-artifacts-to-s3: - docker: - - image: cimg/python:3.10 - working_directory: /mnt/ramdisk/ - steps: - - attach_workspace: - at: /mnt/ramdisk/ - - run: - command: | - cd /mnt/ramdisk/mattermost-server/dist/ - rm -rf mattermost - - aws-s3/copy: - from: /mnt/ramdisk/mattermost-server/dist/ - to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/" - arguments: --acl public-read --cache-control "no-cache" --recursive - - aws-s3/copy: - from: /mnt/ramdisk/mattermost-server/dist/ - to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/ - arguments: --acl public-read --cache-control "no-cache" --recursive - build-docker: - working_directory: /mnt/ramdisk/ - docker: - - image: cimg/base:stable - steps: - - attach_workspace: - at: /mnt/ramdisk/ - - setup_remote_docker: - version: 20.10.7 - - run: - command: | - export TAG="${CIRCLE_SHA1:0:7}" - cd mattermost-server/build - export DOCKER_CLI_EXPERIMENTAL=enabled - echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin - export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz - docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} . - # Dedicate job for sentry it does not need anything only the server code for that - # and to make more clear when the job fails - sentry: - docker: - - image: getsentry/sentry-cli:1.64.1 - working_directory: /mnt/ramdisk/ - steps: - - checkout - - run: - command: apk add git - - run: - command: | - sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD` - sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD` - test: - executor: - name: ubuntu - parameters: - dbdriver: - type: string - dbsource: - type: string - racemode: - type: string - steps: - - attach_workspace: - at: ~/mattermost - - run: - name: Run Docker compose - command: | - cd mattermost-server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps - background: true - - run: - name: Wait for docker compose - command: | - until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; - docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" - - run: - name: Run Tests - command: | - mkdir -p mattermost-server/client/plugins - cd mattermost-server/build - docker run -it --net circleci_mm-test \ - --env-file=dotenv/test.env \ - --env MM_SQLSETTINGS_DATASOURCE="<>" \ - --env MM_SQLSETTINGS_DRIVERNAME=<> \ - --env CIRCLECI=true \ - -v ~/mattermost:/mattermost \ - -w /mattermost/mattermost-server \ - mattermost/mattermost-build-server:20230118_golang-1.19.5 \ - bash -c "ulimit -n 8096; make test-server<< parameters.racemode >> BUILD_NUMBER=$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM TESTFLAGS= TESTFLAGSEE=" \ - bash -c scripts/diff-email-templates.sh - no_output_timeout: 2h - - run: - name: Capture docker logs - when: always - command: | - cd mattermost-server/build - # Capture docker logs - mkdir -p logs - docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs - docker ps -a --no-trunc > logs/docker_ps - docker stats -a --no-stream > logs/docker_stats - tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats - - store_artifacts: - path: ~/mattermost/mattermost-server/build/logs - - run: - when: always - command: | - cd mattermost-server - mkdir -p test-results - cp report.xml test-results - - store_test_results: - path: ~/mattermost/mattermost-server/test-results/ - - store_artifacts: - path: ~/mattermost/mattermost-server/test-results/ - - test-schema: - executor: - name: ubuntu_xlarge - steps: - - attach_workspace: - at: ~/mattermost - - run: - name: Run Docker compose - command: | - cd mattermost-server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps - background: true - - run: - name: Wait for docker compose - command: | - until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; - docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" - - run: - name: Postgres schema migration validation - command: | - cd mattermost-server/build - mkdir -p mattermost-server/client/plugins - echo "Creating databases" - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' - echo "Importing postgres dump from version 6.0.0" - docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.0.sql - docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" - docker run -it --net circleci_mm-test \ - --env-file="dotenv/test-schema-validation.env" \ - --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \ - --env MM_SQLSETTINGS_DRIVERNAME=postgres \ - -v ~/mattermost:/mattermost \ - -w /mattermost/mattermost-server \ - mattermost/mattermost-build-server:20230118_golang-1.19.5 \ - bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli" - echo "Generating dump" - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql - docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql - echo "Removing databases created for db comparison" - docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' - echo "Generating diff" - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) - no_output_timeout: 1h - - run: - name: MySQL schema migration validation - command: | - cd mattermost-server/build - mkdir -p mattermost-server/client/plugins - echo "Creating databases" - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser" - echo "Importing mysql dump from version 6.0.0" - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql - docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" - docker run -it --net circleci_mm-test \ - --env-file="dotenv/test-schema-validation.env" \ - --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ - --env MM_SQLSETTINGS_DRIVERNAME=mysql \ - -v ~/mattermost:/mattermost \ - -w /mattermost/mattermost-server \ - mattermost/mattermost-build-server:20230118_golang-1.19.5 \ - bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli" - echo "Generating dump" - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql - docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql - echo "Removing databases created for db comparison" - docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' - echo "Generating diff" - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) - no_output_timeout: 1h - -workflows: - version: 2 - untagged-build: - jobs: - - setup-multi-product-repositories: - context: - - matterbuild-github-token - filters: - branches: - ignore: - - master - - /^release-.*/ - - setup-mattermost-webapp: - filters: - branches: - ignore: - - master - - /^release-.*/ - - setup-focalboard-product: - filters: - branches: - ignore: - - master - - /^release-.*/ - requires: - - setup-multi-product-repositories - - check-go-mod-tidy: - requires: - - setup-multi-product-repositories - - check-golangci-lint: - requires: - - setup-multi-product-repositories - - check-app-layers: - requires: - - setup-multi-product-repositories - - check-generate-worktemplates: - requires: - - setup-multi-product-repositories - - check-store-layers: - requires: - - setup-multi-product-repositories - - check-mocks: - requires: - - setup-multi-product-repositories - - check-email-templates: - requires: - - setup-multi-product-repositories - - check-gen-serialized: - requires: - - setup-multi-product-repositories - - check-mattermost-vet: - requires: - - setup-multi-product-repositories - - check-migrations: - requires: - - setup-multi-product-repositories - - build-api-spec: - requires: - - setup-multi-product-repositories - - build-mattermost-server: - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - build-focalboard: - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - test: - name: test-mysql - dbdriver: mysql - dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true - racemode: "" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - test: - name: test-postgres-normal - dbdriver: postgres - dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 - racemode: "" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - test: - name: test-postgres-binary - dbdriver: postgres - dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes - racemode: "" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - test-schema: - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - upload-artifacts-to-s3: - context: mattermost-ci-pr-builds-s3 - requires: - - build-mattermost-server - - build-focalboard - - build-docker: - context: matterbuild-docker - requires: - - upload-artifacts-to-s3 - release-build: - jobs: - - setup-multi-product-repositories: - context: - - matterbuild-github-token - filters: - branches: - only: - - master - - /^release-.*/ - - cloud - - setup-mattermost-webapp: - filters: - branches: - only: - - master - - /^release-.*/ - - cloud - - setup-focalboard-product: - filters: - branches: - only: - - master - - /^release-.*/ - - cloud - requires: - - setup-multi-product-repositories - - check-go-mod-tidy: - requires: - - setup-multi-product-repositories - - check-golangci-lint: - requires: - - setup-multi-product-repositories - - check-app-layers: - requires: - - setup-multi-product-repositories - - check-generate-worktemplates: - requires: - - setup-multi-product-repositories - - check-store-layers: - requires: - - setup-multi-product-repositories - - check-mocks: - requires: - - setup-multi-product-repositories - - check-email-templates: - requires: - - setup-multi-product-repositories - - check-gen-serialized: - requires: - - setup-multi-product-repositories - - check-mattermost-vet: - requires: - - setup-multi-product-repositories - - check-migrations: - requires: - - setup-multi-product-repositories - - build-api-spec: - requires: - - setup-multi-product-repositories - - sentry: - context: matterbuild-sentry - requires: - - setup-multi-product-repositories - - build-mattermost-server: - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - setup-focalboard-product - - build-focalboard: - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - setup-mattermost-webapp - - test: - name: test-mysql - dbdriver: mysql - dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true - racemode: "-race" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - test: - name: test-postgres-normal - dbdriver: postgres - dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 - racemode: "-race" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec - - test: - name: test-postgres-binary - dbdriver: postgres - dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes - racemode: "" - requires: - - check-go-mod-tidy - - check-golangci-lint - - check-app-layers - - check-generate-worktemplates - - check-store-layers - - check-mocks - - check-email-templates - - check-gen-serialized - - check-mattermost-vet - - check-migrations - - build-api-spec diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..36665a43fc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,291 @@ +name: mattermost-build +on: + pull_request: + push: + branches: + - master + - cloud + - release-* +jobs: + check-mocks: + name: Check mocks + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Generate mocks + run: make mocks + - name: Check mocks + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi + check-go-mod-tidy: + name: Check go mod tidy + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Run go mod tidy + run: make modules-tidy + - name: Check modules + run: if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi + check-gen-serialized: + name: Check serialization methods for hot structs + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Run make-gen-serialized + run: make gen-serialized + - name: Check serialized + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi + check-mattermost-vet: + name: Check style + runs-on: ubuntu-latest-8-cores + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Checkout focalboard + run: | + cd .. + git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git + cd focalboard + git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable + echo $(git rev-parse HEAD) + cd ../mattermost-server + make setup-go-work + - name: Reset config + run: make config-reset + - name: Run plugin-checker + run: make plugin-checker + - name: Run mattermost-vet + run: make vet BUILD_NUMBER='${GITHUB_HEAD_REF}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' + check-migrations: + name: Check migration files + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Extract migrations files + run: make migrations-extract + - name: Check migration files + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi + build-api-spec: + name: Build API specification + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Checkout mattermost-api-reference + run: | + cd .. + git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git + cd mattermost-api-reference + echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server" + git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || true + make build + cd ../mattermost-server + check-generate-work-templates: + name: Generate work templates + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Generate work templates + run: make generate-worktemplates + - name: Check generated work templates + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi + check-email-templates: + name: Generate email templates + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Generate email templates + run: | + sudo npm install -g mjml@4.9.0 + make build-templates + - name: Check generated email templates + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi + check-store-layers: + name: Check store layers + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Generate store layers + run: make store-layers + - name: Check generated code + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi + check-app-layers: + name: Check app layers + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Generate app layers + run: make app-layers + - name: Check generated code + run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi + test-postgres-binary: + name: Run tests on postgres with binary parameters + needs: check-mattermost-vet + uses: ./.github/workflows/test.yml + with: + datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes + drivername: postgres + test-postgres-normal: + name: Run tests on postgres + needs: check-mattermost-vet + uses: ./.github/workflows/test.yml + with: + datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 + drivername: postgres + test-mysql: + name: Run tests on mysql + needs: check-mattermost-vet + uses: ./.github/workflows/test.yml + with: + datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true + drivername: mysql + build-mattermost-server: + name: Build mattermost-server + runs-on: ubuntu-latest-8-cores + needs: check-mattermost-vet + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Checkout mattermost-webapp + run: | + cd .. + git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git + cd mattermost-webapp + git checkout $GITHUB_HEAD_REF || git checkout master + export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) + echo "$WEBAPP_GIT_COMMIT" + FILE_DIST=dist.tar.gz + runtime="2 minute" + endtime=$(date -ud "$runtime" +%s) + while [[ $(date -u +%s) -le $endtime ]]; do + if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then + break + fi + echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`" + sleep 5 + done + if [[ -f "$FILE_DIST" ]]; then + echo "Precompiled version of web app found" + mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1 + else + echo "Building web app from source" + make dist + fi + cd ../mattermost-server + - name: Checkout and build focalboard + run: | + cd .. + git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git + cd focalboard + git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable + echo $(git rev-parse HEAD) + make server-linux + echo "Building Boards product for web app" + # make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile + make prebuild + cd mattermost-plugin/webapp + npm run build:product + cd ../../../mattermost-server + make setup-go-work + - name: Build + run: | + make config-reset + make build-cmd BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}' + make package BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}' + - name: Persist dist artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: server-dist-artifact + path: dist/ + retention-days: 14 + - name: Persist build artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: server-build-artifact + path: build/ + retention-days: 14 + upload-s3: + name: Upload to S3 bucket + runs-on: ubuntu-22.04 + needs: + - build-mattermost-server + - test-mysql + - test-postgres-binary + - test-postgres-normal + env: + REPO_NAME: ${{ github.event.repository.name }} + steps: + - name: Download dist artifacts + uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b # v3.0.2 + with: + name: server-dist-artifact + path: dist/ + - name: Configure AWS + uses: aws-actions/configure-aws-credentials@07c2f971bac433df982ccc261983ae443861db49 # v1-node16 + with: + aws-region: us-east-1 + aws-access-key-id: ${{ secrets.MM_SERVER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.MM_SERVER_AWS_SECRET_ACCESS_KEY }} + # We need to sanitize the branch name before using it + - name: ci/sanitize-branch-name + id: branch + uses: transferwise/sanitize-branch-name@b10b4d524ac5a7b645b43a3527db3a6cca017b9d # v1 + # Search for the string "pull" and replace it with "PR" in branch-name + - name: ci/sanitize-branch-name-replace-pull-with-PR- + run: echo "BRANCH_NAME=$(echo ${{ steps.branch.outputs.sanitized-branch-name }} | sed 's/^pull\//PR-/g')" >> $GITHUB_ENV + - name: ci/artifact-upload + run: | + aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/$BRANCH_NAME/ --acl public-read --cache-control "no-cache" --recursive + aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/commit/${{ github.sha }}/ --acl public-read --cache-control "no-cache" --recursive + build-docker: + name: Build docker image + runs-on: ubuntu-22.04 + needs: upload-s3 + steps: + - name: Download build artifacts + uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b # v3.0.2 + with: + name: server-build-artifact + path: build/ + - name: Login to Docker Hub + uses: docker/login-action@3da7dc6e2b31f99ef2cb9fb4c50fb0971e0d0139 # v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@11e8a2e2910826a92412015c515187a2d6750279 # v2.4 + - name: Docker build and push + env: + DOCKER_CLI_EXPERIMENTAL: enabled + run: | + export TAG=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) + cd build + export DOCKER_CLI_EXPERIMENTAL=enabled + export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${GITHUB_SHA}/mattermost-team-linux-amd64.tar.gz + docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} . + sentry: + name: Send build info to sentry + runs-on: ubuntu-22.04 + needs: + - test-postgres-binary + - test-postgres-normal + - test-mysql + if: ${{ github.event_name == 'push' }} + env: + SENTRY_CLI_VERSION: 1.64.1 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Install sentry-cli + run: curl -sL https://sentry.io/get-cli/ | sh + - name: Run sentry-cli + run: | + sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD` + sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD` diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..5b221dcf20 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,48 @@ +name: Test +on: + workflow_call: + inputs: + datasource: + required: true + type: string + drivername: + required: true + type: string +jobs: + run-tests: + runs-on: ubuntu-latest-8-cores + env: + COMPOSE_PROJECT_NAME: ghactions + BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5 + steps: + - name: Checkout mattermost-server + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Run docker compose + run: | + cd build + docker-compose --no-ansi run --rm start_dependencies + cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker-compose --no-ansi ps + cd .. + - name: Wait for docker compose + run: | + until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; + docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" + docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" + - name: Run Tests + run: | + if [[ ${{ github.event_name }} == 'push' ]]; then + export RACE_MODE="-race" + fi + mkdir -p client/plugins + cd build + docker run --net ghactions_mm-test \ + --ulimit nofile=8096:8096 \ + --env-file=dotenv/test.env \ + --env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \ + --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \ + -v ~/work/mattermost-server:/mattermost-server \ + -w /mattermost-server/mattermost-server \ + $BUILD_IMAGE \ + make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE=