Migrate to CI pipeline to GitHub actions (#22392)

* migrate to CI pipeline to github actions

* simplfy workflow

---------

Co-authored-by: Akis Maziotis <36594156+phoinixgrr@users.noreply.github.com>

.circleci/config.yml

@@ -1,801 +0,0 @@
-version: 2.1
-
-orbs:
-  aws-s3: circleci/aws-s3@3.0.0
-
-executors:
-  ubuntu:
-    working_directory: ~/mattermost/
-    resource_class: 2xlarge
-    machine:
-      image: "ubuntu-2004:202201-02"
-    environment:
-      COMPOSE_PROJECT_NAME: "circleci"
-  ubuntu_xlarge:
-    working_directory: ~/mattermost/
-    resource_class: xlarge
-    machine:
-      image: "ubuntu-2004:202201-02"
-    environment:
-      COMPOSE_PROJECT_NAME: "circleci"
-
-jobs:
-  setup-multi-product-repositories:
-    parameters:
-      target-branch:
-        type: string
-        default: '$(curl -s $(echo https://api.github.com/repos/${CIRCLE_PULL_REQUEST:19} | sed "s/\/pull\//\/pulls\//") | jq ".base.ref" | tr -d "\042" )'
-    working_directory: /mnt/ramdisk/mattermost-server
-    docker:
-      - image: cimg/go:1.18
-    resource_class: large
-    steps:
-      - checkout
-      - run: |
-          cd ..
-          git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
-          cd focalboard
-          git checkout $CIRCLE_BRANCH || git checkout <<parameters.target-branch>> || git checkout rolling-stable
-          echo $(git rev-parse HEAD)
-          cd ../mattermost-server
-          make setup-go-work
-          cd ../focalboard
-          EXCLUDE_ENTERPRISE=TRUE make templates-archive
-      - persist_to_workspace:
-          root: /mnt/ramdisk
-          paths:
-            - mattermost-server
-            - focalboard
-  setup-mattermost-webapp:
-    working_directory: /mnt/ramdisk/
-    docker:
-      - image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d
-    resource_class: xlarge
-    steps:
-      - run: |
-          git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
-          cd mattermost-webapp
-          git checkout $CIRCLE_BRANCH || git checkout master
-          export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
-          echo "$WEBAPP_GIT_COMMIT"
-          FILE_DIST=dist.tar.gz
-          curl --version
-          runtime="2 minute"
-          endtime=$(date -ud "$runtime" +%s)
-          while [[ $(date -u +%s) -le $endtime ]]; do
-              if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then
-                break
-              fi
-              echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`"
-              sleep 5
-          done
-          if [[ -f "$FILE_DIST" ]]; then
-            echo "Precompiled version of web app found"
-            mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1
-          else
-            echo "Building web app from source"
-            make dist
-          fi
-      - persist_to_workspace:
-          root: /mnt/ramdisk
-          paths:
-            - mattermost-webapp
-  setup-focalboard-product:
-    working_directory: /mnt/ramdisk/
-    docker:
-      - image: cimg/go:1.18-node
-    resource_class: large
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run: |
-          cd focalboard
-          echo "Building Boards product for web app"
-          # make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile
-          make prebuild
-          cd mattermost-plugin/webapp
-          npm run build:product
-          cd ../..
-      - persist_to_workspace:
-          root: /mnt/ramdisk
-          paths:
-            - focalboard/mattermost-plugin/webapp/dist
-
-  check-app-layers:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: medium
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make app-layers
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
-
-  check-generate-worktemplates:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: medium
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make generate-worktemplates
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi
-  check-go-mod-tidy:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: medium+
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make modules-tidy
-            if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi
-  check-store-layers:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: medium
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make store-layers
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
-  check-mocks:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: xlarge
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make mocks
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
-  check-email-templates:
-    docker:
-      - image: cimg/go:1.18-node
-    resource_class: medium+
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            sudo npm install -g mjml@4.9.0
-            make build-templates
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
-  check-gen-serialized:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: small
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make gen-serialized
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
-  check-migrations:
-    docker:
-      - image: cimg/go:1.18
-    resource_class: small
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make migrations-extract
-            if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
-  # Dedicated job for golangci-lint it does not need anything only the server code for that
-  # and to make more clear when the job fails
-  check-golangci-lint:
-    docker:
-      # Keep the version in sync with the command in Makefile
-      - image: golangci/golangci-lint:v1.50.1
-    resource_class: xlarge
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            go vet ./...
-            make golangci-lint
-  # Dedicate job for mattermost-vet to make more clear when the job fails
-  check-mattermost-vet:
-    docker:
-      - image: mattermost/mattermost-build-server:20230118_golang-1.19.5
-    resource_class: xlarge
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            export GOBIN=${PWD}/mattermost-server/bin
-            export PATH=$PATH:$GOBIN
-            cd mattermost-server
-            make config-reset
-            make plugin-checker
-            make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
-  build-api-spec:
-    docker:
-      - image: cimg/node:lts
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git
-            cd mattermost-api-reference
-            echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server"
-            git checkout ${CIRCLE_BRANCH} || true
-            make build
-  build-focalboard:
-    docker:
-      - image: mattermost/mattermost-build-server:20230118_golang-1.19.5
-    resource_class: medium
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd focalboard
-            make server-linux
-  build-mattermost-server:
-    docker:
-      - image: mattermost/mattermost-build-server:20230118_golang-1.19.5
-    resource_class: xlarge
-    working_directory: /mnt/ramdisk
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk
-      - run:
-          command: |
-            cd mattermost-server
-            make config-reset
-            make build-cmd BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
-            make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
-      - store_artifacts:
-          path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz
-      - store_artifacts:
-          path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz
-      - store_artifacts:
-          path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-arm64.tar.gz
-      - store_artifacts:
-          path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-arm64.tar.gz
-      - store_artifacts:
-          path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-windows-amd64.zip
-      - persist_to_workspace:
-          root: /mnt/ramdisk
-          paths:
-            - mattermost-server/dist
-  upload-artifacts-to-s3:
-    docker:
-      - image: cimg/python:3.10
-    working_directory: /mnt/ramdisk/
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk/
-      - run:
-          command: |
-            cd /mnt/ramdisk/mattermost-server/dist/
-            rm -rf mattermost
-      - aws-s3/copy:
-          from: /mnt/ramdisk/mattermost-server/dist/
-          to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/"
-          arguments: --acl public-read --cache-control "no-cache" --recursive
-      - aws-s3/copy:
-          from: /mnt/ramdisk/mattermost-server/dist/
-          to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/
-          arguments: --acl public-read --cache-control "no-cache" --recursive
-  build-docker:
-    working_directory: /mnt/ramdisk/
-    docker:
-      - image: cimg/base:stable
-    steps:
-      - attach_workspace:
-          at: /mnt/ramdisk/
-      - setup_remote_docker:
-          version: 20.10.7
-      - run:
-          command: |
-            export TAG="${CIRCLE_SHA1:0:7}"
-            cd mattermost-server/build
-            export DOCKER_CLI_EXPERIMENTAL=enabled
-            echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
-            export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
-            docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} .
-  # Dedicate job for sentry it does not need anything only the server code for that
-  # and to make more clear when the job fails
-  sentry:
-    docker:
-      - image: getsentry/sentry-cli:1.64.1
-    working_directory: /mnt/ramdisk/
-    steps:
-      - checkout
-      - run:
-          command: apk add git
-      - run:
-          command: |
-            sentry-cli --log-level=debug releases new  --finalize -p mattermost-server `git rev-parse HEAD`
-            sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`
-  test:
-    executor:
-      name: ubuntu
-    parameters:
-      dbdriver:
-        type: string
-      dbsource:
-        type: string
-      racemode:
-        type: string
-    steps:
-      - attach_workspace:
-          at: ~/mattermost
-      - run:
-          name: Run Docker compose
-          command: |
-            cd mattermost-server/build
-            docker-compose --no-ansi run --rm start_dependencies
-            cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
-            docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
-            docker-compose --no-ansi ps
-          background: true
-      - run:
-          name: Wait for docker compose
-          command: |
-            until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
-            docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
-            docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
-      - run:
-          name: Run Tests
-          command: |
-            mkdir -p mattermost-server/client/plugins
-            cd mattermost-server/build
-            docker run -it --net circleci_mm-test \
-              --env-file=dotenv/test.env \
-              --env MM_SQLSETTINGS_DATASOURCE="<<parameters.dbsource>>" \
-              --env MM_SQLSETTINGS_DRIVERNAME=<<parameters.dbdriver>> \
-              --env CIRCLECI=true \
-              -v ~/mattermost:/mattermost \
-              -w /mattermost/mattermost-server \
-              mattermost/mattermost-build-server:20230118_golang-1.19.5 \
-              bash -c "ulimit -n 8096; make test-server<< parameters.racemode >> BUILD_NUMBER=$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM TESTFLAGS= TESTFLAGSEE=" \
-              bash -c scripts/diff-email-templates.sh
-          no_output_timeout: 2h
-      - run:
-          name: Capture docker logs
-          when: always
-          command: |
-            cd mattermost-server/build
-            # Capture docker logs
-            mkdir -p logs
-            docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
-            docker ps -a --no-trunc > logs/docker_ps
-            docker stats -a --no-stream > logs/docker_stats
-            tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
-      - store_artifacts:
-          path: ~/mattermost/mattermost-server/build/logs
-      - run:
-          when: always
-          command: |
-            cd mattermost-server
-            mkdir -p test-results
-            cp report.xml test-results
-      - store_test_results:
-          path: ~/mattermost/mattermost-server/test-results/
-      - store_artifacts:
-          path: ~/mattermost/mattermost-server/test-results/
-
-  test-schema:
-    executor:
-      name: ubuntu_xlarge
-    steps:
-      - attach_workspace:
-          at: ~/mattermost
-      - run:
-          name: Run Docker compose
-          command: |
-            cd mattermost-server/build
-            docker-compose --no-ansi run --rm start_dependencies
-            cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
-            docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
-            docker-compose --no-ansi ps
-          background: true
-      - run:
-          name: Wait for docker compose
-          command: |
-            until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
-            docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
-            docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
-      - run:
-          name: Postgres schema migration validation
-          command: |
-            cd mattermost-server/build
-            mkdir -p mattermost-server/client/plugins
-            echo "Creating databases"
-            docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
-            echo "Importing postgres dump from version 6.0.0"
-            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.0.sql
-            docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
-            docker run -it --net circleci_mm-test \
-              --env-file="dotenv/test-schema-validation.env" \
-              --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
-              --env MM_SQLSETTINGS_DRIVERNAME=postgres \
-              -v ~/mattermost:/mattermost \
-              -w /mattermost/mattermost-server \
-              mattermost/mattermost-build-server:20230118_golang-1.19.5 \
-              bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli"
-            echo "Generating dump"
-            docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
-            docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
-            echo "Removing databases created for db comparison"
-            docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
-            echo "Generating diff"
-            diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
-          no_output_timeout: 1h
-      - run:
-          name: MySQL schema migration validation
-          command: |
-            cd mattermost-server/build
-            mkdir -p mattermost-server/client/plugins
-            echo "Creating databases"
-            docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
-            echo "Importing mysql dump from version 6.0.0"
-            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
-            docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
-            docker run -it --net circleci_mm-test \
-              --env-file="dotenv/test-schema-validation.env" \
-              --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
-              --env MM_SQLSETTINGS_DRIVERNAME=mysql \
-              -v ~/mattermost:/mattermost \
-              -w /mattermost/mattermost-server \
-              mattermost/mattermost-build-server:20230118_golang-1.19.5 \
-              bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli"
-            echo "Generating dump"
-            docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
-            docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
-            echo "Removing databases created for db comparison"
-            docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
-            echo "Generating diff"
-            diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
-          no_output_timeout: 1h
-
-workflows:
-  version: 2
-  untagged-build:
-    jobs:
-      - setup-multi-product-repositories:
-          context:
-            - matterbuild-github-token
-          filters:
-            branches:
-              ignore:
-                - master
-                - /^release-.*/
-      - setup-mattermost-webapp:
-          filters:
-            branches:
-              ignore:
-                - master
-                - /^release-.*/
-      - setup-focalboard-product:
-          filters:
-            branches:
-              ignore:
-                - master
-                - /^release-.*/
-          requires:
-            - setup-multi-product-repositories
-      - check-go-mod-tidy:
-          requires:
-            - setup-multi-product-repositories
-      - check-golangci-lint:
-          requires:
-            - setup-multi-product-repositories
-      - check-app-layers:
-          requires:
-            - setup-multi-product-repositories
-      - check-generate-worktemplates:
-          requires:
-            - setup-multi-product-repositories
-      - check-store-layers:
-          requires:
-            - setup-multi-product-repositories
-      - check-mocks:
-          requires:
-            - setup-multi-product-repositories
-      - check-email-templates:
-          requires:
-            - setup-multi-product-repositories
-      - check-gen-serialized:
-          requires:
-            - setup-multi-product-repositories
-      - check-mattermost-vet:
-          requires:
-            - setup-multi-product-repositories
-      - check-migrations:
-          requires:
-            - setup-multi-product-repositories
-      - build-api-spec:
-          requires:
-            - setup-multi-product-repositories
-      - build-mattermost-server:
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - build-focalboard:
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-      - test:
-          name: test-mysql
-          dbdriver: mysql
-          dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
-          racemode: ""
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - test:
-          name: test-postgres-normal
-          dbdriver: postgres
-          dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
-          racemode: ""
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - test:
-          name: test-postgres-binary
-          dbdriver: postgres
-          dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
-          racemode: ""
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - test-schema:
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - upload-artifacts-to-s3:
-          context: mattermost-ci-pr-builds-s3
-          requires:
-            - build-mattermost-server
-            - build-focalboard
-      - build-docker:
-          context: matterbuild-docker
-          requires:
-            - upload-artifacts-to-s3
-  release-build:
-    jobs:
-      - setup-multi-product-repositories:
-          context:
-            - matterbuild-github-token
-          filters:
-            branches:
-              only:
-                - master
-                - /^release-.*/
-                - cloud
-      - setup-mattermost-webapp:
-          filters:
-            branches:
-              only:
-                - master
-                - /^release-.*/
-                - cloud
-      - setup-focalboard-product:
-          filters:
-            branches:
-              only:
-                - master
-                - /^release-.*/
-                - cloud
-          requires:
-            - setup-multi-product-repositories
-      - check-go-mod-tidy:
-          requires:
-            - setup-multi-product-repositories
-      - check-golangci-lint:
-          requires:
-            - setup-multi-product-repositories
-      - check-app-layers:
-          requires:
-            - setup-multi-product-repositories
-      - check-generate-worktemplates:
-          requires:
-            - setup-multi-product-repositories
-      - check-store-layers:
-          requires:
-            - setup-multi-product-repositories
-      - check-mocks:
-          requires:
-            - setup-multi-product-repositories
-      - check-email-templates:
-          requires:
-            - setup-multi-product-repositories
-      - check-gen-serialized:
-          requires:
-            - setup-multi-product-repositories
-      - check-mattermost-vet:
-          requires:
-            - setup-multi-product-repositories
-      - check-migrations:
-          requires:
-            - setup-multi-product-repositories
-      - build-api-spec:
-          requires:
-            - setup-multi-product-repositories
-      - sentry:
-          context: matterbuild-sentry
-          requires:
-            - setup-multi-product-repositories
-      - build-mattermost-server:
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-            - setup-focalboard-product
-      - build-focalboard:
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-            - setup-mattermost-webapp
-      - test:
-          name: test-mysql
-          dbdriver: mysql
-          dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
-          racemode: "-race"
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-      - test:
-          name: test-postgres-normal
-          dbdriver: postgres
-          dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
-          racemode: "-race"
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec
-      - test:
-          name: test-postgres-binary
-          dbdriver: postgres
-          dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
-          racemode: ""
-          requires:
-            - check-go-mod-tidy
-            - check-golangci-lint
-            - check-app-layers
-            - check-generate-worktemplates
-            - check-store-layers
-            - check-mocks
-            - check-email-templates
-            - check-gen-serialized
-            - check-mattermost-vet
-            - check-migrations
-            - build-api-spec

.github/workflows/ci.yml

@@ -0,0 +1,291 @@
+name: mattermost-build
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - cloud
+      - release-*
+jobs:
+  check-mocks:
+    name: Check mocks
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Generate mocks
+        run: make mocks
+      - name: Check mocks
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
+  check-go-mod-tidy:
+    name: Check go mod tidy
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Run go mod tidy
+        run: make modules-tidy
+      - name: Check modules
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi
+  check-gen-serialized:
+    name: Check serialization methods for hot structs
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Run make-gen-serialized
+        run: make gen-serialized
+      - name: Check serialized
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
+  check-mattermost-vet:
+    name: Check style
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Checkout focalboard
+        run: |
+          cd ..
+          git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git  
+          cd focalboard
+          git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
+          echo $(git rev-parse HEAD)
+          cd ../mattermost-server
+          make setup-go-work
+      - name: Reset config
+        run: make config-reset
+      - name: Run plugin-checker
+        run: make plugin-checker
+      - name: Run mattermost-vet
+        run: make vet BUILD_NUMBER='${GITHUB_HEAD_REF}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
+  check-migrations:
+    name: Check migration files
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Extract migrations files
+        run: make migrations-extract
+      - name: Check migration files
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
+  build-api-spec:
+    name: Build API specification
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Checkout mattermost-api-reference
+        run: |
+          cd ..
+          git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git
+          cd mattermost-api-reference
+          echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server"
+          git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || true
+          make build
+          cd ../mattermost-server
+  check-generate-work-templates:
+    name: Generate work templates
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Generate work templates
+        run: make generate-worktemplates
+      - name: Check generated work templates
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi
+  check-email-templates:
+    name: Generate email templates
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Generate email templates
+        run: |
+          sudo npm install -g mjml@4.9.0
+          make build-templates
+      - name: Check generated email templates
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
+  check-store-layers:
+    name: Check store layers
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Generate store layers
+        run: make store-layers
+      - name: Check generated code
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
+  check-app-layers:
+    name: Check app layers
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Generate app layers
+        run: make app-layers
+      - name: Check generated code
+        run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
+  test-postgres-binary:
+    name: Run tests on postgres with binary parameters
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/test.yml
+    with:
+      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
+      drivername: postgres
+  test-postgres-normal:
+    name: Run tests on postgres
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/test.yml
+    with:
+      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
+      drivername: postgres
+  test-mysql:
+    name: Run tests on mysql
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/test.yml
+    with:
+      datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
+      drivername: mysql
+  build-mattermost-server:
+    name: Build mattermost-server
+    runs-on: ubuntu-latest-8-cores
+    needs: check-mattermost-vet
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Checkout mattermost-webapp
+        run: |
+          cd ..
+          git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
+          cd mattermost-webapp
+          git checkout $GITHUB_HEAD_REF || git checkout master
+          export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
+          echo "$WEBAPP_GIT_COMMIT"
+          FILE_DIST=dist.tar.gz
+          runtime="2 minute"
+          endtime=$(date -ud "$runtime" +%s)
+          while [[ $(date -u +%s) -le $endtime ]]; do
+            if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then
+              break
+            fi
+            echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`"
+            sleep 5
+          done
+          if [[ -f "$FILE_DIST" ]]; then
+            echo "Precompiled version of web app found"
+            mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1
+          else
+          echo "Building web app from source"
+          make dist
+          fi
+          cd ../mattermost-server
+      - name: Checkout and build focalboard
+        run: |
+          cd ..
+          git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
+          cd focalboard
+          git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
+          echo $(git rev-parse HEAD)
+          make server-linux
+          echo "Building Boards product for web app"
+          # make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile
+          make prebuild
+          cd mattermost-plugin/webapp
+          npm run build:product
+          cd ../../../mattermost-server
+          make setup-go-work
+      - name: Build
+        run: |
+          make config-reset
+          make build-cmd BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}'
+          make package BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}'
+      - name: Persist dist artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: server-dist-artifact
+          path: dist/
+          retention-days: 14
+      - name: Persist build artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: server-build-artifact
+          path: build/
+          retention-days: 14 
+  upload-s3:
+    name: Upload to S3 bucket
+    runs-on: ubuntu-22.04
+    needs: 
+      - build-mattermost-server
+      - test-mysql
+      - test-postgres-binary
+      - test-postgres-normal 
+    env:
+      REPO_NAME: ${{ github.event.repository.name }}
+    steps:
+    - name: Download dist artifacts
+      uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b  # v3.0.2
+      with:
+        name: server-dist-artifact
+        path: dist/
+    - name: Configure AWS
+      uses: aws-actions/configure-aws-credentials@07c2f971bac433df982ccc261983ae443861db49 # v1-node16
+      with:
+        aws-region: us-east-1
+        aws-access-key-id: ${{ secrets.MM_SERVER_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.MM_SERVER_AWS_SECRET_ACCESS_KEY }}
+    # We need to sanitize the branch name before using it
+    - name: ci/sanitize-branch-name
+      id: branch
+      uses: transferwise/sanitize-branch-name@b10b4d524ac5a7b645b43a3527db3a6cca017b9d # v1
+    # Search for the string "pull" and replace it with "PR" in branch-name
+    - name: ci/sanitize-branch-name-replace-pull-with-PR-
+      run: echo "BRANCH_NAME=$(echo ${{ steps.branch.outputs.sanitized-branch-name }} | sed 's/^pull\//PR-/g')" >> $GITHUB_ENV
+    - name: ci/artifact-upload
+      run: |
+        aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/$BRANCH_NAME/ --acl public-read --cache-control "no-cache" --recursive
+        aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/commit/${{ github.sha }}/ --acl public-read --cache-control "no-cache" --recursive
+  build-docker:
+    name: Build docker image
+    runs-on: ubuntu-22.04
+    needs: upload-s3
+    steps:
+    - name: Download build artifacts
+      uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b  # v3.0.2
+      with:
+        name: server-build-artifact
+        path: build/
+    - name: Login to Docker Hub
+      uses: docker/login-action@3da7dc6e2b31f99ef2cb9fb4c50fb0971e0d0139 # v2.1.0
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Setup Docker Buildx
+      uses: docker/setup-buildx-action@11e8a2e2910826a92412015c515187a2d6750279 # v2.4
+    - name: Docker build and push
+      env:
+        DOCKER_CLI_EXPERIMENTAL: enabled
+      run: |
+        export TAG=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7)
+        cd build
+        export DOCKER_CLI_EXPERIMENTAL=enabled
+        export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${GITHUB_SHA}/mattermost-team-linux-amd64.tar.gz
+        docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} .
+  sentry:
+    name: Send build info to sentry
+    runs-on: ubuntu-22.04
+    needs:
+      - test-postgres-binary
+      - test-postgres-normal
+      - test-mysql
+    if:  ${{ github.event_name == 'push' }}
+    env:
+      SENTRY_CLI_VERSION: 1.64.1
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Install sentry-cli
+        run: curl -sL https://sentry.io/get-cli/ | sh
+      - name: Run sentry-cli
+        run: |
+          sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD`
+          sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+name: Test
+on:
+  workflow_call:
+    inputs:
+      datasource:
+        required: true
+        type: string
+      drivername:
+        required: true
+        type: string
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest-8-cores
+    env:
+      COMPOSE_PROJECT_NAME: ghactions
+      BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5
+    steps:
+      - name: Checkout mattermost-server
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: Run docker compose
+        run: |
+          cd build
+          docker-compose --no-ansi run --rm start_dependencies
+          cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+          docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
+          docker-compose --no-ansi ps
+          cd ..
+      - name: Wait for docker compose
+        run: |
+          until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
+          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
+          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
+      - name: Run Tests
+        run: |
+          if [[ ${{ github.event_name }} == 'push' ]]; then
+            export RACE_MODE="-race"
+          fi
+          mkdir -p client/plugins
+          cd build
+          docker run --net ghactions_mm-test \
+            --ulimit nofile=8096:8096 \
+            --env-file=dotenv/test.env \
+            --env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \
+            --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
+            -v ~/work/mattermost-server:/mattermost-server \
+            -w /mattermost-server/mattermost-server \
+            $BUILD_IMAGE \
+            make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE=