mostlymatter/.github/workflows/e2e-fulltests-ci.yml

---
name: E2E Tests
on:
  # For PRs, this workflow gets triggered from the Argo Events platform.
  # Check the following repo for details: https://github.com/mattermost/delivery-platform
  workflow_dispatch:
    inputs:
      ref:
        type: string
        description: Git ref to test. Must be a full commit SHA for PR testing, and a tag for release testing. Ignored for daily tests.
        required: false
      PR_NUMBER:
        type: string
        description: PR number (if applicable)
        required: false
      ROLLING_RELEASE_FROM_TAG:
        type: string
        description: Mattermost release git tag for RollingRelease tests. Optional.
        required: false
      MM_ENV:
        type: string
        required: false
        description: A comma-separated list of environment variables to set for the server. Spaces are not supported.
      MM_SERVICE_OVERRIDES:
        type: string
        required: false
        description: A comma-separated list of service overrides. E.g. "-elasticsearch,+opensearch"
      REPORT_TYPE:
        type: choice
        description: The context this report is being generated in
        options:
          - PR
          - RELEASE
          - RELEASE_CLOUD
          - MASTER
          - MASTER_UNSTABLE
          - CLOUD
          - CLOUD_UNSTABLE
          - NONE
        default: NONE
      RUN_CYPRESS:
        type: boolean
        description: Enable Cypress run
        default: true
      RUN_PLAYWRIGHT:
        type: boolean
        description: Enable Playwright run
        default: true

concurrency:
  group: "${{ github.workflow }}-${{ inputs.REPORT_TYPE }}-${{ inputs.PR_NUMBER || inputs.ref }}-${{ inputs.MM_ENV }}"
  cancel-in-progress: true

jobs:
  generate-test-variables:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    defaults:
      run:
        shell: bash
    outputs:
      commit_sha: "${{ steps.generate.outputs.commit_sha }}"
      BRANCH: "${{ steps.generate.outputs.BRANCH }}"
      SERVER_IMAGE: "${{ steps.generate.outputs.SERVER_IMAGE }}"
      status_check_context: "${{ steps.generate.outputs.status_check_context }}"
      workers_number: "${{ steps.generate.outputs.workers_number }}"
      server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection
      SERVER: "${{ steps.generate.outputs.SERVER }}"
      ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
      TEST_FILTER_CYPRESS: "${{ steps.generate.outputs.TEST_FILTER_CYPRESS }}"
      TEST_FILTER_PLAYWRIGHT: "tests/ --project=chrome" # Note: Run on chrome but eventually will enable to all projects which include firefox and ipad.
      BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
      TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
      REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
      ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
      ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
      WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
      CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
    env:
      GH_TOKEN: "${{ github.token }}"
      REF: "${{ inputs.ref || github.sha }}"
      PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
      REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
      ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}"
      AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
      # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
      TEST_FILTER_CYPRESS_PR: >-
        --stage="@prod"
        --excludeGroup="@te_only,@cloud_only,@high_availability"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
      TEST_FILTER_CYPRESS_PROD_ONPREM: >-
        --stage="@prod"
        --excludeGroup="@te_only,@cloud_only,@high_availability"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
      TEST_FILTER_CYPRESS_PROD_CLOUD: >-
        --stage="@prod"
        --excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
      MM_SERVICE_OVERRIDES: "${{ inputs.MM_SERVICE_OVERRIDES }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: "${{ inputs.ref || github.sha }}"
          fetch-depth: 0
      - name: ci/generate-test-variables
        id: generate
        run: |
          MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
          if grep -q CLOUD <<<"$REPORT_TYPE"; then
            SERVER=cloud
          else
            SERVER=onprem
          fi
          case "$REPORT_TYPE" in
            NONE | PR)
              ### Populate support variables
              _COMMIT_SHA_COMPUTED=$(git rev-parse --verify "$REF") # NB: not actually used for resolving the commit; it's only to double check the value of 'inputs.ref'
              ### For image tag generation: utilize 'inputs.ref', assume that it is a full commit SHA
              COMMIT_SHA="${REF}"
              BRANCH="server-pr-${PR_NUMBER}"   # For reference, the real branch name may be retrievable with command: 'jq -r .head.ref <pr.json'
              SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
              SERVER_IMAGE_ORG=mattermostdevelopment
              BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent"
              WORKERS_NUMBER=20
              TEST_FILTER_CYPRESS="$TEST_FILTER_CYPRESS_PR"
              COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
              ### Run sanity assertions after variable generations
              [ "$REF" = "${_COMMIT_SHA_COMPUTED}" ]             # 'inputs.ref' must be a full commit hash, and the commit must exist
              [ "$REPORT_TYPE" != "PR" ] || [ "$PR_NUMBER" -gt "0" ] # If report type is PR, then PR_NUMBER must be set to a number
              ;;
            MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE)
              ### Populate support variables
              _IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # The variable's value is 'UNSTABLE' if report type is for unstable tests, otherwise it's empty
              _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
              ### For ref and image tag generation: ignore 'inputs.ref', and use master branch directly. Note that 'COMMIT_SHA' will be used for reporting the test result, and for checking out the testing scripts and test cases
              COMMIT_SHA="$(git rev-parse --verify origin/master)"
              BRANCH=master
              SERVER_IMAGE_TAG=master
              SERVER_IMAGE_ORG=mattermostdevelopment
              BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
              BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
              WORKERS_NUMBER=10   # Daily tests are not time critical, and it's more efficient to run on fewer workers
              TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
              TM4J_ENABLE=true
              COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
              ;;
            RELEASE | RELEASE_CLOUD)
              ### Populate support variables
              _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
              ### For ref and image tag generation: assume the 'inputs.ref' is a tag, and use the first two digits to construct the branch name
              COMMIT_SHA="$(git rev-parse --verify HEAD)"
              BRANCH=$(sed -E "s/v([0-9]+)\.([0-9]+)\..+$/release-\1.\2/g" <<<$REF)
              SERVER_IMAGE_TAG="$(cut -c2- <<<$REF)"   # Remove the leading 'v' from the given tag name, to generate the docker image tag
              SERVER_IMAGE_ORG=mattermost
              BUILD_ID_SUFFIX="release-${SERVER}-ent"
              BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
              WORKERS_NUMBER=20
              TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE}"
              TM4J_ENABLE=true
              COMPUTED_REPORT_TYPE=RELEASE
              ### Run sanity assertions after variable generations
              git show-ref --verify "refs/tags/${REF}"              # 'inputs.ref' must be a tag, for release report types
              git show-ref --verify "refs/remotes/origin/${BRANCH}" # The release branch computed from the given tag must exist
              ;;
            *)
              echo "Fatal: unimplemented test type. Aborting."
              exit 1
          esac
          if [ -n "$ROLLING_RELEASE_FROM_TAG" ]; then
            ROLLING_RELEASE_COMMIT_SHA=$(git rev-parse --verify "$ROLLING_RELEASE_FROM_TAG")
            ROLLING_RELEASE_SERVER_IMAGE_TAG=$(echo "$ROLLING_RELEASE_FROM_TAG" | sed 's/^v//') # Remove the leading 'v' from the given tag name, to generate the docker image tag
            ROLLING_RELEASE_SERVER_IMAGE="mattermost/mattermost-enterprise-edition:${ROLLING_RELEASE_SERVER_IMAGE_TAG}"
            BUILD_ID_SUFFIX="rolling${ROLLING_RELEASE_FROM_TAG/-/_}-$BUILD_ID_SUFFIX"
            BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
            WORKERS_NUMBER=10   # Rolling release tests are particularly impacted by increased parallelism. It's more efficient to run on fewer workers
            ### Run sanity assertions after variable generations
            git show-ref --verify "refs/tags/${ROLLING_RELEASE_FROM_TAG}" # 'inputs.ROLLING_RELEASE_FROM_TAG' must be a tag, for release report types
          fi
          ENABLED_DOCKER_SERVICES="postgres inbucket minio openldap elasticsearch keycloak"
          for SVC_OP in $(tr , ' '<<<"$MM_SERVICE_OVERRIDES"); do
            OP=$(cut -c1 <<<$SVC_OP)
            SVC=$(cut -c2- <<<$SVC_OP)
            case "$OP" in
              "+") ENABLED_DOCKER_SERVICES="$ENABLED_DOCKER_SERVICES $SVC" ;;
              "-") ENABLED_DOCKER_SERVICES=$(sed -E "s:(^| )${SVC}( |\$): :g" <<<"$ENABLED_DOCKER_SERVICES") ;;
              *) echo "Invalid MM_SERVICE_OVERRIDE value: $SVC_OP"; exit 1 ;;
            esac
          done
          # BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
          # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
          BUILD_ID="${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}"
          echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
          echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
          echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/mattermost-enterprise-edition:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
          echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
          echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
          echo "ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES}" >> $GITHUB_OUTPUT
          echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
          echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
          echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
          echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
          echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
          echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
          echo "ROLLING_RELEASE_SERVER_IMAGE=${ROLLING_RELEASE_SERVER_IMAGE}" >> $GITHUB_OUTPUT
          echo "BUILD_ID=${BUILD_ID}" >> $GITHUB_OUTPUT
          # User notification variables
          echo "WORKFLOW_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{github.run_id}}" >> $GITHUB_OUTPUT
          echo "CYCLE_URL=${AUTOMATION_DASHBOARD_URL%%/api}/cycle/${BUILD_ID}" >> $GITHUB_OUTPUT
      - name: ci/notify-user
        env:
          COMMIT_SHA: "${{steps.generate.outputs.commit_sha}}"
          STATUS_CHECK_CONTEXT: "${{steps.generate.outputs.status_check_context}}"
          WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
          CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
          RUN_CYPRESS: "${{inputs.RUN_CYPRESS || ''}}"
          RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT || ''}}"
        run: |
          if [ -n "$PR_NUMBER" ]; then
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
          E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}${MM_SERVICE_OVERRIDES:+, Cypress service overrides \`$MM_SERVICE_OVERRIDES\`}.
          To check the run progress:
          - Cypress: ${RUN_CYPRESS:+look for commit status \`$STATUS_CHECK_CONTEXT\` or the access the [Automation Dashboard Cycle URL]($CYCLE_URL)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "will not run").
          - Playwright: ${RUN_PLAYWRIGHT:+look for commit status \`$STATUS_CHECK_CONTEXT-playwright\`}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "will not run").
          
          You can also look at the [E2E test's Workflow Run URL]($WORKFLOW_RUN_URL) (run ID \`${{ github.run_id }}\`).
          EOF
          fi

  e2e-fulltest-cypress:
    needs:
      - generate-test-variables
    uses: ./.github/workflows/e2e-tests-ci-template.yml
    if: ${{ inputs.RUN_CYPRESS }}
    with:
      commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
      status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
      workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
      testcase_failure_fatal: false
      run_preflight_checks: false
      enable_reporting: true
      SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
      SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
      ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
      TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_CYPRESS }}"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
      BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
      BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
      REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
      ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
      ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
    secrets:
      MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
      AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
      AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
      PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
      REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
      REPORT_TM4J_API_KEY: "${{ needs.generate-test-variables.outputs.TM4J_ENABLE == 'true' && secrets.MM_E2E_TM4J_API_KEY || '' }}"
      REPORT_TM4J_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"
      CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
      CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"

  e2e-fulltest-playwright:
    needs:
      - generate-test-variables
    uses: ./.github/workflows/e2e-tests-ci-template.yml
    if: ${{ inputs.RUN_PLAYWRIGHT }}
    with:
      commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
      status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
      workers_number: "1"
      testcase_failure_fatal: false # NB: do not enable until Playwright test stability is reached
      run_preflight_checks: false
      enable_reporting: true
      SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
      SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
      TEST: playwright
      TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_PLAYWRIGHT }}"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
      BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
      BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
      REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
      ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
      ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
    secrets:
      MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
      PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
      REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
      CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
      CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"

  notify-user:
    runs-on: ubuntu-latest
    if: always()
    needs:
      - generate-test-variables
      - e2e-fulltest-cypress
      - e2e-fulltest-playwright
    permissions:
      issues: write
      pull-requests: write
    defaults:
      run:
        shell: bash
    env:
      GH_TOKEN: "${{ github.token }}"
      PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
      COMMIT_SHA: "${{ needs.generate-test-variables.outputs.commit_sha }}"
      STATUS_CHECK_CONTEXT: "${{ needs.generate-test-variables.outputs.status_check_context }}"
      WORKFLOW_RUN_URL: "${{ needs.generate-test-variables.outputs.WORKFLOW_RUN_URL }}"
      CYCLE_URL: "${{ needs.generate-test-variables.outputs.CYCLE_URL }}"
      RUN_CYPRESS: "${{inputs.RUN_CYPRESS || ''}}"
      RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT || ''}}"
    steps:
      - name: ci/notify-user-test-completion
        run: |
          if [ -n "$PR_NUMBER" ]; then
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
          E2E test has completed for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
          Results summary:
          - Cypress: ${RUN_CYPRESS:+pass rate is \`${{ needs.e2e-fulltest-cypress.outputs.pass_rate || 'unknown' }}\` (see [Automation Dashboard]($CYCLE_URL) and commit status check \`$STATUS_CHECK_CONTEXT\`)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "did not run").
          - Playwright: ${RUN_PLAYWRIGHT:+pass rate is \`${{ needs.e2e-fulltest-playwright.outputs.pass_rate || 'unknown' }}\` (see commit status check \`$STATUS_CHECK_CONTEXT-playwright\`)}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "did not run").
          
          The run summary artifacts are available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
          EOF
          fi