CLD-8395 Integrate playwright into E2E tests (#28492)

* Finalize Playwright E2E scripts integration, add videos * Add suggestion by Saturn Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com> --------- Co-authored-by: Mattermost Build <build@mattermost.com> Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>
2024-10-09 13:05:11 +02:00
--- a/.github/workflows/e2e-fulltests-ci.yml
+++ b/.github/workflows/e2e-fulltests-ci.yml
@@ -56,7 +56,8 @@ jobs:
      server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection
      SERVER: "${{ steps.generate.outputs.SERVER }}"
      ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
-      TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}"
+      TEST_FILTER_CYPRESS: "${{ steps.generate.outputs.TEST_FILTER_CYPRESS }}"
+      TEST_FILTER_PLAYWRIGHT: "tests/ --project=chrome" # Note: Run on chrome but eventually will enable to all projects which include firefox and ipad.
      BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
      TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
      REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
@@ -72,17 +73,17 @@ jobs:
      ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}"
      AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
      # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
-      TEST_FILTER_PR: >-
+      TEST_FILTER_CYPRESS_PR: >-
        --stage="@prod"
        --excludeGroup="@te_only,@cloud_only,@high_availability"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
-      TEST_FILTER_PROD_ONPREM: >-
+      TEST_FILTER_CYPRESS_PROD_ONPREM: >-
        --stage="@prod"
        --excludeGroup="@te_only,@cloud_only,@high_availability"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
-      TEST_FILTER_PROD_CLOUD: >-
+      TEST_FILTER_CYPRESS_PROD_CLOUD: >-
        --stage="@prod"
        --excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal"
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
@@ -114,7 +115,7 @@ jobs:
              SERVER_IMAGE_ORG=mattermostdevelopment
              BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent"
              WORKERS_NUMBER=20
-              TEST_FILTER="$TEST_FILTER_PR"
+              TEST_FILTER_CYPRESS="$TEST_FILTER_CYPRESS_PR"
              COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
              ### Run sanity assertions after variable generations
              [ "$REF" = "${_COMMIT_SHA_COMPUTED}" ]             # 'inputs.ref' must be a full commit hash, and the commit must exist
@@ -123,7 +124,7 @@ jobs:
            MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE)
              ### Populate support variables
              _IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # The variable's value is 'UNSTABLE' if report type is for unstable tests, otherwise it's empty
-              _TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}"
+              _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
              ### For ref and image tag generation: ignore 'inputs.ref', and use master branch directly. Note that 'COMMIT_SHA' will be used for reporting the test result, and for checking out the testing scripts and test cases
              COMMIT_SHA="$(git rev-parse --verify origin/master)"
              BRANCH=master
@@ -132,13 +133,13 @@ jobs:
              BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
              BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
              WORKERS_NUMBER=10   # Daily tests are not time critical, and it's more efficient to run on fewer workers
-              TEST_FILTER="${!_TEST_FILTER_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
+              TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
              TM4J_ENABLE=true
              COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
              ;;
            RELEASE | RELEASE_CLOUD)
              ### Populate support variables
-              _TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}"
+              _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
              ### For ref and image tag generation: assume the 'inputs.ref' is a tag, and use the first two digits to construct the branch name
              COMMIT_SHA="$(git rev-parse --verify HEAD)"
              BRANCH=$(sed -E "s/v([0-9]+)\.([0-9]+)\..+$/release-\1.\2/g" <<<$REF)
@@ -147,7 +148,7 @@ jobs:
              BUILD_ID_SUFFIX="release-${SERVER}-ent"
              BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
              WORKERS_NUMBER=20
-              TEST_FILTER="${!_TEST_FILTER_VARIABLE}"
+              TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE}"
              TM4J_ENABLE=true
              COMPUTED_REPORT_TYPE=RELEASE
              ### Run sanity assertions after variable generations
@@ -179,7 +180,7 @@ jobs:
          echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
          echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
          echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
-          echo "TEST_FILTER=${TEST_FILTER}" >> $GITHUB_OUTPUT
+          echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
          echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
          echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
          echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
@@ -199,13 +200,13 @@ jobs:
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
          E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
          You can check its progress by either:
-          - Looking at the corresponding commit status, which will be available in a few moments: \`${STATUS_CHECK_CONTEXT}\`.
+          - Looking at the corresponding commit statuses: \`${STATUS_CHECK_CONTEXT}\` and \`${STATUS_CHECK_CONTEXT}-playwright\`.
          - Looking at the [E2E test's Workflow Run]($WORKFLOW_RUN_URL), with Run ID \`${{ github.run_id }}\`.
-          - Looking at the [E2E test's Cycle on the Automation Dashboard]($CYCLE_URL).
+          - Looking at the [E2E test's Cycle on the Automation Dashboard]($CYCLE_URL). (Cypress only)
          EOF
          fi

-  e2e-fulltest:
+  e2e-fulltest-cypress:
    needs:
      - generate-test-variables
    uses: ./.github/workflows/e2e-tests-ci-template.yml
@@ -223,7 +224,7 @@ jobs:
      SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
      SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
      ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
-      TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}"
+      TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_CYPRESS }}"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
      BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
      BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
@@ -241,12 +242,45 @@ jobs:
      CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
      CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"

+  e2e-fulltest-playwright:
+    needs:
+      - generate-test-variables
+    uses: ./.github/workflows/e2e-tests-ci-template.yml
+    strategy:
+      matrix:
+        type:
+        - name: PR
+    with:
+      commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
+      status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
+      workers_number: "1"
+      testcase_failure_fatal: false # NB: do not enable until Playwright test stability is reached
+      run_preflight_checks: false
+      enable_reporting: true
+      SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
+      SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
+      TEST: playwright
+      TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_PLAYWRIGHT }}"
+      MM_ENV: "${{ inputs.MM_ENV || '' }}"
+      BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
+      BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
+      REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
+      ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
+      ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
+    secrets:
+      MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
+      PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
+      REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
+      CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
+      CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"
+
  notify-user:
    runs-on: ubuntu-latest
    if: always()
    needs:
      - generate-test-variables
-      - e2e-fulltest
+      - e2e-fulltest-cypress
+      - e2e-fulltest-playwright
    permissions:
      issues: write
      pull-requests: write
@@ -267,7 +301,9 @@ jobs:
          if [ -n "$PR_NUMBER" ]; then
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
          E2E test has completed for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
-          You can check its result on the [Automation Dashboard]($CYCLE_URL), or by checking out the status check named \`${STATUS_CHECK_CONTEXT}\` in this PR.
-          The run summary artifact is available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
+          Results summary:
+          - Cypress pass rate: ${{ needs.e2e-fulltest-cypress.outputs.pass_rate || 'unknown' }} (see [Automation Dashboard]($CYCLE_URL) and commit status check \`${STATUS_CHECK_CONTEXT}\`)
+          - Playwright pass rate: ${{ needs.e2e-fulltest-playwright.outputs.pass_rate || 'unknown' }} (see commit status check \`${STATUS_CHECK_CONTEXT}-playwright\`)
+          The run summary artifacts are available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
          EOF
          fi
--- a/.github/workflows/e2e-tests-ci-template.yml
+++ b/.github/workflows/e2e-tests-ci-template.yml
@@ -40,6 +40,10 @@ on:
      ENABLED_DOCKER_SERVICES:
        type: string
        required: false
+      TEST: # Valid values are: cypress, playwright
+        type: string
+        required: false
+        default: "cypress"
      TEST_FILTER:
        type: string
        required: false
@@ -80,6 +84,15 @@ on:
        required: false
      CWS_EXTRA_HTTP_HEADERS:
        required: false
+    outputs:
+      passed:
+        value: "${{ jobs.report.outputs.passed }}"
+      failed:
+        value: "${{ jobs.report.outputs.failed }}"
+      failed_expected:
+        value: "${{ jobs.report.outputs.failed_expected }}"
+      pass_rate:
+        value: "${{ jobs.report.outputs.pass_rate }}"

 jobs:
  update-initial-status:
@@ -190,6 +203,7 @@ jobs:
        shell: bash
    outputs:
      workers: "${{ steps.generate.outputs.workers }}"
+      node-cache-dependency-path: "${{ steps.generate.outputs.node-cache-dependency-path }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -200,14 +214,16 @@ jobs:
        id: generate
        env:
          WORKERS: ${{ inputs.workers_number }}
+          TEST: ${{ inputs.TEST }}
        run: |
          [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
          echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
+          echo "node-cache-dependency-path=e2e-tests/${TEST}/package-lock.json" >> $GITHUB_OUTPUT

  generate-test-cycle:
    runs-on: ubuntu-latest
    needs:
-      - update-initial-status
+      - generate-build-variables
    defaults:
      run:
        shell: bash
@@ -226,7 +242,7 @@ jobs:
        with:
          node-version-file: ".nvmrc"
          cache: npm
-          cache-dependency-path: "e2e-tests/cypress/package-lock.json"
+          cache-dependency-path: "e2e-tests/cypress/package-lock.json" # NB: the generate-cycle script is cypress-specific operation for now
      - name: ci/e2e-test-gencycle
        id: e2e-test-gencycle
        env:
@@ -234,6 +250,7 @@ jobs:
          AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
          BRANCH: "${{ inputs.BRANCH }}"
          BUILD_ID: "${{ inputs.BUILD_ID }}"
+          TEST: "${{ inputs.TEST }}"
          TEST_FILTER: "${{ inputs.TEST_FILTER }}"
        run: |
          set -e -o pipefail
@@ -280,6 +297,7 @@ jobs:
      SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
      MM_LICENSE: "${{ secrets.MM_LICENSE }}"
      ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
+      TEST: "${{ inputs.TEST }}"
      TEST_FILTER: "${{ inputs.TEST_FILTER }}"
      MM_ENV: "${{ inputs.MM_ENV }}"
      BRANCH: "${{ inputs.BRANCH }}"
@@ -311,13 +329,13 @@ jobs:
        with:
          node-version-file: ".nvmrc"
          cache: npm
-          cache-dependency-path: "e2e-tests/cypress/package-lock.json"
+          cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }}
      - name: ci/e2e-test
        run: |
          make cloud-init
          if [ -n "$ROLLING_RELEASE_SERVER_IMAGE" ]; then
            echo "RollingRelease: checking out E2E test cases from revision ${ROLLING_RELEASE_COMMIT_SHA}, for initial smoketest"
-            git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- cypress/ && git status
+            git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- "${TEST}/" && git status
            (
            echo "RollingRelease: running initial smoketest against image $ROLLING_RELEASE_SERVER_IMAGE"
            export SERVER_IMAGE="$ROLLING_RELEASE_SERVER_IMAGE"
@@ -326,12 +344,12 @@ jobs:
            make
            )
            echo "RollingRelease: asserting smoketest result has zero failures."
-            FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
+            FAILURES=$(jq -r '.failed' "${TEST}/results/summary.json")
            if [ "$FAILURES" -ne "0" ]; then
              echo "RollingRelease: initial smoketest for rolling release E2E run has nonzero ($FAILURES) failures. Aborting test run." >&2
              exit 1
            fi
-            rm -rfv cypress/results
+            rm -rfv "${TEST}/{results,logs}"
            echo "RollingRelease: reset the E2E test cases to the revision to test"
            git reset --hard HEAD && git status
            echo "RollingRelease: smoketest completed. Starting full E2E tests."
@@ -340,11 +358,12 @@ jobs:
          make cloud-teardown
      - name: ci/e2e-test-store-results
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        if: always()
        with:
-          name: e2e-test-results-${{ matrix.os }}-${{ matrix.worker_index }}
+          name: e2e-test-results-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }}
          path: |
-            e2e-tests/cypress/logs/
-            e2e-tests/cypress/results/
+            e2e-tests/${{ inputs.TEST }}/logs/
+            e2e-tests/${{ inputs.TEST }}/results/
          retention-days: 1

  report:
@@ -357,10 +376,11 @@ jobs:
        shell: bash
        working-directory: e2e-tests
    outputs:
-      successes: "${{ steps.calculate-results.outputs.successes }}"
-      failures: "${{ steps.calculate-results.outputs.failures }}"
-      failures_expected: "${{ steps.calculate-results.outputs.failures_expected }}"
-      total_specs: "${{ steps.calculate-results.outputs.total_specs }}"
+      passed: "${{ steps.calculate-results.outputs.passed }}"
+      failed: "${{ steps.calculate-results.outputs.failed }}"
+      failed_expected: "${{ steps.calculate-results.outputs.failed_expected }}"
+      pass_rate: "${{ steps.calculate-results.outputs.pass_rate }}"
+      commit_status_message: "${{ steps.calculate-results.outputs.commit_status_message }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -370,15 +390,15 @@ jobs:
      - name: ci/download-artifacts
        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
        with:
-          path: e2e-tests/cypress/
+          path: e2e-tests/${{ inputs.TEST }}/
          merge-multiple: true
      - name: ci/upload-report-global
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
-          name: e2e-test-results
+          name: e2e-test-results-${{ inputs.TEST }}
          path: |
-            e2e-tests/cypress/logs/
-            e2e-tests/cypress/results/
+            e2e-tests/${{ inputs.TEST }}/logs/
+            e2e-tests/${{ inputs.TEST }}/results/
      - name: ci/setup-node
        if: "${{ inputs.enable_reporting }}"
        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -386,11 +406,12 @@ jobs:
        with:
          node-version-file: ".nvmrc"
          cache: npm
-          cache-dependency-path: "e2e-tests/cypress/package-lock.json"
+          cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }}
      - name: ci/publish-report
        if: "${{ inputs.enable_reporting }}"
        env:
          TYPE: "${{ inputs.REPORT_TYPE }}"
+          TEST: "${{ inputs.TEST }}"
          SERVER: "${{ inputs.SERVER }}"
          SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
          AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
@@ -407,38 +428,43 @@ jobs:
        if: "${{ inputs.enable_reporting }}"
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
-          name: e2e-test-results
+          name: e2e-test-results-${{ inputs.TEST }}
          path: |
-            e2e-tests/cypress/logs/
-            e2e-tests/cypress/results/
+            e2e-tests/${{ inputs.TEST }}/logs/
+            e2e-tests/${{ inputs.TEST }}/results/
          overwrite: true
      - name: ci/report-calculate-results
        id: calculate-results
        env:
-          AD_CYCLE_FILE: "cypress/results/ad_cycle.json"
+          TEST: "${{ inputs.TEST }}"
        run: |
+          AD_CYCLE_FILE="${TEST}/results/ad_cycle.json"
          if [ -f "$AD_CYCLE_FILE" ]; then
            # Prefer using the Automation Dashboard's results to calculate failures
-            SUCCESSES=$(jq -r .pass "$AD_CYCLE_FILE")
-            FAILURES=$(jq -r .fail "$AD_CYCLE_FILE")
-            FAILURES_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE")
+            export PASSED=$(jq -r .pass "$AD_CYCLE_FILE")
+            export FAILED=$(jq -r .fail "$AD_CYCLE_FILE")
+            export FAILED_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE")
          else
-            # Otherwise, utilize the test specs to calculate the failures
-            SUCCESSES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.passes' | jq -s add)
-            FAILURES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
-            FAILURES_EXPECTED="0"
+            # Otherwise, utilize summary.json to calculate the failures
+            # NB: in this job, this file only makes sense if a single worker is used, as with Playwright
+            export PASSED=$(jq '.passed' "${TEST}/results/summary.json")
+            export FAILED=$(jq '.failed' "${TEST}/results/summary.json")
+            export FAILED_EXPECTED=$(jq '.failed_expected' "${TEST}/results/summary.json")
          fi
-          TOTAL_SPECS=$(( FAILURES + SUCCESSES ))
-          echo "successes=${SUCCESSES:?}" >> $GITHUB_OUTPUT
-          echo "failures=${FAILURES:?}" >> $GITHUB_OUTPUT
-          echo "failures_expected=${FAILURES_EXPECTED:?}" >> $GITHUB_OUTPUT
-          echo "total_specs=${TOTAL_SPECS:?}" >> $GITHUB_OUTPUT
-          echo "Cypress run completed:  ${SUCCESSES}/${TOTAL_SPECS} passing specs (plus ${FAILURES_EXPECTED} unrelated failures)"
+          export TOTAL_SPECS=$(( PASSED + FAILED ))
+          export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
+          export COMMIT_STATUS_MESSAGE="Completed with pass rate: ${PASS_RATE}% (plus ${FAILED_EXPECTED} unrelated failures)"
+          echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
+          echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
+          echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT
+          echo "pass_rate=${PASS_RATE:?}%" >> $GITHUB_OUTPUT
+          echo "commit_status_message=${COMMIT_STATUS_MESSAGE:?}" >> $GITHUB_OUTPUT
+          echo "$COMMIT_STATUS_MESSAGE"
      - name: ci/e2e-test-assert-results
        if: "${{ inputs.testcase_failure_fatal }}"
        run: |
          # Assert that the run contained 0 failures
-          [ "${{ steps.calculate-results.outputs.failures }}" = "0" ]
+          [ "${{ steps.calculate-results.outputs.failed }}" = "0" ]

  update-failure-final-status:
    runs-on: ubuntu-latest
@@ -455,8 +481,7 @@ jobs:
          repository_full_name: ${{ github.repository }}
          commit_sha: ${{ inputs.commit_sha }}
          context: ${{ inputs.status_check_context }}
-          description: |
-            Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures)
+          description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
          status: failure
          target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"

@@ -475,7 +500,6 @@ jobs:
          repository_full_name: ${{ github.repository }}
          commit_sha: ${{ inputs.commit_sha }}
          context: ${{ inputs.status_check_context }}
-          description: |
-            Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures)
+          description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
          status: success
          target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"