Enhance reporting, add shellcheck (#28240)

* Enhance reporting, add shellcheck

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

.github/workflows/e2e-tests-ci-template.yml

@@ -163,6 +163,24 @@ jobs:
         run: |
           npm run check
 
+  shell-check:
+    runs-on: ubuntu-latest
+    needs:
+      - update-initial-status
+    defaults:
+      run:
+        working-directory: e2e-tests
+    steps:
+      - name: ci/checkout-repo
+        if: "${{ inputs.run_preflight_checks }}"
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ inputs.commit_sha }}
+          fetch-depth: 0
+      - name: ci/shell-check
+        if: "${{ inputs.run_preflight_checks }}"
+        run: make check-shell
+
   generate-build-variables:
     runs-on: ubuntu-latest
     needs:
@@ -248,6 +266,7 @@ jobs:
     needs:
       - cypress-check
       - playwright-check
+      - shell-check
       - generate-build-variables
       - generate-test-cycle
     defaults:
@@ -338,7 +357,10 @@ jobs:
         shell: bash
         working-directory: e2e-tests
     outputs:
-      failures: "${{ steps.calculate-failures.outputs.failures }}"
+      successes: "${{ steps.calculate-results.outputs.successes }}"
+      failures: "${{ steps.calculate-results.outputs.failures }}"
+      failures_expected: "${{ steps.calculate-results.outputs.failures_expected }}"
+      total_specs: "${{ steps.calculate-results.outputs.total_specs }}"
     steps:
       - name: ci/checkout-repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -357,17 +379,6 @@ jobs:
           path: |
             e2e-tests/cypress/logs/
             e2e-tests/cypress/results/
-      - name: ci/report-calculate-failures
-        id: calculate-failures
-        run: |
-          FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
-          echo "failures=${FAILURES}" >> $GITHUB_OUTPUT
-          echo "Cypress run completed with $FAILURES failures"
-      - name: ci/e2e-test-assert-results
-        if: "${{ inputs.testcase_failure_fatal }}"
-        run: |
-          # Assert that the run contained 0 failures
-          [ "${{ steps.calculate-failures.outputs.failures }}" = "0" ]
       - name: ci/setup-node
         if: "${{ inputs.enable_reporting }}"
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@@ -390,7 +401,44 @@ jobs:
           TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}"
           TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}"
         run: |
-          make publish-report
+          make report
+      # The results dir may have been modified as part of the reporting: re-upload
+      - name: ci/upload-report-global
+        if: "${{ inputs.enable_reporting }}"
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: e2e-test-results
+          path: |
+            e2e-tests/cypress/logs/
+            e2e-tests/cypress/results/
+          overwrite: true
+      - name: ci/report-calculate-results
+        id: calculate-results
+        env:
+          AD_CYCLE_FILE: "cypress/results/ad_cycle.json"
+        run: |
+          if [ -f "$AD_CYCLE_FILE" ]; then
+            # Prefer using the Automation Dashboard's results to calculate failures
+            SUCCESSES=$(jq -r .pass "$AD_CYCLE_FILE")
+            FAILURES=$(jq -r .fail "$AD_CYCLE_FILE")
+            FAILURES_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE")
+          else
+            # Otherwise, utilize the test specs to calculate the failures
+            SUCCESSES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.passes' | jq -s add)
+            FAILURES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
+            FAILURES_EXPECTED="0"
+          fi
+          TOTAL_SPECS=$(( FAILURES + SUCCESSES ))
+          echo "successes=${SUCCESSES:?}" >> $GITHUB_OUTPUT
+          echo "failures=${FAILURES:?}" >> $GITHUB_OUTPUT
+          echo "failures_expected=${FAILURES_EXPECTED:?}" >> $GITHUB_OUTPUT
+          echo "total_specs=${TOTAL_SPECS:?}" >> $GITHUB_OUTPUT
+          echo "Cypress run completed:  ${SUCCESSES}/${TOTAL_SPECS} passing specs (plus ${FAILURES_EXPECTED} unrelated failures)"
+      - name: ci/e2e-test-assert-results
+        if: "${{ inputs.testcase_failure_fatal }}"
+        run: |
+          # Assert that the run contained 0 failures
+          [ "${{ steps.calculate-results.outputs.failures }}" = "0" ]
 
   update-failure-final-status:
     runs-on: ubuntu-latest
@@ -408,7 +456,7 @@ jobs:
           commit_sha: ${{ inputs.commit_sha }}
           context: ${{ inputs.status_check_context }}
           description: |
-            Completed with ${{ needs.report.outputs.failures }} failures
+            Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures)
           status: failure
           target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"
 
@@ -428,6 +476,6 @@ jobs:
           commit_sha: ${{ inputs.commit_sha }}
           context: ${{ inputs.status_check_context }}
           description: |
-            Completed with ${{ needs.report.outputs.failures }} failures
+            Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures)
           status: success
           target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"