Daily E2E test fixes (#29580)

* [skip ci] Stabilize testcases, clean reporting for nighly unstables

* [skip ci] Better output naming

* [skip ci] Fix testcase failure fatal condition

* [skip ci] Make linter happy

* [skip ci] Revert global beforeEach change

* [skip ci] Fix workflow file, skip unstable tests

* [skip ci] Fix testcase_failure_fatal templating

.github/workflows/e2e-fulltests-ci.yml

@@ -74,6 +74,7 @@ jobs:
       BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
       TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
       REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
+      TESTCASE_FAILURE_FATAL: "${{ steps.generate.outputs.TESTCASE_FAILURE_FATAL }}"
       ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
       ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
       WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
@@ -113,6 +114,7 @@ jobs:
         id: generate
         run: |
           MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
+          TESTCASE_FAILURE_FATAL="true"
           if grep -q CLOUD <<<"$REPORT_TYPE"; then
             SERVER=cloud
           else
@@ -150,6 +152,7 @@ jobs:
               TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
               TM4J_ENABLE=true
               COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
+              [ -z "$_IS_TEST_UNSTABLE" ] || TESTCASE_FAILURE_FATAL="" # Assert that tests are stable. If they are not, the status check will be always green
               ;;
             RELEASE | RELEASE_CLOUD)
               ### Populate support variables
@@ -205,6 +208,7 @@ jobs:
           echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
           echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
           echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
+          echo "TESTCASE_FAILURE_FATAL=${TESTCASE_FAILURE_FATAL}" >> $GITHUB_OUTPUT
           echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
           echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
           echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
@@ -242,7 +246,7 @@ jobs:
       commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
       status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
       workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
-      testcase_failure_fatal: true
+      testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
       run_preflight_checks: false
       enable_reporting: true
       SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
@@ -275,7 +279,7 @@ jobs:
       commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
       status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
       workers_number: "1"
-      testcase_failure_fatal: true
+      testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
       run_preflight_checks: false
       enable_reporting: true
       SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"