Daily E2E test fixes (#29580)

* [skip ci] Stabilize testcases, clean reporting for nighly unstables

* [skip ci] Better output naming

* [skip ci] Fix testcase failure fatal condition

* [skip ci] Make linter happy

* [skip ci] Revert global beforeEach change

* [skip ci] Fix workflow file, skip unstable tests

* [skip ci] Fix testcase_failure_fatal templating

.github/workflows/e2e-fulltests-ci.yml

@@ -74,6 +74,7 @@ jobs:
       BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
       TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
       REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
+      TESTCASE_FAILURE_FATAL: "${{ steps.generate.outputs.TESTCASE_FAILURE_FATAL }}"
       ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
       ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
       WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
@@ -113,6 +114,7 @@ jobs:
         id: generate
         run: |
           MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
+          TESTCASE_FAILURE_FATAL="true"
           if grep -q CLOUD <<<"$REPORT_TYPE"; then
             SERVER=cloud
           else
@@ -150,6 +152,7 @@ jobs:
               TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
               TM4J_ENABLE=true
               COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
+              [ -z "$_IS_TEST_UNSTABLE" ] || TESTCASE_FAILURE_FATAL="" # Assert that tests are stable. If they are not, the status check will be always green
               ;;
             RELEASE | RELEASE_CLOUD)
               ### Populate support variables
@@ -205,6 +208,7 @@ jobs:
           echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
           echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
           echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
+          echo "TESTCASE_FAILURE_FATAL=${TESTCASE_FAILURE_FATAL}" >> $GITHUB_OUTPUT
           echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
           echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
           echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
@@ -242,7 +246,7 @@ jobs:
       commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
       status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
       workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
-      testcase_failure_fatal: true
+      testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
       run_preflight_checks: false
       enable_reporting: true
       SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
@@ -275,7 +279,7 @@ jobs:
       commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
       status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
       workers_number: "1"
-      testcase_failure_fatal: true
+      testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
       run_preflight_checks: false
       enable_reporting: true
       SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"

e2e-tests/cypress/tests/integration/channels/enterprise/oauth/oauth_spec.ts

@@ -352,7 +352,7 @@ describe('Integrations page', () => {
         });
     });
 
-    it('MM-T652 Regenerate Secret', () => {
+    it.skip('MM-T652 Regenerate Secret', () => {
         cy.apiLogin(user1);
         cy.visit(testChannelUrl1);
 

e2e-tests/cypress/tests/integration/channels/integrations/builtin_commands/user_status_commands_spec.js

@@ -27,6 +27,11 @@ describe('Integrations', () => {
         });
     });
 
+    beforeEach(() => {
+        // Ensure that the user is set to 'online' before starting each testcase
+        cy.apiUpdateUserStatus('online');
+    });
+
     it('I18456 Built-in slash commands: change user status via post', () => {
         cy.apiSaveMessageDisplayPreference('compact');
         cy.visit(offTopicUrl);

e2e-tests/cypress/tests/integration/channels/mark_as_unread/leave_channel_unread_spec.js

@@ -24,6 +24,7 @@ describe('Leaving channel', () => {
     let post1;
 
     beforeEach(() => {
+        cy.visit('/');
         cy.apiAdminLogin();
         cy.apiInitSetup().then(({team, channel, user}) => {
             testUser = user;

e2e-tests/cypress/tests/integration/playbooks/channels/channel_header_spec.js

@@ -71,7 +71,7 @@ describe('channels > channel header', {testIsolation: true}, () => {
             cy.apiLogin(testUser);
         });
 
-        it('webapp should show the Playbook channel header button', () => {
+        it.skip('webapp should show the Playbook channel header button', () => {
             // # Navigate directly to a non-playbook run channel
             cy.visit(`/${testTeam.name}/channels/town-square`);
 
@@ -94,7 +94,7 @@ describe('channels > channel header', {testIsolation: true}, () => {
             cy.get('#playbooksChannelHeaderButton').contains('Playbooks');
         });
 
-        it('webapp should make the Playbook channel header button active when opened', () => {
+        it.skip('webapp should make the Playbook channel header button active when opened', () => {
             // # Navigate directly to a non-playbook run channel
             cy.visit(`/${testTeam.name}/channels/town-square`);