Enable Cypress and Playwright retries (#29056)

* Enable Cypress and Playwright retries
* Improve E2E commit status messages
* Demote unstable testcases, increase task timeout

.github/workflows/e2e-tests-ci-template.yml

@@ -453,7 +453,11 @@ jobs:
           fi
           export TOTAL_SPECS=$(( PASSED + FAILED ))
           export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
-          export COMMIT_STATUS_MESSAGE="Completed with pass rate: ${PASS_RATE}% (plus ${FAILED_EXPECTED} unrelated failures)"
+          if [ "$FAILED" = "0" ]; then
+            export COMMIT_STATUS_MESSAGE="All test cases passed"
+          else
+            export COMMIT_STATUS_MESSAGE="${FAILED} test cases failed. Please check the workflow logs"
+          fi
           echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
           echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
           echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT

e2e-tests/.ci/server.run_playwright.sh

@@ -43,7 +43,7 @@ ${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/p
 # Documentation on the results.json file: https://playwright.dev/docs/api/class-testcase#test-case-expected-status
 # NB: the following line is needed only for compatibility reasons, to support RollingRelease tests for versions prior to v10.1.0
 #     It can be removed after releases <=v10.0.x are phased out
-mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json || true
+mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json 2>/dev/null || true
 jq -f /dev/stdin ../playwright/results/results.json > ../playwright/results/summary.json <<EOF
 {
   passed: .stats.expected,

e2e-tests/cypress/cypress.config.ts

@@ -9,8 +9,9 @@ export default defineConfig({
     downloadsFolder: 'tests/downloads',
     fixturesFolder: 'tests/fixtures',
     numTestsKeptInMemory: 0,
+    retries: 2,
     screenshotsFolder: 'tests/screenshots',
-    taskTimeout: 20000,
+    taskTimeout: 60000,
     video: true,
     videoCompression: true,
     videosFolder: 'tests/videos',

e2e-tests/cypress/tests/integration/channels/notifications/direct_messages_do_not_add_indicator_spec.js

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @notifications
 
 describe('Notifications', () => {

e2e-tests/cypress/tests/integration/channels/performance/channel_switch_spec.js

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @channel
 
 import {measurePerformance} from './utils.js';

e2e-tests/cypress/tests/integration/playbooks/channels/rhs/status_update_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 /* eslint-disable no-only-tests/no-only-tests */

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/commands_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 import {switchToChannel} from '../../../channels/mark_as_unread/helpers';

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/todo_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 describe('channels > slash command > todo', {testIsolation: true}, () => {

e2e-tests/playwright/playwright.config.ts

@@ -12,6 +12,7 @@ export default defineConfig({
     globalSetup: require.resolve('./global_setup'),
     forbidOnly: testConfig.isCI,
     outputDir: './results/tests',
+    retries: testConfig.isCI ? 2 : 0,
     testDir: 'tests',
     timeout: duration.one_min,
     workers: testConfig.workers,

e2e-tests/playwright/tests/accessibility/channels/intro_channel.spec.ts

@@ -3,7 +3,7 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('Base channel accessibility', async ({pw, pages, axe}) => {
+test.fixme('Base channel accessibility', async ({pw, pages, axe}) => {
     // # Create and sign in a new user
     const {user} = await pw.initSetup();
 

e2e-tests/playwright/tests/functional/channels/gif_picker/gif_picker_search.spec.ts

@@ -3,10 +3,9 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox', async ({
-    pw,
-    pages,
-}) => {
+test.fixme(
+    'MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox',
+    async ({pw, pages}) => {
         const {user} = await pw.initSetup();
 
         // # Log in as a user in new browser context
@@ -39,12 +38,12 @@ test('MM-T5445 Should search, select and post correct Gif when Gif picker is ope
         const lastPost = await channelPage.centerView.getLastPost();
         await lastPost.toBeVisible();
         await expect(lastPost.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
-});
+    },
+);
 
-test('MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox', async ({
-    pw,
-    pages,
-}) => {
+test.fixme(
+    'MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox',
+    async ({pw, pages}) => {
         const {user} = await pw.initSetup();
 
         // # Log in as a user in new browser context
@@ -94,4 +93,5 @@ test('MM-T5446 Should search, select and post correct Gif when Gif picker is ope
         const lastPostInRHS = await sidebarRight.getLastPost();
         await lastPostInRHS.toBeVisible();
         await expect(lastPostInRHS.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
-});
+    },
+);

e2e-tests/playwright/tests/visual/channels/intro_channel.spec.ts

@@ -3,7 +3,7 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
+test.fixme('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
     // Create and sign in a new user
     const {user} = await pw.initSetup();