Enable Cypress and Playwright retries (#29056)

* Enable Cypress and Playwright retries
* Improve E2E commit status messages
* Demote unstable testcases, increase task timeout

.github/workflows/e2e-tests-ci-template.yml

@@ -453,7 +453,11 @@ jobs:
           fi
           export TOTAL_SPECS=$(( PASSED + FAILED ))
           export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
-          export COMMIT_STATUS_MESSAGE="Completed with pass rate: ${PASS_RATE}% (plus ${FAILED_EXPECTED} unrelated failures)"
+          if [ "$FAILED" = "0" ]; then
+            export COMMIT_STATUS_MESSAGE="All test cases passed"
+          else
+            export COMMIT_STATUS_MESSAGE="${FAILED} test cases failed. Please check the workflow logs"
+          fi
           echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
           echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
           echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT

e2e-tests/.ci/server.run_playwright.sh

@@ -43,7 +43,7 @@ ${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/p
 # Documentation on the results.json file: https://playwright.dev/docs/api/class-testcase#test-case-expected-status
 # NB: the following line is needed only for compatibility reasons, to support RollingRelease tests for versions prior to v10.1.0
 #     It can be removed after releases <=v10.0.x are phased out
-mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json || true
+mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json 2>/dev/null || true
 jq -f /dev/stdin ../playwright/results/results.json > ../playwright/results/summary.json <<EOF
 {
   passed: .stats.expected,

e2e-tests/cypress/cypress.config.ts

@@ -9,8 +9,9 @@ export default defineConfig({
     downloadsFolder: 'tests/downloads',
     fixturesFolder: 'tests/fixtures',
     numTestsKeptInMemory: 0,
+    retries: 2,
     screenshotsFolder: 'tests/screenshots',
-    taskTimeout: 20000,
+    taskTimeout: 60000,
     video: true,
     videoCompression: true,
     videosFolder: 'tests/videos',

e2e-tests/cypress/tests/integration/channels/notifications/direct_messages_do_not_add_indicator_spec.js

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @notifications
 
 describe('Notifications', () => {

e2e-tests/cypress/tests/integration/channels/performance/channel_switch_spec.js

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @channel
 
 import {measurePerformance} from './utils.js';

e2e-tests/cypress/tests/integration/playbooks/channels/rhs/status_update_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 /* eslint-disable no-only-tests/no-only-tests */

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/commands_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 import {switchToChannel} from '../../../channels/mark_as_unread/helpers';

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/todo_spec.js

@@ -6,7 +6,6 @@
 // - [*] indicates an assertion (e.g. * Check the title)
 // ***************************************************************
 
-// Stage: @prod
 // Group: @playbooks
 
 describe('channels > slash command > todo', {testIsolation: true}, () => {

e2e-tests/playwright/playwright.config.ts

@@ -12,6 +12,7 @@ export default defineConfig({
     globalSetup: require.resolve('./global_setup'),
     forbidOnly: testConfig.isCI,
     outputDir: './results/tests',
+    retries: testConfig.isCI ? 2 : 0,
     testDir: 'tests',
     timeout: duration.one_min,
     workers: testConfig.workers,

e2e-tests/playwright/tests/accessibility/channels/intro_channel.spec.ts

@@ -3,7 +3,7 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('Base channel accessibility', async ({pw, pages, axe}) => {
+test.fixme('Base channel accessibility', async ({pw, pages, axe}) => {
     // # Create and sign in a new user
     const {user} = await pw.initSetup();
 

e2e-tests/playwright/tests/functional/channels/gif_picker/gif_picker_search.spec.ts

@@ -3,95 +3,95 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox', async ({
+test.fixme(
-    pw,
+    'MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox',
-    pages,
+    async ({pw, pages}) => {
-}) => {
+        const {user} = await pw.initSetup();
-    const {user} = await pw.initSetup();
 
-    // # Log in as a user in new browser context
+        // # Log in as a user in new browser context
-    const {page} = await pw.testBrowser.login(user);
+        const {page} = await pw.testBrowser.login(user);
 
-    // # Visit default channel page
+        // # Visit default channel page
-    const channelPage = new pages.ChannelsPage(page);
+        const channelPage = new pages.ChannelsPage(page);
-    await channelPage.goto();
+        await channelPage.goto();
-    await channelPage.toBeVisible();
+        await channelPage.toBeVisible();
 
-    // # Open emoji/gif picker
+        // # Open emoji/gif picker
-    await channelPage.centerView.postCreate.openEmojiPicker();
+        await channelPage.centerView.postCreate.openEmojiPicker();
-    await channelPage.emojiGifPickerPopup.toBeVisible();
+        await channelPage.emojiGifPickerPopup.toBeVisible();
 
-    // # Open gif tab
+        // # Open gif tab
-    await channelPage.emojiGifPickerPopup.openGifTab();
+        await channelPage.emojiGifPickerPopup.openGifTab();
 
-    // # Search for gif
+        // # Search for gif
-    await channelPage.emojiGifPickerPopup.searchGif('hello');
+        await channelPage.emojiGifPickerPopup.searchGif('hello');
 
-    // # Select the first gif
+        // # Select the first gif
-    const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
+        const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
-        await channelPage.emojiGifPickerPopup.getNthGif(0);
+            await channelPage.emojiGifPickerPopup.getNthGif(0);
-    await firstSearchGifResult.click();
+        await firstSearchGifResult.click();
 
-    // # Send the selected gif as a message
+        // # Send the selected gif as a message
-    await channelPage.centerView.postCreate.sendMessage();
+        await channelPage.centerView.postCreate.sendMessage();
 
-    // * Verify that last message has the gif
+        // * Verify that last message has the gif
-    const lastPost = await channelPage.centerView.getLastPost();
+        const lastPost = await channelPage.centerView.getLastPost();
-    await lastPost.toBeVisible();
+        await lastPost.toBeVisible();
-    await expect(lastPost.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
+        await expect(lastPost.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
-});
+    },
+);
 
-test('MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox', async ({
+test.fixme(
-    pw,
+    'MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox',
-    pages,
+    async ({pw, pages}) => {
-}) => {
+        const {user} = await pw.initSetup();
-    const {user} = await pw.initSetup();
 
-    // # Log in as a user in new browser context
+        // # Log in as a user in new browser context
-    const {page} = await pw.testBrowser.login(user);
+        const {page} = await pw.testBrowser.login(user);
 
-    // # Visit default channel page
+        // # Visit default channel page
-    const channelPage = new pages.ChannelsPage(page);
+        const channelPage = new pages.ChannelsPage(page);
-    await channelPage.goto();
+        await channelPage.goto();
-    await channelPage.toBeVisible();
+        await channelPage.toBeVisible();
 
-    // # Send a message
+        // # Send a message
-    await channelPage.centerView.postCreate.postMessage('Message to open RHS');
+        await channelPage.centerView.postCreate.postMessage('Message to open RHS');
 
-    // # Open the last post sent in RHS
+        // # Open the last post sent in RHS
-    const lastPost = await channelPage.centerView.getLastPost();
+        const lastPost = await channelPage.centerView.getLastPost();
-    await lastPost.hover();
+        await lastPost.hover();
-    await lastPost.postMenu.toBeVisible();
+        await lastPost.postMenu.toBeVisible();
-    await lastPost.postMenu.reply();
+        await lastPost.postMenu.reply();
 
-    const sidebarRight = channelPage.sidebarRight;
+        const sidebarRight = channelPage.sidebarRight;
-    await sidebarRight.toBeVisible();
+        await sidebarRight.toBeVisible();
 
-    // # Send a message in the thread
+        // # Send a message in the thread
-    await sidebarRight.postCreate.toBeVisible();
+        await sidebarRight.postCreate.toBeVisible();
-    await sidebarRight.postCreate.writeMessage('Replying to a thread');
+        await sidebarRight.postCreate.writeMessage('Replying to a thread');
-    await sidebarRight.postCreate.sendMessage();
+        await sidebarRight.postCreate.sendMessage();
 
-    // # Open emoji/gif picker
+        // # Open emoji/gif picker
-    await sidebarRight.postCreate.openEmojiPicker();
+        await sidebarRight.postCreate.openEmojiPicker();
-    await channelPage.emojiGifPickerPopup.toBeVisible();
+        await channelPage.emojiGifPickerPopup.toBeVisible();
 
-    // # Open gif tab
+        // # Open gif tab
-    await channelPage.emojiGifPickerPopup.openGifTab();
+        await channelPage.emojiGifPickerPopup.openGifTab();
 
-    // # Search for gif
+        // # Search for gif
-    await channelPage.emojiGifPickerPopup.searchGif('hello');
+        await channelPage.emojiGifPickerPopup.searchGif('hello');
 
-    // # Select the first gif
+        // # Select the first gif
-    const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
+        const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
-        await channelPage.emojiGifPickerPopup.getNthGif(0);
+            await channelPage.emojiGifPickerPopup.getNthGif(0);
-    await firstSearchGifResult.click();
+        await firstSearchGifResult.click();
 
-    // # Send the selected gif as a message in the thread
+        // # Send the selected gif as a message in the thread
-    await sidebarRight.postCreate.sendMessage();
+        await sidebarRight.postCreate.sendMessage();
 
-    // * Verify that last message has the gif
+        // * Verify that last message has the gif
-    const lastPostInRHS = await sidebarRight.getLastPost();
+        const lastPostInRHS = await sidebarRight.getLastPost();
-    await lastPostInRHS.toBeVisible();
+        await lastPostInRHS.toBeVisible();
-    await expect(lastPostInRHS.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
+        await expect(lastPostInRHS.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
-});
+    },
+);

e2e-tests/playwright/tests/visual/channels/intro_channel.spec.ts

@@ -3,7 +3,7 @@
 
 import {expect, test} from '@e2e-support/test_fixture';
 
-test('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
+test.fixme('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
     // Create and sign in a new user
     const {user} = await pw.initSetup();