Support E2E stress tests in run_tests.js (#29682)

* [skip ci] Adapt run_tests.js for stress testing
* [skip ci] Make channel_read_after_permalink_spec.js retry-able
* [skip ci] Fix ui_commands.ts fix
* [skip ci] Improve postMessageAndWait
* [skip ci] Fix unread_on_public_channel_spec.js
* [skip ci] Demote unread_on_public_channel_spec.js
* Update e2e-tests/README.md

Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

* [skip ci] Fix mesage_reply_bot_post_spec.js

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

e2e-tests/.ci/.e2erc

@@ -100,6 +100,7 @@ case "${TEST:-$TEST_DEFAULT}" in
   playwright )
     export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;;
   * )
+    export TEST_FILTER_DEFAULT='' ;;
 esac
 # OS specific defaults overrides
 case $MME2E_OSTYPE in

e2e-tests/.ci/server.generate.sh

@@ -62,6 +62,7 @@ services:
       MM_SERVICEENVIRONMENT: "test"
       MM_FEATUREFLAGS_MOVETHREADSENABLED: "true"
       MM_LOGSETTINGS_ENABLEDIAGNOSTICS: "false"
+      MM_LOGSETTINGS_CONSOLELEVEL: "DEBUG"
     network_mode: host
     depends_on:
 $(for service in $ENABLED_DOCKER_SERVICES; do

e2e-tests/README.md

@@ -61,3 +61,14 @@ Notes:
 
 ##### For code changes:
 * `make fmt-ci` to format and check yaml files and shell scripts.
+
+##### For test stressing an E2E testcase
+
+For Cypress:
+1. Enter the `cypress/` subdirectory
+2. Identify which test files you want to run, and how many times each. For instance: suppose you want to run `create_a_team_spec.js` and `demoted_user_spec.js` (which you can locate with the `find` command, under `cypress/tests/`), each run 3 times
+3. Run the chosen testcases the desired amount of times: `node run_tests.js --include-file=create_a_team_spec.js,demoted_user_spec.js --invert --stress-test-count=3`
+  * Your system needs to be setup for Cypress usage, to be able to run this command. Refer to the [E2E testing developer documentation](https://developers.mattermost.com/contribute/more-info/webapp/e2e-testing/) for this.
+4. The `cypress/results/testPasses.json` file will count, for each of the testfiles, how many times it was run, and how many times each of the testcases contained in it passed. If the attempts and passes numbers do not match, that specific testcase may be flaky.
+
+For Playwright: WIP

e2e-tests/cypress/run_tests.js

@@ -32,6 +32,8 @@
  *      Exclude spec files with matching directory or filename pattern. Uses `find` command under the hood. It can be of multiple values separated by comma.
  *      E.g. "--exclude-file='channel'" will exclude files recursively under `channel` directory/s.
  *      E.g. "--exclude-file='*channel*'" will exclude files and files under directory/s recursively that matches the name with `*channel*`.
+ *   --stress-test-count=[number of repeated executions]
+ *      The amount of times to run each spec file for. If greater than 1, report how the percentage of successes over total runs for each spec.
 
  *
  * Environment:
@@ -83,8 +85,11 @@ async function runTests() {
     const browser = BROWSER || 'chrome';
     const headless = typeof HEADLESS === 'undefined' ? true : HEADLESS === 'true';
     const platform = os.platform();
+    const stressTestCount = argv.stressTestCount > 1 ? argv.stressTestCount : 1;
+    const testPasses = {};
 
-    const {sortedFiles} = getSortedTestFiles(platform, browser, headless);
+    const sortedFilesObject = getSortedTestFiles(platform, browser, headless);
+    const sortedFiles = sortedFilesObject.sortedFiles.flatMap((el) => Array(stressTestCount).fill(el));
     const numberOfTestFiles = sortedFiles.length;
 
     if (!numberOfTestFiles) {
@@ -102,6 +107,13 @@ async function runTests() {
         printMessage(sortedFiles, i, j + 1, count);
 
         const testFile = sortedFiles[i];
+        var testFileAttempt = 1;
+        if (testFile in testPasses === false) {
+            testPasses[testFile] = {attempt: 1};
+        } else {
+            testPasses[testFile].attempt += 1;
+            testFileAttempt = testPasses[testFile].attempt;
+        }
 
         const result = await cypress.run({
             browser,
@@ -111,6 +123,7 @@ async function runTests() {
                 screenshotsFolder: `${MOCHAWESOME_REPORT_DIR}/screenshots`,
                 videosFolder: `${MOCHAWESOME_REPORT_DIR}/videos`,
                 trashAssetsBeforeRuns: false,
+                retries: stressTestCount > 1 ? 0 : 2,
             },
             env: {
                 firstTest: j === 0,
@@ -135,11 +148,22 @@ async function runTests() {
                         headless,
                         branch: BRANCH,
                         buildId: BUILD_ID,
+                        testFileAttempt,
                     },
                 },
             },
         });
 
+        for (var testCase of result.runs[0].tests) {
+            var testCaseTitle = testCase.title.join(' - ');
+            if (testCaseTitle in testPasses[testFile] === false) {
+                testPasses[testFile][testCaseTitle] = 0;
+            }
+            if (testCase.state === 'passed') {
+                testPasses[testFile][testCaseTitle] += 1;
+            }
+        }
+
         // Write test environment details once only
         if (i === 0) {
             const environment = {
@@ -155,6 +179,7 @@ async function runTests() {
             writeJsonToFile(environment, 'environment.json', RESULTS_DIR);
         }
     }
+    writeJsonToFile(testPasses, 'testPasses.json', RESULTS_DIR);
 }
 
 function printMessage(testFiles, overallIndex, currentItem, lastItem) {

e2e-tests/cypress/tests/integration/channels/messaging/channel_read_after_permalink_spec.js

@@ -18,7 +18,7 @@ describe('Messaging', () => {
     let testUser;
     let otherUser;
 
-    before(() => {
+    beforeEach(() => {
         cy.apiInitSetup().then(({team, channel, user}) => {
             testTeam = team;
             testChannel = channel;

e2e-tests/cypress/tests/integration/channels/messaging/message_reply_bot_post_spec.js

@@ -144,7 +144,7 @@ describe('Messaging', () => {
                     cy.uiCloseRHS();
 
                     // * Verify that the reply is in the channel view with matching text
-                    cy.get(`#post_${replyId}`).within(() => {
+                    cy.get(`#post_${replyId}`).scrollIntoView().within(() => {
                         cy.findByTestId('post-link').should('be.visible').and('have.text', 'Commented on ' + bot.username + 'BOT\'s message: Hello message from ' + bot.username);
                         cy.get(`#postMessageText_${replyId}`).should('be.visible').and('have.text', replyMessage);
                     });

e2e-tests/cypress/tests/integration/channels/notifications/unread_on_public_channel_spec.js

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @notifications
 
 import * as TIMEOUTS from '../../../fixtures/timeouts';
@@ -18,7 +17,7 @@ describe('Notifications', () => {
     let testTeam;
 
     beforeEach(() => {
-        cy.apiInitSetup().then(({team, user}) => {
+        cy.apiAdminLogin().apiInitSetup().then(({team, user}) => {
             testUser = user;
             testTeam = team;
 
@@ -48,11 +47,6 @@ describe('Notifications', () => {
                     // # Go to test channel
                     cy.visit(`/${team.name}/channels/${channel.name}`);
 
-                    // # Scroll above the last few messages
-                    cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible').
-                        scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}).
-                        wait(TIMEOUTS.ONE_SEC);
-
                     // # scroll to the last channel
                     cy.get('#sidebar-left li').last().scrollIntoView();
                 });
@@ -66,7 +60,11 @@ describe('Notifications', () => {
 
         // # Get last post and reply a message
         cy.getLastPostId().then((postId) => {
+            // # Open the RHS reply form, then scroll above the last few messages
             cy.clickPostCommentIcon(postId);
+            cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible').
+                scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}).
+                wait(TIMEOUTS.ONE_SEC);
             const replyMessage = 'A reply to an older post';
             cy.postMessageReplyInRHS(replyMessage);
         });

e2e-tests/cypress/tests/support/ui_commands.ts

@@ -98,7 +98,7 @@ function postMessageAndWait(textboxSelector: string, message: string, isComment
         waitForCommentDraft(message);
     }
 
-    cy.get(textboxSelector).should('have.value', message).type('{enter}').wait(TIMEOUTS.HALF_SEC);
+    cy.get(textboxSelector).should('have.value', message).focus().type('{enter}').wait(TIMEOUTS.HALF_SEC);
 
     cy.get(textboxSelector).invoke('val').then((value: string) => {
         if (value.length > 0 && value === message) {
@@ -325,12 +325,12 @@ function clickPostHeaderItem(postId: string, location: string, item: string) {
     }
 
     if (postId) {
-        cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true});
-        cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${postId}`).click({force: true});
+        cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true}).
+            get(`#${location}_${item}_${postId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true});
     } else {
         cy.getLastPostId().then((lastPostId) => {
-            cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true});
-            cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${lastPostId}`).click({force: true});
+            cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true}).
+                get(`#${location}_${item}_${lastPostId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true});
         });
     }
 }