Fix E2E test reporting message for webhook (#29304)

* Fix testcases failing  Webpack compilation
* Fix E2E test report message for webhook
* Demote more unstable tests
* Fix playwright reporting message, better test demotion

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @channel
 
 describe('Archived channels', () => {
@@ -24,7 +23,7 @@ describe('Archived channels', () => {
         });
     });
 
-    it.skip('MM-T1721 Archive channel posts menu should have copy link and reply options', () => {
+    it('MM-T1721 Archive channel posts menu should have copy link and reply options', () => {
         // # Click to add a channel description
         //   Note that it is invisible until the mouse hovers it, which is unfeasible in Cypress
         //   https://docs.cypress.io/api/commands/hover

e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.ts

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @bot_accounts
 
 import {Bot} from '@mattermost/types/bots';
@@ -46,7 +45,7 @@ describe('Bot display name', () => {
         });
     });
 
-    it.skip('MM-T1813 Display name for bots stays current', () => {
+    it('MM-T1813 Display name for bots stays current', () => {
         cy.makeClient({user: otherSysadmin}).then((client) => {
             // # Create a bot and get bot user id
             cy.apiCreateBot().then(({bot}) => {

e2e-tests/cypress/tests/integration/channels/collapsed_reply_threads/channel_notifications_spec.ts

@@ -7,7 +7,6 @@
 // - Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @collapsed_reply_threads
 
 import {Team} from '@mattermost/types/teams';

e2e-tests/cypress/tests/integration/channels/enterprise/system_console/about/edition_and_license_spec.js

@@ -13,7 +13,7 @@
 import * as TIMEOUTS from '../../../../../fixtures/timeouts';
 import {getAdminAccount} from '../../../../../support/env';
 
-import {promoteToChannelOrTeamAdmin} from '../channel_moderation/helpers.js';
+import {promoteToChannelOrTeamAdmin} from '../channel_moderation/helpers.ts';
 
 describe('System console', () => {
     const sysadmin = getAdminAccount();

e2e-tests/cypress/tests/integration/channels/notifications/desktop_notifications_1_spec.js

@@ -115,7 +115,7 @@ describe('Desktop notifications', () => {
         });
     });
 
-    it('MM-T495 Desktop Notifications - Can set to DND and no notification fires on DM', () => {
+    it.skip('MM-T495 Desktop Notifications - Can set to DND and no notification fires on DM', () => {
         cy.apiCreateDirectChannel([otherUser.id, testUser.id]).then(({channel}) => {
             // # Ensure notifications are set up to fire a desktop notification if you receive a DM
             cy.apiPatchUser(testUser.id, {notify_props: {...testUser.notify_props, desktop: 'all'}});

e2e-tests/cypress/tests/integration/channels/team_settings/manage_members_spec.js

@@ -12,7 +12,7 @@
 
 import {
     promoteToChannelOrTeamAdmin,
-} from '../enterprise/system_console/channel_moderation/helpers.js';
+} from '../enterprise/system_console/channel_moderation/helpers.ts';
 
 describe('Manage Members', () => {
     let testTeam;

e2e-tests/cypress/tests/integration/channels/toast/new_messages_toast_spec.js

@@ -7,7 +7,6 @@
 // Use element ID when selecting an element. Create one if none.
 // ***************************************************************
 
-// Stage: @prod
 // Group: @channels @toast
 
 import {getRandomId} from '../../../utils';

e2e-tests/cypress/utils/constants.js

@@ -3,8 +3,10 @@
 
 const RESULTS_DIR = 'results';
 const MOCHAWESOME_REPORT_DIR = 'results/mochawesome-report';
+const AD_CYCLE_FILE = 'results/ad_cycle.json';
 
 module.exports = {
     MOCHAWESOME_REPORT_DIR,
     RESULTS_DIR,
+    AD_CYCLE_FILE,
 };

e2e-tests/cypress/utils/report.js

@@ -9,7 +9,7 @@ const dayjs = require('dayjs');
 const duration = require('dayjs/plugin/duration');
 dayjs.extend(duration);
 
-const {MOCHAWESOME_REPORT_DIR} = require('./constants');
+const {MOCHAWESOME_REPORT_DIR, AD_CYCLE_FILE} = require('./constants');
 
 const MAX_FAILED_TITLES = 5;
 
@@ -76,6 +76,16 @@ function generateShortSummary(report) {
     const failedFullTitles = tests.filter((t) => t.fail).map((t) => t.fullTitle);
     const statsFieldValue = generateStatsFieldValue(stats, failedFullTitles);
 
+    // If AD Cycle file is found, we have data from the Automation Dashboard available
+    // We are able to override the run stats with enriched information
+    const adCycle = readJsonFromFile(AD_CYCLE_FILE);
+    if (!(adCycle instanceof Error)) {
+        stats.passes = adCycle.pass;
+        stats.failures = adCycle.fail;
+        stats.tests = adCycle.pass + adCycle.fail;
+        stats.passPercent = 100 * (stats.passes / stats.tests);
+    }
+
     return {
         stats,
         statsFieldValue,

e2e-tests/playwright/report.webhookgen.js

@@ -13,7 +13,7 @@ const summaryFile = 'results/summary.json';
 const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
 const summary = JSON.parse(fs.readFileSync(summaryFile, 'utf8'));
 const passRate = summary.passed / (summary.passed + summary.failed);
-const totalSpecs = summary.passed + summary.failed + summary.failed_expected;
+const totalSpecs = summary.passed + summary.failed;
 const playwrightVersion = results.config.version;
 const playwrightDuration = dayjs.duration(results.stats.duration, 'seconds').format('HH:mm:ss');
 

e2e-tests/playwright/tests/functional/channels/scheduled_drafts/create_scheduled_draft.spec.ts

@@ -3,7 +3,7 @@ import {test} from '@e2e-support/test_fixture';
 import {ChannelsPage, ScheduledDraftPage} from '@e2e-support/ui/pages';
 import {duration, wait} from '@e2e-support/util';
 
-test('MM-T5643_1 should create a scheduled message from a channel', async ({pw, pages}) => {
+test.fixme('MM-T5643_1 should create a scheduled message from a channel', async ({pw, pages}) => {
     test.setTimeout(120000);
 
     const draftMessage = 'Scheduled Draft';
@@ -38,7 +38,7 @@ test('MM-T5643_1 should create a scheduled message from a channel', async ({pw,
     await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage);
 });
 
-test('MM-T5643_6 should create a scheduled message under a thread post ', async ({pw, pages}) => {
+test.fixme('MM-T5643_6 should create a scheduled message under a thread post ', async ({pw, pages}) => {
     test.setTimeout(120000);
 
     const draftMessage = 'Scheduled Threaded Message';
@@ -113,7 +113,7 @@ test('MM-T5643_6 should create a scheduled message under a thread post ', async
     await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage);
 });
 
-test('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => {
+test.fixme('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => {
     const draftMessage = 'Scheduled Draft';
     await pw.skipIfNoLicense();
 
@@ -147,7 +147,7 @@ test('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => {
     await compareMessageTimestamps(rescheduledDraftChannelInfo, scheduledDraftPageInfo, scheduledDraftPage);
 });
 
-test('MM-T5645 should delete a scheduled message', async ({pw, pages}) => {
+test.fixme('MM-T5645 should delete a scheduled message', async ({pw, pages}) => {
     const draftMessage = 'Scheduled Draft';
     await pw.skipIfNoLicense();
 
@@ -168,7 +168,7 @@ test('MM-T5645 should delete a scheduled message', async ({pw, pages}) => {
     await expect(scheduledDraftPage.noscheduledDraftIcon).toBeVisible();
 });
 
-test('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages}) => {
+test.fixme('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages}) => {
     const draftMessage = 'Scheduled Draft';
     await pw.skipIfNoLicense();
 
@@ -193,7 +193,7 @@ test('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages
     await expect(await channelPage.getLastPost()).toHaveText(draftMessage);
 });
 
-test('MM-T5643_3 should create a scheduled message from a DM', async ({pw, pages}) => {
+test.fixme('MM-T5643_3 should create a scheduled message from a DM', async ({pw, pages}) => {
     test.setTimeout(120000);
 
     const draftMessage = 'Scheduled Draft';
@@ -265,7 +265,7 @@ test('MM-T5648 should create a draft and then schedule it', async ({pw, pages})
     await scheduledDraftPage.assertscheduledDraftBody(draftMessage);
 });
 
-test('MM-T5644 should edit scheduled message', async ({pw, pages}) => {
+test.fixme('MM-T5644 should edit scheduled message', async ({pw, pages}) => {
     test.setTimeout(120000);
 
     const draftMessage = 'Scheduled Draft';
@@ -303,7 +303,7 @@ test('MM-T5644 should edit scheduled message', async ({pw, pages}) => {
     await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage);
 });
 
-test('MM-T5650 should copy scheduled message', async ({pw, pages, browserName}) => {
+test.fixme('MM-T5650 should copy scheduled message', async ({pw, pages, browserName}) => {
     test.setTimeout(120000);
 
     // Skip this test in Firefox clipboard permissions are not supported