From cf5b4610b11f0ee98c4af63e0b1fd192d253aef2 Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Mon, 18 Nov 2024 09:49:17 +0100 Subject: [PATCH] Fix E2E test reporting message for webhook (#29304) * Fix testcases failing Webpack compilation * Fix E2E test report message for webhook * Demote more unstable tests * Fix playwright reporting message, better test demotion --------- Co-authored-by: Mattermost Build --- .../channels/archived_channel/post_menu_spec.ts | 3 +-- .../channels/bot_accounts/display_name_spec.ts | 3 +-- .../channel_notifications_spec.ts | 1 - .../about/edition_and_license_spec.js | 2 +- .../desktop_notifications_1_spec.js | 2 +- .../team_settings/manage_members_spec.js | 2 +- .../channels/toast/new_messages_toast_spec.js | 1 - e2e-tests/cypress/utils/constants.js | 2 ++ e2e-tests/cypress/utils/report.js | 12 +++++++++++- e2e-tests/playwright/report.webhookgen.js | 2 +- .../create_scheduled_draft.spec.ts | 16 ++++++++-------- 11 files changed, 27 insertions(+), 19 deletions(-) diff --git a/e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts b/e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts index 8dc08ac77b..896b59fdcd 100644 --- a/e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts +++ b/e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts @@ -7,7 +7,6 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Stage: @prod // Group: @channels @channel describe('Archived channels', () => { @@ -24,7 +23,7 @@ describe('Archived channels', () => { }); }); - it.skip('MM-T1721 Archive channel posts menu should have copy link and reply options', () => { + it('MM-T1721 Archive channel posts menu should have copy link and reply options', () => { // # Click to add a channel description // Note that it is invisible until the mouse hovers it, which is unfeasible in Cypress // https://docs.cypress.io/api/commands/hover diff --git a/e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.ts b/e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.ts index 38f81411db..334ee6c9a5 100644 --- a/e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.ts +++ b/e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.ts @@ -7,7 +7,6 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Stage: @prod // Group: @channels @bot_accounts import {Bot} from '@mattermost/types/bots'; @@ -46,7 +45,7 @@ describe('Bot display name', () => { }); }); - it.skip('MM-T1813 Display name for bots stays current', () => { + it('MM-T1813 Display name for bots stays current', () => { cy.makeClient({user: otherSysadmin}).then((client) => { // # Create a bot and get bot user id cy.apiCreateBot().then(({bot}) => { diff --git a/e2e-tests/cypress/tests/integration/channels/collapsed_reply_threads/channel_notifications_spec.ts b/e2e-tests/cypress/tests/integration/channels/collapsed_reply_threads/channel_notifications_spec.ts index 474234ad34..266c630ac2 100644 --- a/e2e-tests/cypress/tests/integration/channels/collapsed_reply_threads/channel_notifications_spec.ts +++ b/e2e-tests/cypress/tests/integration/channels/collapsed_reply_threads/channel_notifications_spec.ts @@ -7,7 +7,6 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Stage: @prod // Group: @channels @collapsed_reply_threads import {Team} from '@mattermost/types/teams'; diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/about/edition_and_license_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/about/edition_and_license_spec.js index 8ebba6a80c..8b8068d5be 100644 --- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/about/edition_and_license_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/about/edition_and_license_spec.js @@ -13,7 +13,7 @@ import * as TIMEOUTS from '../../../../../fixtures/timeouts'; import {getAdminAccount} from '../../../../../support/env'; -import {promoteToChannelOrTeamAdmin} from '../channel_moderation/helpers.js'; +import {promoteToChannelOrTeamAdmin} from '../channel_moderation/helpers.ts'; describe('System console', () => { const sysadmin = getAdminAccount(); diff --git a/e2e-tests/cypress/tests/integration/channels/notifications/desktop_notifications_1_spec.js b/e2e-tests/cypress/tests/integration/channels/notifications/desktop_notifications_1_spec.js index e823387c89..8d6e0ea3b6 100644 --- a/e2e-tests/cypress/tests/integration/channels/notifications/desktop_notifications_1_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/notifications/desktop_notifications_1_spec.js @@ -115,7 +115,7 @@ describe('Desktop notifications', () => { }); }); - it('MM-T495 Desktop Notifications - Can set to DND and no notification fires on DM', () => { + it.skip('MM-T495 Desktop Notifications - Can set to DND and no notification fires on DM', () => { cy.apiCreateDirectChannel([otherUser.id, testUser.id]).then(({channel}) => { // # Ensure notifications are set up to fire a desktop notification if you receive a DM cy.apiPatchUser(testUser.id, {notify_props: {...testUser.notify_props, desktop: 'all'}}); diff --git a/e2e-tests/cypress/tests/integration/channels/team_settings/manage_members_spec.js b/e2e-tests/cypress/tests/integration/channels/team_settings/manage_members_spec.js index 0cff4b281b..5de377c63c 100644 --- a/e2e-tests/cypress/tests/integration/channels/team_settings/manage_members_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/team_settings/manage_members_spec.js @@ -12,7 +12,7 @@ import { promoteToChannelOrTeamAdmin, -} from '../enterprise/system_console/channel_moderation/helpers.js'; +} from '../enterprise/system_console/channel_moderation/helpers.ts'; describe('Manage Members', () => { let testTeam; diff --git a/e2e-tests/cypress/tests/integration/channels/toast/new_messages_toast_spec.js b/e2e-tests/cypress/tests/integration/channels/toast/new_messages_toast_spec.js index 27c2197a0c..0cda6d0a0b 100644 --- a/e2e-tests/cypress/tests/integration/channels/toast/new_messages_toast_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/toast/new_messages_toast_spec.js @@ -7,7 +7,6 @@ // Use element ID when selecting an element. Create one if none. // *************************************************************** -// Stage: @prod // Group: @channels @toast import {getRandomId} from '../../../utils'; diff --git a/e2e-tests/cypress/utils/constants.js b/e2e-tests/cypress/utils/constants.js index 491d1f78fb..e20fbb2102 100644 --- a/e2e-tests/cypress/utils/constants.js +++ b/e2e-tests/cypress/utils/constants.js @@ -3,8 +3,10 @@ const RESULTS_DIR = 'results'; const MOCHAWESOME_REPORT_DIR = 'results/mochawesome-report'; +const AD_CYCLE_FILE = 'results/ad_cycle.json'; module.exports = { MOCHAWESOME_REPORT_DIR, RESULTS_DIR, + AD_CYCLE_FILE, }; diff --git a/e2e-tests/cypress/utils/report.js b/e2e-tests/cypress/utils/report.js index 5589b11254..bcd15d5334 100644 --- a/e2e-tests/cypress/utils/report.js +++ b/e2e-tests/cypress/utils/report.js @@ -9,7 +9,7 @@ const dayjs = require('dayjs'); const duration = require('dayjs/plugin/duration'); dayjs.extend(duration); -const {MOCHAWESOME_REPORT_DIR} = require('./constants'); +const {MOCHAWESOME_REPORT_DIR, AD_CYCLE_FILE} = require('./constants'); const MAX_FAILED_TITLES = 5; @@ -76,6 +76,16 @@ function generateShortSummary(report) { const failedFullTitles = tests.filter((t) => t.fail).map((t) => t.fullTitle); const statsFieldValue = generateStatsFieldValue(stats, failedFullTitles); + // If AD Cycle file is found, we have data from the Automation Dashboard available + // We are able to override the run stats with enriched information + const adCycle = readJsonFromFile(AD_CYCLE_FILE); + if (!(adCycle instanceof Error)) { + stats.passes = adCycle.pass; + stats.failures = adCycle.fail; + stats.tests = adCycle.pass + adCycle.fail; + stats.passPercent = 100 * (stats.passes / stats.tests); + } + return { stats, statsFieldValue, diff --git a/e2e-tests/playwright/report.webhookgen.js b/e2e-tests/playwright/report.webhookgen.js index d59d15f92a..a7f46f5925 100755 --- a/e2e-tests/playwright/report.webhookgen.js +++ b/e2e-tests/playwright/report.webhookgen.js @@ -13,7 +13,7 @@ const summaryFile = 'results/summary.json'; const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8')); const summary = JSON.parse(fs.readFileSync(summaryFile, 'utf8')); const passRate = summary.passed / (summary.passed + summary.failed); -const totalSpecs = summary.passed + summary.failed + summary.failed_expected; +const totalSpecs = summary.passed + summary.failed; const playwrightVersion = results.config.version; const playwrightDuration = dayjs.duration(results.stats.duration, 'seconds').format('HH:mm:ss'); diff --git a/e2e-tests/playwright/tests/functional/channels/scheduled_drafts/create_scheduled_draft.spec.ts b/e2e-tests/playwright/tests/functional/channels/scheduled_drafts/create_scheduled_draft.spec.ts index 7e02299ce3..4269941bc0 100644 --- a/e2e-tests/playwright/tests/functional/channels/scheduled_drafts/create_scheduled_draft.spec.ts +++ b/e2e-tests/playwright/tests/functional/channels/scheduled_drafts/create_scheduled_draft.spec.ts @@ -3,7 +3,7 @@ import {test} from '@e2e-support/test_fixture'; import {ChannelsPage, ScheduledDraftPage} from '@e2e-support/ui/pages'; import {duration, wait} from '@e2e-support/util'; -test('MM-T5643_1 should create a scheduled message from a channel', async ({pw, pages}) => { +test.fixme('MM-T5643_1 should create a scheduled message from a channel', async ({pw, pages}) => { test.setTimeout(120000); const draftMessage = 'Scheduled Draft'; @@ -38,7 +38,7 @@ test('MM-T5643_1 should create a scheduled message from a channel', async ({pw, await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage); }); -test('MM-T5643_6 should create a scheduled message under a thread post ', async ({pw, pages}) => { +test.fixme('MM-T5643_6 should create a scheduled message under a thread post ', async ({pw, pages}) => { test.setTimeout(120000); const draftMessage = 'Scheduled Threaded Message'; @@ -113,7 +113,7 @@ test('MM-T5643_6 should create a scheduled message under a thread post ', async await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage); }); -test('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => { +test.fixme('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => { const draftMessage = 'Scheduled Draft'; await pw.skipIfNoLicense(); @@ -147,7 +147,7 @@ test('MM-T5644 should rechedule a scheduled message', async ({pw, pages}) => { await compareMessageTimestamps(rescheduledDraftChannelInfo, scheduledDraftPageInfo, scheduledDraftPage); }); -test('MM-T5645 should delete a scheduled message', async ({pw, pages}) => { +test.fixme('MM-T5645 should delete a scheduled message', async ({pw, pages}) => { const draftMessage = 'Scheduled Draft'; await pw.skipIfNoLicense(); @@ -168,7 +168,7 @@ test('MM-T5645 should delete a scheduled message', async ({pw, pages}) => { await expect(scheduledDraftPage.noscheduledDraftIcon).toBeVisible(); }); -test('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages}) => { +test.fixme('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages}) => { const draftMessage = 'Scheduled Draft'; await pw.skipIfNoLicense(); @@ -193,7 +193,7 @@ test('MM-T5643_9 should send a scheduled message immediately', async ({pw, pages await expect(await channelPage.getLastPost()).toHaveText(draftMessage); }); -test('MM-T5643_3 should create a scheduled message from a DM', async ({pw, pages}) => { +test.fixme('MM-T5643_3 should create a scheduled message from a DM', async ({pw, pages}) => { test.setTimeout(120000); const draftMessage = 'Scheduled Draft'; @@ -265,7 +265,7 @@ test('MM-T5648 should create a draft and then schedule it', async ({pw, pages}) await scheduledDraftPage.assertscheduledDraftBody(draftMessage); }); -test('MM-T5644 should edit scheduled message', async ({pw, pages}) => { +test.fixme('MM-T5644 should edit scheduled message', async ({pw, pages}) => { test.setTimeout(120000); const draftMessage = 'Scheduled Draft'; @@ -303,7 +303,7 @@ test('MM-T5644 should edit scheduled message', async ({pw, pages}) => { await verifyNoscheduledDraftsPending(channelPage, team, scheduledDraftPage, draftMessage); }); -test('MM-T5650 should copy scheduled message', async ({pw, pages, browserName}) => { +test.fixme('MM-T5650 should copy scheduled message', async ({pw, pages, browserName}) => { test.setTimeout(120000); // Skip this test in Firefox clipboard permissions are not supported