CLD-8395 Integrate playwright into E2E tests (#28492)

* Finalize Playwright E2E scripts integration, add videos
* Add suggestion by Saturn

Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

e2e-tests/playwright/.eslintrc.json

@@ -9,6 +9,10 @@
     "rules": {
         "@typescript-eslint/explicit-module-boundary-types": "off",
         "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-var-requires": "off",
         "no-console": "error"
+    },
+    "env": {
+        "node": true
     }
 }

e2e-tests/playwright/package-lock.json

@@ -14,6 +14,7 @@
                 "async-wait-until": "2.0.12",
                 "axe-core": "4.10.0",
                 "chalk": "4.1.2",
+                "dayjs": "^1.11.13",
                 "deepmerge": "4.3.1",
                 "dotenv": "16.4.5",
                 "form-data-encoder": "4.0.2",
@@ -1035,6 +1036,11 @@
                 "node": ">= 8"
             }
         },
+        "node_modules/dayjs": {
+            "version": "1.11.13",
+            "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz",
+            "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg=="
+        },
         "node_modules/debug": {
             "version": "4.3.4",
             "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3158,6 +3164,11 @@
                 "which": "^2.0.1"
             }
         },
+        "dayjs": {
+            "version": "1.11.13",
+            "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz",
+            "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg=="
+        },
         "debug": {
             "version": "4.3.4",
             "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",

e2e-tests/playwright/package.json

@@ -22,6 +22,7 @@
         "async-wait-until": "2.0.12",
         "axe-core": "4.10.0",
         "chalk": "4.1.2",
+        "dayjs": "1.11.13",
         "deepmerge": "4.3.1",
         "dotenv": "16.4.5",
         "form-data-encoder": "4.0.2",

e2e-tests/playwright/playwright.config.ts

@@ -6,12 +6,12 @@ import {defineConfig, devices} from '@playwright/test';
 import {duration} from '@e2e-support/util';
 import testConfig from '@e2e-test.config';
 
-const defaultOutputFolder = './playwright-report';
+const defaultOutputFolder = './results';
 
 export default defineConfig({
     globalSetup: require.resolve('./global_setup'),
     forbidOnly: testConfig.isCI,
-    outputDir: './test-results',
+    outputDir: './results/tests',
     testDir: 'tests',
     timeout: duration.one_min,
     workers: testConfig.workers,
@@ -40,7 +40,7 @@ export default defineConfig({
         screenshot: 'only-on-failure',
         timezoneId: 'America/Los_Angeles',
         trace: 'off',
-        video: 'on-first-retry',
+        video: 'retain-on-failure',
         actionTimeout: duration.half_min,
         storageState: {
             cookies: [],

e2e-tests/playwright/report.webhookgen.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const dayjs = require('dayjs');
+const duration = require('dayjs/plugin/duration');
+dayjs.extend(duration);
+
+const {TYPE, SERVER_TYPE, BRANCH, PULL_REQUEST, BUILD_ID, MM_ENV, MM_DOCKER_IMAGE, MM_DOCKER_TAG, RELEASE_DATE} =
+    process.env;
+
+const resultsFile = 'results/results.json';
+const summaryFile = 'results/summary.json';
+const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
+const summary = JSON.parse(fs.readFileSync(summaryFile, 'utf8'));
+const passRate = summary.passed / (summary.passed + summary.failed);
+const totalSpecs = summary.passed + summary.failed + summary.failed_expected;
+const playwrightVersion = results.config.version;
+const playwrightDuration = dayjs.duration(results.stats.duration, 'seconds').format('HH:mm:ss');
+
+function generateTitle() {
+    let dockerImageLink = '';
+    let releaseDate = '';
+    if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) {
+        dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
+    }
+    if (RELEASE_DATE) {
+        releaseDate = ` for ${RELEASE_DATE}`;
+    }
+
+    let title = '';
+    switch (TYPE) {
+        case 'PR':
+            title = `E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})${dockerImageLink}`;
+            break;
+        case 'RELEASE':
+            title = `E2E for Release Build${dockerImageLink}${releaseDate}`;
+            break;
+        case 'MASTER':
+            title = `E2E for Master Nightly Build (Prod tests)${dockerImageLink}`;
+            break;
+        case 'MASTER_UNSTABLE':
+            title = `E2E for Master Nightly Build (Unstable tests)${dockerImageLink}`;
+            break;
+        case 'CLOUD':
+            title = `E2E for Cloud Build (Prod tests)${dockerImageLink}${releaseDate}`;
+            break;
+        case 'CLOUD_UNSTABLE':
+            title = `E2E for Cloud Build (Unstable tests)${dockerImageLink}`;
+            break;
+        default:
+            title = `E2E for Build${dockerImageLink}`;
+    }
+    return title;
+}
+
+function generateWebhookBody() {
+    let testResult;
+    const testResults = [
+        {status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'},
+        {status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'},
+        {status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'},
+        {status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'},
+    ];
+    for (let i = 0; i < testResults.length; i++) {
+        if (passRate >= testResults[i].cutOff) {
+            testResult = testResults[i];
+            break;
+        }
+    }
+
+    const summaryField = `${(passRate * 100).toFixed(2)}% (${summary.passed}/${totalSpecs}) | ${playwrightDuration} | playwright@${playwrightVersion}`;
+    const serverTypeField = SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : '';
+    const mmEnvField = MM_ENV ? '\nTest server override: ' + MM_ENV : '';
+    const rollingReleaseMatchRegex = BUILD_ID?.match(/-rolling(?<version>[^-]+)-/);
+    const rollingReleaseFrom = rollingReleaseMatchRegex?.groups?.version;
+    const rollingReleaseFromField = rollingReleaseFrom ? `\nRolling release upgrade from: ${rollingReleaseFrom}` : '';
+    return {
+        username: 'Playwright UI Test',
+        icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+        attachments: [
+            {
+                color: testResult.color,
+                author_name: 'Webapp End-to-end Testing (Playwright)',
+                author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+                author_link: 'https://www.mattermost.com',
+                title: generateTitle(),
+                text: `${summaryField}${serverTypeField}${rollingReleaseFromField}${mmEnvField}`,
+            },
+        ],
+    };
+}
+
+let webhookBody = generateWebhookBody();
+process.stdout.write(JSON.stringify(webhookBody));