Move /e2e -> /e2e-tests

e2e-tests/cypress/utils/artifacts.js

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console,consistent-return */
+
+const fs = require('fs');
+
+const path = require('path');
+
+const async = require('async');
+const AWS = require('aws-sdk');
+const mime = require('mime-types');
+const readdir = require('recursive-readdir');
+
+const {MOCHAWESOME_REPORT_DIR} = require('./constants');
+
+require('dotenv').config();
+
+const {
+    AWS_S3_BUCKET,
+    AWS_ACCESS_KEY_ID,
+    AWS_SECRET_ACCESS_KEY,
+    BUILD_ID,
+    BRANCH,
+    BUILD_TAG,
+} = process.env;
+
+const s3 = new AWS.S3({
+    signatureVersion: 'v4',
+    accessKeyId: AWS_ACCESS_KEY_ID,
+    secretAccessKey: AWS_SECRET_ACCESS_KEY,
+});
+
+function getFiles(dirPath) {
+    return fs.existsSync(dirPath) ? readdir(dirPath) : [];
+}
+
+async function saveArtifacts() {
+    if (!AWS_S3_BUCKET || !AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
+        console.log('No AWS credentials found. Test artifacts not uploaded to S3.');
+
+        return;
+    }
+
+    const s3Folder = `${BUILD_ID}-${BRANCH}-${BUILD_TAG}`.replace(/\./g, '-');
+    const uploadPath = path.resolve(__dirname, `../${MOCHAWESOME_REPORT_DIR}`);
+    const filesToUpload = await getFiles(uploadPath);
+
+    return new Promise((resolve, reject) => {
+        async.eachOfLimit(
+            filesToUpload,
+            10,
+            async.asyncify(async (file) => {
+                const Key = file.replace(uploadPath, s3Folder);
+                const contentType = mime.lookup(file);
+                const charset = mime.charset(contentType);
+
+                return new Promise((res, rej) => {
+                    s3.upload(
+                        {
+                            Key,
+                            Bucket: AWS_S3_BUCKET,
+                            Body: fs.readFileSync(file),
+                            ContentType: `${contentType}${charset ? '; charset=' + charset : ''}`,
+                        },
+                        (err) => {
+                            if (err) {
+                                console.log('Failed to upload artifact:', file);
+                                return rej(new Error(err));
+                            }
+                            res({success: true});
+                        },
+                    );
+                });
+            }),
+            (err) => {
+                if (err) {
+                    console.log('Failed to upload artifacts');
+                    return reject(new Error(err));
+                }
+
+                const reportLink = `https://${AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/mochawesome.html`;
+                resolve({success: true, reportLink});
+            },
+        );
+    });
+}
+
+module.exports = {saveArtifacts};

e2e-tests/cypress/utils/constants.js

@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+const RESULTS_DIR = 'results';
+const MOCHAWESOME_REPORT_DIR = 'results/mochawesome-report';
+
+module.exports = {
+    MOCHAWESOME_REPORT_DIR,
+    RESULTS_DIR,
+};

e2e-tests/cypress/utils/dashboard.js

@@ -0,0 +1,167 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console */
+
+/*
+ * Environment:
+ *   AUTOMATION_DASHBOARD_URL=[url]
+ *   AUTOMATION_DASHBOARD_TOKEN=[token]
+ */
+
+const fs = require('fs');
+
+const readFile = require('util').promisify(fs.readFile);
+
+const axios = require('axios');
+const axiosRetry = require('axios-retry');
+const chalk = require('chalk');
+const mime = require('mime-types');
+
+require('dotenv').config();
+
+const maxRetry = 5;
+const timeout = 60 * 1000;
+
+axiosRetry(axios, {
+    retries: maxRetry,
+    retryDelay: axiosRetry.exponentialDelay,
+});
+
+const {
+    AUTOMATION_DASHBOARD_URL,
+    AUTOMATION_DASHBOARD_TOKEN,
+} = process.env;
+
+const connectionErrors = ['ECONNABORTED', 'ECONNREFUSED'];
+
+async function createAndStartCycle(data) {
+    const response = await axios({
+        url: `${AUTOMATION_DASHBOARD_URL}/cycles/start`,
+        headers: {
+            Authorization: `Bearer ${AUTOMATION_DASHBOARD_TOKEN}`,
+        },
+        method: 'post',
+        timeout,
+        data,
+    });
+
+    return response.data;
+}
+
+async function getSpecToTest({repo, branch, build, server}) {
+    try {
+        const response = await axios({
+            url: `${AUTOMATION_DASHBOARD_URL}/executions/specs/start?repo=${repo}&branch=${branch}&build=${build}`,
+            headers: {
+                Authorization: `Bearer ${AUTOMATION_DASHBOARD_TOKEN}`,
+            },
+            method: 'post',
+            timeout,
+            data: {server},
+        });
+
+        return response.data;
+    } catch (err) {
+        console.log(chalk.red('Failed to get spec to test'));
+        if (connectionErrors.includes(err.code) || !err.response) {
+            console.log(chalk.red(`Error code: ${err.code}`));
+            return {code: err.code};
+        }
+
+        return err.response && err.response.data;
+    }
+}
+
+async function recordSpecResult(specId, spec, tests) {
+    try {
+        const response = await axios({
+            url: `${AUTOMATION_DASHBOARD_URL}/executions/specs/end?id=${specId}`,
+            headers: {
+                Authorization: `Bearer ${AUTOMATION_DASHBOARD_TOKEN}`,
+            },
+            method: 'post',
+            timeout,
+            data: {spec, tests},
+        });
+
+        console.log(chalk.green('Successfully recorded!'));
+        return response.data;
+    } catch (err) {
+        console.log(chalk.red('Failed to record spec result'));
+        if (connectionErrors.includes(err.code) || !err.response) {
+            console.log(chalk.red(`Error code: ${err.code}`));
+            return {code: err.code};
+        }
+
+        return err.response && err.response.data;
+    }
+}
+
+async function updateCycle(id, cyclePatch) {
+    try {
+        const response = await axios({
+            url: `${AUTOMATION_DASHBOARD_URL}/cycles/${id}`,
+            headers: {
+                Authorization: `Bearer ${AUTOMATION_DASHBOARD_TOKEN}`,
+            },
+            method: 'put',
+            timeout,
+            data: cyclePatch,
+        });
+
+        console.log(chalk.green('Successfully updated the cycle with test environment data!'));
+        return response.data;
+    } catch (err) {
+        console.log(chalk.red('Failed to update cycle'));
+        if (connectionErrors.includes(err.code) || !err.response) {
+            console.log(chalk.red(`Error code: ${err.code}`));
+            return {code: err.code};
+        }
+
+        return err.response && err.response.data;
+    }
+}
+
+async function uploadScreenshot(filePath, repo, branch, build) {
+    try {
+        const contentType = mime.lookup(filePath);
+        const extension = mime.extension(contentType);
+
+        const {data} = await axios({
+            url: `${AUTOMATION_DASHBOARD_URL}/upload-request`,
+            headers: {
+                Authorization: `Bearer ${AUTOMATION_DASHBOARD_TOKEN}`,
+            },
+            method: 'get',
+            timeout,
+            data: {repo, branch, build, extension},
+        });
+
+        const file = await readFile(filePath);
+
+        await axios({
+            url: data.upload_url,
+            method: 'put',
+            headers: {'Content-Type': contentType},
+            data: file,
+        });
+
+        return data.object_url;
+    } catch (err) {
+        if (connectionErrors.includes(err.code) || !err.response) {
+            console.log(chalk.red(`Error code: ${err.code}`));
+            return {code: err.code};
+        }
+
+        return {error: 'Failed to upload a screenshot.'};
+    }
+}
+
+module.exports = {
+    createAndStartCycle,
+    getSpecToTest,
+    recordSpecResult,
+    updateCycle,
+    uploadScreenshot,
+};

e2e-tests/cypress/utils/even_distribution.js

@@ -0,0 +1,48 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+function* distributeItems(total, divider) {
+    if (divider === 0) {
+        yield 0;
+    } else {
+        let rest = total % divider;
+        const result = total / divider;
+
+        for (let i = 0; i < divider; i++) {
+            if (rest-- > 0) {
+                yield Math.ceil(result);
+            } else {
+                yield Math.floor(result);
+            }
+        }
+    }
+}
+
+function getTestFilesIdentifier(numberOfTestFiles, part, of) {
+    const PART = parseInt(part, 10) || 1;
+    const OF = parseInt(of, 10) || 1;
+    if (PART > OF) {
+        throw new Error(`"--part=${PART}" should not be greater than "--of=${OF}"`);
+    }
+
+    const distributions = [];
+    for (const member of distributeItems(numberOfTestFiles, OF)) {
+        distributions.push(member);
+    }
+
+    const indexedPart = (PART - 1);
+
+    let start = 0;
+    for (let i = 0; i < indexedPart; i++) {
+        start += distributions[i];
+    }
+
+    const end = distributions[indexedPart] + start;
+    const count = distributions[indexedPart];
+
+    return {start, end, count};
+}
+
+module.exports = {
+    getTestFilesIdentifier,
+};

e2e-tests/cypress/utils/even_distribution.test.js

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import {getTestFilesIdentifier} from './even_distribution';
+
+describe('getTestFilesIdentifier', () => {
+    it('should return expected output', () => {
+        const testCases = [
+            {numberOfTestFiles: 5, part: 1, of: 4, outStart: 0, outEnd: 2, outCount: 2},
+            {numberOfTestFiles: 5, part: 2, of: 4, outStart: 2, outEnd: 3, outCount: 1},
+            {numberOfTestFiles: 5, part: 3, of: 4, outStart: 3, outEnd: 4, outCount: 1},
+            {numberOfTestFiles: 5, part: 4, of: 4, outStart: 4, outEnd: 5, outCount: 1},
+
+            {numberOfTestFiles: 10, part: 1, of: 4, outStart: 0, outEnd: 3, outCount: 3},
+            {numberOfTestFiles: 10, part: 2, of: 4, outStart: 3, outEnd: 6, outCount: 3},
+            {numberOfTestFiles: 10, part: 3, of: 4, outStart: 6, outEnd: 8, outCount: 2},
+            {numberOfTestFiles: 10, part: 4, of: 4, outStart: 8, outEnd: 10, outCount: 2},
+
+            {numberOfTestFiles: 410, part: 1, of: 8, outStart: 0, outEnd: 52, outCount: 52},
+            {numberOfTestFiles: 410, part: 2, of: 8, outStart: 52, outEnd: 104, outCount: 52},
+            {numberOfTestFiles: 410, part: 3, of: 8, outStart: 104, outEnd: 155, outCount: 51},
+            {numberOfTestFiles: 410, part: 4, of: 8, outStart: 155, outEnd: 206, outCount: 51},
+            {numberOfTestFiles: 410, part: 5, of: 8, outStart: 206, outEnd: 257, outCount: 51},
+            {numberOfTestFiles: 410, part: 6, of: 8, outStart: 257, outEnd: 308, outCount: 51},
+            {numberOfTestFiles: 410, part: 7, of: 8, outStart: 308, outEnd: 359, outCount: 51},
+            {numberOfTestFiles: 410, part: 8, of: 8, outStart: 359, outEnd: 410, outCount: 51},
+        ];
+
+        testCases.forEach((testCase) => {
+            const actual = getTestFilesIdentifier(testCase.numberOfTestFiles, testCase.part, testCase.of);
+
+            expect(testCase.outStart).toEqual(actual.start);
+            expect(testCase.outEnd).toEqual(actual.end);
+            expect(testCase.outCount).toEqual(actual.count);
+        });
+    });
+});

e2e-tests/cypress/utils/file.js

@@ -0,0 +1,259 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+
+const chalk = require('chalk');
+const intersection = require('lodash.intersection');
+const without = require('lodash.without');
+const shell = require('shelljs');
+const argv = require('yargs').
+    default('includeFile', '').
+    default('excludeFile', '').
+    argv;
+
+const TEST_DIR = 'tests';
+
+const grepCommand = (word = '') => {
+    // -r, recursive search on subdirectories
+    // -I, ignore binary
+    // -l, only names of files to stdout/return
+    // -w, expression is searched for as a word
+    return `grep -rIlw '${word}' ${TEST_DIR}`;
+};
+
+const grepFiles = (command) => {
+    return shell.exec(command, {silent: true}).stdout.
+        split('\n').
+        filter((f) => f.includes('spec.js') || f.includes('spec.ts'));
+};
+
+const findFiles = (pattern) => {
+    function diveOnFiles(dirPath, filesArr) {
+        const files = fs.readdirSync(dirPath);
+        let arrayOfFiles = filesArr || [];
+
+        files.forEach((file) => {
+            const filePath = `${dirPath}/${file}`;
+            if (fs.statSync(filePath).isDirectory()) {
+                arrayOfFiles = diveOnFiles(filePath, arrayOfFiles);
+            } else {
+                arrayOfFiles.push(filePath);
+            }
+        });
+
+        return arrayOfFiles;
+    }
+
+    return shell.exec(`find ${TEST_DIR}/integration -name "${pattern}"`, {silent: true}).stdout.
+        split('\n').
+        filter((matched) => Boolean(matched)).
+        map((fileOrDir) => {
+            if (fs.statSync(`./${fileOrDir}`).isDirectory(fileOrDir)) {
+                return diveOnFiles(`./${fileOrDir}`);
+            }
+            return fileOrDir;
+        }).
+        flat().
+        filter((file) => file.includes('spec.js') || file.includes('spec.ts')).
+        map((file) => file.replace('./', ''));
+};
+
+function getBaseTestFiles() {
+    const {invert, group, stage} = argv;
+
+    const allFiles = grepFiles(grepCommand());
+    const stageFiles = getFilesByMetadata(stage);
+    const groupFiles = getFilesByMetadata(group);
+
+    if (invert) {
+        // Return no test file if no stage and withGroup, but inverted
+        if (!stage && !group) {
+            return [];
+        }
+
+        // Return all excluding stage files
+        if (stage && !group) {
+            return without(allFiles, ...stageFiles);
+        }
+
+        // Return all excluding group files
+        if (!stage && group) {
+            return without(allFiles, ...groupFiles);
+        }
+
+        // Return all excluding group and stage files
+        return without(allFiles, ...intersection(stageFiles, groupFiles));
+    }
+
+    // Return all files if no stage and group flags
+    if (!stage && !group) {
+        return allFiles;
+    }
+
+    // Return stage files if no group flag
+    if (stage && !group) {
+        return stageFiles;
+    }
+
+    // Return group files if no stage flag
+    if (!stage && group) {
+        return groupFiles;
+    }
+
+    // Return files if both in stage and group
+    return intersection(stageFiles, groupFiles);
+}
+
+function getWeightedFiles(metadata, sortFirst = true) {
+    let weightedFiles = [];
+    if (metadata) {
+        metadata.split(',').forEach((word, i, arr) => {
+            const files = getFilesByMetadata(word).map((file) => {
+                return {
+                    file,
+                    sortWeight: sortFirst ? (i - arr.length) : (i + 1),
+                };
+            });
+            weightedFiles.push(...files);
+        });
+    }
+
+    if (sortFirst) {
+        weightedFiles = weightedFiles.reverse();
+    }
+
+    return weightedFiles.reduce((acc, f) => {
+        acc[f.file] = f;
+        return acc;
+    }, {});
+}
+
+function reorderFiles(files = {}, filesToReorder = {}) {
+    const testFilesObject = Object.assign({}, files);
+
+    const validFiles = intersection(Object.keys(testFilesObject), Object.keys(filesToReorder));
+    Object.entries(filesToReorder).forEach(([k, v]) => {
+        if (validFiles.includes(k)) {
+            testFilesObject[k] = v;
+        }
+    });
+
+    return testFilesObject;
+}
+
+function removeFromFiles(files = {}, filesToRemove = []) {
+    const testFilesObject = Object.assign({}, files);
+
+    const removedFiles = intersection(Object.keys(testFilesObject), filesToRemove);
+    removedFiles.forEach((file) => {
+        if (testFilesObject.hasOwnProperty(file)) {
+            delete testFilesObject[file];
+        }
+    });
+
+    return {testFilesObject, removedFiles};
+}
+
+function getSortedTestFiles(platform, browser, headless) {
+    // Get test files based on stage, group and/or invert
+    const baseTestFiles = getBaseTestFiles();
+
+    // Add files matched by spec metadata
+    const includeFilesByGroup = getFilesByMetadata(argv.includeGroup);
+    if (includeFilesByGroup.length) {
+        printMessage(includeFilesByGroup, `\nIncluded test files due to --include-group="${argv.includeGroup}"`);
+    }
+
+    // Add files matched by filename
+    const includeFilesByFilename = argv.includeFile.split(',').
+        map((pattern) => findFiles(pattern)).
+        reduce((acc, files) => acc.concat(files), []);
+    if (includeFilesByFilename.length) {
+        printMessage(includeFilesByFilename, `\nIncluded test files due to --include-file="${argv.includeFile}"`);
+    }
+
+    let testFilesObject = baseTestFiles.
+        concat(includeFilesByGroup).
+        concat(includeFilesByFilename).
+        reduce((acc, file) => {
+            acc[file] = {file, sortWeight: 0};
+            return acc;
+        }, {});
+
+    // Remove skipped files due to test environment
+    let removedFiles;
+    const skippedFiles = getSkippedFiles(platform, browser, headless);
+    ({testFilesObject, removedFiles} = removeFromFiles(testFilesObject, skippedFiles));
+    printMessage(removedFiles, `\nSkipped test files due to ${platform}/${browser} (${headless ? 'headless' : 'headed'})`);
+
+    // Remove files matched by spec metadata
+    const excludeFilesByGroup = getFilesByMetadata(argv.excludeGroup);
+    ({testFilesObject, removedFiles} = removeFromFiles(testFilesObject, excludeFilesByGroup));
+    if (excludeFilesByGroup.length) {
+        printMessage(removedFiles, `\nExcluded test files due to --exclude-group="${argv.excludeGroup}"`);
+    }
+
+    // Remove files matched by filename
+    const excludeFilesByFilename = argv.excludeFile.split(',').
+        map((pattern) => findFiles(pattern)).
+        reduce((acc, files) => acc.concat(files), []);
+
+    ({testFilesObject, removedFiles} = removeFromFiles(testFilesObject, excludeFilesByFilename));
+    if (excludeFilesByFilename.length) {
+        printMessage(removedFiles, `\nExcluded test files due to --exclude-file="${argv.excludeFile}"`);
+    }
+
+    // Get files to be sorted first
+    const firstFilesObject = getWeightedFiles(argv.sortFirst, true);
+    testFilesObject = reorderFiles(testFilesObject, firstFilesObject);
+
+    // Get files to be sorted last
+    const lastFilesObject = getWeightedFiles(argv.sortLast, false);
+    testFilesObject = reorderFiles(testFilesObject, lastFilesObject);
+
+    const sortedFiles = Object.values(testFilesObject).
+        sort((a, b) => {
+            if (a.sortWeight > b.sortWeight) {
+                return 1;
+            } else if (a.sortWeight < b.sortWeight) {
+                return -1;
+            }
+
+            return a.file.localeCompare(b.file);
+        }).
+        map((sortedObj) => sortedObj.file);
+
+    return {sortedFiles, skippedFiles, weightedTestFiles: Object.values(testFilesObject)};
+}
+
+function getFilesByMetadata(metadata) {
+    if (!metadata) {
+        return [];
+    }
+
+    const egc = grepCommand(metadata.split(',').join('\\|'));
+    return grepFiles(egc);
+}
+
+function printMessage(files = [], message) {
+    console.log(chalk.cyan(`\n${message}:`));
+
+    files.forEach((file, index) => {
+        console.log(chalk.cyan(`- [${index + 1}] ${file}`));
+    });
+}
+
+function getSkippedFiles(platform, browser, headless) {
+    const platformFiles = getFilesByMetadata(`@${platform}`);
+    const browserFiles = getFilesByMetadata(`@${browser}`);
+    const headlessFiles = getFilesByMetadata(`@${headless ? 'headless' : 'headed'}`);
+
+    return platformFiles.concat(browserFiles, headlessFiles);
+}
+
+module.exports = {
+    getSortedTestFiles,
+};

e2e-tests/cypress/utils/report.js

@@ -0,0 +1,330 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console, camelcase */
+
+const axios = require('axios');
+const fse = require('fs-extra');
+const dayjs = require('dayjs');
+const duration = require('dayjs/plugin/duration');
+dayjs.extend(duration);
+
+const {MOCHAWESOME_REPORT_DIR} = require('./constants');
+
+const MAX_FAILED_TITLES = 5;
+
+let incrementalDuration = 0;
+
+function getAllTests(results) {
+    const tests = [];
+    results.forEach((result) => {
+        result.tests.forEach((test) => {
+            incrementalDuration += test.duration;
+            tests.push({...test, incrementalDuration});
+        });
+
+        if (result.suites.length > 0) {
+            getAllTests(result.suites).forEach((test) => tests.push(test));
+        }
+    });
+
+    return tests;
+}
+
+function generateStatsFieldValue(stats, failedFullTitles) {
+    const startAt = dayjs(stats.start);
+    const endAt = dayjs(stats.end);
+    const statsDuration = dayjs.duration(endAt.diff(startAt)).format('H:mm:ss');
+
+    let statsFieldValue = `
+| Key | Value |
+|:---|:---|
+| Passing Rate | ${stats.passPercent.toFixed(2)}% |
+| Duration | ${statsDuration} |
+| Suites | ${stats.suites} |
+| Tests | ${stats.tests} |
+| :white_check_mark: Passed | ${stats.passes} |
+| :x: Failed | ${stats.failures} |
+| :fast_forward: Skipped | ${stats.skipped} |
+`;
+
+    // If present, add full title of failing tests.
+    // Only show per maximum number of failed titles with the last item as "more..." if failing tests are more than that.
+    let failedTests;
+    if (failedFullTitles && failedFullTitles.length > 0) {
+        const re = /[:'"\\]/gi;
+        const failed = failedFullTitles;
+        if (failed.length > MAX_FAILED_TITLES) {
+            failedTests = failed.slice(0, MAX_FAILED_TITLES - 1).map((f) => `- ${f.replace(re, '')}`).join('\n');
+            failedTests += '\n- more...';
+        } else {
+            failedTests = failed.map((f) => `- ${f.replace(re, '')}`).join('\n');
+        }
+    }
+
+    if (failedTests) {
+        statsFieldValue += '###### Failed Tests:\n' + failedTests;
+    }
+
+    return statsFieldValue;
+}
+
+function generateShortSummary(report) {
+    const {results, stats} = report;
+    const tests = getAllTests(results);
+
+    const failedFullTitles = tests.filter((t) => t.fail).map((t) => t.fullTitle);
+    const statsFieldValue = generateStatsFieldValue(stats, failedFullTitles);
+
+    return {
+        stats,
+        statsFieldValue,
+    };
+}
+
+function removeOldGeneratedReports() {
+    [
+        'all.json',
+        'summary.json',
+        'mochawesome.html',
+    ].forEach((file) => fse.removeSync(`${MOCHAWESOME_REPORT_DIR}/${file}`));
+}
+
+function writeJsonToFile(jsonObject, filename, dir) {
+    fse.writeJson(`${dir}/${filename}`, jsonObject).
+        then(() => console.log('Successfully written:', filename)).
+        catch((err) => console.error(err));
+}
+
+function readJsonFromFile(file) {
+    try {
+        return fse.readJsonSync(file);
+    } catch (err) {
+        return {err};
+    }
+}
+
+const result = [
+    {status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'},
+    {status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'},
+    {status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'},
+    {status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'},
+];
+
+function generateTestReport(summary, isUploadedToS3, reportLink, environment, testCycleKey) {
+    const {
+        FULL_REPORT,
+        TEST_CYCLE_LINK_PREFIX,
+        MM_ENV,
+        SERVER_TYPE,
+    } = process.env;
+    const {statsFieldValue, stats} = summary;
+    const {
+        cypress_version,
+        browser_name,
+        browser_version,
+        headless,
+        os_name,
+        os_version,
+        node_version,
+    } = environment;
+
+    let testResult;
+    for (let i = 0; i < result.length; i++) {
+        if (stats.passPercent >= result[i].cutOff) {
+            testResult = result[i];
+            break;
+        }
+    }
+
+    const title = generateTitle();
+    const runnerEnvValue = `cypress@${cypress_version} | node@${node_version} | ${browser_name}@${browser_version}${headless ? ' (headless)' : ''} | ${os_name}@${os_version}`;
+
+    if (FULL_REPORT === 'true') {
+        let reportField;
+        if (isUploadedToS3) {
+            reportField = {
+                short: false,
+                title: 'Test Report',
+                value: `[Link to the report](${reportLink})`,
+            };
+        }
+
+        let testCycleField;
+        if (testCycleKey) {
+            testCycleField = {
+                short: false,
+                title: 'Test Execution',
+                value: `[Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})`,
+            };
+        }
+
+        let serverEnvField;
+        if (MM_ENV) {
+            serverEnvField = {
+                short: false,
+                title: 'Test Server Override',
+                value: MM_ENV,
+            };
+        }
+
+        let serverTypeField;
+        if (SERVER_TYPE) {
+            serverTypeField = {
+                short: false,
+                title: 'Test Server',
+                value: SERVER_TYPE,
+            };
+        }
+
+        return {
+            username: 'Cypress UI Test',
+            icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            attachments: [{
+                color: testResult.color,
+                author_name: 'Webapp End-to-end Testing',
+                author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+                author_link: 'https://www.mattermost.com',
+                title,
+                fields: [
+                    {
+                        short: false,
+                        title: 'Environment',
+                        value: runnerEnvValue,
+                    },
+                    serverTypeField,
+                    serverEnvField,
+                    reportField,
+                    testCycleField,
+                    {
+                        short: false,
+                        title: `Key metrics (required support: ${testResult.priority})`,
+                        value: statsFieldValue,
+                    },
+                ],
+            }],
+        };
+    }
+
+    let quickSummary = `${stats.passPercent.toFixed(2)}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`;
+    if (isUploadedToS3) {
+        quickSummary = `[${quickSummary}](${reportLink})`;
+    }
+
+    let testCycleLink = '';
+    if (testCycleKey) {
+        testCycleLink = testCycleKey ? `| [Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})` : '';
+    }
+
+    const startAt = dayjs(stats.start);
+    const endAt = dayjs(stats.end);
+    const statsDuration = dayjs.duration(endAt.diff(startAt)).format('H:mm:ss');
+
+    return {
+        username: 'Cypress UI Test',
+        icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+        attachments: [{
+            color: testResult.color,
+            author_name: 'Webapp End-to-end Testing',
+            author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            author_link: 'https://www.mattermost.com/',
+            title,
+            text: `${quickSummary} | ${statsDuration} ${testCycleLink}\n${runnerEnvValue}${SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : ''}${MM_ENV ? '\nTest server override: ' + MM_ENV : ''}`,
+        }],
+    };
+}
+
+function generateTitle() {
+    const {
+        BRANCH,
+        MM_DOCKER_IMAGE,
+        MM_DOCKER_TAG,
+        PULL_REQUEST,
+        RELEASE_DATE,
+        TYPE,
+    } = process.env;
+
+    let dockerImageLink = '';
+    if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) {
+        dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermost/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
+    }
+
+    let releaseDate = '';
+    if (RELEASE_DATE) {
+        releaseDate = ` for ${RELEASE_DATE}`;
+    }
+
+    let title;
+
+    switch (TYPE) {
+    case 'PR':
+        title = `E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})${dockerImageLink}`;
+        break;
+    case 'RELEASE':
+        title = `E2E for Release Build${dockerImageLink}${releaseDate}`;
+        break;
+    case 'MASTER':
+        title = `E2E for Master Nightly Build (Prod tests)${dockerImageLink}`;
+        break;
+    case 'MASTER_UNSTABLE':
+        title = `E2E for Master Nightly Build (Unstable tests)${dockerImageLink}`;
+        break;
+    case 'CLOUD':
+        title = `E2E for Cloud Build (Prod tests)${dockerImageLink}${releaseDate}`;
+        break;
+    case 'CLOUD_UNSTABLE':
+        title = `E2E for Cloud Build (Unstable tests)${dockerImageLink}`;
+        break;
+    default:
+        title = `E2E for Build${dockerImageLink}`;
+    }
+
+    return title;
+}
+
+function generateDiagnosticReport(summary, serverInfo) {
+    const {BRANCH, BUILD_ID} = process.env;
+
+    return {
+        username: 'Cypress UI Test',
+        icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+        attachments: [{
+            color: '#43A047',
+            author_name: 'Cypress UI Test',
+            author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            author_link: 'https://community.mattermost.com/core/channels/ui-test-automation',
+            title: `Cypress UI Test Automation #${BUILD_ID}, **${BRANCH}** branch`,
+            fields: [{
+                short: false,
+                value: `Start: **${summary.stats.start}**\nEnd: **${summary.stats.end}**\nUser ID: **${serverInfo.userId}**\nTeam ID: **${serverInfo.teamId}**`,
+            }],
+        }],
+    };
+}
+
+async function sendReport(name, url, data) {
+    const requestOptions = {method: 'POST', url, data};
+
+    try {
+        const response = await axios(requestOptions);
+
+        if (response.data) {
+            console.log(`Successfully sent ${name}.`);
+        }
+        return response;
+    } catch (er) {
+        console.log(`Something went wrong while sending ${name}.`, er);
+        return false;
+    }
+}
+
+module.exports = {
+    generateDiagnosticReport,
+    generateShortSummary,
+    generateTestReport,
+    getAllTests,
+    removeOldGeneratedReports,
+    sendReport,
+    readJsonFromFile,
+    writeJsonToFile,
+};

e2e-tests/cypress/utils/test_cases.js

@@ -0,0 +1,201 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-console */
+
+// See reference: https://support.smartbear.com/tm4j-cloud/api-docs/
+
+const axios = require('axios');
+const chalk = require('chalk');
+
+const {getAllTests} = require('./report');
+
+const status = {
+    passed: 'Pass',
+    failed: 'Fail',
+    pending: 'Pending',
+    skipped: 'Skip',
+};
+
+const environment = {
+    chrome: 'Chrome',
+    firefox: 'Firefox',
+};
+
+function getStepStateResult(steps = []) {
+    return steps.reduce((acc, item) => {
+        if (acc[item.state]) {
+            acc[item.state] += 1;
+        } else {
+            acc[item.state] = 1;
+        }
+
+        return acc;
+    }, {});
+}
+
+function getStepStateSummary(steps = []) {
+    const result = getStepStateResult(steps);
+
+    return Object.entries(result).map(([key, value]) => `${value} ${key}`).join(',');
+}
+
+function getTM4JTestCases(report) {
+    return getAllTests(report.results).
+        filter((item) => /^(MM-T)\w+/g.test(item.title)). // eslint-disable-line wrap-regex
+        map((item) => {
+            return {
+                title: item.title,
+                duration: item.duration,
+                incrementalDuration: item.incrementalDuration,
+                state: item.state,
+                pass: item.pass,
+                fail: item.fail,
+                pending: item.pending,
+            };
+        }).
+        reduce((acc, item) => {
+            // Extract the key to exactly match with "MM-T[0-9]+"
+            const key = item.title.match(/(MM-T\d+)/)[0];
+
+            if (acc[key]) {
+                acc[key].push(item);
+            } else {
+                acc[key] = [item];
+            }
+
+            return acc;
+        }, {});
+}
+
+function saveToEndpoint(url, data) {
+    return axios({
+        method: 'POST',
+        url,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8',
+            Authorization: process.env.TM4J_API_KEY,
+        },
+        data,
+    }).catch((error) => {
+        console.log('Something went wrong:', error.response.data.message);
+        return error.response.data;
+    });
+}
+
+async function createTestCycle(startDate, endDate) {
+    const {
+        BRANCH,
+        BUILD_ID,
+        JIRA_PROJECT_KEY,
+        TM4J_CYCLE_NAME,
+        TM4J_FOLDER_ID,
+    } = process.env;
+
+    const testCycle = {
+        projectKey: JIRA_PROJECT_KEY,
+        name: TM4J_CYCLE_NAME ? `${TM4J_CYCLE_NAME} (${BUILD_ID}-${BRANCH})` : `${BUILD_ID}-${BRANCH}`,
+        description: `Cypress automated test with ${BRANCH}`,
+        plannedStartDate: startDate,
+        plannedEndDate: endDate,
+        statusName: 'Done',
+        folderId: TM4J_FOLDER_ID,
+    };
+
+    const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle);
+    return response.data;
+}
+
+async function createTestExecutions(report, testCycle) {
+    const {
+        BROWSER,
+        JIRA_PROJECT_KEY,
+        TM4J_ENVIRONMENT_NAME,
+    } = process.env;
+
+    const testCases = getTM4JTestCases(report);
+    const startDate = new Date(report.stats.start);
+    const startTime = startDate.getTime();
+
+    const promises = [];
+    Object.entries(testCases).forEach(([key, steps], index) => {
+        const testScriptResults = steps.
+            sort((a, b) => a.title.localeCompare(b.title)).
+            map((item) => {
+                return {
+                    statusName: status[item.state],
+                    actualEndDate: new Date(startTime + item.incrementalDuration).toISOString(),
+                    actualResult: 'Cypress automated test completed',
+                };
+            });
+
+        const stateResult = getStepStateResult(steps);
+
+        const testExecution = {
+            projectKey: JIRA_PROJECT_KEY,
+            testCaseKey: key,
+            testCycleKey: testCycle.key,
+            statusName: stateResult.passed && stateResult.passed === steps.length ? 'Pass' : 'Fail',
+            testScriptResults,
+            environmentName: TM4J_ENVIRONMENT_NAME || environment[BROWSER] || 'Chrome',
+            actualEndDate: testScriptResults[testScriptResults.length - 1].actualEndDate,
+            executionTime: steps.reduce((acc, prev) => {
+                acc += prev.duration; // eslint-disable-line no-param-reassign
+                return acc;
+            }, 0),
+            comment: `Cypress automated test - ${getStepStateSummary(steps)}`,
+        };
+
+        // Temporarily log to verify cases that were being saved.
+        console.log(index, key); // eslint-disable-line no-console
+
+        promises.push(saveTestExecution(testExecution, index));
+    });
+
+    await Promise.all(promises);
+    console.log('Successfully saved test cases into the Test Management System');
+}
+
+const saveTestCases = async (allReport) => {
+    const {start, end} = allReport.stats;
+
+    const testCycle = await createTestCycle(start, end);
+
+    await createTestExecutions(allReport, testCycle);
+};
+
+const RETRY = [];
+
+async function saveTestExecution(testExecution, index) {
+    await axios({
+        method: 'POST',
+        url: 'https://api.zephyrscale.smartbear.com/v2/testexecutions',
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8',
+            Authorization: process.env.TM4J_API_KEY,
+        },
+        data: testExecution,
+    }).then(() => {
+        console.log(chalk.green('Success:', index, testExecution.testCaseKey));
+    }).catch((error) => {
+        // Retry on 500 error code / internal server error
+        if (!error.response || error.response.data.errorCode === 500) {
+            if (RETRY[testExecution.testCaseKey]) {
+                RETRY[testExecution.testCaseKey] += 1;
+            } else {
+                RETRY[testExecution.testCaseKey] = 1;
+            }
+
+            saveTestExecution(testExecution, index);
+            console.log(chalk.magenta('Retry:', index, testExecution.testCaseKey, `(${RETRY[testExecution.testCaseKey]}x)`));
+        } else {
+            console.log(chalk.red('Error:', index, testExecution.testCaseKey, error.response.data.message));
+        }
+    });
+}
+
+module.exports = {
+    createTestCycle,
+    saveTestCases,
+    createTestExecutions,
+};

e2e-tests/cypress/utils/webhook_utils.js

@@ -0,0 +1,270 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+function getFullDialog(triggerId, webhookBaseUrl) {
+    return {
+        trigger_id: triggerId,
+        url: `${webhookBaseUrl}/dialog_submit`,
+        dialog: {
+            callback_id: 'somecallbackid',
+            title: 'Title for Full Dialog Test',
+            icon_url:
+                'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            elements: [
+                {
+                    display_name: 'Display Name',
+                    name: 'realname',
+                    type: 'text',
+                    subtype: '',
+                    default: 'default text',
+                    placeholder: 'placeholder',
+                    help_text:
+                        'This a regular input in an interactive dialog triggered by a test integration.',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: '',
+                    options: null,
+                },
+                {
+                    display_name: 'Email',
+                    name: 'someemail',
+                    type: 'text',
+                    subtype: 'email',
+                    default: '',
+                    placeholder: 'placeholder@bladekick.com',
+                    help_text:
+                        'This a regular email input in an interactive dialog triggered by a test integration.',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: '',
+                    options: null,
+                },
+                {
+                    display_name: 'Number',
+                    name: 'somenumber',
+                    type: 'text',
+                    subtype: 'number',
+                    default: '',
+                    placeholder: '',
+                    help_text: '',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: '',
+                    options: null,
+                },
+                {
+                    display_name: 'Password',
+                    name: 'somepassword',
+                    type: 'text',
+                    subtype: 'password',
+                    default: 'p@ssW0rd',
+                    placeholder: 'placeholder',
+                    help_text:
+                        'This a password input in an interactive dialog triggered by a test integration.',
+                    optional: true,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: '',
+                    options: null,
+                },
+                {
+                    display_name: 'Display Name Long Text Area',
+                    name: 'realnametextarea',
+                    type: 'textarea',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'placeholder',
+                    help_text: '',
+                    optional: true,
+                    min_length: 5,
+                    max_length: 100,
+                    data_source: '',
+                    options: null,
+                },
+                {
+                    display_name: 'User Selector',
+                    name: 'someuserselector',
+                    type: 'select',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'Select a user...',
+                    help_text: '',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: 'users',
+                    options: null,
+                },
+                {
+                    display_name: 'Channel Selector',
+                    name: 'somechannelselector',
+                    type: 'select',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'Select a channel...',
+                    help_text: 'Choose a channel from the list.',
+                    optional: true,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: 'channels',
+                    options: null,
+                },
+                {
+                    display_name: 'Option Selector',
+                    name: 'someoptionselector',
+                    type: 'select',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'Select an option...',
+                    help_text: '',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: '',
+                    options: [
+                        {
+                            text: 'Option1',
+                            value: 'opt1',
+                        },
+                        {
+                            text: 'Option2',
+                            value: 'opt2',
+                        },
+                        {
+                            text: 'Option3',
+                            value: 'opt3',
+                        },
+                    ],
+                },
+                {
+                    display_name: 'Radio Option Selector',
+                    name: 'someradiooptions',
+                    type: 'radio',
+                    help_text: '',
+                    optional: false,
+                    options: [
+                        {
+                            text: 'Engineering',
+                            value: 'engineering',
+                        },
+                        {
+                            text: 'Sales',
+                            value: 'sales',
+                        },
+                    ],
+                },
+                {
+                    display_name: 'Boolean Selector',
+                    placeholder: 'Was this modal helpful?',
+                    name: 'boolean_input',
+                    type: 'bool',
+                    default: 'True',
+                    optional: true,
+                    help_text: 'This is the help text',
+                },
+            ],
+            submit_label: 'Submit',
+            notify_on_cancel: true,
+            state: 'somestate',
+        },
+    };
+}
+
+function getSimpleDialog(triggerId, webhookBaseUrl) {
+    return {
+        trigger_id: triggerId,
+        url: `${webhookBaseUrl}/dialog_submit`,
+        dialog: {
+            callback_id: 'somecallbackid',
+            title: 'Title for Dialog Test without elements',
+            icon_url:
+                'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            submit_label: 'Submit Test',
+            notify_on_cancel: true,
+            state: 'somestate',
+        },
+    };
+}
+
+function getUserAndChannelDialog(triggerId, webhookBaseUrl) {
+    return {
+        trigger_id: triggerId,
+        url: `${webhookBaseUrl}/dialog_submit`,
+        dialog: {
+            callback_id: 'somecallbackid',
+            title: 'Title for Dialog Test with user and channel element',
+            icon_url:
+                'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            submit_label: 'Submit Test',
+            notify_on_cancel: true,
+            state: 'somestate',
+            elements: [
+                {
+                    display_name: 'User Selector',
+                    name: 'someuserselector',
+                    type: 'select',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'Select a user...',
+                    help_text: '',
+                    optional: false,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: 'users',
+                    options: null,
+                },
+                {
+                    display_name: 'Channel Selector',
+                    name: 'somechannelselector',
+                    type: 'select',
+                    subtype: '',
+                    default: '',
+                    placeholder: 'Select a channel...',
+                    help_text: 'Choose a channel from the list.',
+                    optional: true,
+                    min_length: 0,
+                    max_length: 0,
+                    data_source: 'channels',
+                    options: null,
+                },
+            ],
+        },
+    };
+}
+
+function getBooleanDialog(triggerId, webhookBaseUrl) {
+    return {
+        trigger_id: triggerId,
+        url: `${webhookBaseUrl}/dialog_submit`,
+        dialog: {
+            callback_id: 'somecallbackid',
+            title: 'Title for Dialog Test with boolean element',
+            icon_url:
+                'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
+            submit_label: 'Submit Test',
+            notify_on_cancel: true,
+            state: 'somestate',
+            elements: [
+                {
+                    display_name: 'Boolean Selector',
+                    placeholder: 'Was this modal helpful?',
+                    name: 'boolean_input',
+                    type: 'bool',
+                    default: 'True',
+                    optional: true,
+                    help_text: 'This is the help text',
+                },
+            ],
+        },
+    };
+}
+
+module.exports = {
+    getFullDialog,
+    getSimpleDialog,
+    getUserAndChannelDialog,
+    getBooleanDialog,
+};