CLD-8395 Integrate playwright into E2E tests (#28492)
* Finalize Playwright E2E scripts integration, add videos * Add suggestion by Saturn Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com> --------- Co-authored-by: Mattermost Build <build@mattermost.com> Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>
Этот коммит содержится в:
Mario Vitale
коммит произвёл
GitHub
GitHub
родитель
2467c39224
Коммит
429c41747c
@@ -90,10 +90,17 @@ export BROWSER_DEFAULT="chrome"
|
||||
export SERVER_DEFAULT="onprem"
|
||||
export TEST_DEFAULT="cypress"
|
||||
export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
|
||||
export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
|
||||
export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown")
|
||||
export BUILD_ID_DEFAULT=$(date +%s)
|
||||
export CI_BASE_URL_DEFAULT="localhost"
|
||||
# Test tool default overrides
|
||||
case "${TEST:-$TEST_DEFAULT}" in
|
||||
cypress )
|
||||
export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' ;;
|
||||
playwright )
|
||||
export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;;
|
||||
* )
|
||||
esac
|
||||
# OS specific defaults overrides
|
||||
case $MME2E_OSTYPE in
|
||||
darwin )
|
||||
|
||||
@@ -3,13 +3,16 @@ set -e -u -o pipefail
|
||||
cd "$(dirname "$0")"
|
||||
. .e2erc
|
||||
|
||||
mme2e_log "Loading variables from .env.cypress"
|
||||
if ! [ -f .env.cypress ]; then
|
||||
mme2e_log "Error: .env.cypress is required to exist, for the test cycle to be generated. Aborting." >&2
|
||||
TEST_ENV_FILE=".env.$TEST"
|
||||
|
||||
mme2e_log "Loading variables from $TEST_ENV_FILE"
|
||||
if ! [ -f "$TEST_ENV_FILE" ]; then
|
||||
mme2e_log "Error: $TEST_ENV_FILE is required to exist, for the test cycle to be generated. Aborting." >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
. .env.cypress
|
||||
# shellcheck disable=SC1090
|
||||
. "$TEST_ENV_FILE"
|
||||
|
||||
if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
|
||||
mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation."
|
||||
@@ -17,7 +20,7 @@ if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
|
||||
fi
|
||||
|
||||
mme2e_log "Generating the test cycle on the Automation Dashboard"
|
||||
cd ../cypress
|
||||
cd "../$TEST"
|
||||
npm i
|
||||
# shellcheck disable=SC2086
|
||||
exec node --trace-warnings generate_test_cycle.js $TEST_FILTER
|
||||
|
||||
@@ -8,8 +8,8 @@ set -e -u -o pipefail
|
||||
cd "$(dirname "$0")"
|
||||
. .e2erc
|
||||
|
||||
# The collected data will be written to the "e2e-tests/cypress/results/" directory
|
||||
cd ../cypress/
|
||||
# The collected data will be written to the "e2e-tests/$TEST/results/" directory
|
||||
cd "../$TEST/"
|
||||
if [ ! -d "results/" ]; then
|
||||
mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2
|
||||
exit 1
|
||||
|
||||
@@ -24,7 +24,7 @@ if [ "$TYPE" = "PR" ]; then
|
||||
fi
|
||||
|
||||
# Env vars used during the test. Their values will be included in the report
|
||||
: ${TEST:?} # See E2E tests' readme; usually set to 'cypress'
|
||||
: ${TEST:?} # See E2E tests' readme
|
||||
: ${BRANCH:?} # May be either a ref, a commit hash, or 'server-pr-PR_NUMBER' (if TYPE=PR)
|
||||
: ${BUILD_ID:?}
|
||||
: ${SERVER:?} # May be either 'onprem' or 'cloud'
|
||||
@@ -89,11 +89,23 @@ if [ -n "${AWS_S3_BUCKET:-}" ]; then
|
||||
mme2e_log "S3 report upload enabled."
|
||||
fi
|
||||
|
||||
cd ../cypress/
|
||||
# Double check that the "results/" subdirectory to collect report informations from exists
|
||||
cd "../${TEST}/"
|
||||
if [ ! -d "results/" ]; then
|
||||
mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2
|
||||
mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
npm i
|
||||
node save_report.js
|
||||
case "$TEST" in
|
||||
cypress)
|
||||
npm i
|
||||
node save_report.js
|
||||
;;
|
||||
playwright)
|
||||
if [ -n "$WEBHOOK_URL" ]; then
|
||||
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm i
|
||||
# Utilize environment data and report files to generate the webhook body
|
||||
./report.webhookgen.js | curl -X POST -fsSL -H 'Content-Type: application/json' -d @- "$WEBHOOK_URL"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -224,6 +224,7 @@ $(if mme2e_is_token_in_list "playwright" "$ENABLED_DOCKER_SERVICES"; then
|
||||
- "./.env.playwright"
|
||||
environment:
|
||||
CI: "true"
|
||||
PLAYWRIGHT_SKIP_BROWSER_GC: 1
|
||||
PW_BASE_URL: http://localhost:8065
|
||||
PW_ADMIN_USERNAME: sysadmin
|
||||
PW_ADMIN_PASSWORD: Sys@dmin-sample1
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2038
|
||||
set -e -u -o pipefail
|
||||
cd "$(dirname "$0")"
|
||||
. .e2erc
|
||||
@@ -40,5 +41,14 @@ else
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee "../cypress/logs/${LOGFILE_SUFFIX}_cypress.log"
|
||||
fi
|
||||
|
||||
# Collect run results
|
||||
cat > ../cypress/results/summary.json <<EOF
|
||||
{
|
||||
"passed": $(find ../cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.passes' | jq -s add),
|
||||
"failed": $(find ../cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add),
|
||||
"failed_expected": 0
|
||||
}
|
||||
EOF
|
||||
|
||||
# Collect server logs
|
||||
${MME2E_DC_SERVER} logs --no-log-prefix -- server > "../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log" 2>&1
|
||||
|
||||
@@ -3,22 +3,21 @@ set -e -u -o pipefail
|
||||
cd "$(dirname "$0")"
|
||||
. .e2erc
|
||||
|
||||
# Initialize Playwright report directory
|
||||
mme2e_log "Prepare Playwright: clean and initialize report and logs directory"
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<EOF
|
||||
cd e2e-tests/playwright
|
||||
rm -rf logs playwright-report test-results storage_state
|
||||
mkdir -p logs
|
||||
rm -rf logs results storage_state
|
||||
mkdir -p logs results
|
||||
touch logs/mattermost.log
|
||||
EOF
|
||||
|
||||
# Install webapp dependencies
|
||||
mme2e_log "Prepare Playwright: install webapp dependencies"
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash -c "cd webapp && npm install --cache /tmp/empty-cache"
|
||||
|
||||
# Install Playwright dependencies
|
||||
mme2e_log "Prepare Playwright: install dependencies"
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && rm -rf node_modules && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --cache /tmp/empty-cache"
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<EOF
|
||||
cd webapp/
|
||||
npm install --cache /tmp/empty-cache
|
||||
cd ../e2e-tests/playwright
|
||||
npm install --cache /tmp/empty-cache
|
||||
EOF
|
||||
|
||||
mme2e_log "Prepare Playwright: environment info"
|
||||
${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<"EOF"
|
||||
@@ -37,8 +36,21 @@ EOF
|
||||
# export DEBUG=pw:protocol,pw:browser,pw:api
|
||||
|
||||
# Run Playwright test
|
||||
# Note: Run on chrome but eventually will enable to all projects which include firefox and ipad.
|
||||
${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && npm run test -- --project=chrome" | tee ../playwright/logs/playwright.log
|
||||
# NB: do not exit the script if some testcases fail
|
||||
${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && npm run test -- ${TEST_FILTER}" | tee ../playwright/logs/playwright.log || true
|
||||
|
||||
# Collect run results
|
||||
# Documentation on the results.json file: https://playwright.dev/docs/api/class-testcase#test-case-expected-status
|
||||
# NB: the following line is needed only for compatibility reasons, to support RollingRelease tests for versions prior to v10.1.0
|
||||
# It can be removed after releases <=v10.0.x are phased out
|
||||
mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json || true
|
||||
jq -f /dev/stdin ../playwright/results/results.json > ../playwright/results/summary.json <<EOF
|
||||
{
|
||||
passed: .stats.expected,
|
||||
failed: .stats.unexpected,
|
||||
failed_expected: (.stats.skipped + .stats.flaky)
|
||||
}
|
||||
EOF
|
||||
|
||||
# Collect server logs
|
||||
${MME2E_DC_SERVER} logs --no-log-prefix -- server >../playwright/logs/mattermost.log 2>&1
|
||||
|
||||
@@ -26,7 +26,8 @@ Instructions, detailed:
|
||||
* The following variables, will be passed over to the server container: `MM_LICENSE` (no enterprise features will be available if this is unset; required when `SERVER=cloud`), and the exploded `MM_ENV` (a comma-separated list of env var specifications)
|
||||
* The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
|
||||
* The `SERVER_IMAGE` variable can also be set if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used.
|
||||
* The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
|
||||
* The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress/Playwright to run. If not specified, only the smoke tests will run
|
||||
- Its format depends on which tool is used: for Cypress, please check the `e2e-tests/cypress/run_tests.js` file for details. For Playwright, it can simply be populated with arguments you want to give to the `playwright test` command.
|
||||
* More variables may be required to configure reporting and cloud interactions. Check the content of the `.ci/report.*.sh` and `.ci/server.cloud_*.sh` scripts for reference.
|
||||
2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID`
|
||||
* NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`.
|
||||
@@ -41,6 +42,7 @@ Instructions, detailed:
|
||||
* If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make`
|
||||
* If you just want to run a local server instance, without any further testing, you can run `TEST=none make`
|
||||
* If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machines (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running.
|
||||
* This script will also parse the local test results, and write a `e2e-tests/${TEST}/results/summary.json` file containing the following keys: `passed`, `failed` and `failed_expected` (the total number of testcases that were run is the sum of these three numbers)
|
||||
4. `make stop`: tears down the server (and the dashboard, if running)
|
||||
* This will stop and cleanup all of the E2E testing containers, including the database and its persistent volume.
|
||||
* This also implicitly runs `make clean`, which also removes any generated environment or docker-compose files.
|
||||
|
||||
@@ -9,6 +9,10 @@
|
||||
"rules": {
|
||||
"@typescript-eslint/explicit-module-boundary-types": "off",
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-var-requires": "off",
|
||||
"no-console": "error"
|
||||
},
|
||||
"env": {
|
||||
"node": true
|
||||
}
|
||||
}
|
||||
|
||||
11
e2e-tests/playwright/package-lock.json
сгенерированный
11
e2e-tests/playwright/package-lock.json
сгенерированный
@@ -14,6 +14,7 @@
|
||||
"async-wait-until": "2.0.12",
|
||||
"axe-core": "4.10.0",
|
||||
"chalk": "4.1.2",
|
||||
"dayjs": "^1.11.13",
|
||||
"deepmerge": "4.3.1",
|
||||
"dotenv": "16.4.5",
|
||||
"form-data-encoder": "4.0.2",
|
||||
@@ -1035,6 +1036,11 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/dayjs": {
|
||||
"version": "1.11.13",
|
||||
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz",
|
||||
"integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg=="
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
@@ -3158,6 +3164,11 @@
|
||||
"which": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"dayjs": {
|
||||
"version": "1.11.13",
|
||||
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz",
|
||||
"integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg=="
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"async-wait-until": "2.0.12",
|
||||
"axe-core": "4.10.0",
|
||||
"chalk": "4.1.2",
|
||||
"dayjs": "1.11.13",
|
||||
"deepmerge": "4.3.1",
|
||||
"dotenv": "16.4.5",
|
||||
"form-data-encoder": "4.0.2",
|
||||
|
||||
@@ -6,12 +6,12 @@ import {defineConfig, devices} from '@playwright/test';
|
||||
import {duration} from '@e2e-support/util';
|
||||
import testConfig from '@e2e-test.config';
|
||||
|
||||
const defaultOutputFolder = './playwright-report';
|
||||
const defaultOutputFolder = './results';
|
||||
|
||||
export default defineConfig({
|
||||
globalSetup: require.resolve('./global_setup'),
|
||||
forbidOnly: testConfig.isCI,
|
||||
outputDir: './test-results',
|
||||
outputDir: './results/tests',
|
||||
testDir: 'tests',
|
||||
timeout: duration.one_min,
|
||||
workers: testConfig.workers,
|
||||
@@ -40,7 +40,7 @@ export default defineConfig({
|
||||
screenshot: 'only-on-failure',
|
||||
timezoneId: 'America/Los_Angeles',
|
||||
trace: 'off',
|
||||
video: 'on-first-retry',
|
||||
video: 'retain-on-failure',
|
||||
actionTimeout: duration.half_min,
|
||||
storageState: {
|
||||
cookies: [],
|
||||
|
||||
94
e2e-tests/playwright/report.webhookgen.js
Исполняемый файл
94
e2e-tests/playwright/report.webhookgen.js
Исполняемый файл
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const dayjs = require('dayjs');
|
||||
const duration = require('dayjs/plugin/duration');
|
||||
dayjs.extend(duration);
|
||||
|
||||
const {TYPE, SERVER_TYPE, BRANCH, PULL_REQUEST, BUILD_ID, MM_ENV, MM_DOCKER_IMAGE, MM_DOCKER_TAG, RELEASE_DATE} =
|
||||
process.env;
|
||||
|
||||
const resultsFile = 'results/results.json';
|
||||
const summaryFile = 'results/summary.json';
|
||||
const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
|
||||
const summary = JSON.parse(fs.readFileSync(summaryFile, 'utf8'));
|
||||
const passRate = summary.passed / (summary.passed + summary.failed);
|
||||
const totalSpecs = summary.passed + summary.failed + summary.failed_expected;
|
||||
const playwrightVersion = results.config.version;
|
||||
const playwrightDuration = dayjs.duration(results.stats.duration, 'seconds').format('HH:mm:ss');
|
||||
|
||||
function generateTitle() {
|
||||
let dockerImageLink = '';
|
||||
let releaseDate = '';
|
||||
if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) {
|
||||
dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
|
||||
}
|
||||
if (RELEASE_DATE) {
|
||||
releaseDate = ` for ${RELEASE_DATE}`;
|
||||
}
|
||||
|
||||
let title = '';
|
||||
switch (TYPE) {
|
||||
case 'PR':
|
||||
title = `E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})${dockerImageLink}`;
|
||||
break;
|
||||
case 'RELEASE':
|
||||
title = `E2E for Release Build${dockerImageLink}${releaseDate}`;
|
||||
break;
|
||||
case 'MASTER':
|
||||
title = `E2E for Master Nightly Build (Prod tests)${dockerImageLink}`;
|
||||
break;
|
||||
case 'MASTER_UNSTABLE':
|
||||
title = `E2E for Master Nightly Build (Unstable tests)${dockerImageLink}`;
|
||||
break;
|
||||
case 'CLOUD':
|
||||
title = `E2E for Cloud Build (Prod tests)${dockerImageLink}${releaseDate}`;
|
||||
break;
|
||||
case 'CLOUD_UNSTABLE':
|
||||
title = `E2E for Cloud Build (Unstable tests)${dockerImageLink}`;
|
||||
break;
|
||||
default:
|
||||
title = `E2E for Build${dockerImageLink}`;
|
||||
}
|
||||
return title;
|
||||
}
|
||||
|
||||
function generateWebhookBody() {
|
||||
let testResult;
|
||||
const testResults = [
|
||||
{status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'},
|
||||
{status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'},
|
||||
{status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'},
|
||||
{status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'},
|
||||
];
|
||||
for (let i = 0; i < testResults.length; i++) {
|
||||
if (passRate >= testResults[i].cutOff) {
|
||||
testResult = testResults[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const summaryField = `${(passRate * 100).toFixed(2)}% (${summary.passed}/${totalSpecs}) | ${playwrightDuration} | playwright@${playwrightVersion}`;
|
||||
const serverTypeField = SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : '';
|
||||
const mmEnvField = MM_ENV ? '\nTest server override: ' + MM_ENV : '';
|
||||
const rollingReleaseMatchRegex = BUILD_ID?.match(/-rolling(?<version>[^-]+)-/);
|
||||
const rollingReleaseFrom = rollingReleaseMatchRegex?.groups?.version;
|
||||
const rollingReleaseFromField = rollingReleaseFrom ? `\nRolling release upgrade from: ${rollingReleaseFrom}` : '';
|
||||
return {
|
||||
username: 'Playwright UI Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [
|
||||
{
|
||||
color: testResult.color,
|
||||
author_name: 'Webapp End-to-end Testing (Playwright)',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://www.mattermost.com',
|
||||
title: generateTitle(),
|
||||
text: `${summaryField}${serverTypeField}${rollingReleaseFromField}${mmEnvField}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
let webhookBody = generateWebhookBody();
|
||||
process.stdout.write(JSON.stringify(webhookBody));
|
||||
Ссылка в новой задаче
Block a user