diff --git a/.github/workflows/e2e-fulltests-ci.yml b/.github/workflows/e2e-fulltests-ci.yml index f5dace9131..3319036a8f 100644 --- a/.github/workflows/e2e-fulltests-ci.yml +++ b/.github/workflows/e2e-fulltests-ci.yml @@ -56,7 +56,8 @@ jobs: server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection SERVER: "${{ steps.generate.outputs.SERVER }}" ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}" - TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}" + TEST_FILTER_CYPRESS: "${{ steps.generate.outputs.TEST_FILTER_CYPRESS }}" + TEST_FILTER_PLAYWRIGHT: "tests/ --project=chrome" # Note: Run on chrome but eventually will enable to all projects which include firefox and ipad. BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}" TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}" REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}" @@ -72,17 +73,17 @@ jobs: ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}" AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}" # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report - TEST_FILTER_PR: >- + TEST_FILTER_CYPRESS_PR: >- --stage="@prod" --excludeGroup="@te_only,@cloud_only,@high_availability" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap" --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal" - TEST_FILTER_PROD_ONPREM: >- + TEST_FILTER_CYPRESS_PROD_ONPREM: >- --stage="@prod" --excludeGroup="@te_only,@cloud_only,@high_availability" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks" --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal" - TEST_FILTER_PROD_CLOUD: >- + TEST_FILTER_CYPRESS_PROD_CLOUD: >- --stage="@prod" --excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks" @@ -114,7 +115,7 @@ jobs: SERVER_IMAGE_ORG=mattermostdevelopment BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent" WORKERS_NUMBER=20 - TEST_FILTER="$TEST_FILTER_PR" + TEST_FILTER_CYPRESS="$TEST_FILTER_CYPRESS_PR" COMPUTED_REPORT_TYPE="${REPORT_TYPE}" ### Run sanity assertions after variable generations [ "$REF" = "${_COMMIT_SHA_COMPUTED}" ] # 'inputs.ref' must be a full commit hash, and the commit must exist @@ -123,7 +124,7 @@ jobs: MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE) ### Populate support variables _IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # The variable's value is 'UNSTABLE' if report type is for unstable tests, otherwise it's empty - _TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}" + _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}" ### For ref and image tag generation: ignore 'inputs.ref', and use master branch directly. Note that 'COMMIT_SHA' will be used for reporting the test result, and for checking out the testing scripts and test cases COMMIT_SHA="$(git rev-parse --verify origin/master)" BRANCH=master @@ -132,13 +133,13 @@ jobs: BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent" BUILD_ID_SUFFIX_IN_STATUS_CHECK=true WORKERS_NUMBER=10 # Daily tests are not time critical, and it's more efficient to run on fewer workers - TEST_FILTER="${!_TEST_FILTER_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}" + TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}" TM4J_ENABLE=true COMPUTED_REPORT_TYPE="${REPORT_TYPE}" ;; RELEASE | RELEASE_CLOUD) ### Populate support variables - _TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}" + _TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}" ### For ref and image tag generation: assume the 'inputs.ref' is a tag, and use the first two digits to construct the branch name COMMIT_SHA="$(git rev-parse --verify HEAD)" BRANCH=$(sed -E "s/v([0-9]+)\.([0-9]+)\..+$/release-\1.\2/g" <<<$REF) @@ -147,7 +148,7 @@ jobs: BUILD_ID_SUFFIX="release-${SERVER}-ent" BUILD_ID_SUFFIX_IN_STATUS_CHECK=true WORKERS_NUMBER=20 - TEST_FILTER="${!_TEST_FILTER_VARIABLE}" + TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE}" TM4J_ENABLE=true COMPUTED_REPORT_TYPE=RELEASE ### Run sanity assertions after variable generations @@ -179,7 +180,7 @@ jobs: echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT - echo "TEST_FILTER=${TEST_FILTER}" >> $GITHUB_OUTPUT + echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT @@ -199,13 +200,13 @@ jobs: gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <> $GITHUB_OUTPUT + echo "node-cache-dependency-path=e2e-tests/${TEST}/package-lock.json" >> $GITHUB_OUTPUT generate-test-cycle: runs-on: ubuntu-latest needs: - - update-initial-status + - generate-build-variables defaults: run: shell: bash @@ -226,7 +242,7 @@ jobs: with: node-version-file: ".nvmrc" cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" + cache-dependency-path: "e2e-tests/cypress/package-lock.json" # NB: the generate-cycle script is cypress-specific operation for now - name: ci/e2e-test-gencycle id: e2e-test-gencycle env: @@ -234,6 +250,7 @@ jobs: AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}" BRANCH: "${{ inputs.BRANCH }}" BUILD_ID: "${{ inputs.BUILD_ID }}" + TEST: "${{ inputs.TEST }}" TEST_FILTER: "${{ inputs.TEST_FILTER }}" run: | set -e -o pipefail @@ -280,6 +297,7 @@ jobs: SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}" MM_LICENSE: "${{ secrets.MM_LICENSE }}" ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}" + TEST: "${{ inputs.TEST }}" TEST_FILTER: "${{ inputs.TEST_FILTER }}" MM_ENV: "${{ inputs.MM_ENV }}" BRANCH: "${{ inputs.BRANCH }}" @@ -311,13 +329,13 @@ jobs: with: node-version-file: ".nvmrc" cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" + cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }} - name: ci/e2e-test run: | make cloud-init if [ -n "$ROLLING_RELEASE_SERVER_IMAGE" ]; then echo "RollingRelease: checking out E2E test cases from revision ${ROLLING_RELEASE_COMMIT_SHA}, for initial smoketest" - git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- cypress/ && git status + git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- "${TEST}/" && git status ( echo "RollingRelease: running initial smoketest against image $ROLLING_RELEASE_SERVER_IMAGE" export SERVER_IMAGE="$ROLLING_RELEASE_SERVER_IMAGE" @@ -326,12 +344,12 @@ jobs: make ) echo "RollingRelease: asserting smoketest result has zero failures." - FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) + FAILURES=$(jq -r '.failed' "${TEST}/results/summary.json") if [ "$FAILURES" -ne "0" ]; then echo "RollingRelease: initial smoketest for rolling release E2E run has nonzero ($FAILURES) failures. Aborting test run." >&2 exit 1 fi - rm -rfv cypress/results + rm -rfv "${TEST}/{results,logs}" echo "RollingRelease: reset the E2E test cases to the revision to test" git reset --hard HEAD && git status echo "RollingRelease: smoketest completed. Starting full E2E tests." @@ -340,11 +358,12 @@ jobs: make cloud-teardown - name: ci/e2e-test-store-results uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + if: always() with: - name: e2e-test-results-${{ matrix.os }}-${{ matrix.worker_index }} + name: e2e-test-results-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }} path: | - e2e-tests/cypress/logs/ - e2e-tests/cypress/results/ + e2e-tests/${{ inputs.TEST }}/logs/ + e2e-tests/${{ inputs.TEST }}/results/ retention-days: 1 report: @@ -357,10 +376,11 @@ jobs: shell: bash working-directory: e2e-tests outputs: - successes: "${{ steps.calculate-results.outputs.successes }}" - failures: "${{ steps.calculate-results.outputs.failures }}" - failures_expected: "${{ steps.calculate-results.outputs.failures_expected }}" - total_specs: "${{ steps.calculate-results.outputs.total_specs }}" + passed: "${{ steps.calculate-results.outputs.passed }}" + failed: "${{ steps.calculate-results.outputs.failed }}" + failed_expected: "${{ steps.calculate-results.outputs.failed_expected }}" + pass_rate: "${{ steps.calculate-results.outputs.pass_rate }}" + commit_status_message: "${{ steps.calculate-results.outputs.commit_status_message }}" steps: - name: ci/checkout-repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -370,15 +390,15 @@ jobs: - name: ci/download-artifacts uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 with: - path: e2e-tests/cypress/ + path: e2e-tests/${{ inputs.TEST }}/ merge-multiple: true - name: ci/upload-report-global uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: e2e-test-results + name: e2e-test-results-${{ inputs.TEST }} path: | - e2e-tests/cypress/logs/ - e2e-tests/cypress/results/ + e2e-tests/${{ inputs.TEST }}/logs/ + e2e-tests/${{ inputs.TEST }}/results/ - name: ci/setup-node if: "${{ inputs.enable_reporting }}" uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 @@ -386,11 +406,12 @@ jobs: with: node-version-file: ".nvmrc" cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" + cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }} - name: ci/publish-report if: "${{ inputs.enable_reporting }}" env: TYPE: "${{ inputs.REPORT_TYPE }}" + TEST: "${{ inputs.TEST }}" SERVER: "${{ inputs.SERVER }}" SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}" AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}" @@ -407,38 +428,43 @@ jobs: if: "${{ inputs.enable_reporting }}" uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: e2e-test-results + name: e2e-test-results-${{ inputs.TEST }} path: | - e2e-tests/cypress/logs/ - e2e-tests/cypress/results/ + e2e-tests/${{ inputs.TEST }}/logs/ + e2e-tests/${{ inputs.TEST }}/results/ overwrite: true - name: ci/report-calculate-results id: calculate-results env: - AD_CYCLE_FILE: "cypress/results/ad_cycle.json" + TEST: "${{ inputs.TEST }}" run: | + AD_CYCLE_FILE="${TEST}/results/ad_cycle.json" if [ -f "$AD_CYCLE_FILE" ]; then # Prefer using the Automation Dashboard's results to calculate failures - SUCCESSES=$(jq -r .pass "$AD_CYCLE_FILE") - FAILURES=$(jq -r .fail "$AD_CYCLE_FILE") - FAILURES_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE") + export PASSED=$(jq -r .pass "$AD_CYCLE_FILE") + export FAILED=$(jq -r .fail "$AD_CYCLE_FILE") + export FAILED_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE") else - # Otherwise, utilize the test specs to calculate the failures - SUCCESSES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.passes' | jq -s add) - FAILURES=$(find cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) - FAILURES_EXPECTED="0" + # Otherwise, utilize summary.json to calculate the failures + # NB: in this job, this file only makes sense if a single worker is used, as with Playwright + export PASSED=$(jq '.passed' "${TEST}/results/summary.json") + export FAILED=$(jq '.failed' "${TEST}/results/summary.json") + export FAILED_EXPECTED=$(jq '.failed_expected' "${TEST}/results/summary.json") fi - TOTAL_SPECS=$(( FAILURES + SUCCESSES )) - echo "successes=${SUCCESSES:?}" >> $GITHUB_OUTPUT - echo "failures=${FAILURES:?}" >> $GITHUB_OUTPUT - echo "failures_expected=${FAILURES_EXPECTED:?}" >> $GITHUB_OUTPUT - echo "total_specs=${TOTAL_SPECS:?}" >> $GITHUB_OUTPUT - echo "Cypress run completed: ${SUCCESSES}/${TOTAL_SPECS} passing specs (plus ${FAILURES_EXPECTED} unrelated failures)" + export TOTAL_SPECS=$(( PASSED + FAILED )) + export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f') + export COMMIT_STATUS_MESSAGE="Completed with pass rate: ${PASS_RATE}% (plus ${FAILED_EXPECTED} unrelated failures)" + echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT + echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT + echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT + echo "pass_rate=${PASS_RATE:?}%" >> $GITHUB_OUTPUT + echo "commit_status_message=${COMMIT_STATUS_MESSAGE:?}" >> $GITHUB_OUTPUT + echo "$COMMIT_STATUS_MESSAGE" - name: ci/e2e-test-assert-results if: "${{ inputs.testcase_failure_fatal }}" run: | # Assert that the run contained 0 failures - [ "${{ steps.calculate-results.outputs.failures }}" = "0" ] + [ "${{ steps.calculate-results.outputs.failed }}" = "0" ] update-failure-final-status: runs-on: ubuntu-latest @@ -455,8 +481,7 @@ jobs: repository_full_name: ${{ github.repository }} commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.status_check_context }} - description: | - Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures) + description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }} status: failure target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}" @@ -475,7 +500,6 @@ jobs: repository_full_name: ${{ github.repository }} commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.status_check_context }} - description: | - Completed: ${{ needs.report.outputs.successes }}/${{ needs.report.outputs.total_specs }} passing specs (plus ${{ needs.report.outputs.failures_expected }} unrelated failures) + description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }} status: success target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}" diff --git a/.gitignore b/.gitignore index c56cb15610..8af02c68bf 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ e2e-tests/playwright/node_modules e2e-tests/playwright/playwright-report e2e-tests/playwright/storage_state e2e-tests/playwright/test-results +e2e-tests/playwright/results e2e-tests/playwright/tests/**/*-darwin.png e2e-tests/playwright/tests/**/*-window.png e2e-tests/playwright/tests/accessibility/**/*-snapshots @@ -155,4 +156,4 @@ docker-compose.override.yaml ## Notice Folders .notice/ -.notice-work/ \ No newline at end of file +.notice-work/ diff --git a/e2e-tests/.ci/.e2erc b/e2e-tests/.ci/.e2erc index fed1ea8653..152eda4bf3 100644 --- a/e2e-tests/.ci/.e2erc +++ b/e2e-tests/.ci/.e2erc @@ -90,10 +90,17 @@ export BROWSER_DEFAULT="chrome" export SERVER_DEFAULT="onprem" export TEST_DEFAULT="cypress" export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket" -export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown") export BUILD_ID_DEFAULT=$(date +%s) export CI_BASE_URL_DEFAULT="localhost" +# Test tool default overrides +case "${TEST:-$TEST_DEFAULT}" in + cypress ) + export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' ;; + playwright ) + export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;; + * ) +esac # OS specific defaults overrides case $MME2E_OSTYPE in darwin ) diff --git a/e2e-tests/.ci/dashboard.generate_test_cycle.sh b/e2e-tests/.ci/dashboard.generate_test_cycle.sh index 603f3edeac..1f0b91fbc2 100644 --- a/e2e-tests/.ci/dashboard.generate_test_cycle.sh +++ b/e2e-tests/.ci/dashboard.generate_test_cycle.sh @@ -3,13 +3,16 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc -mme2e_log "Loading variables from .env.cypress" -if ! [ -f .env.cypress ]; then - mme2e_log "Error: .env.cypress is required to exist, for the test cycle to be generated. Aborting." >&2 +TEST_ENV_FILE=".env.$TEST" + +mme2e_log "Loading variables from $TEST_ENV_FILE" +if ! [ -f "$TEST_ENV_FILE" ]; then + mme2e_log "Error: $TEST_ENV_FILE is required to exist, for the test cycle to be generated. Aborting." >&2 exit 1 fi set -a -. .env.cypress +# shellcheck disable=SC1090 +. "$TEST_ENV_FILE" if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation." @@ -17,7 +20,7 @@ if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then fi mme2e_log "Generating the test cycle on the Automation Dashboard" -cd ../cypress +cd "../$TEST" npm i # shellcheck disable=SC2086 exec node --trace-warnings generate_test_cycle.js $TEST_FILTER diff --git a/e2e-tests/.ci/report.collect.sh b/e2e-tests/.ci/report.collect.sh index 1a9e5198f7..85cc2ef7ab 100755 --- a/e2e-tests/.ci/report.collect.sh +++ b/e2e-tests/.ci/report.collect.sh @@ -8,8 +8,8 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc -# The collected data will be written to the "e2e-tests/cypress/results/" directory -cd ../cypress/ +# The collected data will be written to the "e2e-tests/$TEST/results/" directory +cd "../$TEST/" if [ ! -d "results/" ]; then mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2 exit 1 diff --git a/e2e-tests/.ci/report.publish.sh b/e2e-tests/.ci/report.publish.sh index 4fc4dc045e..31e7076c57 100755 --- a/e2e-tests/.ci/report.publish.sh +++ b/e2e-tests/.ci/report.publish.sh @@ -24,7 +24,7 @@ if [ "$TYPE" = "PR" ]; then fi # Env vars used during the test. Their values will be included in the report -: ${TEST:?} # See E2E tests' readme; usually set to 'cypress' +: ${TEST:?} # See E2E tests' readme : ${BRANCH:?} # May be either a ref, a commit hash, or 'server-pr-PR_NUMBER' (if TYPE=PR) : ${BUILD_ID:?} : ${SERVER:?} # May be either 'onprem' or 'cloud' @@ -89,11 +89,23 @@ if [ -n "${AWS_S3_BUCKET:-}" ]; then mme2e_log "S3 report upload enabled." fi -cd ../cypress/ +# Double check that the "results/" subdirectory to collect report informations from exists +cd "../${TEST}/" if [ ! -d "results/" ]; then - mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2 + mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2 exit 1 fi -npm i -node save_report.js +case "$TEST" in + cypress) + npm i + node save_report.js + ;; + playwright) + if [ -n "$WEBHOOK_URL" ]; then + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm i + # Utilize environment data and report files to generate the webhook body + ./report.webhookgen.js | curl -X POST -fsSL -H 'Content-Type: application/json' -d @- "$WEBHOOK_URL" + fi + ;; +esac diff --git a/e2e-tests/.ci/server.generate.sh b/e2e-tests/.ci/server.generate.sh index b64ee64e44..c6280e5a41 100755 --- a/e2e-tests/.ci/server.generate.sh +++ b/e2e-tests/.ci/server.generate.sh @@ -224,6 +224,7 @@ $(if mme2e_is_token_in_list "playwright" "$ENABLED_DOCKER_SERVICES"; then - "./.env.playwright" environment: CI: "true" + PLAYWRIGHT_SKIP_BROWSER_GC: 1 PW_BASE_URL: http://localhost:8065 PW_ADMIN_USERNAME: sysadmin PW_ADMIN_PASSWORD: Sys@dmin-sample1 diff --git a/e2e-tests/.ci/server.run_cypress.sh b/e2e-tests/.ci/server.run_cypress.sh index 36d6deb51c..8e5ed7598c 100755 --- a/e2e-tests/.ci/server.run_cypress.sh +++ b/e2e-tests/.ci/server.run_cypress.sh @@ -1,4 +1,5 @@ #!/bin/bash +# shellcheck disable=SC2038 set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc @@ -40,5 +41,14 @@ else ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee "../cypress/logs/${LOGFILE_SUFFIX}_cypress.log" fi +# Collect run results +cat > ../cypress/results/summary.json < "../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log" 2>&1 diff --git a/e2e-tests/.ci/server.run_playwright.sh b/e2e-tests/.ci/server.run_playwright.sh index 7f7bd43257..63061b9251 100755 --- a/e2e-tests/.ci/server.run_playwright.sh +++ b/e2e-tests/.ci/server.run_playwright.sh @@ -3,22 +3,21 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc -# Initialize Playwright report directory mme2e_log "Prepare Playwright: clean and initialize report and logs directory" ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash < ../playwright/results/summary.json <../playwright/logs/mattermost.log 2>&1 diff --git a/e2e-tests/README.md b/e2e-tests/README.md index e00324a9b8..8715204021 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -26,7 +26,8 @@ Instructions, detailed: * The following variables, will be passed over to the server container: `MM_LICENSE` (no enterprise features will be available if this is unset; required when `SERVER=cloud`), and the exploded `MM_ENV` (a comma-separated list of env var specifications) * The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` * The `SERVER_IMAGE` variable can also be set if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used. - * The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format. + * The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress/Playwright to run. If not specified, only the smoke tests will run + - Its format depends on which tool is used: for Cypress, please check the `e2e-tests/cypress/run_tests.js` file for details. For Playwright, it can simply be populated with arguments you want to give to the `playwright test` command. * More variables may be required to configure reporting and cloud interactions. Check the content of the `.ci/report.*.sh` and `.ci/server.cloud_*.sh` scripts for reference. 2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID` * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`. @@ -41,6 +42,7 @@ Instructions, detailed: * If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make` * If you just want to run a local server instance, without any further testing, you can run `TEST=none make` * If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machines (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running. + * This script will also parse the local test results, and write a `e2e-tests/${TEST}/results/summary.json` file containing the following keys: `passed`, `failed` and `failed_expected` (the total number of testcases that were run is the sum of these three numbers) 4. `make stop`: tears down the server (and the dashboard, if running) * This will stop and cleanup all of the E2E testing containers, including the database and its persistent volume. * This also implicitly runs `make clean`, which also removes any generated environment or docker-compose files. diff --git a/e2e-tests/playwright/.eslintrc.json b/e2e-tests/playwright/.eslintrc.json index 20064e2aa4..35d4515698 100644 --- a/e2e-tests/playwright/.eslintrc.json +++ b/e2e-tests/playwright/.eslintrc.json @@ -9,6 +9,10 @@ "rules": { "@typescript-eslint/explicit-module-boundary-types": "off", "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-var-requires": "off", "no-console": "error" + }, + "env": { + "node": true } } diff --git a/e2e-tests/playwright/package-lock.json b/e2e-tests/playwright/package-lock.json index 4e7b8b0d8e..2460214dd8 100644 --- a/e2e-tests/playwright/package-lock.json +++ b/e2e-tests/playwright/package-lock.json @@ -14,6 +14,7 @@ "async-wait-until": "2.0.12", "axe-core": "4.10.0", "chalk": "4.1.2", + "dayjs": "^1.11.13", "deepmerge": "4.3.1", "dotenv": "16.4.5", "form-data-encoder": "4.0.2", @@ -1035,6 +1036,11 @@ "node": ">= 8" } }, + "node_modules/dayjs": { + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", + "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==" + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -3158,6 +3164,11 @@ "which": "^2.0.1" } }, + "dayjs": { + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", + "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==" + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", diff --git a/e2e-tests/playwright/package.json b/e2e-tests/playwright/package.json index af586d28d2..04feebaacb 100644 --- a/e2e-tests/playwright/package.json +++ b/e2e-tests/playwright/package.json @@ -22,6 +22,7 @@ "async-wait-until": "2.0.12", "axe-core": "4.10.0", "chalk": "4.1.2", + "dayjs": "1.11.13", "deepmerge": "4.3.1", "dotenv": "16.4.5", "form-data-encoder": "4.0.2", diff --git a/e2e-tests/playwright/playwright.config.ts b/e2e-tests/playwright/playwright.config.ts index 0bdc2dae54..ffe751164c 100644 --- a/e2e-tests/playwright/playwright.config.ts +++ b/e2e-tests/playwright/playwright.config.ts @@ -6,12 +6,12 @@ import {defineConfig, devices} from '@playwright/test'; import {duration} from '@e2e-support/util'; import testConfig from '@e2e-test.config'; -const defaultOutputFolder = './playwright-report'; +const defaultOutputFolder = './results'; export default defineConfig({ globalSetup: require.resolve('./global_setup'), forbidOnly: testConfig.isCI, - outputDir: './test-results', + outputDir: './results/tests', testDir: 'tests', timeout: duration.one_min, workers: testConfig.workers, @@ -40,7 +40,7 @@ export default defineConfig({ screenshot: 'only-on-failure', timezoneId: 'America/Los_Angeles', trace: 'off', - video: 'on-first-retry', + video: 'retain-on-failure', actionTimeout: duration.half_min, storageState: { cookies: [], diff --git a/e2e-tests/playwright/report.webhookgen.js b/e2e-tests/playwright/report.webhookgen.js new file mode 100755 index 0000000000..d59d15f92a --- /dev/null +++ b/e2e-tests/playwright/report.webhookgen.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const dayjs = require('dayjs'); +const duration = require('dayjs/plugin/duration'); +dayjs.extend(duration); + +const {TYPE, SERVER_TYPE, BRANCH, PULL_REQUEST, BUILD_ID, MM_ENV, MM_DOCKER_IMAGE, MM_DOCKER_TAG, RELEASE_DATE} = + process.env; + +const resultsFile = 'results/results.json'; +const summaryFile = 'results/summary.json'; +const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8')); +const summary = JSON.parse(fs.readFileSync(summaryFile, 'utf8')); +const passRate = summary.passed / (summary.passed + summary.failed); +const totalSpecs = summary.passed + summary.failed + summary.failed_expected; +const playwrightVersion = results.config.version; +const playwrightDuration = dayjs.duration(results.stats.duration, 'seconds').format('HH:mm:ss'); + +function generateTitle() { + let dockerImageLink = ''; + let releaseDate = ''; + if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) { + dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`; + } + if (RELEASE_DATE) { + releaseDate = ` for ${RELEASE_DATE}`; + } + + let title = ''; + switch (TYPE) { + case 'PR': + title = `E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})${dockerImageLink}`; + break; + case 'RELEASE': + title = `E2E for Release Build${dockerImageLink}${releaseDate}`; + break; + case 'MASTER': + title = `E2E for Master Nightly Build (Prod tests)${dockerImageLink}`; + break; + case 'MASTER_UNSTABLE': + title = `E2E for Master Nightly Build (Unstable tests)${dockerImageLink}`; + break; + case 'CLOUD': + title = `E2E for Cloud Build (Prod tests)${dockerImageLink}${releaseDate}`; + break; + case 'CLOUD_UNSTABLE': + title = `E2E for Cloud Build (Unstable tests)${dockerImageLink}`; + break; + default: + title = `E2E for Build${dockerImageLink}`; + } + return title; +} + +function generateWebhookBody() { + let testResult; + const testResults = [ + {status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'}, + {status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'}, + {status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'}, + {status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'}, + ]; + for (let i = 0; i < testResults.length; i++) { + if (passRate >= testResults[i].cutOff) { + testResult = testResults[i]; + break; + } + } + + const summaryField = `${(passRate * 100).toFixed(2)}% (${summary.passed}/${totalSpecs}) | ${playwrightDuration} | playwright@${playwrightVersion}`; + const serverTypeField = SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : ''; + const mmEnvField = MM_ENV ? '\nTest server override: ' + MM_ENV : ''; + const rollingReleaseMatchRegex = BUILD_ID?.match(/-rolling(?[^-]+)-/); + const rollingReleaseFrom = rollingReleaseMatchRegex?.groups?.version; + const rollingReleaseFromField = rollingReleaseFrom ? `\nRolling release upgrade from: ${rollingReleaseFrom}` : ''; + return { + username: 'Playwright UI Test', + icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png', + attachments: [ + { + color: testResult.color, + author_name: 'Webapp End-to-end Testing (Playwright)', + author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png', + author_link: 'https://www.mattermost.com', + title: generateTitle(), + text: `${summaryField}${serverTypeField}${rollingReleaseFromField}${mmEnvField}`, + }, + ], + }; +} + +let webhookBody = generateWebhookBody(); +process.stdout.write(JSON.stringify(webhookBody));