CLD-8395 Integrate playwright into E2E tests (#28492)

* Finalize Playwright E2E scripts integration, add videos
* Add suggestion by Saturn

Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

e2e-tests/.ci/.e2erc

@@ -90,10 +90,17 @@ export BROWSER_DEFAULT="chrome"
 export SERVER_DEFAULT="onprem"
 export TEST_DEFAULT="cypress"
 export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
-export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
 export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown")
 export BUILD_ID_DEFAULT=$(date +%s)
 export CI_BASE_URL_DEFAULT="localhost"
+# Test tool default overrides
+case "${TEST:-$TEST_DEFAULT}" in
+  cypress )
+    export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' ;;
+  playwright )
+    export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;;
+  * )
+esac
 # OS specific defaults overrides
 case $MME2E_OSTYPE in
   darwin )

e2e-tests/.ci/dashboard.generate_test_cycle.sh

@@ -3,13 +3,16 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-mme2e_log "Loading variables from .env.cypress"
-if ! [ -f .env.cypress ]; then
-  mme2e_log "Error: .env.cypress is required to exist, for the test cycle to be generated. Aborting." >&2
+TEST_ENV_FILE=".env.$TEST"
+
+mme2e_log "Loading variables from $TEST_ENV_FILE"
+if ! [ -f "$TEST_ENV_FILE" ]; then
+  mme2e_log "Error: $TEST_ENV_FILE is required to exist, for the test cycle to be generated. Aborting." >&2
   exit 1
 fi
 set -a
-. .env.cypress
+# shellcheck disable=SC1090
+. "$TEST_ENV_FILE"
 
 if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
   mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation."
@@ -17,7 +20,7 @@ if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
 fi
 
 mme2e_log "Generating the test cycle on the Automation Dashboard"
-cd ../cypress
+cd "../$TEST"
 npm i
 # shellcheck disable=SC2086
 exec node --trace-warnings generate_test_cycle.js $TEST_FILTER

e2e-tests/.ci/report.collect.sh

@@ -8,8 +8,8 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-# The collected data will be written to the "e2e-tests/cypress/results/" directory
-cd ../cypress/
+# The collected data will be written to the "e2e-tests/$TEST/results/" directory
+cd "../$TEST/"
 if [ ! -d "results/" ]; then
   mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2
   exit 1

e2e-tests/.ci/report.publish.sh

@@ -24,7 +24,7 @@ if [ "$TYPE" = "PR" ]; then
 fi
 
 # Env vars used during the test. Their values will be included in the report
-: ${TEST:?} # See E2E tests' readme; usually set to 'cypress'
+: ${TEST:?} # See E2E tests' readme
 : ${BRANCH:?} # May be either a ref, a commit hash, or 'server-pr-PR_NUMBER' (if TYPE=PR)
 : ${BUILD_ID:?}
 : ${SERVER:?} # May be either 'onprem' or 'cloud'
@@ -89,11 +89,23 @@ if [ -n "${AWS_S3_BUCKET:-}" ]; then
   mme2e_log "S3 report upload enabled."
 fi
 
-cd ../cypress/
+# Double check that the "results/" subdirectory to collect report informations from exists
+cd "../${TEST}/"
 if [ ! -d "results/" ]; then
-  mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2
+  mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2
   exit 1
 fi
 
-npm i
-node save_report.js
+case "$TEST" in
+  cypress)
+    npm i
+    node save_report.js
+    ;;
+  playwright)
+    if [ -n "$WEBHOOK_URL" ]; then
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm i
+      # Utilize environment data and report files to generate the webhook body
+      ./report.webhookgen.js | curl -X POST -fsSL -H 'Content-Type: application/json' -d @- "$WEBHOOK_URL"
+    fi
+    ;;
+esac

e2e-tests/.ci/server.generate.sh

@@ -224,6 +224,7 @@ $(if mme2e_is_token_in_list "playwright" "$ENABLED_DOCKER_SERVICES"; then
       - "./.env.playwright"
     environment:
       CI: "true"
+      PLAYWRIGHT_SKIP_BROWSER_GC: 1
       PW_BASE_URL: http://localhost:8065
       PW_ADMIN_USERNAME: sysadmin
       PW_ADMIN_PASSWORD: Sys@dmin-sample1

e2e-tests/.ci/server.run_cypress.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# shellcheck disable=SC2038
 set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
@@ -40,5 +41,14 @@ else
   ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee "../cypress/logs/${LOGFILE_SUFFIX}_cypress.log"
 fi
 
+# Collect run results
+cat > ../cypress/results/summary.json <<EOF
+{
+  "passed": $(find ../cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.passes' | jq -s add),
+  "failed": $(find ../cypress/results/mochawesome-report/json/tests/ -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add),
+  "failed_expected": 0
+}
+EOF
+
 # Collect server logs
 ${MME2E_DC_SERVER} logs --no-log-prefix -- server > "../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log" 2>&1

e2e-tests/.ci/server.run_playwright.sh

@@ -3,22 +3,21 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-# Initialize Playwright report directory
 mme2e_log "Prepare Playwright: clean and initialize report and logs directory"
 ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<EOF
 cd e2e-tests/playwright
-rm -rf logs playwright-report test-results storage_state
-mkdir -p logs
+rm -rf logs results storage_state
+mkdir -p logs results
 touch logs/mattermost.log
 EOF
 
-# Install webapp dependencies
-mme2e_log "Prepare Playwright: install webapp dependencies"
-${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash -c "cd webapp && npm install --cache /tmp/empty-cache"
-
-# Install Playwright dependencies
 mme2e_log "Prepare Playwright: install dependencies"
-${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && rm -rf node_modules && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --cache /tmp/empty-cache"
+${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<EOF
+cd webapp/
+npm install --cache /tmp/empty-cache
+cd ../e2e-tests/playwright
+npm install --cache /tmp/empty-cache
+EOF
 
 mme2e_log "Prepare Playwright: environment info"
 ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- playwright bash <<"EOF"
@@ -37,8 +36,21 @@ EOF
 # export DEBUG=pw:protocol,pw:browser,pw:api
 
 # Run Playwright test
-# Note: Run on chrome but eventually will enable to all projects which include firefox and ipad.
-${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && npm run test -- --project=chrome" | tee ../playwright/logs/playwright.log
+# NB: do not exit the script if some testcases fail
+${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/playwright && npm run test -- ${TEST_FILTER}" | tee ../playwright/logs/playwright.log || true
+
+# Collect run results
+# Documentation on the results.json file: https://playwright.dev/docs/api/class-testcase#test-case-expected-status
+# NB: the following line is needed only for compatibility reasons, to support RollingRelease tests for versions prior to v10.1.0
+#     It can be removed after releases <=v10.0.x are phased out
+mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json || true
+jq -f /dev/stdin ../playwright/results/results.json > ../playwright/results/summary.json <<EOF
+{
+  passed: .stats.expected,
+  failed: .stats.unexpected,
+  failed_expected: (.stats.skipped + .stats.flaky)
+}
+EOF
 
 # Collect server logs
 ${MME2E_DC_SERVER} logs --no-log-prefix -- server >../playwright/logs/mattermost.log 2>&1