From 43c3003e9d55450b0f15308af623061e55d6ce2a Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Tue, 9 Jan 2024 11:40:33 +0100 Subject: [PATCH] CLD-6867 E2E Tests - Decouple cycle creation from test running (#25367) * Separate test cycle generation from test running --------- Co-authored-by: Mattermost Build --- e2e-tests/.ci/.e2erc | 2 ++ e2e-tests/.ci/dashboard.generate_test_cycle.sh | 17 +++++++++++++++++ e2e-tests/.ci/server.generate.sh | 9 ++++----- e2e-tests/.ci/server.run_cypress.sh | 6 ------ e2e-tests/.ci/server.start.sh | 6 ------ e2e-tests/Makefile | 4 +++- e2e-tests/README.md | 12 +++++++----- e2e-tests/cypress/generate_test_cycle.js | 1 - 8 files changed, 33 insertions(+), 24 deletions(-) create mode 100644 e2e-tests/.ci/dashboard.generate_test_cycle.sh diff --git a/e2e-tests/.ci/.e2erc b/e2e-tests/.ci/.e2erc index 95c1aad2e3..907657ceff 100644 --- a/e2e-tests/.ci/.e2erc +++ b/e2e-tests/.ci/.e2erc @@ -16,6 +16,7 @@ export BROWSER_DEFAULT="chrome" export SERVER_DEFAULT="self-hosted" export TEST_DEFAULT="cypress" export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket" +export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' # OS specific defaults overrides case $MME2E_OSTYPE in darwin ) @@ -29,6 +30,7 @@ export BROWSER=${BROWSER:-$BROWSER_DEFAULT} export SERVER=${SERVER:-$SERVER_DEFAULT} export TEST=${TEST:-$TEST_DEFAULT} export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT} +export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT} # Function definitions mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; } diff --git a/e2e-tests/.ci/dashboard.generate_test_cycle.sh b/e2e-tests/.ci/dashboard.generate_test_cycle.sh new file mode 100644 index 0000000000..fea409663b --- /dev/null +++ b/e2e-tests/.ci/dashboard.generate_test_cycle.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e -u -o pipefail +cd "$(dirname "$0")" +. .e2erc + +mme2e_log "Loading variables from .env.cypress" +if ! [ -f .env.cypress ]; then + mme2e_log "Error: .env.cypress is required to exist, for the test cycle to be generated. Aborting." >&2 + exit 1 +fi +set -a +. .env.cypress + +mme2e_log "Generating the test cycle on the Automation Dashboard" +cd ../cypress +# shellcheck disable=SC2086 +exec node --trace-warnings generate_test_cycle.js $TEST_FILTER diff --git a/e2e-tests/.ci/server.generate.sh b/e2e-tests/.ci/server.generate.sh index 402100b8af..427fc058ec 100755 --- a/e2e-tests/.ci/server.generate.sh +++ b/e2e-tests/.ci/server.generate.sh @@ -172,9 +172,6 @@ $(if mme2e_is_token_in_list "cypress" "$ENABLED_DOCKER_SERVICES"; then env_file: - "../../e2e-tests/.ci/.env.cypress" environment: - REPO: "mattermost" - # Cypress configuration - HEADLESS: "true" CYPRESS_baseUrl: "http://localhost:8065" CYPRESS_dbConnection: "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10" CYPRESS_smtpUrl: "http://localhost:9001" @@ -290,6 +287,8 @@ generate_env_files() { export BRANCH=${BRANCH:-$BRANCH_DEFAULT} export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT} export CI_BASE_URL="${CI_BASE_URL:-localhost}" + export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run + export HEADLESS=true # Static, but declared here for making generate_test_cycle.js easier to run case "$TEST" in cypress) mme2e_log "Cypress: Generating .env.cypress" @@ -298,8 +297,8 @@ generate_env_files() { BUILD_ID CI_BASE_URL BROWSER - AUTOMATION_DASHBOARD_URL - AUTOMATION_DASHBOARD_TOKEN + HEADLESS + REPO EOF # Adding service-specific cypress variables for SERVICE in $ENABLED_DOCKER_SERVICES; do diff --git a/e2e-tests/.ci/server.run_cypress.sh b/e2e-tests/.ci/server.run_cypress.sh index 115f2d6a24..951156e899 100755 --- a/e2e-tests/.ci/server.run_cypress.sh +++ b/e2e-tests/.ci/server.run_cypress.sh @@ -3,10 +3,6 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc -# Set required variables -TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' -TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT} - # Print run information mme2e_log "Printing Cypress container informations" ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node -p 'module.paths' @@ -36,8 +32,6 @@ EOF # shellcheck disable=SC2016 if ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress bash -c '[ -n "${AUTOMATION_DASHBOARD_URL}" ]'; then mme2e_log "AUTOMATION_DASHBOARD_URL is set. Using run_test_cycle.js for the cypress run" - # shellcheck disable=SC2086 - ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node --trace-warnings generate_test_cycle.js $TEST_FILTER ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee ../cypress/logs/cypress.log else mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Using run_tests.js for the cypress run" diff --git a/e2e-tests/.ci/server.start.sh b/e2e-tests/.ci/server.start.sh index e1ab708935..a28460354b 100755 --- a/e2e-tests/.ci/server.start.sh +++ b/e2e-tests/.ci/server.start.sh @@ -3,12 +3,6 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc -BRANCH_DEFAULT=$(git branch --show-current) -export BRANCH=${BRANCH:-$BRANCH_DEFAULT} -BUILD_ID_DEFAULT=$(date +%s) -export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT} -export CI_BASE_URL="${CI_BASE_URL:-localhost}" - # Cleanup old containers, if any mme2e_log "Stopping leftover E2E containers, if any are running" ${MME2E_DC_SERVER} down -v --remove-orphans diff --git a/e2e-tests/Makefile b/e2e-tests/Makefile index 8b0d5c0eb2..5ac5248c86 100644 --- a/e2e-tests/Makefile +++ b/e2e-tests/Makefile @@ -25,9 +25,11 @@ stop-server: generate-server bash ./.ci/server.stop.sh restart-server: stop-server start-server -.PHONY: start-dashboard stop-dashboard +.PHONY: start-dashboard generate-test-cycle stop-dashboard start-dashboard: bash ./.ci/dashboard.start.sh +generate-test-cycle: requirecmd-node generate-server + bash ./.ci/dashboard.generate_test_cycle.sh stop-dashboard: bash ./.ci/dashboard.stop.sh diff --git a/e2e-tests/README.md b/e2e-tests/README.md index c1db06e87f..2a8f34ce20 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -10,7 +10,7 @@ Please refer to the [dedicated developer documentation](https://developers.matte ##### For pipeline debugging -The E2E testing pipeline's scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`) +The E2E testing pipeline's scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, `node` and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`) Instructions, tl;dr: create a local branch with your E2E test changes, then open a PR to the `mattermost-server` repo targeting the `master` branch (so that CI will produce the image that docker-compose needs), then run `make` in this directory. @@ -27,8 +27,9 @@ Instructions, detailed: * The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` * The `SERVER_IMAGE` variable can also be set, if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used. * The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format. -2. (optional) `make start-dashboard`: start the automation-dashboard in the background - * This also sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container +2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation-dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID` + * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`. + * This also automatically sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container * Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your `.ci/env` file, the latter variables will take precedence. * The dashboard is used for orchestrating specs with parallel test run and is typically used in CI. * Only Cypress is currently using the dashboard; Playwright is not. @@ -43,11 +44,12 @@ Instructions, detailed: Notes: - Setting a variable in `.ci/env` is functionally equivalent to exporting variables in your current shell's environment, before invoking the makefile. - The `.ci/.env.*` files are auto generated by the pipeline scripts, and aren't meant to be modified manually. The only file you should edit to control the containers' environment is `.ci/env`, as specified in the instructions above. -- Aside from some exceptions (e.g. `TEST_FILTER`), most of the variables in `.ci/env` must be set before the `make generate-server` command is run. Modifying that file afterwards has no effect, because the containers' env files are generated in that step. +- All of the variables in `.ci/env` must be set before the `make generate-server` command is run (or, if using the dashboard, before the `make generate-test-cycle` command). Modifying that file afterwards has no effect, because the containers' env files are generated in that step. - If you restart the dashboard at any point, you must also restart the server containers, so that it picks up the new IP of the dashboard from the newly generated `.env.dashboard` file -- If new variables need to be passed to any of the containers: +- If new variables need to be passed to any of the containers, here are the general principles to follow when deciding where to populate it: * If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container. * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably. + * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality) ##### For code changes: * `make fmt-ci` to format and check yaml files and shell scripts. diff --git a/e2e-tests/cypress/generate_test_cycle.js b/e2e-tests/cypress/generate_test_cycle.js index 03a8a53e0c..a1fc70a13d 100644 --- a/e2e-tests/cypress/generate_test_cycle.js +++ b/e2e-tests/cypress/generate_test_cycle.js @@ -44,7 +44,6 @@ * BROWSER : Chrome by default. Set to run test on other browser such as chrome, edge, electron and firefox. * The environment should have the specified browser to successfully run. * HEADLESS : Headless by default (true) or false to run on headed mode. - * CI_BASE_URL : Test server base URL in CI * * Example: * 1. "node generate_test_cycle.js"