CLD-5783 Implement rolling upgrade tests (#27842)

* Make server.prepare idempotent, don't cleanup containers on server.start
* Implement rolling upgrade E2E tests
* Apply Akis' suggestions from code review
* Apply Saturn's suggestions from review
* Communicate RollingRelease on GHA and test automation results channel
* Implement E2E test case versioning for rolling release tests

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Akis Maziotis <akis.maziotis@mattermost.com>

e2e-tests/.ci/report.publish.sh

@@ -35,6 +35,7 @@ export BUILD_TAG="${SERVER_IMAGE##*/}"
 export MM_DOCKER_IMAGE="${BUILD_TAG%%:*}" # NB: the 'mattermostdevelopment/' prefix is assumed
 export MM_DOCKER_TAG="${BUILD_TAG##*:}"
 export SERVER_TYPE="${SERVER}"
+export AUTOMATION_DASHBOARD_FRONTEND_URL="${AUTOMATION_DASHBOARD_URL:+${AUTOMATION_DASHBOARD_URL%%/api}}"
 
 if [ -n "${TM4J_API_KEY:-}" ]; then
   # Set additional variables required for Zephyr reporting

e2e-tests/.ci/server.cloud_init.sh

@@ -2,10 +2,14 @@
 set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
+[ -f .env.cloud ] && . .env.cloud
 
 if [ "$SERVER" != "cloud" ]; then
   mme2e_log "Skipping cloud instance initialization: operation supported only for cloud server, but running with SERVER='$SERVER'"
   exit 0
+elif [ -n "${MM_CUSTOMER_ID:-}" ]; then
+  mme2e_log "Skipping cloud user creation: customer with ID '$MM_CUSTOMER_ID' is already configured. Please run 'make cloud-teardown' before creating a new user."
+  exit 0
 fi
 
 mme2e_log "Initializing cloud tests"

e2e-tests/.ci/server.cloud_teardown.sh

@@ -2,9 +2,10 @@
 set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
+[ -f .env.cloud ] && . .env.cloud
 
-if [ "$SERVER" != "cloud" ]; then
-  mme2e_log "Skipping cloud instance teardown: operation supported only for cloud server, but running with SERVER='$SERVER'"
+if [ -z "${MM_CUSTOMER_ID:-}" ]; then
+  mme2e_log "Skipping cloud instance teardown: MM_CUSTOMER_ID variable is empty, no cloud user to cleanup."
   exit 0
 fi
 
@@ -20,5 +21,6 @@ MME2E_ENVCHECK_MSG="variable required for tearing down cloud tests, but is empty
 
 mme2e_log "Deleting customer $MM_CUSTOMER_ID."
 curl -fsSL -X DELETE -H @- "${CWS_URL}/api/v1/tests/customers/$MM_CUSTOMER_ID/payment-customer" <<<"${CWS_EXTRA_HTTP_HEADERS:-}"
+rm -fv .env.cloud
 
 mme2e_log "Test cloud customer deleted, MM_CUSTOMER_ID: $MM_CUSTOMER_ID."

e2e-tests/.ci/server.prepare.sh

@@ -26,16 +26,33 @@ if [ "$TEST" = "cypress" ]; then
   ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress npm i
   ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress cypress install
   mme2e_log "Prepare Cypress: populating fixtures"
-  ${MME2E_DC_SERVER} exec -T -- server curl -L --silent https://github.com/mattermost/mattermost-plugin-gitlab/releases/download/v1.3.0/com.github.manland.mattermost-plugin-gitlab-1.3.0.tar.gz | ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress tee tests/fixtures/com.github.manland.mattermost-plugin-gitlab-1.3.0.tar.gz >/dev/null
-  ${MME2E_DC_SERVER} exec -T -- server curl -L --silent https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.9.0/com.mattermost.demo-plugin-0.9.0.tar.gz | ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress tee tests/fixtures/com.mattermost.demo-plugin-0.9.0.tar.gz >/dev/null
-  ${MME2E_DC_SERVER} exec -T -- server curl -L --silent https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.8.0/com.mattermost.demo-plugin-0.8.0.tar.gz | ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress tee tests/fixtures/com.mattermost.demo-plugin-0.8.0.tar.gz >/dev/null
   ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress tee tests/fixtures/keycloak.crt >/dev/null <../../server/build/docker/keycloak/keycloak.crt
+  for PLUGIN_URL in \
+    "https://github.com/mattermost/mattermost-plugin-gitlab/releases/download/v1.3.0/com.github.manland.mattermost-plugin-gitlab-1.3.0.tar.gz" \
+    "https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.9.0/com.mattermost.demo-plugin-0.9.0.tar.gz" \
+    "https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.8.0/com.mattermost.demo-plugin-0.8.0.tar.gz"
+  do
+    PLUGIN_NAME="${PLUGIN_URL##*/}"
+    PLUGIN_PATH="tests/fixtures/$PLUGIN_NAME"
+    if ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress test -f "$PLUGIN_PATH"; then
+      mme2e_log "Skipping installation of plugin $PLUGIN_NAME: file exists"
+      continue
+    fi
+    mme2e_log "Downloading $PLUGIN_NAME to fixtures directory"
+    ${MME2E_DC_SERVER} exec -T -- server curl -L --silent "${PLUGIN_URL}" | ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress tee "$PLUGIN_PATH" >/dev/null
+  done
 fi
 
 # Run service-specific initialization steps
 for SERVICE in $ENABLED_DOCKER_SERVICES; do
   case "$SERVICE" in
   openldap)
+    LDIF_FILE=../../server/tests/test-data.ldif
+    LDIF_CANARY=$(sed -n -E 's/^dn:[[:space:]]*(.*)$/\1/p' ${LDIF_FILE} | tail -n1)
+    if ${MME2E_DC_SERVER} exec -T -- openldap bash -c "ldapsearch -x -D \"cn=admin,dc=mm,dc=test,dc=com\" -w mostest -b \"$LDIF_CANARY\" >/dev/null"; then
+      mme2e_log "Skipping configuration for the $SERVICE container: already initialized"
+      continue
+    fi
     mme2e_log "Configuring the $SERVICE container"
     ${MME2E_DC_SERVER} exec -T -- openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' <../../server/tests/test-data.ldif
     ;;

e2e-tests/.ci/server.start.sh

@@ -3,10 +3,6 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-# Cleanup old containers, if any
-mme2e_log "Stopping leftover E2E containers, if any are running"
-${MME2E_DC_SERVER} down -v --remove-orphans
-
 # Wait for the required server image
 mme2e_log "Waiting for server image to be available"
 mme2e_wait_image "$SERVER_IMAGE" 4 30

e2e-tests/Makefile

@@ -2,11 +2,11 @@ SHELL := /bin/bash
 
 .PHONY: all run stop clean
 all: run
-run: cloud-init generate-server start-server prepare-server run-test cloud-teardown
+run: generate-server start-server prepare-server run-test
 stop: stop-server stop-dashboard clean
 clean:
 	rm -fv .ci/server.yml
-	rm -fv .ci/.env*
+	rm -fv .ci/.env.{server,dashboard,cypress,playwright}
 
 .PHONY: generate-server start-server prepare-server run-test stop-server restart-server
 generate-server:

e2e-tests/README.md

@@ -37,11 +37,12 @@ Instructions, detailed:
   * Only Cypress is currently using the dashboard; Playwright is not.
 3. `make`: start and prepare the server, then run the Cypress smoke tests
   * You can track the progress of the run in the `http://localhost:4000/cycles` dashboard if you launched it locally
-  * When running with `SERVER=cloud`, this will automatically create a cloud customer against the specified `CWS_URL` service, and delete that user after the run is complete.
+  * For `SERVER=cloud` runs, you'll need to first create a cloud customer against the specified `CWS_URL` service by running `make cloud-init`. The user isn't automatically removed, and may be reused across multiple runs until you run `make cloud-teardown` to delete it.
   * If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make`
   * If you just want to run a local server instance, without any further testing, you can run `TEST=none make`
   * If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machines (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running.
 4. `make stop`: tears down the server (and the dashboard, if running)
+  * This will stop and cleanup all of the E2E testing containers, including the database and its persistent volume.
   * This also implicitly runs `make clean`, which also removes any generated environment or docker-compose files.
 
 Notes:
@@ -54,6 +55,7 @@ Notes:
   * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability and simplifies the question of what container has access to which variable considerably.
   * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
 - The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
+- `make start-server` won't cleanup containers that don't change across runs. This means that you can use it to emulate a Mattermost server upgrade while retaining your database data by simply changing the `SERVER_IMAGE` variable on your machine, and then re-runing `make start-server` (and `make prepare-server` if you intend to run cypress again, since its container is also recreated). But this also means that if you want to run a clean local environment, you may have to manually run `make stop` to cleanup any running containers and their volumes, which include e.g. the database.
 
 ##### For code changes:
 * `make fmt-ci` to format and check yaml files and shell scripts.

e2e-tests/cypress/save_report.js

@@ -23,6 +23,7 @@
  *      - TM4J_FOLDER_ID=[folder_id], e.g. 847997
  *   For sending hooks to Mattermost channels
  *      - FULL_REPORT, WEBHOOK_URL and DIAGNOSTIC_WEBHOOK_URL
+ *      - If AUTOMATION_DASHBOARD_FRONTEND_URL is set, a link to the corresponding cycle will be included in the report message
  *   Test type
  *      - TYPE=[type]; valid values: "PR", "RELEASE", "MASTER", "MASTER_UNSTABLE", "CLOUD", "CLOUD_UNSTABLE", "NONE"
  *   Server type

e2e-tests/cypress/utils/report.js

@@ -117,6 +117,8 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te
         TEST_CYCLE_LINK_PREFIX,
         MM_ENV,
         SERVER_TYPE,
+        BUILD_ID,
+        AUTOMATION_DASHBOARD_FRONTEND_URL,
     } = process.env;
     const {statsFieldValue, stats} = summary;
     const {
@@ -216,6 +218,12 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te
         testCycleLink = testCycleKey ? `| [Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})` : '';
     }
 
+    const automationDashboardField = AUTOMATION_DASHBOARD_FRONTEND_URL ? `| [Automation Dashboard](${AUTOMATION_DASHBOARD_FRONTEND_URL}/cycle/${BUILD_ID})` : '';
+
+    const rollingReleaseMatchRegex = BUILD_ID.match(/-rolling(?<version>[^-]+)-/);
+    const rollingReleaseFrom = rollingReleaseMatchRegex?.groups?.version;
+    const rollingReleaseFromField = rollingReleaseFrom ? `\nRolling release upgrade from: ${rollingReleaseFrom}` : '';
+
     const startAt = dayjs(stats.start);
     const endAt = dayjs(stats.end);
     const statsDuration = dayjs.duration(endAt.diff(startAt)).format('H:mm:ss');
@@ -229,7 +237,7 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te
             author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
             author_link: 'https://www.mattermost.com/',
             title,
-            text: `${quickSummary} | ${statsDuration} ${testCycleLink}\n${runnerEnvValue}${SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : ''}${MM_ENV ? '\nTest server override: ' + MM_ENV : ''}`,
+            text: `${quickSummary} | ${statsDuration} ${testCycleLink} ${automationDashboardField}\n${runnerEnvValue}${SERVER_TYPE ? '\nTest server: ' + SERVER_TYPE : ''}${rollingReleaseFromField}${MM_ENV ? '\nTest server override: ' + MM_ENV : ''}`,
         }],
     };
 }