CLD-7074 Implement full E2E tests on Github Actions (#26093)

* Move license setting from env var to mmctl upload
* Extract common E2E steps
* Add E2E fulltests
* Fix plugin_startup_fail_spec.js, timeout runners, fix local dashboard run
* Implement reporting script and job
* Bump artifact related actions
* Fix E2E variable generator script
* Skip reporting steps if not required
* Get rid of deprecation warnings
* Fix inbucket hostname parameter
* Support arbitrary refs in test template
* Fix cycle BRANCH var for non-PR commits
---------

Co-authored-by: Mattermost Build <build@mattermost.com>

e2e-tests/.ci/.e2erc

@@ -1,37 +1,5 @@
 # shellcheck disable=SC2148,SC2155
 
-# Utility variables
-# NB: these assume you `source` them from the directory this file is in
-# Set up base docker compose file and export
-export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml"
-export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml"
-export MME2E_UID=$(id -u)
-export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}')
-export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}')
-export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc)
-
-# Default values for optional variables
-export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)"
-export BROWSER_DEFAULT="chrome"
-export SERVER_DEFAULT="self-hosted"
-export TEST_DEFAULT="cypress"
-export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
-export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
-# OS specific defaults overrides
-case $MME2E_OSTYPE in
-  darwin )
-    BROWSER_DEFAULT="electron" ;;
-  * )
-esac
-
-# Populate the optional variables that are used in the docker-compose file
-export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
-export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
-export SERVER=${SERVER:-$SERVER_DEFAULT}
-export TEST=${TEST:-$TEST_DEFAULT}
-export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT}
-export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
-
 # Function definitions
 mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; }
 mme2e_get_current_shopt_arg () {
@@ -98,10 +66,48 @@ mme2e_is_token_in_list() {
   grep -qE "(^| )$TOKEN( |$)" <<<"$LIST"
 }
 
-# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose' in your shell
-# NB: this won't work in the script
+# Call prerequisite utility functions
+mme2e_load_env_file
+
+# Utility variables
+# NB: these assume you `source` them from the directory this file is in
+# Set up base docker compose file and export
+export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml"
+export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml"
+export MME2E_UID=$(id -u)
+export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}')
+export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}')
+export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc)
+
+# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose-mmserver' in your shell
+# NB: this only works in interactive shells
 alias docker-compose-mmserver='${MME2E_DC_SERVER}'
 alias docker-compose-mmdashboard='${MME2E_DC_DASHBOARD}'
 
-# Call prerequisite utility functions
-mme2e_load_env_file
+# Default values for optional variables
+export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)"
+export BROWSER_DEFAULT="chrome"
+export SERVER_DEFAULT="self-hosted"
+export TEST_DEFAULT="cypress"
+export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
+export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
+export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown")
+export BUILD_ID_DEFAULT=$(date +%s)
+# OS specific defaults overrides
+case $MME2E_OSTYPE in
+  darwin )
+    BROWSER_DEFAULT="electron" ;;
+  * )
+esac
+
+# Populate the optional variables that are used in the docker-compose file
+export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
+export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
+export SERVER=${SERVER:-$SERVER_DEFAULT}
+export TEST=${TEST:-$TEST_DEFAULT}
+export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT}
+export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
+export BRANCH_UNSANITIZED=${BRANCH:-$BRANCH_DEFAULT}
+export BRANCH=${BRANCH_UNSANITIZED::50} # The automation dashboard only accepts branch names up to 50 characters
+export BUILD_ID_UNSANITIZED=${BUILD_ID:-$BUILD_ID_DEFAULT}
+export BUILD_ID=${BUILD_ID_UNSANITIZED::64} # The automation dashboard only accepts build IDs up to 64 characters

e2e-tests/.ci/dashboard.generate_test_cycle.sh

@@ -11,7 +11,13 @@ fi
 set -a
 . .env.cypress
 
+if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
+  mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation."
+  exit 0
+fi
+
 mme2e_log "Generating the test cycle on the Automation Dashboard"
 cd ../cypress
+npm i
 # shellcheck disable=SC2086
 exec node --trace-warnings generate_test_cycle.js $TEST_FILTER

e2e-tests/.ci/report.publish.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# SC2034: <variable> appears unused.
+# https://www.shellcheck.net/wiki/SC2034
+# shellcheck disable=SC2034
+
+set -e -u -o pipefail
+cd "$(dirname "$0")"
+. .e2erc
+
+# Default required variables, assert that they are set, and document optional variables
+: ${FULL_REPORT:=false} # Valid values: true, false
+: ${TYPE:=NONE}         # Valid values: PR, RELEASE, MASTER, MASTER_UNSTABLE, CLOUD, CLOUD_UNSTABLE, NONE (which is the same as omitting it)
+: ${WEBHOOK_URL:-}      # Optional. Mattermost webhook to post the report back to
+: ${RELEASE_DATE:-}     # Optional. If set, its value will be included in the report as the release date of the tested artifact
+
+# Env vars used during the test. Their values will be included in the report
+: ${BRANCH:?}
+: ${BUILD_ID:?}
+: ${MM_ENV:-}
+
+# Populate intermediate variables
+export BUILD_TAG="${SERVER_IMAGE##*/}"
+export MM_DOCKER_IMAGE="${BUILD_TAG%%:*}" # NB: the 'mattermostdevelopment/' prefix is assumed
+export MM_DOCKER_TAG="${BUILD_TAG##*:}"
+export SERVER_TYPE="${SERVER}"
+# NB: assume that BRANCH follows the convention 'server-pr-${PR_NUMBER}'. If multiple PRs match, the last one is used to generate the link
+# Only needed if TYPE=PR
+export PULL_REQUEST="https://github.com/mattermost/mattermost/pull/${BRANCH##*-}"
+
+if [ -n "${TM4J_API_KEY:-}" ]; then
+  export TM4J_ENABLE=true
+  export JIRA_PROJECT_KEY=MM
+  export TM4J_ENVIRONMENT_NAME="${TEST}/${BROWSER}/${SERVER}"
+  case "${SERVER}" in
+    cloud)
+      export TM4J_FOLDER_ID="2014474" ;;
+    *)
+      export TM4J_FOLDER_ID="2014475" ;;
+  esac
+  : ${TEST_CYCLE_LINK_PREFIX:?}
+  : ${TM4J_CYCLE_KEY:-}
+  : ${TM4J_CYCLE_NAME:-}
+  mme2e_log "TMJ4 integration enabled."
+fi
+
+if [ -n "${DIAGNOSTIC_WEBHOOK_URL:-}" ]; then
+  : ${DIAGNOSTIC_USER_ID:?}
+  : ${DIAGNOSTIC_TEAM_ID:?}
+  mme2e_log "Diagnostic report upload enabled."
+fi
+
+if [ -n "${AWS_S3_BUCKET:-}" ]; then
+  : ${AWS_ACCESS_KEY_ID:?}
+  : ${AWS_SECRET_ACCESS_KEY:?}
+  mme2e_log "S3 report upload enabled."
+fi
+
+cd ../cypress/
+if [ ! -d "results/" ]; then
+  mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2
+  exit 1
+fi
+
+npm i
+node save_report.js

e2e-tests/.ci/server.generate.sh

@@ -61,7 +61,7 @@ services:
       MM_TEAMSETTINGS_ENABLEOPENSERVER: "true"
       MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
       MM_SQLSETTINGS_DRIVERNAME: "postgres"
-      MM_EMAILSETTINGS_SMTPSERVER: "inbucket"
+      MM_EMAILSETTINGS_SMTPSERVER: "localhost"
       MM_CLUSTERSETTINGS_READONLYCONFIG: "false"
       MM_SERVICESETTINGS_ENABLEONBOARDINGFLOW: "false"
       MM_FEATUREFLAGS_ONBOARDINGTOURTIPS: "false"
@@ -259,9 +259,7 @@ EOL
 generate_env_files() {
   # Generate .env.server
   mme2e_log "Generating .env.server"
-  mme2e_generate_envfile_from_var_names >.env.server <<-EOF
-	MM_LICENSE
-	EOF
+  truncate --size=0 .env.server
 
   # Setting SERVER-specific variables
   case "$SERVER" in
@@ -282,16 +280,15 @@ generate_env_files() {
   done
 
   # Generating TEST-specific env files
-  BRANCH_DEFAULT=$(git branch --show-current)
-  BUILD_ID_DEFAULT=$(date +%s)
-  export BRANCH=${BRANCH:-$BRANCH_DEFAULT}
-  export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT}
-  export CI_BASE_URL="${CI_BASE_URL:-localhost}"
+  # Some are defaulted in .e2erc due to being needed to other scripts as well
+  export CI_BASE_URL="${CI_BASE_URL:-http://localhost:8065}"
   export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run
   export HEADLESS=true   # Static, but declared here for making generate_test_cycle.js easier to run
   case "$TEST" in
   cypress)
     mme2e_log "Cypress: Generating .env.cypress"
+    truncate --size=0 .env.cypress
+
     mme2e_generate_envfile_from_var_names >.env.cypress <<-EOF
 	BRANCH
 	BUILD_ID
@@ -299,6 +296,7 @@ generate_env_files() {
 	BROWSER
         HEADLESS
         REPO
+        CYPRESS_pushNotificationServer
 	EOF
     # Adding service-specific cypress variables
     for SERVICE in $ENABLED_DOCKER_SERVICES; do
@@ -327,8 +325,14 @@ generate_env_files() {
       echo "CYPRESS_serverEdition=E20" >>.env.cypress
       ;;
     esac
-    # If the dashboard is running, load .env.dashboard into .env.cypress
-    if DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then
+    # Add Automation Dashboard related variables to cypress container
+    if [ -n "${AUTOMATION_DASHBOARD_URL:-}" ]; then
+      mme2e_log "Automation dashboard URL is set: loading related variables into the Cypress container"
+      mme2e_generate_envfile_from_var_names >>.env.cypress <<-EOF
+	AUTOMATION_DASHBOARD_URL
+	AUTOMATION_DASHBOARD_TOKEN
+	EOF
+    elif DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then
       mme2e_log "Detected a running automation dashboard: loading its access variables into the Cypress container"
       cat >>.env.cypress <.env.dashboard
     fi

e2e-tests/.ci/server.prepare.sh

@@ -3,6 +3,13 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
+if [ -n "${MM_LICENSE:-}" ]; then
+  # We prefer uploading the license here, instead of setting the env var for the server
+  # This is to retain the flexibility of being able to remove it programmatically, if the tests require it
+  mme2e_log "Uploading license to server"
+  ${MME2E_DC_SERVER} exec -T -- server mmctl --local license upload-string "$MM_LICENSE"
+fi
+
 if [ "$TEST" = "cypress" ]; then
   mme2e_log "Prepare Cypress: install dependencies"
   ${MME2E_DC_SERVER} exec -T -u 0 -- cypress bash -c "id $MME2E_UID || useradd -u $MME2E_UID -m nodeci" # Works around the node image's assumption that the app files are owned by user 1000

e2e-tests/Makefile

@@ -39,6 +39,10 @@ cloud-init: requirecmd-jq requirecmd-curl
 cloud-teardown:
 	bash ./.ci/server.cloud_teardown.sh
 
+.PHONY: publish-report
+publish-report: requirecmd-node
+	bash ./.ci/report.publish.sh
+
 .PHONY: fmt-node fmt-shell fmt
 requirecmd-%:
 	@which "$(*)" >/dev/null || { echo "Error, missing required CLI tool: $(*). Aborting." >&2; exit 1; }

e2e-tests/README.md

@@ -38,6 +38,7 @@ Instructions, detailed:
   * When running with `SERVER=cloud`, this will automatically create a cloud customer against the specified `CWS_URL` service, and delete that user after the run is complete.
   * If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make`
   * If you just want to run a local server instance, without any further testing, you can run `TEST=none make`
+  * If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machiness (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running.
 4. `make stop`: tears down the server (and the dashboard, if running)
   * `make clean` will also remove any generated environment or docker-compose files, in addition to stopping the containers.
 
@@ -50,6 +51,7 @@ Notes:
   * If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container.
   * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably.
   * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
+- The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
 
 ##### For code changes:
 * `make fmt-ci` to format and check yaml files and shell scripts.

e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js

@@ -33,8 +33,8 @@ describe('If plugins fail to start, they can be disabled', () => {
 
     it('MM-T2391 If plugins fail to start, they can be disabled', () => {
         const mimeType = 'application/gzip';
-        cy.fixture(gitlabPlugin.filename, 'binary').
-            then(Cypress.Blob.binaryStringToBlob).
+        cy.fixture(gitlabPlugin.filename, null).
+            then(Cypress.Blob.arrayBufferToBlob).
             then((fileContent) => {
                 cy.get('input[type=file]').attachFile({fileContent, fileName: gitlabPlugin.filename, mimeType});
             });

e2e-tests/cypress/utils/report.js

@@ -246,7 +246,7 @@ function generateTitle() {
 
     let dockerImageLink = '';
     if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) {
-        dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermost/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
+        dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
     }
 
     let releaseDate = '';