CLD-5704 Migrate daily master/cloud tests (#27393)

* [skip ci] Support Cloud/daily tests and Zephyr integration

* [skip ci] Fix workflow file

* [skip ci] Fix typo in workflow input name

* Fix cloud variable passing

* [skip ci] Fix typo

* Utilize master branch image for daily tests

* Apply Saturn's suggestion, fixes and improvements

e2e-tests/.ci/.e2erc

@@ -87,7 +87,7 @@ alias docker-compose-mmdashboard='${MME2E_DC_DASHBOARD}'
 # Default values for optional variables
 export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)"
 export BROWSER_DEFAULT="chrome"
-export SERVER_DEFAULT="self-hosted"
+export SERVER_DEFAULT="onprem"
 export TEST_DEFAULT="cypress"
 export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
 export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'

e2e-tests/.ci/report.publish.sh

@@ -8,15 +8,26 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-# Default required variables, assert that they are set, and document optional variables
+# Default or assert variables required by the save_report.js script, and document optional variables
 : ${FULL_REPORT:=false} # Valid values: true, false
-: ${TYPE:=NONE}         # Valid values: PR, RELEASE, MASTER, MASTER_UNSTABLE, CLOUD, CLOUD_UNSTABLE, NONE (which is the same as omitting it)
+: ${TYPE:=NONE}         # Valid values: PR, RELEASE, MASTER, MASTER_UNSTABLE, CLOUD, CLOUD_UNSTABLE, NONE (which is the same as omitting it); also known as REPORT_TYPE
 : ${WEBHOOK_URL:-}      # Optional. Mattermost webhook to post the report back to
 : ${RELEASE_DATE:-}     # Optional. If set, its value will be included in the report as the release date of the tested artifact
+if [ "$TYPE" = "PR" ]; then
+  # In this case, we expect the PR number to be present in the BRANCH variable
+  BRANCH_REGEX='^server-pr-[0-9]+$'
+  if ! grep -qE "${BRANCH_REGEX}"<<<"$BRANCH"; then
+    mme2e_log "Error: when using TYPE=PR, the BRANCH variable should respect regex '$BRANCH_REGEX'. Aborting." >&2
+    exit 1
+  fi
+  export PULL_REQUEST="https://github.com/mattermost/mattermost/pull/${BRANCH##*-}"
+fi
 
 # Env vars used during the test. Their values will be included in the report
-: ${BRANCH:?}
+: ${TEST:?} # See E2E tests' readme; usually set to 'cypress'
+: ${BRANCH:?} # May be either a ref, a commit hash, or 'server-pr-PR_NUMBER' (if TYPE=PR)
 : ${BUILD_ID:?}
+: ${SERVER:?} # May be either 'onprem' or 'cloud'
 : ${MM_ENV:-}
 
 # Populate intermediate variables
@@ -24,26 +35,45 @@ export BUILD_TAG="${SERVER_IMAGE##*/}"
 export MM_DOCKER_IMAGE="${BUILD_TAG%%:*}" # NB: the 'mattermostdevelopment/' prefix is assumed
 export MM_DOCKER_TAG="${BUILD_TAG##*:}"
 export SERVER_TYPE="${SERVER}"
-# NB: assume that BRANCH follows the convention 'server-pr-${PR_NUMBER}'. If multiple PRs match, the last one is used to generate the link
-# Only needed if TYPE=PR
-export PULL_REQUEST="https://github.com/mattermost/mattermost/pull/${BRANCH##*-}"
 
 if [ -n "${TM4J_API_KEY:-}" ]; then
+  # Set additional variables required for Zephyr reporting
+  if grep -q 'UNSTABLE' <<<"$TYPE"; then
+    export TEST_SUITE=prod
+  else
+    export TEST_SUITE=unstable
+  fi
+  if [ "$TYPE" = "RELEASE" ]; then
+    export SERVER_ARTIFACT_TYPE=release
+  else
+    export SERVER_ARTIFACT_TYPE=master
+    export TEST_IS_DAILY=yes
+  fi
   export TM4J_ENABLE=true
   export JIRA_PROJECT_KEY=MM
-  export TM4J_ENVIRONMENT_NAME="${TEST}/${BROWSER}/${SERVER}"
-  case "${SERVER}" in
-  cloud)
-    export TM4J_FOLDER_ID="2014474"
-    ;;
+  export TM4J_ENVIRONMENT_NAME="${TEST@u}/${BROWSER@u}/${SERVER_ARTIFACT_TYPE}-${SERVER_TYPE}-ent${TEST_IS_DAILY:+-${TEST_SUITE}}"
+  # Assert that the test type is among the ones supported by Zephyr, and select the corresponding folderId
+  case "${SERVER}-${TYPE}" in
+  onprem-RELEASE)
+    export TM4J_FOLDER_ID="2014475" ;;
+  onprem-MASTER)
+    export TM4J_FOLDER_ID="2014476" ;;
+  onprem-MASTER_UNSTABLE)
+    export TM4J_FOLDER_ID="2014478" ;;
+  cloud-RELEASE)
+    export TM4J_FOLDER_ID="2014474" ;;
+  cloud-CLOUD)
+    export TM4J_FOLDER_ID="2014479" ;;
+  cloud-CLOUD_UNSTABLE)
+    export TM4J_FOLDER_ID="2014481" ;;
   *)
-    export TM4J_FOLDER_ID="2014475"
-    ;;
+    mme2e_log "Error: unsupported Zephyr environment for the requested report (SERVER=${SERVER}, TYPE=${TYPE}). Aborting." >&2
+    exit 1
   esac
   : ${TEST_CYCLE_LINK_PREFIX:?}
-  : ${TM4J_CYCLE_KEY:-}
-  : ${TM4J_CYCLE_NAME:-}
-  mme2e_log "TMJ4 integration enabled."
+  : ${TM4J_CYCLE_KEY:-}  # Optional. Populated automatically by the reporting script
+  : ${TM4J_CYCLE_NAME:-} # Optional. Populated automatically by the reporting script
+  mme2e_log "TMJ4 integration enabled. Environment: $TM4J_ENVIRONMENT_NAME"
 fi
 
 if [ -n "${DIAGNOSTIC_WEBHOOK_URL:-}" ]; then

e2e-tests/.ci/server.cloud_init.sh

@@ -15,7 +15,7 @@ MME2E_ENVCHECK_MSG="variable required for initializing cloud tests, but is empty
 : "${CWS_URL:?$MME2E_ENVCHECK_MSG}"
 : "${MM_LICENSE:?$MME2E_ENVCHECK_MSG}"
 
-response=$(curl -X POST "${CWS_URL}/api/v1/internal/tests/create-customer?sku=cloud-enterprise&is_paid=true")
+response=$(curl -fsSL -X POST -H @- "${CWS_URL}/api/v1/tests/create-customer?sku=cloud-enterprise&is_paid=true" <<<"${CWS_EXTRA_HTTP_HEADERS:-}")
 MM_CUSTOMER_ID=$(echo "$response" | jq -r .customer_id)
 MM_CLOUD_API_KEY=$(echo "$response" | jq -r .api_key)
 MM_CLOUD_INSTALLATION_ID=$(echo "$response" | jq -r .installation_id)

e2e-tests/.ci/server.cloud_teardown.sh

@@ -19,6 +19,6 @@ MME2E_ENVCHECK_MSG="variable required for tearing down cloud tests, but is empty
 : "${MM_CUSTOMER_ID:?$MME2E_ENVCHECK_MSG}"
 
 mme2e_log "Deleting customer $MM_CUSTOMER_ID."
-curl -X DELETE "${CWS_URL}/api/v1/internal/tests/customers/$MM_CUSTOMER_ID/payment-customer"
+curl -fsSL -X DELETE -H @- "${CWS_URL}/api/v1/tests/customers/$MM_CUSTOMER_ID/payment-customer" <<<"${CWS_EXTRA_HTTP_HEADERS:-}"
 
 mme2e_log "Test cloud customer deleted, MM_CUSTOMER_ID: $MM_CUSTOMER_ID."

e2e-tests/README.md

@@ -16,7 +16,7 @@ Instructions, tl;dr: create a local branch with your E2E test changes, then open
 
 Instructions, detailed:
 1. (optional, undefined variables are set to sane defaults) Create the `.ci/env` file, and populate it with the variables you need out of the following list:
-  * `SERVER`: either `self-hosted` (default) or `cloud`.
+  * `SERVER`: either `onprem` (default) or `cloud`.
   * `CWS_URL` (mandatory when `SERVER=cloud`, only used in such case): when spinning up a cloud-like test server that communicates with a test instance of a customer web server.
   * `TEST`: either `cypress` (default), `playwright`, or `none` (to avoid creating the cypress/playwright sidecar containers, e.g. if you only want to launch a server instance)
   * `ENABLED_DOCKER_SERVICES`: a space-separated list of services to start alongside the server. Default to `postgres inbucket`, for smoke test purposes and for lightweight and faster start-up time. Depending on the test requirement being worked on, you may want to override as needed, as such:
@@ -27,6 +27,7 @@ Instructions, detailed:
   * The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
   * The `SERVER_IMAGE` variable can also be set if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used.
   * The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
+  * More variables may be required to configure reporting and cloud interactions. Check the content of the `.ci/report.publish.sh` and `.ci/server.cloud_*.sh` scripts for reference.
 2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID`
   * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`.
   * Note that part of the dashboard functionality assumes the `BUILD_ID` to have a certain format (see [here](https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23) for details). This is not relevant for local running, but it's important to note in the testing pipelines.

e2e-tests/cypress/save_report.js

@@ -24,7 +24,9 @@
  *   For sending hooks to Mattermost channels
  *      - FULL_REPORT, WEBHOOK_URL and DIAGNOSTIC_WEBHOOK_URL
  *   Test type
- *      - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "CLOUD"
+ *      - TYPE=[type]; valid values: "PR", "RELEASE", "MASTER", "MASTER_UNSTABLE", "CLOUD", "CLOUD_UNSTABLE", "NONE"
+ *   Server type
+ *      - SERVER_TYPE=[type]; used for the 'Test Server' field in the webhook. Any string representing the server type is valid, common values are "onprem" and "cloud"
  */
 
 const {merge} = require('mochawesome-merge');