Enhance reporting, add shellcheck (#28240)

* Enhance reporting, add shellcheck

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

e2e-tests/.ci/report.collect.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# SC2034: <variable> appears unused.
+# https://www.shellcheck.net/wiki/SC2034
+# shellcheck disable=SC2034
+# shellcheck disable=SC2086,SC2223
+
+set -e -u -o pipefail
+cd "$(dirname "$0")"
+. .e2erc
+
+# The collected data will be written to the "e2e-tests/cypress/results/" directory
+cd ../cypress/
+if [ ! -d "results/" ]; then
+  mme2e_log "Error: 'results/' directory does not exist. Aborting report data collection." >&2
+  exit 1
+fi
+
+# If the Automation Dashboard is used, try to collect run data from it
+if [ -n "${AUTOMATION_DASHBOARD_URL:-}" ]; then
+  mme2e_log "Automation Dashboard usage detected: retrieving cycle results"
+  # Assume that we're in the cypress dir and that the 'results/' directory exists
+  : ${BUILD_ID:?}
+  AD_CYCLE_FILE="results/ad_cycle.json"
+  AD_SPECS_FILE="results/ad_specs.json"
+  curl -o "$AD_CYCLE_FILE" -fsSL "${AUTOMATION_DASHBOARD_URL}/cycle?build=${BUILD_ID}"
+  curl -o "$AD_SPECS_FILE" -fsSL "${AUTOMATION_DASHBOARD_URL}/executions/specs?cycle_id=$(jq -r .id "$AD_CYCLE_FILE")"
+fi

e2e-tests/.ci/server.prepare.sh

@@ -11,7 +11,8 @@ for SETTING in \
     PluginSettings.AutomaticPrepackagedPlugins=true
   do
   mme2e_log "Configuring parameter: $SETTING"
-  ${MME2E_DC_SERVER} exec -T -- server mmctl --local config set $(tr = ' ' <<<$SETTING)
+  # shellcheck disable=SC2046
+  ${MME2E_DC_SERVER} exec -T -- server mmctl --local config set $(tr '=' ' ' <<<$SETTING)
 done
 if [ -n "${MM_LICENSE:-}" ]; then
   # We prefer uploading the license here, instead of setting the env var for the server

e2e-tests/.ci/server.run_cypress.sh

@@ -33,12 +33,12 @@ LOGFILE_SUFFIX="${CI_BASE_URL//\//_}" # Remove slashes from CI_BASE_URL to produ
 # shellcheck disable=SC2016
 if ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress bash -c '[ -n "${AUTOMATION_DASHBOARD_URL}" ]'; then
   mme2e_log "AUTOMATION_DASHBOARD_URL is set. Using run_test_cycle.js for the cypress run"
-  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee ../cypress/logs/${LOGFILE_SUFFIX}_cypress.log
+  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee "../cypress/logs/${LOGFILE_SUFFIX}_cypress.log"
 else
   mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Using run_tests.js for the cypress run"
   # shellcheck disable=SC2086
-  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee ../cypress/logs/${LOGFILE_SUFFIX}_cypress.log
+  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee "../cypress/logs/${LOGFILE_SUFFIX}_cypress.log"
 fi
 
 # Collect server logs
-${MME2E_DC_SERVER} logs --no-log-prefix -- server >../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log 2>&1
+${MME2E_DC_SERVER} logs --no-log-prefix -- server > "../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log" 2>&1

e2e-tests/.ci/server.start.sh

@@ -15,6 +15,7 @@ if ! mme2e_wait_service_healthy server 60 10; then
   mme2e_log "Mattermost container not healthy, retry attempts exhausted. Giving up." >&2
   exit 1
 fi
+# shellcheck disable=SC2043
 for MIGRATION in migration_advanced_permissions_phase_2; do
   # Query explanation: if it doesn't find the migration in the table, there are 0 results and the command fails with a divide-by-zero error. Otherwise the command succeeds
   MIGRATION_CHECK_COMMAND="${MME2E_DC_SERVER} exec -T -- postgres psql -U mmuser mattermost_test -c \"select 1 / (select count(*) from Systems where name = '${MIGRATION}' and value = 'true');\""

e2e-tests/Makefile

@@ -4,6 +4,7 @@ SHELL := /bin/bash
 all: run
 run: generate-server start-server run-test
 stop: stop-server stop-dashboard clean
+report: collect-report-data publish-report
 clean:
 	rm -fv .ci/server.yml
 	rm -fv .ci/.env.{server,dashboard,cypress,playwright}
@@ -34,17 +35,20 @@ cloud-init: requirecmd-jq requirecmd-curl
 cloud-teardown:
 	bash ./.ci/server.cloud_teardown.sh
 
-.PHONY: publish-report
+.PHONY: collect-report-data publish-report
+collect-report-data:
+	bash ./.ci/report.collect.sh
 publish-report: requirecmd-node
 	bash ./.ci/report.publish.sh
 
-.PHONY: fmt-node fmt-shell fmt
+.PHONY: check-shell fmt-shell fmt-node fmt
 requirecmd-%:
 	@which "$(*)" >/dev/null || { echo "Error, missing required CLI tool: $(*). Aborting." >&2; exit 1; }
+check-shell: requirecmd-shellcheck
+	shellcheck ./.ci/*.sh ./.ci/.e2erc* # Install with https://webinstall.dev/shfmt/
+fmt-shell: requirecmd-shfmt
+	shfmt -w -s -i 2 ./.ci/*.sh         # Install with https://webinstall.dev/shellcheck/
 fmt-node: requirecmd-npx
 	# Formats yaml files
 	npx prettier ./.ci "!./.ci/dashboard" --write --cache
-fmt-shell: requirecmd-shfmt requirecmd-shellcheck
-	shfmt -w -s -i 2 ./.ci/*.sh         # Install with https://webinstall.dev/shellcheck/
-	shellcheck ./.ci/*.sh ./.ci/.e2erc* # Install with https://webinstall.dev/shfmt/
 fmt: fmt-node fmt-shell

e2e-tests/README.md

@@ -27,7 +27,7 @@ Instructions, detailed:
   * The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
   * The `SERVER_IMAGE` variable can also be set if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used.
   * The `TEST_FILTER` variable can also be set, to customize which tests you want Cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
-  * More variables may be required to configure reporting and cloud interactions. Check the content of the `.ci/report.publish.sh` and `.ci/server.cloud_*.sh` scripts for reference.
+  * More variables may be required to configure reporting and cloud interactions. Check the content of the `.ci/report.*.sh` and `.ci/server.cloud_*.sh` scripts for reference.
 2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID`
   * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`.
   * Note that part of the dashboard functionality assumes the `BUILD_ID` to have a certain format (see [here](https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23) for details). This is not relevant for local running, but it's important to note in the testing pipelines.
@@ -54,7 +54,7 @@ Notes:
   * If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container.
   * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability and simplifies the question of what container has access to which variable considerably.
   * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
-- The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
+- The `report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
 - `make start-server` won't cleanup containers that don't change across runs. This means that you can use it to emulate a Mattermost server upgrade while retaining your database data by simply changing the `SERVER_IMAGE` variable on your machine, and then re-running `make start-server`. But this also means that if you want to run a clean local environment, you may have to manually run `make stop` to cleanup any running containers and their volumes, which include e.g. the database.
 
 ##### For code changes: