CLD-7841 Optimize E2E testing costs (#27883)

* Utilize free runners for E2E tests, report back to PR on run completion
* Increase test stability on smaller instances
* Merge worker reports
* merge start+prepare steps
* upgrade keycloak
* increase test timeouts
* Add video recording to report artifacts

Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

---------

Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>

.github/workflows/e2e-fulltests-ci.yml

@@ -62,12 +62,15 @@ jobs:
       REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
       ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
       ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
+      WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
+      CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
     env:
       GH_TOKEN: "${{ github.token }}"
       REF: "${{ inputs.ref || github.sha }}"
       PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
       REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
       ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}"
+      AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
       # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
       TEST_FILTER_PR: >-
         --stage="@prod"
@@ -161,9 +164,13 @@ jobs:
             ROLLING_RELEASE_SERVER_IMAGE="mattermost/mattermost-enterprise-edition:${ROLLING_RELEASE_SERVER_IMAGE_TAG}"
             BUILD_ID_SUFFIX="rolling${ROLLING_RELEASE_FROM_TAG/-/_}-$BUILD_ID_SUFFIX"
             BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
+            WORKERS_NUMBER=10   # Rolling release tests are particularly impacted by increased parallelism. It's more efficient to run on fewer workers
             ### Run sanity assertions after variable generations
             git show-ref --verify "refs/tags/${ROLLING_RELEASE_FROM_TAG}" # 'inputs.ROLLING_RELEASE_FROM_TAG' must be a tag, for release report types
           fi
+          # BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
+          # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
+          BUILD_ID="${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}"
           echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
           echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
           echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/mattermost-enterprise-edition:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
@@ -177,18 +184,22 @@ jobs:
           echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
           echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
           echo "ROLLING_RELEASE_SERVER_IMAGE=${ROLLING_RELEASE_SERVER_IMAGE}" >> $GITHUB_OUTPUT
-          # BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
+          echo "BUILD_ID=${BUILD_ID}" >> $GITHUB_OUTPUT
-          # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
+          # User notification variables
-          echo "BUILD_ID=${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "WORKFLOW_RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{github.run_id}}" >> $GITHUB_OUTPUT
+          echo "CYCLE_URL=${AUTOMATION_DASHBOARD_URL%%/api}/cycle/${BUILD_ID}" >> $GITHUB_OUTPUT
       - name: ci/notify-user
+        env:
+          COMMIT_SHA: "${{steps.generate.outputs.commit_sha}}"
+          STATUS_CHECK_CONTEXT: "${{steps.generate.outputs.status_check_context}}"
+          WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
+          CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
         run: |
           if [ -n "$PR_NUMBER" ]; then
-            WORKFLOW_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            CYCLE_URL="https://automation-dashboard.vercel.app/cycle/${{ github.run_id }}"
             gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
-          E2E test run is starting for commit \`${{ steps.generate.outputs.commit_sha }}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
+          E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
           You can check its progress by either:
-          - Looking at the corresponding commit status, which will be available in a few moments: \`${{ steps.generate.outputs.status_check_context }}\`.
+          - Looking at the corresponding commit status, which will be available in a few moments: \`${STATUS_CHECK_CONTEXT}\`.
           - Looking at the [E2E test's Workflow Run]($WORKFLOW_RUN_URL), with Run ID \`${{ github.run_id }}\`.
           - Looking at the [E2E test's Cycle on the Automation Dashboard]($CYCLE_URL).
           EOF
@@ -229,3 +240,34 @@ jobs:
       REPORT_TM4J_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"
       CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
       CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"
+
+  notify-user:
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - generate-test-variables
+      - e2e-fulltest
+    permissions:
+      issues: write
+      pull-requests: write
+    defaults:
+      run:
+        shell: bash
+    env:
+      GH_TOKEN: "${{ github.token }}"
+      PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
+      MM_ENV: "${{ inputs.MM_ENV || '' }}"
+      COMMIT_SHA: "${{ needs.generate-test-variables.outputs.commit_sha }}"
+      STATUS_CHECK_CONTEXT: "${{ needs.generate-test-variables.outputs.status_check_context }}"
+      WORKFLOW_RUN_URL: "${{ needs.generate-test-variables.outputs.WORKFLOW_RUN_URL }}"
+      CYCLE_URL: "${{ needs.generate-test-variables.outputs.CYCLE_URL }}"
+    steps:
+      - name: ci/notify-user-test-completion
+        run: |
+          if [ -n "$PR_NUMBER" ]; then
+            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
+          E2E test has completed for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
+          You can check its result on the [Automation Dashboard]($CYCLE_URL), or by checking out the status check named \`${STATUS_CHECK_CONTEXT}\` in this PR.
+          The run summary artifact is available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
+          EOF
+          fi

.github/workflows/e2e-tests-ci-template.yml

@@ -240,11 +240,11 @@ jobs:
         # - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
         # - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
         #
-        #os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl]
+        #os: [ubuntu-latest, windows-2022, macos-12-xl]
-        os: [ubuntu-latest-8-cores]
+        os: [ubuntu-latest]
         worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
     runs-on: "${{ matrix.os }}"
-    timeout-minutes: 90
+    timeout-minutes: 120
     needs:
       - cypress-check
       - playwright-check
@@ -265,7 +265,7 @@ jobs:
       MM_ENV: "${{ inputs.MM_ENV }}"
       BRANCH: "${{ inputs.BRANCH }}"
       BUILD_ID: "${{ inputs.BUILD_ID }}"
-      CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}"
+      CI_BASE_URL: "${{ matrix.os }}-${{ matrix.worker_index }}"
       CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
       CWS_URL: "${{ secrets.CWS_URL }}"
       CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
@@ -326,6 +326,7 @@ jobs:
           path: |
             e2e-tests/cypress/logs/
             e2e-tests/cypress/results/
+          retention-days: 1
 
   report:
     runs-on: ubuntu-latest
@@ -349,6 +350,13 @@ jobs:
         with:
           path: e2e-tests/cypress/
           merge-multiple: true
+      - name: ci/upload-report-global
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: e2e-test-results
+          path: |
+            e2e-tests/cypress/logs/
+            e2e-tests/cypress/results/
       - name: ci/report-calculate-failures
         id: calculate-failures
         run: |

e2e-tests/.ci/.e2erc

@@ -93,6 +93,7 @@ export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
 export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
 export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown")
 export BUILD_ID_DEFAULT=$(date +%s)
+export CI_BASE_URL_DEFAULT="localhost"
 # OS specific defaults overrides
 case $MME2E_OSTYPE in
   darwin )
@@ -111,3 +112,4 @@ export BRANCH_UNSANITIZED=${BRANCH:-$BRANCH_DEFAULT}
 export BRANCH=${BRANCH_UNSANITIZED::50} # The automation dashboard only accepts branch names up to 50 characters
 export BUILD_ID_UNSANITIZED=${BUILD_ID:-$BUILD_ID_DEFAULT}
 export BUILD_ID=${BUILD_ID_UNSANITIZED::64} # The automation dashboard only accepts build IDs up to 64 characters
+export CI_BASE_URL=${CI_BASE_URL:-$CI_BASE_URL_DEFAULT}

e2e-tests/.ci/server.generate.sh

@@ -276,7 +276,6 @@ generate_env_files() {
 
   # Generating TEST-specific env files
   # Some are defaulted in .e2erc due to being needed to other scripts as well
-  export CI_BASE_URL="${CI_BASE_URL:-http://localhost:8065}"
   export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run
   export HEADLESS=true   # Static, but declared here for making generate_test_cycle.js easier to run
   case "$TEST" in

e2e-tests/.ci/server.run_cypress.sh

@@ -29,15 +29,16 @@ EOF
 
 # Run cypress test
 # No need to collect its exit status: if it's nonzero, this script will terminate since we use '-e'
+LOGFILE_SUFFIX="${CI_BASE_URL//\//_}" # Remove slashes from CI_BASE_URL to produce a usable filename
 # shellcheck disable=SC2016
 if ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress bash -c '[ -n "${AUTOMATION_DASHBOARD_URL}" ]'; then
   mme2e_log "AUTOMATION_DASHBOARD_URL is set. Using run_test_cycle.js for the cypress run"
-  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee ../cypress/logs/cypress.log
+  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee ../cypress/logs/${LOGFILE_SUFFIX}_cypress.log
 else
   mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Using run_tests.js for the cypress run"
   # shellcheck disable=SC2086
-  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee ../cypress/logs/cypress.log
+  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_tests.js $TEST_FILTER | tee ../cypress/logs/${LOGFILE_SUFFIX}_cypress.log
 fi
 
 # Collect server logs
-${MME2E_DC_SERVER} logs --no-log-prefix -- server >../cypress/logs/mattermost.log 2>&1
+${MME2E_DC_SERVER} logs --no-log-prefix -- server >../cypress/logs/${LOGFILE_SUFFIX}_mattermost.log 2>&1

e2e-tests/Makefile

@@ -2,18 +2,17 @@ SHELL := /bin/bash
 
 .PHONY: all run stop clean
 all: run
-run: generate-server start-server prepare-server run-test
+run: generate-server start-server run-test
 stop: stop-server stop-dashboard clean
 clean:
 	rm -fv .ci/server.yml
 	rm -fv .ci/.env.{server,dashboard,cypress,playwright}
 
-.PHONY: generate-server start-server prepare-server run-test stop-server restart-server
+.PHONY: generate-server start-server run-test stop-server restart-server
 generate-server:
 	bash ./.ci/server.generate.sh
 start-server: generate-server
 	bash ./.ci/server.start.sh
-prepare-server:
 	bash ./.ci/server.prepare.sh
 run-test:
 	bash ./.ci/server.run_test.sh

e2e-tests/README.md

@@ -55,7 +55,7 @@ Notes:
   * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability and simplifies the question of what container has access to which variable considerably.
   * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
 - The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
-- `make start-server` won't cleanup containers that don't change across runs. This means that you can use it to emulate a Mattermost server upgrade while retaining your database data by simply changing the `SERVER_IMAGE` variable on your machine, and then re-runing `make start-server` (and `make prepare-server` if you intend to run cypress again, since its container is also recreated). But this also means that if you want to run a clean local environment, you may have to manually run `make stop` to cleanup any running containers and their volumes, which include e.g. the database.
+- `make start-server` won't cleanup containers that don't change across runs. This means that you can use it to emulate a Mattermost server upgrade while retaining your database data by simply changing the `SERVER_IMAGE` variable on your machine, and then re-running `make start-server`. But this also means that if you want to run a clean local environment, you may have to manually run `make stop` to cleanup any running containers and their volumes, which include e.g. the database.
 
 ##### For code changes:
 * `make fmt-ci` to format and check yaml files and shell scripts.

e2e-tests/cypress/cypress.config.ts

@@ -5,13 +5,15 @@ import {defineConfig} from 'cypress';
 
 export default defineConfig({
     chromeWebSecurity: false,
-    defaultCommandTimeout: 20000,
+    defaultCommandTimeout: 30000,
     downloadsFolder: 'tests/downloads',
     fixturesFolder: 'tests/fixtures',
     numTestsKeptInMemory: 0,
     screenshotsFolder: 'tests/screenshots',
     taskTimeout: 20000,
-    video: false,
+    video: true,
+    videoCompression: true,
+    videosFolder: 'tests/videos',
     viewportWidth: 1300,
     env: {
         adminEmail: 'sysadmin@sample.mattermost.com',

e2e-tests/cypress/package-lock.json

@@ -38,7 +38,7 @@
         "chalk": "4.1.2",
         "client-oauth2": "github:larkox/js-client-oauth2#e24e2eb5dfcbbbb3a59d095e831dbe0012b0ac49",
         "cross-env": "7.0.3",
-        "cypress": "13.7.3",
+        "cypress": "13.13.2",
         "cypress-file-upload": "5.0.8",
         "cypress-multi-reporters": "1.6.4",
         "cypress-plugin-tab": "1.0.5",
@@ -4518,13 +4518,13 @@
       "dev": true
     },
     "node_modules/cypress": {
-      "version": "13.7.3",
+      "version": "13.13.2",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.7.3.tgz",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.13.2.tgz",
-      "integrity": "sha512-uoecY6FTCAuIEqLUYkTrxamDBjMHTYak/1O7jtgwboHiTnS1NaMOoR08KcTrbRZFCBvYOiS4tEkQRmsV+xcrag==",
+      "integrity": "sha512-PvJQU33933NvS1StfzEb8/mu2kMy4dABwCF+yd5Bi7Qly1HOVf+Bufrygee/tlmty/6j5lX+KIi8j9Q3JUMbhA==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@cypress/request": "^3.0.0",
+        "@cypress/request": "^3.0.1",
         "@cypress/xvfb": "^1.2.4",
         "@types/sinonjs__fake-timers": "8.1.1",
         "@types/sizzle": "^2.3.2",
@@ -4563,7 +4563,7 @@
         "request-progress": "^3.0.0",
         "semver": "^7.5.3",
         "supports-color": "^8.1.1",
-        "tmp": "~0.2.1",
+        "tmp": "~0.2.3",
         "untildify": "^4.0.0",
         "yauzl": "^2.10.0"
       },
@@ -10429,30 +10429,12 @@
       "dev": true
     },
     "node_modules/tmp": {
-      "version": "0.2.1",
+      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
-      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
       "dev": true,
-      "dependencies": {
-        "rimraf": "^3.0.0"
-      },
       "engines": {
-        "node": ">=8.17.0"
+        "node": ">=14.14"
-      }
-    },
-    "node_modules/tmp/node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
-      "dev": true,
-      "dependencies": {
-        "glob": "^7.1.3"
-      },
-      "bin": {
-        "rimraf": "bin.js"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/to-fast-properties": {
@@ -14844,12 +14826,12 @@
       "dev": true
     },
     "cypress": {
-      "version": "13.7.3",
+      "version": "13.13.2",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.7.3.tgz",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.13.2.tgz",
-      "integrity": "sha512-uoecY6FTCAuIEqLUYkTrxamDBjMHTYak/1O7jtgwboHiTnS1NaMOoR08KcTrbRZFCBvYOiS4tEkQRmsV+xcrag==",
+      "integrity": "sha512-PvJQU33933NvS1StfzEb8/mu2kMy4dABwCF+yd5Bi7Qly1HOVf+Bufrygee/tlmty/6j5lX+KIi8j9Q3JUMbhA==",
       "dev": true,
       "requires": {
-        "@cypress/request": "^3.0.0",
+        "@cypress/request": "^3.0.1",
         "@cypress/xvfb": "^1.2.4",
         "@types/sinonjs__fake-timers": "8.1.1",
         "@types/sizzle": "^2.3.2",
@@ -14888,7 +14870,7 @@
         "request-progress": "^3.0.0",
         "semver": "^7.5.3",
         "supports-color": "^8.1.1",
-        "tmp": "~0.2.1",
+        "tmp": "~0.2.3",
         "untildify": "^4.0.0",
         "yauzl": "^2.10.0"
       },
@@ -19310,24 +19292,10 @@
       "dev": true
     },
     "tmp": {
-      "version": "0.2.1",
+      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
-      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
-      "dev": true,
+      "dev": true
-      "requires": {
-        "rimraf": "^3.0.0"
-      },
-      "dependencies": {
-        "rimraf": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
-          "dev": true,
-          "requires": {
-            "glob": "^7.1.3"
-          }
-        }
-      }
     },
     "to-fast-properties": {
       "version": "2.0.0",

e2e-tests/cypress/package.json

@@ -33,7 +33,7 @@
     "chalk": "4.1.2",
     "client-oauth2": "github:larkox/js-client-oauth2#e24e2eb5dfcbbbb3a59d095e831dbe0012b0ac49",
     "cross-env": "7.0.3",
-    "cypress": "13.7.3",
+    "cypress": "13.13.2",
     "cypress-file-upload": "5.0.8",
     "cypress-multi-reporters": "1.6.4",
     "cypress-plugin-tab": "1.0.5",

e2e-tests/cypress/run_test_cycle.js

@@ -63,6 +63,7 @@ async function runCypressTest(specExecution) {
         spec: specExecution.file,
         config: {
             screenshotsFolder: `${MOCHAWESOME_REPORT_DIR}/screenshots`,
+            videosFolder: `${MOCHAWESOME_REPORT_DIR}/videos`,
             trashAssetsBeforeRuns: false,
         },
         reporter: 'cypress-multi-reporters',

e2e-tests/cypress/run_tests.js

@@ -109,6 +109,7 @@ async function runTests() {
             spec: testFile,
             config: {
                 screenshotsFolder: `${MOCHAWESOME_REPORT_DIR}/screenshots`,
+                videosFolder: `${MOCHAWESOME_REPORT_DIR}/videos`,
                 trashAssetsBeforeRuns: false,
             },
             env: {

e2e-tests/cypress/tests/integration/channels/accessibility/accessibility_post_spec.js

@@ -188,6 +188,10 @@ describe('Verify Accessibility Support in Post', () => {
                 cy.get(`#CENTER_flagIcon_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'save message');
                 cy.focused().tab();
 
+                // * Verify focus is on the actions button
+                cy.get(`#CENTER_actions_button_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'actions');
+                cy.focused().tab();
+
                 // * Verify focus is on the comment button
                 cy.get(`#CENTER_commentIcon_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'reply');
                 cy.focused().tab();
@@ -234,6 +238,10 @@ describe('Verify Accessibility Support in Post', () => {
                 cy.get(`#RHS_COMMENT_button_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'more');
                 cy.focused().tab({shift: true});
 
+                // * Verify focus is on the actions button
+                cy.get(`#RHS_COMMENT_actions_button_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'actions');
+                cy.focused().tab({shift: true});
+
                 // * Verify focus is on the save icon
                 cy.get(`#RHS_COMMENT_flagIcon_${postId}`).should('be.focused').and('have.attr', 'aria-label', 'save message');
                 cy.focused().tab({shift: true});
@@ -333,8 +341,9 @@ function verifyPostLabel(elementId, username, labelSuffix) {
     // * Verify reader reads out the post correctly
     cy.get('@lastPost').then((el) => {
         // # Get the post time
-        cy.wrap(el).find('time.post__time').invoke('text').then((time) => {
+        cy.wrap(el).find('time.post__time').invoke('attr', 'datetime').then((time) => {
-            const expectedLabel = `At ${time} ${Cypress.dayjs().format('dddd, MMMM D')}, ${username} ${labelSuffix}`;
+            const parsedTime = Cypress.dayjs(time);
+            const expectedLabel = `At ${parsedTime.format('h:mm A dddd, MMMM D')}, ${username} ${labelSuffix}`;
             cy.wrap(el).should('have.attr', 'aria-label', expectedLabel);
         });
     });

e2e-tests/cypress/tests/integration/channels/archived_channel/post_menu_spec.ts

@@ -26,7 +26,9 @@ describe('Archived channels', () => {
 
     it('MM-T1721 Archive channel posts menu should have copy link and reply options', () => {
         // # Click to add a channel description
-        cy.findByRoleExtended('button', {name: 'Set header'}).should('be.visible').click();
+        //   Note that it is invisible until the mouse hovers it, which is unfeasible in Cypress
+        //   https://docs.cypress.io/api/commands/hover
+        cy.findByText('Add a channel header').click({force: true});
 
         // # Add channel header for system message
         const header = 'this is a header!';

e2e-tests/cypress/tests/integration/channels/bot_accounts/display_name_spec.js

@@ -35,6 +35,9 @@ describe('Bot display name', () => {
                 });
                 cy.visit(`/${team.name}/channels/off-topic`);
                 cy.wait('@resources');
+
+                // # Wait for the page to fully load before continuing
+                cy.get('#sidebar-header-container').should('be.visible').and('have.text', team.display_name);
             });
         });
     });

e2e-tests/cypress/tests/integration/channels/custom_status/custom_status_expiry/custom_status_expiry_1_spec.ts

@@ -31,6 +31,8 @@ describe('MM-T4063 Custom status expiry', () => {
 
     const waitingTime = 30; //minutes
     let expiresAt = dayjs();
+    let expiresAtAcceptableValues = [''];
+    let expiresAtRegexp: RegExp;
     const expiryTimeFormat = 'h:mm A';
     it('MM-T4063_1 should open status dropdown', () => {
         // # Click on the sidebar header to open status dropdown
@@ -77,7 +79,12 @@ describe('MM-T4063 Custom status expiry', () => {
         cy.get('#custom_status_modal').should('not.exist');
 
         // # Setting the time at which the custom status should be expired
+        // # Note that we need to be flexible around accepted values, as this calculation and the server-side one may differ slightly
         expiresAt = dayjs().add(waitingTime, 'minute');
+        expiresAtAcceptableValues = [-1, 0, 1].map((el) =>
+            expiresAt.add(el, 'minute').format(expiryTimeFormat),
+        );
+        expiresAtRegexp = new RegExp(`(${expiresAtAcceptableValues.join('|')})`);
 
         // * Status should be set and the emoji should be visible in the sidebar header
         cy.uiGetProfileHeader().
@@ -97,7 +104,7 @@ describe('MM-T4063 Custom status expiry', () => {
         cy.get('.status-dropdown-menu .custom_status__row span.emoticon').invoke('attr', 'data-emoticon').should('contain', customStatus.emoji);
 
         // * Correct clear time should be displayed in the status dropdown
-        cy.get('.status-dropdown-menu .custom_status__expiry time').should('have.text', expiresAt.format(expiryTimeFormat));
+        cy.get('.status-dropdown-menu .custom_status__expiry time').invoke('text').should('match', expiresAtRegexp);
     });
 
     it('MM-T4063_6 custom status should be cleared after duration of set custom status', () => {
@@ -116,7 +123,7 @@ describe('MM-T4063 Custom status expiry', () => {
 
         // * Should show expiry time of status when current status is selected
         cy.get('#custom_status_modal .statusSuggestion__content').contains('span', customStatus.text).click();
-        cy.get('#custom_status_modal .expiry-value').should('have.text', expiresAt.format(expiryTimeFormat));
+        cy.get('#custom_status_modal .expiry-value').invoke('text').should('match', expiresAtRegexp);
 
         // # Close custom status modal
         cy.get('#custom_status_modal .modal-header .close').click();

e2e-tests/cypress/tests/integration/channels/enterprise/accessibility/accessibility_modals_dialogs_spec.js

@@ -142,7 +142,7 @@ describe('Verify Accessibility Support in Modals & Dialogs', () => {
             });
         }
 
-        // # Visit the test channel
+        // # Visit the test channel, and wait for the page to fully load
         cy.visit(`/${testTeam.name}/channels/${testChannel.name}`);
 
         // # Open Add Members Dialog
@@ -160,6 +160,7 @@ describe('Verify Accessibility Support in Modals & Dialogs', () => {
 
             // # Search for a text and then check up and down arrow
             cy.findByRole('textbox', {name: 'Search for people or groups'}).
+                wait(TIMEOUTS.HALF_SEC).
                 typeWithForce('u').
                 wait(TIMEOUTS.HALF_SEC).
                 typeWithForce('{downarrow}{downarrow}{downarrow}{uparrow}');

e2e-tests/cypress/tests/integration/channels/files_and_attachments/youtube_video_spec.js

@@ -10,6 +10,8 @@
 // Stage: @prod
 // Group: @channels @files_and_attachments
 
+import * as TIMEOUTS from '../../../fixtures/timeouts';
+
 describe('YouTube Video', () => {
     before(() => {
         // # Enable Link Previews
@@ -27,10 +29,12 @@ describe('YouTube Video', () => {
 
     it('MM-T2258 YouTube Video play, collapse', () => {
         // # Post message
-        cy.postMessage('https://www.youtube.com/watch?v=gLNmtUEvI5A');
+        const youtubeUrl = 'https://www.youtube.com/watch?v=gLNmtUEvI5A';
+        cy.postMessage(youtubeUrl);
+        cy.uiWaitUntilMessagePostedIncludes(youtubeUrl);
         cy.getLastPost().within(() => {
             // # Click play button
-            cy.get('.play-button').click();
+            cy.get('.play-button', {timeout: TIMEOUTS.TEN_SEC}).click();
 
             // * Video should be loaded in the iframe
             cy.get('.video-div > iframe').should('exist');

e2e-tests/cypress/tests/integration/channels/integrations/builtin_commands/common_commands_1_spec.js

@@ -87,8 +87,7 @@ describe('Integrations', () => {
         // # Type "/echo message 3"
         cy.uiGetPostTextBox().clear().type(`/echo ${message} 3{enter}`);
 
-        // * Verify that post is not shown after 1 second
+        // * Verify that post is not shown after pressing enter
-        cy.wait(TIMEOUTS.ONE_SEC);
         cy.getLastPost().within(() => {
             cy.findByText(message).should('not.exist');
         });

e2e-tests/cypress/tests/integration/channels/integrations/builtin_commands/invite_command_spec.js

@@ -84,7 +84,8 @@ describe('Integrations', () => {
         loginAndVisitChannel(testUser, testChannelUrl);
 
         // # In a GM use the /invite command to invite a user to a channel you have permission to add them to but place extra white space before the username
-        cy.postMessage(`/groupmsg @${member1.username} @${member2.username} `);
+        cy.postMessage(`/groupmsg @${member1.username},@${member2.username} `);
+        cy.uiGetChannelHeaderButton().contains(member1.username).contains(member2.username);
         cy.postMessage(`/invite        @${userToInviteGM.username} ~${testChannel.name} `);
 
         // * User added to channel as expected
@@ -133,7 +134,9 @@ describe('Integrations', () => {
         loginAndVisitChannel(testUser, testChannelUrl);
 
         // # In a GM Use the /invite command to invite a channel to another channel (e.g., /invite @[channel name])
-        cy.postMessage(`/groupmsg @${member1.username} @${member2.username} `);
+        cy.postMessage(`/groupmsg @${member1.username},@${member2.username} `);
+        cy.uiGetChannelHeaderButton().contains(member1.username).contains(member2.username);
+        cy.reload(); // Required for Cypress to write in the right channel
         cy.postMessage(`/invite @${testChannel.name} `);
 
         // * Error appears: "We couldn't find the user. They may have been deactivated by the System Administrator."
@@ -159,7 +162,9 @@ describe('Integrations', () => {
         loginAndVisitChannel(testUser, testChannelUrl);
 
         // # In a GM use the /invite command to invite someone to a channel they're already a member of
-        cy.postMessage(`/groupmsg @${member1.username} @${member2.username} `);
+        cy.postMessage(`/groupmsg @${member1.username},@${member2.username} `);
+        cy.uiGetChannelHeaderButton().contains(member1.username).contains(member2.username);
+        cy.reload(); // Required for Cypress to write in the right channel
         cy.postMessage(`/invite @${userToInvite.username} ~${testChannel.name} `);
 
         // * Error appears: "[username] is already in the channel"

e2e-tests/cypress/tests/integration/channels/integrations/message_to_channel_via_slash_command_spec.js

@@ -67,6 +67,7 @@ describe('Integrations', () => {
                 click();
 
             // * Verify that only "Hello World" is posted in off-topic channel
+            cy.uiWaitUntilMessagePostedIncludes('Hello World');
             cy.getLastPostId().then((postId) => {
                 cy.get(`#postMessageText_${postId}`).should('be.visible').and('have.text', 'Hello World');
             });
@@ -103,6 +104,7 @@ describe('Integrations', () => {
                 click();
 
             // * Verify that both messages are posted in off-topic channel
+            cy.uiWaitUntilMessagePostedIncludes('Hello World');
             cy.getLastPostId().then((postId) => {
                 cy.get(`#postMessageText_${postId}`).should('be.visible').and('have.text', 'Hello World');
             });

e2e-tests/cypress/tests/integration/channels/mark_as_unread/toast_appears_unread_spec.js

@@ -11,6 +11,7 @@
 // Group: @channels @mark_as_unread
 
 import {markAsUnreadFromPost, switchToChannel} from './helpers';
+import * as TIMEOUTS from '../../../fixtures/timeouts';
 
 describe('Verify unread toast appears after repeated manual marking post as unread', () => {
     let firstPost;
@@ -33,7 +34,10 @@ describe('Verify unread toast appears after repeated manual marking post as unre
                     cy.visit(`/${team.name}/channels/${testChannel.name}`);
                     switchToChannel(offTopicChannel);
 
-                    cy.postMessageAs({
+                    // Ensure that the Off-Topic channel has loaded successfully, before posting messages in the other channel
+                    cy.get('#channelHeaderTitle').should('be.visible').and('contain.text', offTopicChannel.display_name);
+
+                    cy.wait(TIMEOUTS.ONE_SEC).postMessageAs({
                         sender: otherUser,
                         message: 'First message',
                         channelId: testChannel.id,

e2e-tests/cypress/tests/integration/channels/messaging/message_reply_spec.js

@@ -21,6 +21,9 @@ describe('Message Reply', () => {
         cy.apiInitSetup({loginAfter: true}).then(({team, channel}) => {
             newChannel = channel;
             cy.visit(`/${team.name}/channels/${channel.name}`);
+
+            // # Wait for the page to fully load before continuing
+            cy.get('#sidebar-header-container').should('be.visible').and('have.text', team.display_name);
         });
     });
 
@@ -43,9 +46,11 @@ describe('Message Reply', () => {
             cy.clickPostCommentIcon(postId);
 
             // # Reply with the attachment
-            cy.postMessageReplyInRHS('A reply to an older post with attachment');
+            const replyText = 'A reply to an older post with attachment';
+            cy.postMessageReplyInRHS(replyText);
 
             // # Get the latest reply post
+            cy.uiWaitUntilMessagePostedIncludes(replyText);
             cy.getLastPostId().then((replyId) => {
                 // * Verify that the reply is in the channel view with matching text
                 cy.get(`#post_${replyId}`).within(() => {

e2e-tests/cypress/tests/integration/channels/notifications/at_mentions_spec.js

@@ -41,6 +41,10 @@ describe('Notifications', () => {
             // # Login as receiver and visit off-topic channel
             cy.apiLogin(receiver);
             cy.visit(`/${testTeam.name}/channels/${testChannel.name}`);
+
+            // # Wait for the page to fully load before continuing
+            cy.get('#channelHeaderDropdownButton').should('be.visible').and('have.text', testChannel.display_name);
+
             cy.get(`#sidebarItem_${otherChannel.name}`).click();
             cy.get('#sidebarItem_off-topic').click();
         });
@@ -284,4 +288,5 @@ function setNotificationSettings(desiredSettings = {first: true, username: true,
 
     // # Navigate to a channel we are NOT going to post to
     cy.get(`#sidebarItem_${channelName}`).scrollIntoView().click({force: true});
+    cy.get('#loadingSpinner').should('not.exist');
 }

e2e-tests/cypress/tests/integration/channels/system_console/unsaved_changes_spec.js

@@ -11,6 +11,7 @@
 // Group: @channels @not_cloud @system_console
 
 import {fileSizeToString} from '../../../utils';
+import * as TIMEOUTS from '../../../fixtures/timeouts';
 
 describe('Unsaved Changes', () => {
     let defaultMaxFileSize;
@@ -26,7 +27,7 @@ describe('Unsaved Changes', () => {
     it('MM-T955 Warning leaving changed page without saving: Leave page, discard changes', () => {
         // # Make a change on any page.
         cy.visit('/admin_console/environment/file_storage');
-        cy.findByTestId('FileSettings.MaxFileSizenumber').clear().type('150');
+        cy.findByTestId('FileSettings.MaxFileSizenumber', {timeout: TIMEOUTS.FIVE_SEC}).clear().type('150');
 
         // # Click a navigation item in left nav.
         cy.findByText('Database').click();
@@ -38,7 +39,7 @@ describe('Unsaved Changes', () => {
         // * Opens other page, changes discarded.
         cy.url().should('include', '/environment/database');
         cy.visit('/admin_console/environment/file_storage');
-        cy.findByTestId('FileSettings.MaxFileSizenumber').should('have.value', fileSizeToString(defaultMaxFileSize).replace('MB', ''));
+        cy.findByTestId('FileSettings.MaxFileSizenumber', {timeout: TIMEOUTS.FIVE_SEC}).should('have.value', fileSizeToString(defaultMaxFileSize).replace('MB', ''));
     });
 
     it('MM-T956 Warning leaving changed page without saving: Cancel leaving page', () => {
@@ -46,7 +47,7 @@ describe('Unsaved Changes', () => {
 
         // # Make a change on any page.
         cy.visit('/admin_console/environment/file_storage');
-        cy.findByTestId('FileSettings.MaxFileSizenumber').clear().type(draftValue);
+        cy.findByTestId('FileSettings.MaxFileSizenumber', {timeout: TIMEOUTS.FIVE_SEC}).clear().type(draftValue);
 
         // # Click a navigation item in left nav.
         cy.findByText('Database').click();
@@ -57,7 +58,7 @@ describe('Unsaved Changes', () => {
 
         // * Stays on current page, changes kept.
         cy.url().should('include', '/environment/file_storage');
-        cy.findByTestId('FileSettings.MaxFileSizenumber').should('have.value', draftValue);
+        cy.findByTestId('FileSettings.MaxFileSizenumber', {timeout: TIMEOUTS.FIVE_SEC}).should('have.value', draftValue);
     });
 });
 

e2e-tests/cypress/tests/integration/channels/system_console/user_management/users_spec.js

@@ -68,6 +68,7 @@ describe('System Console > User Management > Users', () => {
     it('MM-T932 Users - Change a user\'s password', () => {
         // # Search for the user.
         cy.findByPlaceholderText('Search users').type(testUser.email).wait(TIMEOUTS.HALF_SEC);
+        cy.get('#systemUsersTable-cell-0_emailColumn').should('contain', testUser.email);
 
         // # Open the actions menu.
         cy.get('#systemUsersTable-cell-0_actionsColumn').click().wait(TIMEOUTS.HALF_SEC);

e2e-tests/cypress/tests/integration/channels/system_console/user_management_not_cloud_spec.js

@@ -156,6 +156,7 @@ describe('User Management', () => {
 
         // # Search for the user.
         cy.get('#input_searchTerm').clear().type(user.email, {delay: TIMEOUTS.ONE_HUNDRED_MILLIS}).wait(TIMEOUTS.HALF_SEC);
+        cy.get('#systemUsersTable-cell-0_emailColumn').should('contain', user.email);
 
         cy.get('#actionMenuButton-systemUsersTable-0').click().wait(TIMEOUTS.HALF_SEC);
 

e2e-tests/cypress/tests/integration/channels/system_console/user_management_spec.js

@@ -185,6 +185,7 @@ describe('User Management', () => {
 
             // # Search for the user.
             cy.get('#input_searchTerm').clear().type(gitlabUser.email).wait(TIMEOUTS.HALF_SEC);
+            cy.get('#systemUsersTable-cell-0_emailColumn').should('contain', gitlabUser.email);
 
             // # Open actions menu.
             cy.get('#systemUsersTable-cell-0_actionsColumn').click().wait(TIMEOUTS.HALF_SEC);
@@ -255,6 +256,7 @@ describe('User Management', () => {
 
         // # Search for the user.
         cy.get('#input_searchTerm').clear().type(oldEmail).wait(TIMEOUTS.HALF_SEC);
+        cy.get('#systemUsersTable-cell-0_emailColumn').should('contain', oldEmail);
 
         cy.get('#systemUsersTable-cell-0_actionsColumn').click().wait(TIMEOUTS.HALF_SEC);
         cy.findByText('Update email').click().wait(TIMEOUTS.HALF_SEC);

e2e-tests/cypress/tests/integration/channels/toast/unread_with_bottom_start_toast_spec.js

@@ -38,6 +38,7 @@ describe('unread_with_bottom_start_toast', () => {
         cy.apiCreateChannel(testTeam.id, 'channel-a', 'ChannelA').then(({channel}) => {
             cy.apiAddUserToChannel(channel.id, otherUser.id).then(() => {
                 cy.visit(`/${testTeam.name}/channels/${channel.name}`);
+                cy.get('#loadingSpinner').should('not.exist'); // The channel should fully load once, before it can mark messages as "new"
                 cy.uiClickSidebarItem('off-topic');
                 cy.postMessage('hi');
 
@@ -73,6 +74,7 @@ describe('unread_with_bottom_start_toast', () => {
         cy.apiCreateChannel(testTeam.id, 'channel-b', 'ChannelB').then(({channel}) => {
             cy.apiAddUserToChannel(channel.id, otherUser.id).then(() => {
                 cy.visit(`/${testTeam.name}/channels/${channel.name}`);
+                cy.get('#loadingSpinner').should('not.exist'); // The channel should fully load once, before it can mark messages as "new"
                 cy.uiClickSidebarItem('off-topic');
                 cy.postMessage('hi');
 
@@ -114,6 +116,10 @@ describe('unread_with_bottom_start_toast', () => {
                     cy.postMessageAs({sender: otherUser, message: `test message ${index}`, channelId: channel.id}).wait(50);
                 }
 
+                // # Switch channel and back
+                cy.uiClickSidebarItem('off-topic');
+                cy.uiClickSidebarItem(channel.name);
+
                 cy.wait(TIMEOUTS.ONE_SEC);
 
                 // # Scroll to the top to find the oldest message
@@ -123,6 +129,7 @@ describe('unread_with_bottom_start_toast', () => {
                     // # Mark post as unread
                     cy.uiClickPostDropdownMenu(postId, 'Mark as Unread');
                 });
+                cy.get('div.toast').should('be.visible').contains('30 new messages'); // The toast message should appear right away
 
                 // # Visit off-topic channel and switch back to test channel
                 cy.uiClickSidebarItem('off-topic');

e2e-tests/cypress/tests/integration/playbooks/channels/rhs/status_update_spec.js

@@ -73,7 +73,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
     describe('post update dialog', () => {
         it('renders description correctly', () => {
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -84,7 +84,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
 
         it('description link navigates to run overview', () => {
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -101,7 +101,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
 
         it('prevents posting an update message with only whitespace', () => {
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -157,7 +157,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
                 cy.visit(`/${testTeam.name}/channels/${channelName}`);
 
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -183,7 +183,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
             const reminderTime = '1 day';
 
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get the dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -249,7 +249,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
         describe('prevents user from losing changes', () => {
             it('cancel, go back and save', () => {
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -279,7 +279,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
 
             it('click overview link, go back and save', () => {
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -310,7 +310,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
 
             it('cancel and discard explicitly', () => {
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -333,7 +333,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
 
             it('click overview link and discard explicitly', () => {
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -364,7 +364,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
         describe('shows the last update in update message', () => {
             it('shows the default when we have not made an update before', () => {
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get the dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -383,7 +383,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
                 cy.updateStatus(firstMessage);
 
                 // # Run the `/playbook update` slash command.
-                cy.uiPostMessageQuickly('/playbook update');
+                cy.uiPostMessageQuickly('/playbook update ');
 
                 // # Get the dialog modal.
                 cy.getStatusUpdateDialog().within(() => {
@@ -399,7 +399,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
     describe('the default reminder', () => {
         it('shows the configured default when we have not made a previous update', () => {
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get the dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -418,7 +418,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
             cy.updateStatus(firstMessage, '15 minutes');
 
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get the dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -437,7 +437,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
             cy.updateStatus(firstMessage, '90 minutes');
 
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get the dialog modal.
             cy.getStatusUpdateDialog().within(() => {
@@ -456,7 +456,7 @@ describe('channels > rhs > status update', {testIsolation: true}, () => {
             cy.updateStatus(firstMessage, '7 days');
 
             // # Run the `/playbook update` slash command.
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // # Get the dialog modal.
             cy.getStatusUpdateDialog().within(() => {

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/commands_spec.js

@@ -90,7 +90,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook check 1 1');
+            cy.uiPostMessageQuickly('/playbook check 1 1 ');
 
             // * Verify the task is checked
             cy.get('[data-rbd-droppable-id="1"]').find('.checkbox').eq(1).should('be.checked');
@@ -98,7 +98,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('check add', () => {
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook checkadd 1 new-task');
+            cy.uiPostMessageQuickly('/playbook checkadd 1 new-task ');
 
             // * Verify the task was added
             cy.get('[data-rbd-droppable-id="1"]').contains('new-task');
@@ -106,7 +106,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('check remove', () => {
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook checkremove 1 1');
+            cy.uiPostMessageQuickly('/playbook checkremove 1 1 ');
 
             // * Verify the task was added
             cy.get('[data-rbd-droppable-id="1"]').contains('Step 2').should('not.exist');
@@ -114,7 +114,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('owner', () => {
             // # Run a slash command
-            cy.uiPostMessageQuickly('/playbook owner');
+            cy.uiPostMessageQuickly('/playbook owner ');
 
             // * Verify the message.
             cy.verifyEphemeralMessage(`@${testUser.username} is the current owner for this playbook run.`);
@@ -123,13 +123,13 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.uiPostMessageQuickly(`/playbook owner @${testUser2.username}`);
 
             // * Verify that the owner was set.
-            cy.uiPostMessageQuickly('/playbook owner');
+            cy.uiPostMessageQuickly('/playbook owner ');
             cy.verifyEphemeralMessage(`@${testUser2.username} is the current owner for this playbook run.`);
         });
 
         it('timeline', () => {
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook timeline');
+            cy.uiPostMessageQuickly('/playbook timeline ');
 
             // * Verify the message.
             cy.verifyEphemeralMessage(`Timeline for ${playbookRunName}`);
@@ -137,7 +137,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('finish', () => {
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook finish');
+            cy.uiPostMessageQuickly('/playbook finish ');
 
             // * Verify confirm modal is visible.
             cy.get('#interactiveDialogModalLabel').should('exist');
@@ -274,19 +274,19 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('check', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook check 1 1');
+            cy.uiPostMessageQuickly('/playbook check 1 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects three arguments: the run number, the checklist number and the item number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook check 2 1 1');
+            cy.uiPostMessageQuickly('/playbook check 2 1 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook check 0 1 1');
+            cy.uiPostMessageQuickly('/playbook check 0 1 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Become a participant to interact with this run');
@@ -301,7 +301,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook check 1 1 1');
+            cy.uiPostMessageQuickly('/playbook check 1 1 1 ');
             cy.get('#rhsContainer').within(() => {
                 // * Verify number of runs
                 cy.get('[data-testid="run-list-card"]').should('have.length', 2);
@@ -316,19 +316,19 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('check add', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook checkadd 1');
+            cy.uiPostMessageQuickly('/playbook checkadd 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects two arguments: the run number and the checklist number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook checkadd 2 1 1');
+            cy.uiPostMessageQuickly('/playbook checkadd 2 1 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook checkadd 0 1 new-task');
+            cy.uiPostMessageQuickly('/playbook checkadd 0 1 new-task ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Become a participant to interact with this run');
@@ -343,7 +343,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook checkadd 1 1 new-task');
+            cy.uiPostMessageQuickly('/playbook checkadd 1 1 new-task ');
 
             cy.get('#rhsContainer').within(() => {
                 // * Verify number of runs
@@ -359,19 +359,19 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('check remove', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook checkremove 1 1');
+            cy.uiPostMessageQuickly('/playbook checkremove 1 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects three arguments: the run number, the checklist number and the item number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook checkremove 2 0 1');
+            cy.uiPostMessageQuickly('/playbook checkremove 2 0 1 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook checkremove 0 1 0');
+            cy.uiPostMessageQuickly('/playbook checkremove 0 1 0 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Become a participant to interact with this run');
@@ -386,7 +386,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook checkremove 1 1 1');
+            cy.uiPostMessageQuickly('/playbook checkremove 1 1 1 ');
 
             cy.get('#rhsContainer').within(() => {
                 // * Verify number of runs
@@ -402,19 +402,19 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('owner', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook owner');
+            cy.uiPostMessageQuickly('/playbook owner ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('/playbook owner expects at most one argument.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook owner 2');
+            cy.uiPostMessageQuickly('/playbook owner 2 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook owner 0');
+            cy.uiPostMessageQuickly('/playbook owner 0 ');
 
             // * Verify the message.
             cy.verifyEphemeralMessage(`@${testUser.username} is the current owner for this playbook run.`);
@@ -429,34 +429,34 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly(`/playbook owner 0 @${testUser2.username}`);
+            cy.uiPostMessageQuickly(`/playbook owner 0 @${testUser2.username} `);
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Become a participant to interact with this run');
 
             // # Run a slash command on a run with write access
-            cy.uiPostMessageQuickly(`/playbook owner 1 @${testUser2.username}`);
+            cy.uiPostMessageQuickly(`/playbook owner 1 @${testUser2.username} `);
 
             // * Verify that the owner was set.
-            cy.uiPostMessageQuickly('/playbook owner 1');
+            cy.uiPostMessageQuickly('/playbook owner 1 ');
             cy.verifyEphemeralMessage(`@${testUser2.username} is the current owner for this playbook run.`);
         });
 
         it('finish', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook finish');
+            cy.uiPostMessageQuickly('/playbook finish ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects one argument: the run number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook finish 2');
+            cy.uiPostMessageQuickly('/playbook finish 2 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook finish 0');
+            cy.uiPostMessageQuickly('/playbook finish 0 ');
 
             // * Verify the message.
             cy.verifyEphemeralMessage(`userID ${testUser2.id} is not an admin or channel member`);
@@ -479,7 +479,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             });
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook finish 1');
+            cy.uiPostMessageQuickly('/playbook finish 1 ');
 
             // * Verify confirm modal is visible.
             cy.get('#interactiveDialogModalLabel').should('exist');
@@ -493,19 +493,19 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('timeline', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook timeline');
+            cy.uiPostMessageQuickly('/playbook timeline ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects one argument: the run number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook timeline 2');
+            cy.uiPostMessageQuickly('/playbook timeline 2 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
 
             // # Run a slash command on a run with view access
-            cy.uiPostMessageQuickly('/playbook timeline 0');
+            cy.uiPostMessageQuickly('/playbook timeline 0 ');
 
             // * Verify the message.
             cy.verifyEphemeralMessage(`Timeline for ${playbookRuns[1].name}`);
@@ -513,13 +513,13 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
 
         it('update', () => {
             // # Run a slash command with not enough parameters
-            cy.uiPostMessageQuickly('/playbook update');
+            cy.uiPostMessageQuickly('/playbook update ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Command expects one argument: the run number.');
 
             // # Run a slash command wrong run number
-            cy.uiPostMessageQuickly('/playbook update 2');
+            cy.uiPostMessageQuickly('/playbook update 2 ');
 
             // * Verify the expected error message.
             cy.verifyEphemeralMessage('Invalid run number');
@@ -534,7 +534,7 @@ describe('channels > slash command > owner', {testIsolation: true}, () => {
             cy.findByTestId('post_textbox').clear();
 
             // # Run a slash command with correct parameters
-            cy.uiPostMessageQuickly('/playbook update 1');
+            cy.uiPostMessageQuickly('/playbook update 1 ');
 
             // # Get dialog modal.
             cy.getStatusUpdateDialog().within(() => {

e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/test_spec.js

@@ -85,6 +85,7 @@ describe('channels > slash command > test', {testIsolation: true}, () => {
                     EnableTesting: true,
                 },
             });
+            cy.apiReloadConfig();
         });
 
         beforeEach(() => {
@@ -132,6 +133,7 @@ describe('channels > slash command > test', {testIsolation: true}, () => {
                         EnableTesting: false,
                     },
                 });
+                cy.apiReloadConfig();
             });
 
             beforeEach(() => {
@@ -178,6 +180,7 @@ describe('channels > slash command > test', {testIsolation: true}, () => {
                         EnableTesting: true,
                     },
                 });
+                cy.apiReloadConfig();
             });
 
             beforeEach(() => {

e2e-tests/cypress/tests/integration/playbooks/playbooks/list_spec.js

@@ -175,6 +175,7 @@ describe('playbooks > list', {testIsolation: true}, () => {
             // # Drop loaded fixture onto playbook list
             cy.findByTestId('playbook-list-scroll-container').selectFile(validPlaybookExport, {
                 action: 'drag-drop',
+                force: true,
             });
 
             // * Verify that a new playbook was created.

e2e-tests/cypress/tests/plugins/index.js

@@ -3,6 +3,8 @@
 
 /* eslint-disable no-console */
 
+const fs = require('fs');
+
 const clientRequest = require('./client_request');
 const {
     dbGetActiveUserSessions,
@@ -72,5 +74,19 @@ module.exports = (on, config) => {
         return launchOptions;
     });
 
+    // https://docs.cypress.io/guides/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests
+    on('after:spec', (spec, results) => {
+        if (results && results.video) {
+            // Do we have failures for any retry attempts?
+            const failures = results.tests.some((test) =>
+                test.attempts.some((attempt) => attempt.state === 'failed'),
+            );
+            if (!failures) {
+                // delete the video if the spec passed and no tests retried
+                fs.unlinkSync(results.video);
+            }
+        }
+    });
+
     return config;
 };

e2e-tests/cypress/tests/support/ui/playbooks.js

@@ -2,7 +2,7 @@
 // See LICENSE.txt for license information.
 
 import * as TIMEOUTS from '../../fixtures/timeouts';
-const playbookRunStartCommand = '/playbook run';
+const playbookRunStartCommand = '/playbook run ';
 
 Cypress.Commands.add('startPlaybookRun', (playbookName, playbookRunName) => {
     cy.get('#interactiveDialogModal').should('exist').within(() => {
@@ -167,7 +167,7 @@ Cypress.Commands.add('selectReminderTime', (timeText) => {
  */
 Cypress.Commands.add('updateStatus', (message, reminderQuery) => {
     // # Run the slash command to update status.
-    cy.uiPostMessageQuickly('/playbook update');
+    cy.uiPostMessageQuickly('/playbook update ');
 
     // # Get the interactive dialog modal.
     cy.getStatusUpdateDialog().within(() => {

server/build/docker/keycloak/kc-healthcheck.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eu -o pipefail
+
+/opt/keycloak/bin/kcadm.sh config credentials -x --server http://localhost:8080 --realm master --user "$KEYCLOAK_ADMIN" --password "$KEYCLOAK_ADMIN_PASSWORD"
+/opt/keycloak/bin/kcadm.sh get realms/mattermost >/dev/null

server/build/gitlab-dc.common.yml

@@ -61,18 +61,22 @@ services:
       timeout: 15s
       retries: 12
   keycloak:
-    image: mattermostdevelopment/mirrored-keycloak:10.0.2
+    image: quay.io/keycloak/keycloak:23.0.7
     restart: always
+    entrypoint: /opt/keycloak/bin/kc.sh start --import-realm
     environment:
-      KEYCLOAK_USER: mmuser
+      KEYCLOAK_ADMIN: admin
-      KEYCLOAK_PASSWORD: mostest
+      KEYCLOAK_ADMIN_PASSWORD: admin
-      DB_VENDOR: h2
+      KC_HOSTNAME_STRICT: 'false'
-      KEYCLOAK_IMPORT: /setup/realm.json
+      KC_HOSTNAME_STRICT_HTTPS: 'false'
+      KC_HTTP_ENABLED: 'true'
       JAVA_OPTS: "-Xms64m -Xmx2G -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true"
     volumes:
-     - "./docker/keycloak:/setup"
+     - "./docker/keycloak/realm-export.json:/opt/keycloak/data/import/realm-export.json"
+     - "./docker/keycloak/kc-healthcheck.sh:/usr/local/bin/kc-healthcheck.sh"
     healthcheck:
-      test: [ "CMD", "bash", "-o", "pipefail", "-c", "curl --silent localhost:9990/health | grep -q '\"status\":\"UP\"'" ]
+      # We can't use a simple 'curl --silent localhost:9990/health | grep -q \"status\":\"UP\"' because there's no curl in the image: https://www.keycloak.org/server/health#_using_the_health_checks
+      test: [ "CMD", "/usr/local/bin/kc-healthcheck.sh" ]
       interval: 10s
       timeout: 15s
       retries: 12