CLD-6297 - E2E tests improvements (#24541)

* Improve e2etests readme, remove unused apt install
* Add testing E2E Tests MultiOS
* Get rid of deprecation warning
* Smart selection of docker network driver for different OSes
* Unify makefile invocation for windows as well
* Tune JVM for ES and Keycloak
* fix: Made required changes to run locally on arm MacOS
* fix: FIxed network command and removed dashboard node user as it is not needed
* fix: make dashboard work for Mac
* Make runs without dashboard work again, make BROWSE tunable and document it

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Antonis Stamatiou <stamatiou.antonis@gmail.com>

.github/workflows/e2e-tests-ci.yml

@@ -8,10 +8,6 @@ on:
         type: string
         required: false
 
-defaults:
-  run:
-    shell: bash
-
 jobs:
   update-initial-status:
     runs-on: ubuntu-22.04
@@ -84,25 +80,46 @@ jobs:
           npm run check
 
   smoketests:
-    runs-on: ubuntu-latest-8-cores
+    strategy:
+      matrix:
+        #
+        # Note that smoketests should be run only on ubuntu, for QA purposes.
+        # But it's useful to be able to run and debug the smoketests for different OSes.
+        # Notes:
+        # - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
+        # - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
+        #
+        #os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl]
+        os: [ubuntu-latest-8-cores]
+    runs-on: "${{ matrix.os }}"
     needs:
       - cypress-check
       - playwright-check
     defaults:
       run:
+        shell: bash
         working-directory: e2e-tests
     steps:
       - name: ci/checkout-repo
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           ref: ${{ inputs.commit_sha || github.sha }}
+      - name: ci/setup-macos-docker
+        if: runner.os == 'macos'
+        # https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
+        run: |
+          brew install docker docker-compose
+          colima start
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
       - name: ci/e2e-smoketests
         run: |
           make
       - name: ci/e2e-smoketests-store-results
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
-          name: e2e-smoketests-results
+          name: e2e-smoketests-results-${{ matrix.os }}
           path: |
             e2e-tests/cypress/logs/
             e2e-tests/cypress/results/

e2e-tests/.ci/.e2erc

@@ -3,12 +3,24 @@
 export MME2E_DC_SERVER="docker compose -p mmserver -f $(readlink -f ../../server/build/gitlab-dc.postgres.yml) -f $(readlink -f ./server.override.yml)"
 export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f $(readlink -f ./dashboard/docker/docker-compose.yml) -f $(readlink -f ./dashboard.override.yml)"
 export MME2E_UID=$(id -u)
-
+export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}')
-# Default the optional variables that are used in the docker-compose file
+export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}')
-export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mm-ee-test:$(git rev-parse --short=7 HEAD)"
-export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
 export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc)
 
+# Default values for optional variables
+export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mm-ee-test:$(git rev-parse --short=7 HEAD)"
+export BROWSER_DEFAULT="chrome"
+case $MME2E_OSTYPE in
+  # OS specific defaults overrides
+  darwin )
+    BROWSER_DEFAULT="electron" ;;
+  * )
+esac
+
+# Populate the optional variables that are used in the docker-compose file
+export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
+export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
+
 # Function definitions
 mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; }
 mme2e_get_current_shopt_arg () {
@@ -70,7 +82,14 @@ mme2e_legacy_setup () {
   # These functions are needed until every pipeline depending on the `server/build/gitlab-dc.*.yml` files is adapted to not use external docker networking anymore
   # After that is fixed, this function and the external network in the docker-compose files may be removed
   export COMPOSE_PROJECT_NAME=mmserver_legacy
-  docker network inspect ${COMPOSE_PROJECT_NAME} >/dev/null 2>&1 || docker network create ${COMPOSE_PROJECT_NAME}
+  case $MME2E_OSTYPE in
+    windows )
+      # https://learn.microsoft.com/en-us/virtualization/windowscontainers/container-networking/network-drivers-topologies
+      DOCKER_NETWORK_DRIVER="nat" ;;
+    * )
+      DOCKER_NETWORK_DRIVER="bridge" ;;
+  esac
+  docker network inspect ${COMPOSE_PROJECT_NAME} >/dev/null 2>&1 || docker network create --driver=${DOCKER_NETWORK_DRIVER} ${COMPOSE_PROJECT_NAME}
 }
 
 # Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose' in your shell

e2e-tests/.ci/dashboard.override.yml

@@ -1,24 +1,23 @@
 ---
-version: '3.1'
+version: "3.1"
 
 services:
-    dashboard:
+  dashboard:
-        image: mattermostdevelopment/mirrored-node:16.17.0
+    image: mattermostdevelopment/mirrored-node:18.17
-        environment:
+    environment:
-          PG_URI: postgres://mmuser:mostest@db:5432/automation_dashboard_db
+      PG_URI: postgres://mmuser:mostest@db:5432/automation_dashboard_db
-          JWT_SECRET: s8gGBA3ujKRohSw1L8HLOY7Jjnu2ZYv8 # Generated with e.g. `dd if=/dev/urandom count=24 bs=1 2>/dev/null | base64 -w0`
+      JWT_SECRET: s8gGBA3ujKRohSw1L8HLOY7Jjnu2ZYv8 # Generated with e.g. `dd if=/dev/urandom count=24 bs=1 2>/dev/null | base64 -w0`
-          JWT_USER: cypress-test
+      JWT_USER: cypress-test
-          JWT_ROLE: integration
+      JWT_ROLE: integration
-          JWT_ALG: HS256
+      JWT_ALG: HS256
-          JWT_EXPIRES_IN: 365d
+      JWT_EXPIRES_IN: 365d
-          ALLOWED_USER: cypress-test
+      ALLOWED_USER: cypress-test
-          ALLOWED_ROLE: integration
+      ALLOWED_ROLE: integration
-        user: "${MME2E_UID}"
+    volumes:
-        volumes:
+      - "../:/app"
-          - "../:/app"
+      - "../../dashboard.entrypoint.sh:/usr/local/bin/dashboard.entrypoint.sh:ro"
-          - "../../dashboard.entrypoint.sh:/usr/local/bin/dashboard.entrypoint.sh:ro"
+      - "../../.env.dashboard:/var/local/.env.dashboard:rw"
-          - "../../.env.dashboard:/var/local/.env.dashboard:rw"
+    working_dir: /app
-        working_dir: /app
+    entrypoint: /usr/local/bin/dashboard.entrypoint.sh
-        entrypoint: /usr/local/bin/dashboard.entrypoint.sh
+    ports:
-        ports:
+      - 4000:4000
-            - 4000:4000

e2e-tests/.ci/dashboard.start.sh

@@ -18,13 +18,18 @@ mme2e_log "Starting the dashboard"
 ${MME2E_DC_DASHBOARD} up -d db dashboard
 
 mme2e_log "Generating the dashboard's local URL"
-MME2E_DC_DASHBOARD_NETWORK=$(${MME2E_DC_DASHBOARD} ps -q dashboard | xargs -l docker inspect | jq -r '.[0].NetworkSettings.Networks | (keys|.[0])')
+case $MME2E_OSTYPE in
-MME2E_DC_DASHBOARD_GATEWAY=$(docker network inspect $MME2E_DC_DASHBOARD_NETWORK | jq -r '.[0].IPAM.Config[0].Gateway')
+  darwin )
-AUTOMATION_DASHBOARD_URL="http://${MME2E_DC_DASHBOARD_GATEWAY}:4000/api"
+    AUTOMATION_DASHBOARD_IP=$(ifconfig $(route get 1.1.1.1 | grep interface | awk '{print $2}') | grep -w inet | awk '{print $2}') ;;
+  * )
+    MME2E_DC_DASHBOARD_NETWORK=$(docker inspect $(${MME2E_DC_DASHBOARD} ps -q dashboard) | jq -r '.[0].NetworkSettings.Networks | (keys|.[0])')
+    AUTOMATION_DASHBOARD_IP=$(docker network inspect $MME2E_DC_DASHBOARD_NETWORK | jq -r '.[0].IPAM.Config[0].Gateway')
+esac
+AUTOMATION_DASHBOARD_URL="http://${AUTOMATION_DASHBOARD_IP}:4000/api"
 
 mme2e_log "Generating a signed JWT token for accessing the dashboard"
-${MME2E_DC_DASHBOARD} exec -T -u $MME2E_UID dashboard npm i
+${MME2E_DC_DASHBOARD} exec -T dashboard npm i
-AUTOMATION_DASHBOARD_TOKEN=$(${MME2E_DC_DASHBOARD} exec -T -u $MME2E_UID dashboard node script/sign.js | awk '{ print $2; }') # The token secret is specified in the dashboard.override.yml file
+AUTOMATION_DASHBOARD_TOKEN=$(${MME2E_DC_DASHBOARD} exec -T dashboard node script/sign.js | awk '{ print $2; }') # The token secret is specified in the dashboard.override.yml file
 
 mme2e_log "Generating the .env.dashboard file, to point Cypress to the dashboard URL"
 mme2e_generate_envfile_from_var_names >.env.dashboard <<EOF

e2e-tests/.ci/server.override.yml

@@ -1,9 +1,8 @@
 ---
 # Image hashes in this file are for amd64 systems
 # NB: paths relative to the `server/build` directory, which contains the original compose file that this yaml is overriding
-version: '2.4'
+version: "2.4"
 services:
-
   server:
     image: "${SERVER_IMAGE}"
     restart: always
@@ -28,7 +27,7 @@ services:
       MM_FEATUREFLAGS_ONBOARDINGTOURTIPS: "false"
       MM_SERVICEENVIRONMENT: "test"
     volumes:
-    - "server-config:/mattermost/config"
+      - "server-config:/mattermost/config"
     ports:
       - "8065:8065"
     depends_on:
@@ -54,12 +53,12 @@ services:
         condition: service_healthy
 
   webhook-interactions:
-    image:  mattermostdevelopment/mirrored-node:${NODE_VERSION_REQUIRED}
+    image: mattermostdevelopment/mirrored-node:${NODE_VERSION_REQUIRED}
     command: sh -c "npm install -g axios express client-oauth2@larkox/js-client-oauth2#e24e2eb5dfcbbbb3a59d095e831dbe0012b0ac49 && exec node webhook_serve.js"
     environment:
       NODE_PATH: /usr/local/lib/node_modules/
     healthcheck:
-      test: [ "CMD", "curl", "-s", "-o/dev/null", "127.0.0.1:3000" ]
+      test: ["CMD", "curl", "-s", "-o/dev/null", "127.0.0.1:3000"]
       interval: 10s
       timeout: 15s
       retries: 12
@@ -72,16 +71,17 @@ services:
           - webhook-interactions
 
   cypress:
-    image: "mattermostdevelopment/mirrored-cypress-browsers-public:node-18.16.1-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1"
+    image: "cypress/browsers:node-18.16.1-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1"
-    entrypoint: [ "/bin/bash", "-c" ]
+    ### Temorarily disabling this image, until both the amd64 and arm64 version are mirrored
-    command: [ "until [ -f /var/run/mm_terminate ]; do sleep 5; done" ]
+    # image: "mattermostdevelopment/mirrored-cypress-browsers-public:node-18.16.1-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1"
+    entrypoint: ["/bin/bash", "-c"]
+    command: ["until [ -f /var/run/mm_terminate ]; do sleep 5; done"]
     env_file:
       - "../../e2e-tests/.ci/.env.dashboard"
       - "../../e2e-tests/.ci/.env.cypress"
     environment:
       REPO: "mattermost-server"
       # Cypress configuration
-      BROWSER: "chrome"
       HEADLESS: "true"
       CYPRESS_baseUrl: "http://server:8065"
       CYPRESS_dbConnection: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
@@ -118,14 +118,14 @@ services:
 
   utils:
     image: "mattermost/mattermost-build-server:20230904_golang-1.20.7"
-    entrypoint: [ "/bin/bash", "-c" ]
+    entrypoint: ["/bin/bash", "-c"]
-    command: [ "until [ -f /var/run/mm_terminate ]; do sleep 5; done" ]
+    command: ["until [ -f /var/run/mm_terminate ]; do sleep 5; done"]
     environment:
       MM_SERVICEENVIRONMENT: "test" # Currently required for running the config_generator with the right environment
     working_dir: "/opt/mattermost-server"
     volumes:
-    - "../../:/opt/mattermost-server"
+      - "../../:/opt/mattermost-server"
-    - "server-config:/opt/server-config"
+      - "server-config:/opt/server-config"
 
 volumes:
   server-config:

e2e-tests/.ci/server.start.sh

@@ -29,6 +29,7 @@ mme2e_generate_envfile_from_var_names >.env.cypress <<EOF
 BRANCH
 BUILD_ID
 CI_BASE_URL
+BROWSER
 AUTOMATION_DASHBOARD_URL
 AUTOMATION_DASHBOARD_TOKEN
 EOF
@@ -42,7 +43,6 @@ mme2e_log "Creating E2E containers and generating server config"
 ${MME2E_DC_SERVER} create
 ${MME2E_DC_SERVER} up -d -- utils
 ${MME2E_DC_SERVER} exec -T -w /opt/mattermost-server/server -- utils bash <<EOF
-apt update && apt install -y jq
 OUTPUT_CONFIG=/tmp/config_generated.json go run scripts/config_generator/main.go
 jq "
   .ServiceSettings.SiteURL=\"http://server:8065\"

e2e-tests/Makefile

@@ -7,20 +7,20 @@ stop: stop-server stop-dashboard clean-env-placeholders
 
 .PHONY: start-server prepare-server run-cypress stop-server restart-server
 start-server:
-	./.ci/server.start.sh
+	bash ./.ci/server.start.sh
 prepare-server:
-	./.ci/server.prepare.sh
+	bash ./.ci/server.prepare.sh
 run-cypress:
-	./.ci/server.run_cypress.sh
+	bash ./.ci/server.run_cypress.sh
 stop-server:
-	./.ci/server.stop.sh
+	bash ./.ci/server.stop.sh
 restart-server: stop-server start-server
 
 .PHONY: start-dashboard stop-dashboard
 start-dashboard:
-	./.ci/dashboard.start.sh
+	bash ./.ci/dashboard.start.sh
 stop-dashboard:
-	./.ci/dashboard.stop.sh
+	bash ./.ci/dashboard.stop.sh
 
 .PHONY: print-env-placeholders clean-env-placeholders
 print-env-placeholders:

e2e-tests/README.md

@@ -4,30 +4,35 @@ This directory contains the E2E testing code for the Mattermost web client.
 
 ### How to run locally
 
-The E2E testing scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`)
+##### For test case development
+
+Please refer to the [dedicated developer documentation](https://developers.mattermost.com/contribute/more-info/webapp/e2e-testing/) for instructions.
+
+##### For pipeline debugging
+
+The E2E testing pipeline's scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`)
 
 Instructions, tl;dr: create a local branch with your E2E test changes, then open a PR to the `mattermost-server` repo targeting the `master` branch (so that CI will produce the image that docker-compose needs), then run `make` in this directory.
 
 Instructions, detailed:
-1. Create the `.ci/env` file, and populate the variables you need, keeping in mind that:
+1. (optional, undefined variables are set to sane defaults) Create the `.ci/env` file, and populate it with the variables you need out of the following list:
   * The following variables will be passed over to the server container: `MM_LICENSE` (no enterprise features will be available if this is unset), and the exploded `MM_ENV` (a comma-separated list of env var specifications)
-  * The following variables will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
+  * The following variables will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
-  * The `SERVER_IMAGE` variable can also be set, if you want to select a custom mattermost-server image
+  * The `SERVER_IMAGE` variable can also be set, if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used.
-  * The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run
+  * The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
-  * All variables are optional, and will be set to sane defaults
 2. (optional) `make start-dashboard`: start the automation-dashboard in the background
   * This also sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container
-  * Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your env, the latter variables will take precedence
+  * Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your `.ci/env` file, the latter variables will take precedence
 3. `make`: start and prepare the server, then run the cypress tests
   * You can track the progress of the run in the `http://localhost:4000/cycles` dashboard, if you launched it locally
 4. `make stop`: tears down the server (and the dashboard, if running), then cleans up the env placeholder files
 
 Notes:
+- Setting a variable in `.ci/env` is functionally equivalent to exporting variables in your current shell's environment, before invoking the makefile.
+- The `.ci/.env.*` files are auto generated by the pipeline scripts, and aren't meant to be modified manually. The only file you should edit to control the containers' environment is `.ci/env`, as specified in the instructions above.
 - Aside from some exceptions (e.g. `TEST_FILTER`), most of the variables in `.ci/env` must be set before the `make start-server` command is run. Modifying that file afterwards has no effect, because the containers' env files are generated in that step.
 - If you restart the dashboard at any point, you must also restart the server containers, so that it picks up the new IP of the dashboard from the newly generated `.env.dashboard` file
 - If you started the dashboard locally in the past, but want to point to another dashboard later, you can run `make clean-env-placeholders` to remove references to the local dashboard (you'll likely need to restart the server)
-- Dynamically set variables for the server or cypress should be managed within the `.env.*` files, rather than in the docker-compose files, to streamline their management.
+- If new variables need to be passed to any of the containers:
-
+  * If their value is fixed (e.g. a static server configuration): these may be simply added to the `.ci/server.override.yml` file, to the appropriate container.
-##### How to control which tests to run
+  * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably.
-
-The `TEST_FILTER` variable will control which test files to run Cypress tests against. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.

server/build/gitlab-dc.common.yml

@@ -47,7 +47,7 @@ services:
       http.cors.allow-headers: "X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization"
       http.cors.allow-credentials: "true"
       transport.host: "127.0.0.1"
-      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+      ES_JAVA_OPTS: "-Xms512m -Xmx2G"
     healthcheck:
       test: [ "CMD", "bash", "-o", "pipefail", "-c", "curl --silent localhost:9200/_cat/health | awk '{ print $$4 }' | grep -qE '^green$$'" ]
       interval: 10s
@@ -68,6 +68,7 @@ services:
       KEYCLOAK_PASSWORD: mostest
       DB_VENDOR: h2
       KEYCLOAK_IMPORT: /setup/realm.json
+      JAVA_OPTS: "-Xms64m -Xmx2G -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true"
     volumes:
      - "./docker/keycloak:/setup"
     healthcheck:

server/build/gitlab-dc.mysql.yml

@@ -96,5 +96,5 @@ services:
 
 networks:
   default:
-    external:
+    name: ${COMPOSE_PROJECT_NAME}
-      name: ${COMPOSE_PROJECT_NAME}
+    external: true

server/build/gitlab-dc.postgres.yml

@@ -94,5 +94,5 @@ services:
 
 networks:
   default:
-    external:
+    name: ${COMPOSE_PROJECT_NAME}
-      name: ${COMPOSE_PROJECT_NAME}
+    external: true

server/build/gitlab-dc.schemamysql.yml

@@ -32,5 +32,5 @@ services:
 
 networks:
   default:
-    external:
+    name: ${COMPOSE_PROJECT_NAME}
-      name: ${COMPOSE_PROJECT_NAME}
+    external: true

server/build/gitlab-dc.schemapostgres.yml

@@ -30,5 +30,5 @@ services:
 
 networks:
   default:
-    external:
+    name: ${COMPOSE_PROJECT_NAME}
-      name: ${COMPOSE_PROJECT_NAME}
+    external: true