E2E Tests: unify compose files, support Opensearch (#28716)

* Unify docker-compose files, support opensearch and redis * Support cypress/playwright bools, MM_SERVICE_OVERRIDES input * Remove unused directives from E2E tests * [skip ci] Fix ES tests, PR messages * [skip ci] Fix service pruning on server restart * [skip ci] Make services overrides a CSV, better param descriptions --------- Co-authored-by: Mattermost Build <build@mattermost.com>
2024-10-18 16:51:24 +02:00
--- a/.github/workflows/e2e-fulltests-ci.yml
+++ b/.github/workflows/e2e-fulltests-ci.yml
@@ -11,15 +11,20 @@ on:
        required: false
      PR_NUMBER:
        type: string
-        description: If testing a PR, this must be specified as well.
+        description: The PR number that the ref to test belongs to. Required, If testing a PR.
        required: false
      ROLLING_RELEASE_FROM_TAG:
        type: string
-        description: If specified, the server will be initialized with the given release tag, perform a rolling upgrade, then run the full E2E test.
+        description: Mattermost release git tag to perform a RollingRelease from, before upgrading to the tested ref and perform the full E2E test. Optional.
        required: false
      MM_ENV:
        type: string
        required: false
+        description: A comma-separated list of environment variables to set for the server. Spaces are not supported.
+      MM_SERVICE_OVERRIDES:
+        type: string
+        required: false
+        description: A comma-separated list of service overrides. E.g. "-elasticsearch,+opensearch"
      REPORT_TYPE:
        type: choice
        description: The context this report is being generated in
@@ -33,6 +38,14 @@ on:
          - CLOUD_UNSTABLE
          - NONE
        default: NONE
+      RUN_CYPRESS:
+        type: boolean
+        description: Enable Cypress run
+        default: true
+      RUN_PLAYWRIGHT:
+        type: boolean
+        description: Enable Playwright run
+        default: true

 concurrency:
  group: "${{ github.workflow }}-${{ inputs.REPORT_TYPE }}-${{ inputs.PR_NUMBER || inputs.ref }}-${{ inputs.MM_ENV }}"
@@ -89,6 +102,7 @@ jobs:
        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa"
      MM_ENV: "${{ inputs.MM_ENV || '' }}"
+      MM_SERVICE_OVERRIDES: "${{ inputs.MM_SERVICE_OVERRIDES }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -169,6 +183,16 @@ jobs:
            ### Run sanity assertions after variable generations
            git show-ref --verify "refs/tags/${ROLLING_RELEASE_FROM_TAG}" # 'inputs.ROLLING_RELEASE_FROM_TAG' must be a tag, for release report types
          fi
+          ENABLED_DOCKER_SERVICES="postgres inbucket minio openldap elasticsearch keycloak"
+          for SVC_OP in $(tr , ' '<<"$MM_SERVICE_OVERRIDES"); do
+            OP=$(cut -c1 <<<$SVC_OP)
+            SVC=$(cut -c2- <<<$SVC_OP)
+            case "$OP" in
+              "+") ENABLED_DOCKER_SERVICES="$ENABLED_DOCKER_SERVICES $SVC" ;;
+              "-") ENABLED_DOCKER_SERVICES=$(sed -E "s:(^| )${SVC}( |\$): :g" <<<"$ENABLED_DOCKER_SERVICES") ;;
+              *) echo "Invalid MM_SERVICE_OVERRIDE value: $SVC_OP"; exit 1 ;;
+            esac
+          done
          # BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
          # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
          BUILD_ID="${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}"
@@ -177,7 +201,7 @@ jobs:
          echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/mattermost-enterprise-edition:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
          echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
          echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
-          echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
+          echo "ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES}" >> $GITHUB_OUTPUT
          echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
          echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
          echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
@@ -195,14 +219,17 @@ jobs:
          STATUS_CHECK_CONTEXT: "${{steps.generate.outputs.status_check_context}}"
          WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
          CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
+          RUN_CYPRESS: "${{inputs.RUN_CYPRESS || ''}}"
+          RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT || ''}}"
        run: |
          if [ -n "$PR_NUMBER" ]; then
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
-          E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
-          You can check its progress by either:
-          - Looking at the corresponding commit statuses: \`${STATUS_CHECK_CONTEXT}\` and \`${STATUS_CHECK_CONTEXT}-playwright\`.
-          - Looking at the [E2E test's Workflow Run]($WORKFLOW_RUN_URL), with Run ID \`${{ github.run_id }}\`.
-          - Looking at the [E2E test's Cycle on the Automation Dashboard]($CYCLE_URL). (Cypress only)
+          E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}${MM_SERVICE_OVERRIDES:+, Cypress service overrides \`$MM_SERVICE_OVERRIDES\`}.
+          To check the run progress:
+          - Cypress: ${RUN_CYPRESS:+look for commit status \`$STATUS_CHECK_CONTEXT\` or the access the [Automation Dashboard Cycle URL]($CYCLE_URL)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "will not run").
+          - Playwright: ${RUN_PLAYWRIGHT:+look for commit status \`$STATUS_CHECK_CONTEXT-playwright\`}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "will not run").
+          
+          You can also look at the [E2E test's Workflow Run URL]($WORKFLOW_RUN_URL) (run ID \`${{ github.run_id }}\`).
          EOF
          fi

@@ -210,10 +237,7 @@ jobs:
    needs:
      - generate-test-variables
    uses: ./.github/workflows/e2e-tests-ci-template.yml
-    strategy:
-      matrix:
-        type:
-        - name: PR
+    if: ${{ inputs.RUN_CYPRESS }}
    with:
      commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
      status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
@@ -246,10 +270,7 @@ jobs:
    needs:
      - generate-test-variables
    uses: ./.github/workflows/e2e-tests-ci-template.yml
-    strategy:
-      matrix:
-        type:
-        - name: PR
+    if: ${{ inputs.RUN_PLAYWRIGHT }}
    with:
      commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
      status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
@@ -295,6 +316,8 @@ jobs:
      STATUS_CHECK_CONTEXT: "${{ needs.generate-test-variables.outputs.status_check_context }}"
      WORKFLOW_RUN_URL: "${{ needs.generate-test-variables.outputs.WORKFLOW_RUN_URL }}"
      CYCLE_URL: "${{ needs.generate-test-variables.outputs.CYCLE_URL }}"
+      RUN_CYPRESS: "${{inputs.RUN_CYPRESS || ''}}"
+      RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT || ''}}"
    steps:
      - name: ci/notify-user-test-completion
        run: |
@@ -302,8 +325,9 @@ jobs:
            gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
          E2E test has completed for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
          Results summary:
-          - Cypress pass rate: ${{ needs.e2e-fulltest-cypress.outputs.pass_rate || 'unknown' }} (see [Automation Dashboard]($CYCLE_URL) and commit status check \`${STATUS_CHECK_CONTEXT}\`)
-          - Playwright pass rate: ${{ needs.e2e-fulltest-playwright.outputs.pass_rate || 'unknown' }} (see commit status check \`${STATUS_CHECK_CONTEXT}-playwright\`)
+          - Cypress: ${RUN_CYPRESS:+pass rate is \`${{ needs.e2e-fulltest-cypress.outputs.pass_rate || 'unknown' }}\` (see [Automation Dashboard]($CYCLE_URL) and commit status check \`$STATUS_CHECK_CONTEXT\`)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "did not run").
+          - Playwright: ${RUN_PLAYWRIGHT:+pass rate is \`${{ needs.e2e-fulltest-playwright.outputs.pass_rate || 'unknown' }}\` (see commit status check \`$STATUS_CHECK_CONTEXT-playwright\`)}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "did not run").
+          
          The run summary artifacts are available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
          EOF
          fi