--- name: E2E Tests - Playwright Template on: workflow_call: inputs: # Test configuration test_type: description: "Type of test run (smoke or full)" type: string required: true test_filter: description: "Test filter arguments (e.g., --grep @smoke)" type: string required: true workers: description: "Number of parallel shards" type: number required: false default: 2 enabled_docker_services: description: "Space-separated list of docker services to enable" type: string required: false default: "postgres inbucket" # Common build variables commit_sha: type: string required: true branch: type: string required: true build_id: type: string required: true server_image_tag: description: "Server image tag (e.g., master or short SHA)" type: string required: true server: type: string required: false default: onprem server_edition: description: "Server edition: enterprise (default), fips, or team" type: string required: false default: enterprise server_image_repo: description: "Docker registry: mattermostdevelopment (default) or mattermost" type: string required: false default: mattermostdevelopment server_image_aliases: description: "Comma-separated alias tags for description (e.g., 'release-11.4, release-11')" type: string required: false # Reporting options enable_reporting: type: boolean required: false default: false report_type: type: string required: false ref_branch: description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')" type: string required: false pr_number: type: string required: false # Commit status configuration context_name: description: "GitHub commit status context name" type: string required: true outputs: passed: description: "Number of passed tests" value: ${{ jobs.report.outputs.passed }} failed: description: "Number of failed tests" value: ${{ jobs.report.outputs.failed }} report_url: description: "URL to test report on S3" value: ${{ jobs.report.outputs.report_url }} secrets: MM_LICENSE: required: false REPORT_WEBHOOK_URL: required: false AWS_ACCESS_KEY_ID: required: true AWS_SECRET_ACCESS_KEY: required: true env: SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}" jobs: update-initial-status: runs-on: ubuntu-24.04 steps: - name: ci/set-initial-status uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.context_name }} description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}" status: pending generate-test-variables: runs-on: ubuntu-24.04 outputs: workers: "${{ steps.generate-workers.outputs.workers }}" start_time: "${{ steps.generate-workers.outputs.start_time }}" steps: - name: ci/generate-workers id: generate-workers run: | echo "workers=$(jq -nc '[range(1; ${{ inputs.workers }} + 1)]')" >> $GITHUB_OUTPUT echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT run-tests: runs-on: ubuntu-24.04 timeout-minutes: 30 continue-on-error: true needs: - generate-test-variables if: needs.generate-test-variables.result == 'success' strategy: fail-fast: false matrix: worker_index: ${{ fromJSON(needs.generate-test-variables.outputs.workers) }} defaults: run: working-directory: e2e-tests env: SERVER: "${{ inputs.server }}" MM_LICENSE: "${{ secrets.MM_LICENSE }}" ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}" TEST: playwright TEST_FILTER: "${{ inputs.test_filter }}" PW_SHARD: "${{ format('--shard={0}/{1}', matrix.worker_index, inputs.workers) }}" BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}" BUILD_ID: "${{ inputs.build_id }}" CI_BASE_URL: "${{ inputs.test_type }}-test-${{ matrix.worker_index }}" steps: - name: ci/checkout-repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" cache: npm cache-dependency-path: "e2e-tests/playwright/package-lock.json" - name: ci/get-webapp-node-modules working-directory: webapp run: make node_modules - name: ci/run-tests run: | make cloud-init make - name: ci/cloud-teardown if: always() run: make cloud-teardown - name: ci/upload-results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-${{ matrix.worker_index }} path: | e2e-tests/playwright/logs/ e2e-tests/playwright/results/ retention-days: 5 calculate-results: runs-on: ubuntu-24.04 needs: - generate-test-variables - run-tests if: always() && needs.generate-test-variables.result == 'success' outputs: passed: ${{ steps.calculate.outputs.passed }} failed: ${{ steps.calculate.outputs.failed }} flaky: ${{ steps.calculate.outputs.flaky }} skipped: ${{ steps.calculate.outputs.skipped }} total_specs: ${{ steps.calculate.outputs.total_specs }} failed_specs: ${{ steps.calculate.outputs.failed_specs }} failed_specs_count: ${{ steps.calculate.outputs.failed_specs_count }} failed_tests: ${{ steps.calculate.outputs.failed_tests }} commit_status_message: ${{ steps.calculate.outputs.commit_status_message }} total: ${{ steps.calculate.outputs.total }} pass_rate: ${{ steps.calculate.outputs.pass_rate }} passing: ${{ steps.calculate.outputs.passing }} color: ${{ steps.calculate.outputs.color }} test_duration: ${{ steps.calculate.outputs.test_duration }} end_time: ${{ steps.record-end-time.outputs.end_time }} steps: - name: ci/checkout-repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" cache: npm cache-dependency-path: "e2e-tests/playwright/package-lock.json" - name: ci/download-shard-results uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-* path: e2e-tests/playwright/shard-results/ merge-multiple: true - name: ci/merge-shard-results working-directory: e2e-tests/playwright run: | mkdir -p results/reporter # Merge blob reports using Playwright merge-reports (per docs) npm install --no-save @playwright/test npx playwright merge-reports --config merge.config.mjs ./shard-results/results/blob-report/ - name: ci/calculate id: calculate uses: ./.github/actions/calculate-playwright-results with: original-results-path: e2e-tests/playwright/results/reporter/results.json - name: ci/upload-merged-results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results path: e2e-tests/playwright/results/ retention-days: 5 - name: ci/record-end-time id: record-end-time run: echo "end_time=$(date +%s)" >> $GITHUB_OUTPUT run-failed-tests: runs-on: ubuntu-24.04 timeout-minutes: 30 needs: - run-tests - calculate-results if: >- always() && needs.calculate-results.result == 'success' && needs.calculate-results.outputs.failed != '0' && fromJSON(needs.calculate-results.outputs.failed_specs_count) <= 20 defaults: run: working-directory: e2e-tests env: SERVER: "${{ inputs.server }}" MM_LICENSE: "${{ secrets.MM_LICENSE }}" ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}" TEST: playwright BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}-retest" BUILD_ID: "${{ inputs.build_id }}-retest" steps: - name: ci/checkout-repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" cache: npm cache-dependency-path: "e2e-tests/playwright/package-lock.json" - name: ci/get-webapp-node-modules working-directory: webapp run: make node_modules - name: ci/run-failed-specs env: SPEC_FILES: ${{ needs.calculate-results.outputs.failed_specs }} run: | echo "Retesting failed specs: $SPEC_FILES" make cloud-init make start-server run-specs - name: ci/cloud-teardown if: always() run: make cloud-teardown - name: ci/upload-retest-results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results path: | e2e-tests/playwright/logs/ e2e-tests/playwright/results/ retention-days: 5 report: runs-on: ubuntu-24.04 needs: - generate-test-variables - run-tests - calculate-results - run-failed-tests if: always() && needs.calculate-results.result == 'success' outputs: passed: "${{ steps.final-results.outputs.passed }}" failed: "${{ steps.final-results.outputs.failed }}" commit_status_message: "${{ steps.final-results.outputs.commit_status_message }}" report_url: "${{ steps.upload-to-s3.outputs.report_url }}" duration: "${{ steps.duration.outputs.duration }}" duration_display: "${{ steps.duration.outputs.duration_display }}" retest_display: "${{ steps.duration.outputs.retest_display }}" defaults: run: working-directory: e2e-tests steps: - name: ci/checkout-repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" cache: npm cache-dependency-path: "e2e-tests/playwright/package-lock.json" # Download merged results (uploaded by calculate-results) - name: ci/download-results uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results path: e2e-tests/playwright/results/ # Download retest results (only if retest ran) - name: ci/download-retest-results if: needs.run-failed-tests.result != 'skipped' uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results path: e2e-tests/playwright/retest-results/ # Calculate results (with optional merge of retest results) - name: ci/calculate-results id: final-results uses: ./.github/actions/calculate-playwright-results with: original-results-path: e2e-tests/playwright/results/reporter/results.json retest-results-path: ${{ needs.run-failed-tests.result != 'skipped' && 'e2e-tests/playwright/retest-results/results/reporter/results.json' || '' }} - name: ci/aws-configure uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-region: us-east-1 aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: ci/upload-to-s3 id: upload-to-s3 env: AWS_REGION: us-east-1 AWS_S3_BUCKET: mattermost-cypress-report PR_NUMBER: "${{ inputs.pr_number }}" RUN_ID: "${{ github.run_id }}" COMMIT_SHA: "${{ inputs.commit_sha }}" TEST_TYPE: "${{ inputs.test_type }}" run: | LOCAL_RESULTS_PATH="playwright/results/" # Use PR number if available, otherwise use commit SHA prefix if [ -n "$PR_NUMBER" ]; then S3_PATH="server-pr-${PR_NUMBER}/e2e-reports/playwright-${TEST_TYPE}/${RUN_ID}" else S3_PATH="server-commit-${COMMIT_SHA::7}/e2e-reports/playwright-${TEST_TYPE}/${RUN_ID}" fi if [[ -d "$LOCAL_RESULTS_PATH" ]]; then aws s3 sync "$LOCAL_RESULTS_PATH" "s3://${AWS_S3_BUCKET}/${S3_PATH}/results/" \ --acl public-read --cache-control "no-cache" fi REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PATH}/results/reporter/index.html" echo "report_url=$REPORT_URL" >> "$GITHUB_OUTPUT" - name: ci/compute-duration id: duration env: START_TIME: ${{ needs.generate-test-variables.outputs.start_time }} FIRST_PASS_END_TIME: ${{ needs.calculate-results.outputs.end_time }} RETEST_RESULT: ${{ needs.run-failed-tests.result }} RETEST_SPEC_COUNT: ${{ needs.calculate-results.outputs.failed_specs_count }} TEST_DURATION: ${{ steps.final-results.outputs.test_duration }} run: | NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) MINUTES=$((ELAPSED / 60)) SECONDS=$((ELAPSED % 60)) DURATION="${MINUTES}m ${SECONDS}s" # Compute first-pass and re-run durations FIRST_PASS_ELAPSED=$((FIRST_PASS_END_TIME - START_TIME)) FP_MIN=$((FIRST_PASS_ELAPSED / 60)) FP_SEC=$((FIRST_PASS_ELAPSED % 60)) FIRST_PASS="${FP_MIN}m ${FP_SEC}s" if [ "$RETEST_RESULT" != "skipped" ]; then RERUN_ELAPSED=$((NOW - FIRST_PASS_END_TIME)) RR_MIN=$((RERUN_ELAPSED / 60)) RR_SEC=$((RERUN_ELAPSED % 60)) RUN_BREAKDOWN=" (first-pass: ${FIRST_PASS}, re-run: ${RR_MIN}m ${RR_SEC}s)" else RUN_BREAKDOWN="" fi # Duration icons: >20m high alert, >15m warning, otherwise clock if [ "$MINUTES" -ge 20 ]; then DURATION_DISPLAY=":rotating_light: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}" elif [ "$MINUTES" -ge 15 ]; then DURATION_DISPLAY=":warning: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}" else DURATION_DISPLAY=":clock3: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}" fi # Retest indicator with spec count if [ "$RETEST_RESULT" != "skipped" ]; then RETEST_DISPLAY=":repeat: re-run ${RETEST_SPEC_COUNT} spec(s)" else RETEST_DISPLAY="" fi echo "duration=${DURATION}" >> $GITHUB_OUTPUT echo "duration_display=${DURATION_DISPLAY}" >> $GITHUB_OUTPUT echo "retest_display=${RETEST_DISPLAY}" >> $GITHUB_OUTPUT - name: ci/publish-report if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != '' env: REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }} COLOR: ${{ steps.final-results.outputs.color }} REPORT_URL: ${{ steps.upload-to-s3.outputs.report_url }} TEST_TYPE: ${{ inputs.test_type }} REPORT_TYPE: ${{ inputs.report_type }} COMMIT_SHA: ${{ inputs.commit_sha }} REF_BRANCH: ${{ inputs.ref_branch }} PR_NUMBER: ${{ inputs.pr_number }} DURATION_DISPLAY: ${{ steps.duration.outputs.duration_display }} RETEST_DISPLAY: ${{ steps.duration.outputs.retest_display }} run: | # Capitalize test type TEST_TYPE_CAP=$(echo "$TEST_TYPE" | sed 's/.*/\u&/') # Build source line based on report type COMMIT_SHORT="${COMMIT_SHA::7}" COMMIT_URL="https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}" if [ "$REPORT_TYPE" = "RELEASE_CUT" ]; then SOURCE_LINE=":github_round: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`" elif [ "$REPORT_TYPE" = "MASTER" ] || [ "$REPORT_TYPE" = "RELEASE" ]; then SOURCE_LINE=":git_merge: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`" else SOURCE_LINE=":open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${{ github.repository }}/pull/${PR_NUMBER})" fi # Build retest part for message RETEST_PART="" if [ -n "$RETEST_DISPLAY" ]; then RETEST_PART=" | ${RETEST_DISPLAY}" fi # Build payload with attachments PAYLOAD=$(cat <" echo "${FAILED} failed, ${PASSED} passed" echo "" echo "| Test | File |" echo "|------|------|" echo "${FAILED_TESTS}" echo "" fi echo "" echo "### Calculation Outputs" echo "" echo "| Output | Value |" echo "|--------|-------|" echo "| passed | ${PASSED} |" echo "| failed | ${FAILED} |" echo "| flaky | ${FLAKY} |" echo "| skipped | ${SKIPPED} |" echo "| total_specs | ${TOTAL_SPECS} |" echo "| failed_specs_count | ${FAILED_SPECS_COUNT} |" echo "| commit_status_message | ${COMMIT_STATUS_MESSAGE} |" echo "| failed_specs | ${FAILED_SPECS:-none} |" echo "| duration | ${DURATION_DISPLAY} |" if [ "$RETEST_RESULT" != "skipped" ]; then echo "| retested | Yes |" else echo "| retested | No |" fi echo "" echo "---" echo "[View Full Report](${REPORT_URL})" } >> $GITHUB_STEP_SUMMARY - name: ci/assert-results run: | [ "${{ steps.final-results.outputs.failed }}" = "0" ] update-success-status: runs-on: ubuntu-24.04 if: always() && needs.report.result == 'success' && needs.calculate-results.result == 'success' needs: - calculate-results - report steps: - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.context_name }} description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}" status: success target_url: ${{ needs.report.outputs.report_url }} update-failure-status: runs-on: ubuntu-24.04 if: always() && (needs.report.result != 'success' || needs.calculate-results.result != 'success') needs: - calculate-results - report steps: - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.context_name }} description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}" status: failure target_url: ${{ needs.report.outputs.report_url }}