feat: Decouple reporting from actual testing (#29587)

* feat: Decouple reporting from actual testing

* fix: Added also PR number for comment

* feat: Add also mmctl publishing

.github/workflows/mmctl-test-template.yml

@@ -28,6 +28,10 @@ jobs:
         id: go
         working-directory: ./server
         run: echo GO_VERSION=$(cat .go-version) >> "${GITHUB_OUTPUT}"
+      - name: Store required variables for publishing results
+        run: |
+          echo "${{ inputs.name }}" > server/test-name
+          echo "${{ github.event.pull_request.number }}" > server/pr-number        
       - name: Setup Go
         uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
@@ -80,10 +84,5 @@ jobs:
           path: |
             server/gotestsum.json
             server/report.xml
-      - name: Publish Test Report
+            server/test-name
-        uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0
+            server/pr-number
-        if: success() || failure() # always run even if the previous step fails
-        with:
-          report_paths: server/report.xml
-          check_name: ${{ inputs.name }} (Results)
-          job_name: ${{ inputs.name }}

.github/workflows/server-ci-report.yml

@@ -0,0 +1,95 @@
+name: Server CI Report
+
+on:
+  workflow_run:
+    workflows:
+      - "Server CI PR"
+      - "Server CI Master"
+    types:
+      - completed
+
+jobs:
+  generate-report-matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      REPORT_MATRIX: ${{ steps.report.outputs.REPORT_MATRIX }}
+    steps:
+      - name: report/download-artifacts-from-PR-workflow
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          pattern: "*-test-logs"
+          path: reports
+      
+      - name: report/generate-report-matrix
+        id: report
+        run: |
+          find "reports" -type f -name "test-name" | while read -r test_file; do
+            folder=$(basename "$(dirname "$test_file")")
+            test_name=$(cat "$test_file")
+            echo "{\"artifact\": \"$folder\", \"name\": \"$test_name\"}"
+          done | jq -s '{ "test": . }' | tee /tmp/report-matrix
+          echo REPORT_MATRIX=$(cat /tmp/report-matrix | jq --compact-output --monochrome-output) >> ${GITHUB_OUTPUT}
+      
+  publish-report:
+    runs-on: ubuntu-22.04
+    name: Publish Report ${{ matrix.test.name }}
+    needs: 
+      - generate-report-matrix
+    permissions:
+      pull-requests: write
+      checks: write
+      issues: write
+    strategy:
+      matrix: ${{ fromJson(needs.generate-report-matrix.outputs.REPORT_MATRIX) }}
+    steps:
+      - name: report/download-artifacts-from-PR-workflow
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          name: ${{ matrix.test.artifact }}
+          path: ${{ matrix.test.artifact }}
+      - name: report/fetch-pr-number
+        if: github.event.workflow_run.name == 'Server CI PR'
+        id: incoming-pr
+        run: echo "NUMBER=$(cat ${{ matrix.test.artifact }}/pr-number)" >> ${GITHUB_OUTPUT}
+      - name: Publish test report
+        id: report
+        uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0
+        with:
+          report_paths: ${{ matrix.test.artifact }}/report.xml
+          check_name: ${{ matrix.test.name }} (Results)
+          job_name: ${{ matrix.test.name }}
+          commit: ${{ github.event.workflow_run.head_commit.id }}
+          require_tests: true
+          check_retries: true
+          flaky_summary: true
+          include_passed: true
+          check_annotations: true
+
+      - name: Report retried tests via webhook (master || release-*)
+        if: ${{ steps.report.outputs.flaky_summary != '<table><tr><th>Test</th><th>Retries</th></tr></table>' && github.event.workflow_run.name == 'Server CI Master' }}
+        uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a # v2.0.0
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }}
+          TEXT: |-
+            #### ⚠️ One or more flaky tests detected ⚠️
+            * Failing job: [github.com/mattermost/mattermost:${{ matrix.test.name }}](${{ github.event.workflow_run.html_url }})
+            * Ideally, this would have been caught in a pull request, but now a volunteer is required. If you're willing to help, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.
+            * Finally, reply to this message with a link to the created JIRA ticket.
+  
+      - name: Report retried tests (pull request)
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        if: ${{ steps.report.outputs.flaky_summary != '<table><tr><th>Test</th><th>Retries</th></tr></table>' && github.event.workflow_run.name == 'Server CI PR' }}
+        with:
+          script: |
+            const body = `#### ⚠️  One or more flaky tests detected ⚠️\n* Failing job: [github.com/mattermost/mattermost:${{ matrix.test.name }}](${{ github.event.workflow_run.html_url }})\n* Double check your code to ensure you haven't introduced a flaky test.\n* If this seems to be unrelated to your changes, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\n\n${{ steps.report.outputs.flaky_summary }}`
+
+            await github.rest.issues.createComment({
+              issue_number: ${{ steps.incoming-pr.outputs.NUMBER }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: body
+            })

.github/workflows/server-test-template.yml

@@ -19,10 +19,6 @@ jobs:
   test:
     name: ${{ inputs.name }}
     runs-on: ubuntu-22.04
-    permissions:
-      pull-requests: write
-      checks: write
-      issues: write
     env:
       COMPOSE_PROJECT_NAME: ghactions
     steps:
@@ -37,6 +33,10 @@ jobs:
         with:
           go-version: ${{ steps.go.outputs.GO_VERSION }}
           cache-dependency-path: server/go.sum
+      - name: Store required variables for publishing results
+        run: |
+          echo "${{ inputs.name }}" > server/test-name
+          echo "${{ github.event.pull_request.number }}" > server/pr-number
       - name: Run docker compose
         run: |
           cd server/build
@@ -76,39 +76,5 @@ jobs:
           path: |
             server/gotestsum.json
             server/report.xml
-      - name: Publish test report
+            server/test-name
-        id: report
+            server/pr-number
-        uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0
-        if: success() || failure() # always run even if the previous step fails
-        with:
-          report_paths: server/report.xml
-          check_name: ${{ inputs.name }} (Results)
-          job_name: ${{ inputs.name }}
-          require_tests: true
-          check_retries: true
-          flaky_summary: true
-          include_passed: true
-          check_annotations: true
-      - name: Report retried tests via webhook (master || release-*)
-        if: ${{ steps.report.outputs.flaky_summary != '<table><tr><th>Test</th><th>Retries</th></tr></table>' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }}
-        uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a # v2.0.0
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }}
-          TEXT: |-
-            #### ⚠️ One or more flaky tests detected ⚠️
-            * Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            * Ideally, this would have been caught in a pull request, but now a volunteer is required. If you're willing to help, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.
-            * Finally, reply to this message with a link to the created JIRA ticket.
-      - name: Report retried tests (pull request)
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
-        if: ${{ steps.report.outputs.flaky_summary != '<table><tr><th>Test</th><th>Retries</th></tr></table>' && !(github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }}
-        with:
-          script: |
-            const body = `#### ⚠️  One or more flaky tests detected ⚠️\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\n* Double check your code to ensure you haven't introduced a flaky test.\n* If this seems to be unrelated to your changes, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\n\n${{ steps.report.outputs.flaky_summary }}`
-
-            await github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: body
-            })