diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml
index f806d4c7d2..52c7768567 100644
--- a/.github/workflows/server-test-template.yml
+++ b/.github/workflows/server-test-template.yml
@@ -83,5 +83,18 @@ jobs:
             --fail \
             -X POST \
             -H "Content-Type: application/json" \
-            -d "{\"text\":\"#### ⚠️  One or more flaky tests detected ⚠️\\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\\n* If this is your pull request, double check your code to ensure you haven't introduced a flaky test.\\n* If this occurred on master or a release branch, reply to this message if you're willing to help.\\n* Submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\\n* Finally, reply to this message with a link to the created JIRA ticket.\\n\"}" \
+            -d "{\"text\":\"#### ⚠️  One or more flaky tests detected ⚠️\\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\\n* Ideally, this would have been caught in a pull request, but now a volunteer is required. If you're willing to help, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\\n* Finally, reply to this message with a link to the created JIRA ticket.\\n\"}" \
             ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }}
+      - name: Report retried tests (pull request)
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        if: ${{ steps.report.outputs.retried > 0 && !(github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }}
+        with:
+          script: |
+            const body = `#### ⚠️  One or more flaky tests detected ⚠️\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\n* Double check your code to ensure you haven't introduced a flaky test.\n* If this seems to be unrelated to your changes, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.`
+
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: body
+            })