From c506e01a978936aec79781f74b757d080a33554f Mon Sep 17 00:00:00 2001
From: Jesse Hallam <jesse.hallam@gmail.com>
Date: Fri, 30 Jun 2023 15:00:37 -0300
Subject: [PATCH] server: automatically retry CI testing failures (#23820)

* server: --rerun-fails=3 for test-server(-race)?

* --rerun-fails=3 for IS_CI=true and -race as well
---
 .github/workflows/server-test-template.yml | 16 +++++++++++++---
 server/Makefile                            |  6 +++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml
index 80b2c70e46..d91f0f6541 100644
--- a/.github/workflows/server-test-template.yml
+++ b/.github/workflows/server-test-template.yml
@@ -68,10 +68,20 @@ jobs:
           path: |
             server/gotestsum.json
             server/report.xml
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@c0e4b81aaa0067314a2d0d06e19b512c9d8af4f5 # v3.7.7
+      - name: Publish test report
+        id: report
+        uses: mikepenz/action-junit-report@dfc44cebdda1e40b1e3c3b244a84dc303b952fb0 # v3.7.7 + count retries + check urls from https://github.com/lieut-data/action-junit-report
         if: success() || failure() # always run even if the previous step fails
         with:
           report_paths: server/report.xml
           check_name: ${{ inputs.name }} (Results)
-          job_name: ${{ inputs.name }}
+          require_tests: true
+      - name: Report retried tests via webhook
+        if: ${{ steps.report.outputs.retried > 0 }}
+        run: >
+          curl
+            --fail
+            -X POST
+            -H "Content-Type: application/json"
+            -d "{\"text\":\"#### ⚠️  One or more flaky tests detected ⚠️\\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\\n* If this is your pull request, double check your code to ensure you haven't introduced a flaky test.\\n* If this occurred on master or a release branch, reply to this message if you're willing to help.\\n* Submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\\n* Finally, reply to this message with a link to the created JIRA ticket.\\ncc @devs\"}"
+            ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }}
diff --git a/server/Makefile b/server/Makefile
index eea5f51796..2628381bbf 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -468,9 +468,9 @@ test-server-race: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
 test-server-race: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
 test-server-race: test-server-pre
 ifeq ($(IS_CI),true)
-	GOMAXPROCS=4 $(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
+	GOMAXPROCS=4 $(GOBIN)/gotestsum --rerun-fails=3 --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
 else
-	$(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
+	$(GOBIN)/gotestsum --rerun-fails=3 --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
 endif
 ifneq ($(IS_CI),true)
   ifneq ($(MM_NO_DOCKER),true)
@@ -486,7 +486,7 @@ test-server: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
 test-server: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
 test-server: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
 test-server: test-server-pre
-	$(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -timeout=90m
+	$(GOBIN)/gotestsum --rerun-fails=3 --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -timeout=90m
 ifneq ($(IS_CI),true)
   ifneq ($(MM_NO_DOCKER),true)
     ifneq ($(TEMP_DOCKER_SERVICES),)