MM-27957: Fix flaky test TestRecycleDBConns (#15415)

https://mattermost.atlassian.net/browse/MM-27957

store/sqlstore/supplier_test.go

@@ -269,10 +269,22 @@ func TestRecycleDBConns(t *testing.T) {
 			assert.Equal(t, 0, int(stats.MaxLifetimeClosed), "unexpected number of connections closed due to maxlifetime")
 
 			supplier.RecycleDBConnections(2 * time.Second)
-			// We cannot reliably control exactly how many open connections are there. So we
-			// just do a basic check and confirm that atleast one has been closed.
-			stats = supplier.GetMaster().Db.Stats()
-			assert.Greater(t, int(stats.MaxLifetimeClosed), 0, "unexpected number of connections closed due to maxlifetime")
+			var success bool
+			// We try 3 times to let the connections be closed.
+			// Because sometimes, there can be significant goroutine contention which does not
+			// give enough time for the connection cleaner goroutine to run.
+			for i := 0; i < 3; i++ {
+				// We cannot reliably control exactly how many open connections are there. So we
+				// just do a basic check and confirm that atleast one has been closed.
+				stats = supplier.GetMaster().Db.Stats()
+				if int(stats.MaxLifetimeClosed) == 0 {
+					time.Sleep(2 * time.Second)
+					continue
+				}
+				success = true
+				break
+			}
+			assert.True(t, success, "No connections were closed due to maxlifetime")
 		})
 	}
 }