[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

server/channels/app/platform/busy_test.go

@@ -16,6 +16,8 @@ import (
 )
 
 func TestBusySet(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	cluster := &ClusterMock{Busy: &Busy{}, t: t}
 	busy := NewBusy(cluster)
 
@@ -30,9 +32,9 @@ func TestBusySet(t *testing.T) {
 	require.True(t, compareBusyState(t, busy, cluster.Busy))
 
 	// should automatically expire after 5s.
-	require.Eventually(t, isNotBusy, time.Second*15, time.Millisecond*100)
+	require.Eventually(t, isNotBusy, time.Second*10, time.Millisecond*100)
 	// allow a moment for cluster to sync.
-	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*15, time.Millisecond*20)
+	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*5, time.Millisecond*20)
 
 	// test set after auto expiry.
 	busy.Set(time.Second * 30)
@@ -54,6 +56,8 @@ func TestBusySet(t *testing.T) {
 }
 
 func TestBusyExpires(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	cluster := &ClusterMock{Busy: &Busy{}, t: t}
 	busy := NewBusy(cluster)
 
@@ -86,10 +90,12 @@ func TestBusyExpires(t *testing.T) {
 	// should be time.Time zero value
 	require.Equal(t, time.Time{}.Unix(), expire.Unix())
 	// allow a moment for cluster to sync
-	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*15, time.Millisecond*20)
+	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*5, time.Millisecond*20)
 }
 
 func TestBusyRace(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	cluster := &ClusterMock{Busy: &Busy{}, t: t}
 	busy := NewBusy(cluster)
 
@@ -147,12 +153,15 @@ func (c *ClusterMock) NotifyMsg(buf []byte)                           {}
 func (c *ClusterMock) GetClusterStats(rctx request.CTX) ([]*model.ClusterStats, *model.AppError) {
 	return nil, nil
 }
+
 func (c *ClusterMock) GetLogs(rctx request.CTX, page, perPage int) ([]string, *model.AppError) {
 	return nil, nil
 }
+
 func (c *ClusterMock) QueryLogs(rctx request.CTX, page, perPage int) (map[string][]string, *model.AppError) {
 	return nil, nil
 }
+
 func (c *ClusterMock) GenerateSupportPacket(rctx request.CTX, options *model.SupportPacketOptions) (map[string][]model.FileData, error) {
 	return nil, nil
 }
@@ -164,6 +173,7 @@ func (c *ClusterMock) HealthScore() int { return 0 }
 func (c *ClusterMock) WebConnCountForUser(userID string) (int, *model.AppError) {
 	return 0, nil
 }
+
 func (c *ClusterMock) GetWSQueues(userID, connectionID string, seqNum int64) (map[string]*model.WSQueues, error) {
 	return nil, nil
 }

server/channels/app/platform/cluster_discovery_test.go

@@ -11,6 +11,7 @@ import (
 )
 
 func TestClusterDiscoveryService(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/config_test.go

@@ -18,6 +18,7 @@ import (
 )
 
 func TestConfigListener(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -53,6 +54,7 @@ func TestConfigListener(t *testing.T) {
 }
 
 func TestConfigSave(t *testing.T) {
+	mainHelper.Parallel(t)
 	cm := &mocks.ClusterInterface{}
 	cm.On("SendClusterMessage", mock.AnythingOfType("*model.ClusterMessage")).Return(nil)
 	th := SetupWithCluster(t, cm)
@@ -106,6 +108,7 @@ func TestConfigSave(t *testing.T) {
 }
 
 func TestIsFirstUserAccount(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := SetupWithStoreMock(t)
 	defer th.TearDown()
 	storeMock := th.Service.Store.(*smocks.Store)
@@ -148,6 +151,7 @@ func TestIsFirstUserAccount(t *testing.T) {
 }
 
 func TestIsFirstUserAccountThunderingHerd(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := SetupWithStoreMock(t)
 	defer th.TearDown()
 	storeMock := th.Service.Store.(*smocks.Store)

server/channels/app/platform/helper_test.go

@@ -25,6 +25,7 @@ type TestHelper struct {
 	Context request.CTX
 	Service *PlatformService
 	Suite   SuiteIFace
+	Store   store.Store
 
 	BasicTeam    *model.Team
 	BasicUser    *model.User
@@ -35,15 +36,16 @@ type TestHelper struct {
 	SystemAdminUser *model.User
 }
 
-var initBasicOnce sync.Once
-var userCache struct {
-	SystemAdminUser *model.User
-	BasicUser       *model.User
-	BasicUser2      *model.User
-}
+var (
+	initBasicOnce sync.Once
+	userCache     struct {
+		SystemAdminUser *model.User
+		BasicUser       *model.User
+		BasicUser2      *model.User
+	}
+)
 
-type mockSuite struct {
-}
+type mockSuite struct{}
 
 func (ms *mockSuite) SetStatusLastActivityAt(userID string, activityAt int64) {}
 func (ms *mockSuite) SetStatusOffline(userID string, manual bool, force bool) {}
@@ -58,20 +60,38 @@ func (ms *mockSuite) RolesGrantPermission(roleNames []string, permissionId strin
 func (ms *mockSuite) UserCanSeeOtherUser(c request.CTX, userID string, otherUserId string) (bool, *model.AppError) {
 	return true, nil
 }
+
 func (ms *mockSuite) HasPermissionToReadChannel(c request.CTX, userID string, channel *model.Channel) bool {
 	return true
 }
 
+func setupDBStore(tb testing.TB) (store.Store, *model.SqlSettings) {
+	var dbStore store.Store
+	var dbSettings *model.SqlSettings
+	if mainHelper.Options.RunParallel {
+		dbStore, _, dbSettings, _ = mainHelper.GetNewStores(tb)
+		tb.Cleanup(func() {
+			dbStore.Close()
+		})
+	} else {
+		dbStore = mainHelper.GetStore()
+		dbStore.DropAllTables()
+		dbStore.MarkSystemRanUnitTests()
+		dbSettings = mainHelper.Settings
+		mainHelper.PreloadMigrations()
+	}
+
+	return dbStore, dbSettings
+}
+
 func Setup(tb testing.TB, options ...Option) *TestHelper {
 	if testing.Short() {
 		tb.SkipNow()
 	}
-	dbStore := mainHelper.GetStore()
-	dbStore.DropAllTables()
-	dbStore.MarkSystemRanUnitTests()
-	mainHelper.PreloadMigrations()
 
-	return setupTestHelper(dbStore, false, true, tb, options...)
+	dbStore, dbSettings := setupDBStore(tb)
+
+	return setupTestHelper(dbStore, dbSettings, false, true, tb, options...)
 }
 
 func (th *TestHelper) InitBasic() *TestHelper {
@@ -92,7 +112,7 @@ func (th *TestHelper) InitBasic() *TestHelper {
 	th.BasicUser2 = userCache.BasicUser2.DeepCopy()
 
 	users := []*model.User{th.SystemAdminUser, th.BasicUser, th.BasicUser2}
-	mainHelper.GetSQLStore().User().InsertUsers(users)
+	th.Store.User().InsertUsers(users)
 
 	th.BasicTeam = th.CreateTeam()
 
@@ -106,7 +126,7 @@ func (th *TestHelper) InitBasic() *TestHelper {
 func SetupWithStoreMock(tb testing.TB, options ...Option) *TestHelper {
 	mockStore := testlib.GetMockStoreForSetupFunctions()
 	options = append(options, StoreOverride(mockStore))
-	th := setupTestHelper(mockStore, false, false, tb, options...)
+	th := setupTestHelper(mockStore, &model.SqlSettings{}, false, false, tb, options...)
 	return th
 }
 
@@ -114,18 +134,16 @@ func SetupWithCluster(tb testing.TB, cluster einterfaces.ClusterInterface) *Test
 	if testing.Short() {
 		tb.SkipNow()
 	}
-	dbStore := mainHelper.GetStore()
-	dbStore.DropAllTables()
-	dbStore.MarkSystemRanUnitTests()
-	mainHelper.PreloadMigrations()
 
-	th := setupTestHelper(dbStore, true, true, tb)
+	dbStore, dbSettings := setupDBStore(tb)
+
+	th := setupTestHelper(dbStore, dbSettings, true, true, tb)
 	th.Service.clusterIFace = cluster
 
 	return th
 }
 
-func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer bool, tb testing.TB, options ...Option) *TestHelper {
+func setupTestHelper(dbStore store.Store, dbSettings *model.SqlSettings, enterprise bool, includeCacheLayer bool, tb testing.TB, options ...Option) *TestHelper {
 	tempWorkspace, err := os.MkdirTemp("", "apptest")
 	if err != nil {
 		panic(err)
@@ -134,7 +152,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 	configStore := config.NewTestMemoryStore()
 
 	memoryConfig := configStore.Get()
-	memoryConfig.SqlSettings = *mainHelper.GetSQLSettings()
+	memoryConfig.SqlSettings = *dbSettings
 	*memoryConfig.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
 	*memoryConfig.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
 	*memoryConfig.PluginSettings.AutomaticPrepackagedPlugins = false
@@ -161,6 +179,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 		Context: request.TestContext(tb),
 		Service: ps,
 		Suite:   &mockSuite{},
+		Store:   dbStore,
 	}
 
 	if _, ok := dbStore.(*mocks.Store); ok {

server/channels/app/platform/license_test.go

@@ -18,6 +18,7 @@ import (
 )
 
 func TestLoadLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -26,6 +27,7 @@ func TestLoadLicense(t *testing.T) {
 }
 
 func TestSaveLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -75,6 +77,7 @@ func TestSaveEnterpriseAdvancedLicense(t *testing.T) {
 }
 
 func TestRemoveLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -83,6 +86,7 @@ func TestRemoveLicense(t *testing.T) {
 }
 
 func TestSetLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -104,6 +108,7 @@ func TestSetLicense(t *testing.T) {
 }
 
 func TestGetSanitizedClientLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/log_test.go

@@ -19,6 +19,8 @@ import (
 )
 
 func TestGetMattermostLog(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -72,6 +74,8 @@ func TestGetMattermostLog(t *testing.T) {
 }
 
 func TestGetNotificationLogFile(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -117,6 +121,8 @@ func TestGetNotificationLogFile(t *testing.T) {
 }
 
 func TestGetAdvancedLogs(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/main_test.go

@@ -5,13 +5,18 @@ package platform
 
 import (
 	"flag"
+	"os"
+	"strconv"
 	"testing"
 
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/v8/channels/testlib"
 )
 
-var mainHelper *testlib.MainHelper
-var replicaFlag bool
+var (
+	mainHelper  *testlib.MainHelper
+	replicaFlag bool
+)
 
 func TestMain(m *testing.M) {
 	if f := flag.Lookup("mysql-replica"); f == nil {
@@ -19,10 +24,21 @@ func TestMain(m *testing.M) {
 		flag.Parse()
 	}
 
-	var options = testlib.HelperOptions{
+	var parallelism int
+	if f := flag.Lookup("test.parallel"); f != nil {
+		parallelism, _ = strconv.Atoi(f.Value.String())
+	}
+	runParallel := os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" && parallelism > 1
+	if runParallel {
+		mlog.Info("Fully parallel tests enabled", mlog.Int("parallelism", parallelism))
+	}
+
+	options := testlib.HelperOptions{
 		EnableStore:     true,
 		EnableResources: true,
 		WithReadReplica: replicaFlag,
+		RunParallel:     runParallel,
+		Parallelism:     parallelism,
 	}
 
 	mainHelper = testlib.NewMainHelperWithOptions(&options)

server/channels/app/platform/service_test.go

@@ -59,6 +59,7 @@ func TestSetLicenseOnStart(t *testing.T) {
 }
 
 func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {
+	mainHelper.Parallel(t)
 	cfg := model.Config{}
 	cfg.SetDefaults()
 	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
@@ -131,7 +132,9 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {
 }
 
 func TestMetrics(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("ensure the metrics server is not started by default", func(t *testing.T) {
+		mainHelper.Parallel(t)
 		th := Setup(t)
 		defer th.TearDown()
 
@@ -139,6 +142,7 @@ func TestMetrics(t *testing.T) {
 	})
 
 	t.Run("ensure the metrics server is started", func(t *testing.T) {
+		mainHelper.Parallel(t)
 		th := Setup(t, StartMetrics())
 		defer th.TearDown()
 
@@ -166,6 +170,7 @@ func TestMetrics(t *testing.T) {
 	})
 
 	t.Run("ensure the metrics server is started with advanced metrics", func(t *testing.T) {
+		mainHelper.Parallel(t)
 		th := Setup(t, StartMetrics())
 		defer th.TearDown()
 
@@ -180,6 +185,7 @@ func TestMetrics(t *testing.T) {
 	})
 
 	t.Run("ensure advanced metrics have database metrics", func(t *testing.T) {
+		mainHelper.Parallel(t)
 		mockMetricsImpl := &mocks.MetricsInterface{}
 		mockMetricsImpl.On("Register").Return()
 		mockMetricsImpl.On("ObserveStoreMethodDuration", mock.Anything, mock.Anything, mock.Anything).Return()
@@ -198,6 +204,7 @@ func TestMetrics(t *testing.T) {
 }
 
 func TestShutdown(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("should shutdown gracefully", func(t *testing.T) {
 		th := Setup(t)
 		rand.Seed(time.Now().UnixNano())
@@ -219,6 +226,7 @@ func TestShutdown(t *testing.T) {
 }
 
 func TestSetTelemetryId(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("ensure client config is regenerated after setting the telemetry id", func(t *testing.T) {
 		th := Setup(t)
 		defer th.TearDown()
@@ -235,6 +243,7 @@ func TestSetTelemetryId(t *testing.T) {
 }
 
 func TestDatabaseTypeAndMattermostVersion(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/session_test.go

@@ -19,6 +19,7 @@ const (
 )
 
 func TestCache(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -71,6 +72,7 @@ func TestCache(t *testing.T) {
 }
 
 func TestSetSessionExpireInHours(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -117,6 +119,7 @@ func TestSetSessionExpireInHours(t *testing.T) {
 }
 
 func TestOAuthRevokeAccessToken(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -149,6 +152,7 @@ func TestOAuthRevokeAccessToken(t *testing.T) {
 }
 
 func TestUpdateSessionsIsGuest(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/support_packet_test.go

@@ -25,6 +25,8 @@ import (
 )
 
 func TestGenerateSupportPacket(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	th := Setup(t)
 	defer th.TearDown()
 

server/channels/app/platform/web_broadcast_hook_test.go

@@ -25,6 +25,7 @@ func (h *testBroadcastHook) Process(msg *HookedWebSocketEvent, webConn *WebConn,
 }
 
 func TestRunBroadcastHooks(t *testing.T) {
+	mainHelper.Parallel(t)
 	hub := &Hub{
 		broadcastHooks: map[string]BroadcastHook{
 			broadcastTest: &testBroadcastHook{},

server/channels/app/platform/web_hub_test.go

@@ -60,6 +60,7 @@ func registerDummyWebConn(t *testing.T, th *TestHelper, addr net.Addr, session *
 }
 
 func TestHubStopWithMultipleConnections(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -84,6 +85,7 @@ func TestHubStopWithMultipleConnections(t *testing.T) {
 // TestHubStopRaceCondition verifies that attempts to use the hub after it has shutdown does not
 // block the caller indefinitely.
 func TestHubStopRaceCondition(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.Service.Store.Close()
 	// We do not call TearDown because th.TearDown shuts down the hub again. And hub close is not idempotent.
@@ -131,6 +133,7 @@ func TestHubStopRaceCondition(t *testing.T) {
 }
 
 func TestHubSessionRevokeRace(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -184,6 +187,7 @@ func TestHubSessionRevokeRace(t *testing.T) {
 }
 
 func TestHubConnIndex(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -435,6 +439,7 @@ func TestHubConnIndex(t *testing.T) {
 }
 
 func TestHubConnIndexIncorrectRemoval(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -485,6 +490,7 @@ func TestHubConnIndexIncorrectRemoval(t *testing.T) {
 }
 
 func TestHubConnIndexInactive(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -549,6 +555,7 @@ func TestHubConnIndexInactive(t *testing.T) {
 }
 
 func TestReliableWebSocketSend(t *testing.T) {
+	mainHelper.Parallel(t)
 	testCluster := &testlib.FakeClusterInterface{}
 
 	th := SetupWithCluster(t, testCluster)
@@ -584,6 +591,7 @@ func TestReliableWebSocketSend(t *testing.T) {
 }
 
 func TestHubIsRegistered(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -620,6 +628,7 @@ func TestHubIsRegistered(t *testing.T) {
 }
 
 func TestHubWebConnCount(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -841,6 +850,7 @@ func BenchmarkHubConnIndex(b *testing.B) {
 }
 
 func TestHubConnIndexRemoveMemLeak(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -893,6 +903,7 @@ func BenchmarkGetHubForUserId(b *testing.B) {
 }
 
 func TestClusterBroadcast(t *testing.T) {
+	mainHelper.Parallel(t)
 	testCluster := &testlib.FakeClusterInterface{}
 
 	th := SetupWithCluster(t, testCluster)
@@ -921,6 +932,7 @@ func TestClusterBroadcast(t *testing.T) {
 }
 
 func TestClusterBroadcastHooks(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("should send broadcast hook information across cluster", func(t *testing.T) {
 		testCluster := &testlib.FakeClusterInterface{}