[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

server/channels/app/user_test.go

@@ -30,6 +30,7 @@ import (
 )
 
 func TestCreateOAuthUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -87,6 +88,7 @@ func TestCreateOAuthUser(t *testing.T) {
 }
 
 func TestUpdateDefaultProfileImage(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 	startTime := model.GetMillis()
@@ -109,6 +111,7 @@ func TestUpdateDefaultProfileImage(t *testing.T) {
 }
 
 func TestAdjustProfileImage(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -134,6 +137,7 @@ func TestAdjustProfileImage(t *testing.T) {
 }
 
 func TestUpdateUserToRestrictedDomain(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -194,6 +198,7 @@ func TestUpdateUserToRestrictedDomain(t *testing.T) {
 }
 
 func TestUpdateUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -237,6 +242,7 @@ func TestUpdateUser(t *testing.T) {
 }
 
 func TestUpdateUserMissingFields(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -270,6 +276,7 @@ func TestUpdateUserMissingFields(t *testing.T) {
 }
 
 func TestCreateUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -340,6 +347,7 @@ func TestCreateUser(t *testing.T) {
 }
 
 func TestUpdateUserActive(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -358,6 +366,7 @@ func TestUpdateUserActive(t *testing.T) {
 }
 
 func TestUpdateActiveBotsSideEffect(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -410,6 +419,7 @@ func TestUpdateActiveBotsSideEffect(t *testing.T) {
 }
 
 func TestUpdateOAuthUserAttrs(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -519,6 +529,7 @@ func TestUpdateOAuthUserAttrs(t *testing.T) {
 }
 
 func TestCreateUserConflict(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -559,6 +570,7 @@ func TestCreateUserConflict(t *testing.T) {
 }
 
 func TestUpdateUserEmail(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -739,6 +751,7 @@ func createGitlabUser(t *testing.T, a *App, c request.CTX, id int64, username st
 }
 
 func TestGetUsersByStatus(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t)
 	defer th.TearDown()
 
@@ -869,6 +882,7 @@ func TestGetUsersByStatus(t *testing.T) {
 }
 
 func TestCreateUserWithInviteId(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -897,6 +911,7 @@ func TestCreateUserWithInviteId(t *testing.T) {
 }
 
 func TestCreateUserWithToken(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1091,6 +1106,7 @@ func TestCreateUserWithToken(t *testing.T) {
 }
 
 func TestPermanentDeleteUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic().DeleteBots()
 	defer th.TearDown()
 
@@ -1116,8 +1132,7 @@ func TestPermanentDeleteUser(t *testing.T) {
 	bots1 := []*model.Bot{}
 	bots2 := []*model.Bot{}
 
-	sqlStore := mainHelper.GetSQLStore()
-	err1 := sqlStore.GetMaster().Select(&bots1, "SELECT * FROM Bots")
+	err1 := th.SQLStore.GetMaster().Select(&bots1, "SELECT * FROM Bots")
 	assert.NoError(t, err1)
 	assert.Equal(t, 1, len(bots1))
 
@@ -1128,7 +1143,7 @@ func TestPermanentDeleteUser(t *testing.T) {
 	err = th.App.PermanentDeleteUser(th.Context, retUser1)
 	assert.Nil(t, err)
 
-	err1 = sqlStore.GetMaster().Select(&bots2, "SELECT * FROM Bots")
+	err1 = th.SQLStore.GetMaster().Select(&bots2, "SELECT * FROM Bots")
 	assert.NoError(t, err1)
 	assert.Equal(t, 0, len(bots2))
 
@@ -1187,6 +1202,7 @@ func TestPermanentDeleteUser(t *testing.T) {
 }
 
 func TestPasswordRecovery(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1242,6 +1258,7 @@ func TestPasswordRecovery(t *testing.T) {
 }
 
 func TestInvalidatePasswordRecoveryTokens(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1280,6 +1297,7 @@ func TestInvalidatePasswordRecoveryTokens(t *testing.T) {
 }
 
 func TestPasswordChangeSessionTermination(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1427,6 +1445,7 @@ func TestPasswordChangeSessionTermination(t *testing.T) {
 }
 
 func TestGetViewUsersRestrictions(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1562,6 +1581,7 @@ func TestGetViewUsersRestrictions(t *testing.T) {
 }
 
 func TestPromoteGuestToUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1698,6 +1718,7 @@ func TestPromoteGuestToUser(t *testing.T) {
 }
 
 func TestDemoteUserToGuest(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1883,6 +1904,7 @@ func TestDemoteUserToGuest(t *testing.T) {
 }
 
 func TestDeactivateGuests(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -1907,6 +1929,7 @@ func TestDeactivateGuests(t *testing.T) {
 }
 
 func TestUpdateUserRolesWithUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	// InitBasic is used to let the first CreateUser call not be
 	// a system_admin
 	th := Setup(t).InitBasic()
@@ -1932,6 +1955,7 @@ func TestUpdateUserRolesWithUser(t *testing.T) {
 }
 
 func TestUpdateLastAdminUserRolesWithUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	// InitBasic is used to let the first CreateUser call not be
 	// a system_admin
 	th := Setup(t).InitBasic()
@@ -1970,6 +1994,7 @@ func TestUpdateLastAdminUserRolesWithUser(t *testing.T) {
 }
 
 func TestDeactivateMfa(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("MFA is disabled", func(t *testing.T) {
 		th := Setup(t).InitBasic()
 		defer th.TearDown()
@@ -1985,6 +2010,7 @@ func TestDeactivateMfa(t *testing.T) {
 }
 
 func TestPatchUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -2023,6 +2049,7 @@ func TestPatchUser(t *testing.T) {
 }
 
 func TestUpdateThreadReadForUser(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("Ensure thread membership exists before updating read", func(t *testing.T) {
 		th := Setup(t).InitBasic()
 		defer th.TearDown()
@@ -2051,6 +2078,7 @@ func TestUpdateThreadReadForUser(t *testing.T) {
 }
 
 func TestCreateUserWithInitialPreferences(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -2107,6 +2135,7 @@ func TestCreateUserWithInitialPreferences(t *testing.T) {
 }
 
 func TestSendSubscriptionHistoryEvent(t *testing.T) {
+	mainHelper.Parallel(t)
 	cloudProduct := &model.Product{
 		ID:                "prod_test1",
 		Name:              "name1",
@@ -2188,6 +2217,7 @@ func TestSendSubscriptionHistoryEvent(t *testing.T) {
 }
 
 func TestGetUsersForReporting(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("should throw error on invalid date range", func(t *testing.T) {
 		th := Setup(t).InitBasic()
 		defer th.TearDown()
@@ -2264,6 +2294,7 @@ func TestGetUsersForReporting(t *testing.T) {
 }
 
 func TestCreateUserOrGuest(t *testing.T) {
+	mainHelper.Parallel(t)
 	t.Run("base case - you can create a user", func(t *testing.T) {
 		th := Setup(t)
 		defer th.TearDown()