[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

server/channels/app/email/email_batching_test.go

@@ -16,6 +16,8 @@ import (
 )
 
 func TestHandleNewNotifications(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	th := SetupWithStoreMock(t)
 	defer th.TearDown()
 
@@ -75,6 +77,7 @@ func TestHandleNewNotifications(t *testing.T) {
 }
 
 func TestCheckPendingNotifications(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -194,6 +197,7 @@ func TestCheckPendingNotifications(t *testing.T) {
  * Ensures that email batch interval defaults to 15 minutes for users that haven't explicitly set this preference
  */
 func TestCheckPendingNotificationsDefaultInterval(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 
@@ -237,6 +241,7 @@ func TestCheckPendingNotificationsDefaultInterval(t *testing.T) {
  * Ensures that email batch interval defaults to 15 minutes if user preference is invalid
  */
 func TestCheckPendingNotificationsCantParseInterval(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 

server/channels/app/email/email_test.go

@@ -4,7 +4,6 @@
 package email
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -14,6 +13,8 @@ import (
 )
 
 func TestCondenseSiteURL(t *testing.T) {
+	mainHelper.Parallel(t)
+
 	require.Equal(t, "", condenseSiteURL(""))
 	require.Equal(t, "mattermost.com", condenseSiteURL("mattermost.com"))
 	require.Equal(t, "mattermost.com", condenseSiteURL("mattermost.com/"))
@@ -35,6 +36,7 @@ func TestCondenseSiteURL(t *testing.T) {
 }
 
 func TestSendInviteEmails(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 	th.ConfigureInbucketMail()
@@ -84,13 +86,14 @@ func TestSendInviteEmails(t *testing.T) {
 
 	t.Run("SendInviteEmails can return error when SMTP connection fails", func(t *testing.T) {
 		originalPort := *th.service.config().EmailSettings.SMTPPort
+		originalTimeout := *th.service.config().EmailSettings.SMTPServerTimeout
 		th.UpdateConfig(func(cfg *model.Config) {
-			os.Setenv("MM_EMAILSETTINGS_SMTPPORT", "5432")
 			*cfg.EmailSettings.SMTPPort = "5432"
+			*cfg.EmailSettings.SMTPServerTimeout = 4
 		})
 		defer th.UpdateConfig(func(cfg *model.Config) {
-			os.Setenv("MM_EMAILSETTINGS_SMTPPORT", originalPort)
 			*cfg.EmailSettings.SMTPPort = originalPort
+			*cfg.EmailSettings.SMTPServerTimeout = originalTimeout
 		})
 
 		err := th.service.SendInviteEmails(th.BasicTeam, "test-user", th.BasicUser.Id, []string{emailTo}, "http://testserver", nil, true, false, false)
@@ -123,14 +126,15 @@ func TestSendInviteEmails(t *testing.T) {
 	})
 
 	t.Run("SendGuestInviteEmail can return error when SMTP connection fails", func(t *testing.T) {
+		originalTimeout := *th.service.config().EmailSettings.SMTPServerTimeout
 		originalPort := *th.service.config().EmailSettings.SMTPPort
 		th.UpdateConfig(func(cfg *model.Config) {
-			os.Setenv("MM_EMAILSETTINGS_SMTPPORT", "5432")
 			*cfg.EmailSettings.SMTPPort = "5432"
+			*cfg.EmailSettings.SMTPServerTimeout = 4
 		})
 		defer th.UpdateConfig(func(cfg *model.Config) {
-			os.Setenv("MM_EMAILSETTINGS_SMTPPORT", originalPort)
 			*cfg.EmailSettings.SMTPPort = originalPort
+			*cfg.EmailSettings.SMTPServerTimeout = originalTimeout
 		})
 
 		err := th.service.SendGuestInviteEmails(
@@ -255,6 +259,7 @@ func TestSendInviteEmails(t *testing.T) {
 }
 
 func TestSendCloudWelcomeEmail(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
 	th.ConfigureInbucketMail()
@@ -294,6 +299,7 @@ func TestSendCloudWelcomeEmail(t *testing.T) {
 }
 
 func TestMailServiceConfig(t *testing.T) {
+	mainHelper.Parallel(t)
 	configuredReplyTo := "feedbackexample@test.com"
 	customReplyTo := "customreplyto@test.com"
 

server/channels/app/email/helper_test.go

@@ -41,10 +41,19 @@ func Setup(tb testing.TB) *TestHelper {
 	if testing.Short() {
 		tb.SkipNow()
 	}
-	dbStore := mainHelper.GetStore()
-	dbStore.DropAllTables()
-	dbStore.MarkSystemRanUnitTests()
-	mainHelper.PreloadMigrations()
+
+	var dbStore store.Store
+	if mainHelper.Options.RunParallel {
+		dbStore, _, _, _ = mainHelper.GetNewStores(tb)
+		tb.Cleanup(func() {
+			dbStore.Close()
+		})
+	} else {
+		dbStore = mainHelper.GetStore()
+		dbStore.DropAllTables()
+		dbStore.MarkSystemRanUnitTests()
+		mainHelper.PreloadMigrations()
+	}
 
 	return setupTestHelper(dbStore, tb)
 }

server/channels/app/email/main_test.go

@@ -5,13 +5,18 @@ package email
 
 import (
 	"flag"
+	"os"
+	"strconv"
 	"testing"
 
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/v8/channels/testlib"
 )
 
-var mainHelper *testlib.MainHelper
-var replicaFlag bool
+var (
+	mainHelper  *testlib.MainHelper
+	replicaFlag bool
+)
 
 func TestMain(m *testing.M) {
 	if f := flag.Lookup("mysql-replica"); f == nil {
@@ -19,10 +24,21 @@ func TestMain(m *testing.M) {
 		flag.Parse()
 	}
 
-	var options = testlib.HelperOptions{
+	var parallelism int
+	if f := flag.Lookup("test.parallel"); f != nil {
+		parallelism, _ = strconv.Atoi(f.Value.String())
+	}
+	runParallel := os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" && parallelism > 1
+	if runParallel {
+		mlog.Info("Fully parallel tests enabled", mlog.Int("parallelism", parallelism))
+	}
+
+	options := testlib.HelperOptions{
 		EnableStore:     true,
 		EnableResources: true,
 		WithReadReplica: replicaFlag,
+		RunParallel:     runParallel,
+		Parallelism:     parallelism,
 	}
 
 	mainHelper = testlib.NewMainHelperWithOptions(&options)

server/channels/app/email/notification_email_test.go

@@ -12,6 +12,7 @@ import (
 )
 
 func TestProcessMessageAttachments(t *testing.T) {
+	mainHelper.Parallel(t)
 	th := Setup(t).InitBasic()
 	defer th.TearDown()