[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

server/channels/testlib/helper.go

@@ -31,16 +31,20 @@ type MainHelper struct {
 	SQLStore         *sqlstore.SqlStore
 	ClusterInterface *FakeClusterInterface
 	Logger           *mlog.Logger
+	Options          HelperOptions
 
 	status           int
 	testResourcePath string
 	replicas         []string
+	storePool        *sqlstore.TestPool
 }
 
 type HelperOptions struct {
 	EnableStore     bool
 	EnableResources bool
 	WithReadReplica bool
+	RunParallel     bool
+	Parallelism     int
 }
 
 func NewMainHelper() *MainHelper {
@@ -85,6 +89,8 @@ func NewMainHelperWithOptions(options *HelperOptions) *MainHelper {
 	}
 
 	if options != nil {
+		mainHelper.Options = *options
+
 		if options.EnableStore && !testing.Short() {
 			mainHelper.setupStore(options.WithReadReplica)
 		}
@@ -92,6 +98,20 @@ func NewMainHelperWithOptions(options *HelperOptions) *MainHelper {
 		if options.EnableResources {
 			mainHelper.setupResources()
 		}
+
+		if options.RunParallel && options.EnableStore {
+			driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+			if driverName == "" {
+				driverName = model.DatabaseDriverPostgres
+			}
+			// NOTE: we use a poolSize higher than the parallelism value (coming from -test.parallel flag) as we need a bit of extra buffer to cover
+			// for subtests that might also run in parallel and initialize a new store.
+			storePool, err := sqlstore.NewTestPool(mainHelper.Logger, driverName, options.Parallelism*2)
+			if err != nil {
+				panic(err)
+			}
+			mainHelper.storePool = storePool
+		}
 	}
 
 	return &mainHelper
@@ -127,6 +147,36 @@ func (h *MainHelper) Main(m *testing.M) {
 	h.status = m.Run()
 }
 
+func (h *MainHelper) GetNewStores(tb testing.TB) (store.Store, *sqlstore.SqlStore, *model.SqlSettings, *searchengine.Broker) {
+	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+	if driverName == "" {
+		driverName = model.DatabaseDriverPostgres
+	}
+
+	config := &model.Config{}
+	config.SetDefaults()
+
+	searchEngine := searchengine.NewBroker(config)
+
+	storePoolEntry := h.storePool.Get(tb)
+	if storePoolEntry == nil {
+		panic("no store available in the pool")
+	}
+	settings := storePoolEntry.Settings
+	sqlStore := storePoolEntry.Store
+	sqlStore.DropAllTables()
+
+	store := searchlayer.NewSearchLayer(&TestStore{
+		sqlStore,
+	}, searchEngine, config)
+
+	store.MarkSystemRanUnitTests()
+
+	preloadMigrations(driverName, sqlStore)
+
+	return store, sqlStore, settings, searchEngine
+}
+
 func (h *MainHelper) setupStore(withReadReplica bool) {
 	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
 	if driverName == "" {
@@ -204,11 +254,11 @@ func (h *MainHelper) setupResources() {
 // pg_dump -a -h localhost -U mmuser -d <> --no-comments --inserts -t roles -t systems
 // mysqldump -u root -p <> --no-create-info --extended-insert=FALSE Systems Roles
 // And keep only the permission related rows in the systems table output.
-func (h *MainHelper) PreloadMigrations() {
+func preloadMigrations(driverName string, sqlStore *sqlstore.SqlStore) {
 	var buf []byte
 	var err error
 
-	switch *h.Settings.DriverName {
+	switch driverName {
 	case model.DatabaseDriverPostgres:
 		finalPath := filepath.Join(server.GetPackagePath(), "channels", "testlib", "testdata", "postgres_migration_warmup.sql")
 		buf, err = os.ReadFile(finalPath)
@@ -222,13 +272,17 @@ func (h *MainHelper) PreloadMigrations() {
 			panic(fmt.Errorf("cannot read file: %v", err))
 		}
 	}
-	handle := h.SQLStore.GetMaster()
+	handle := sqlStore.GetMaster()
 	_, err = handle.Exec(string(buf))
 	if err != nil {
 		panic(errors.Wrap(err, "Error preloading migrations. Check if you have &multiStatements=true in your DSN if you are using MySQL. Or perhaps the schema changed? If yes, then update the warmup files accordingly"))
 	}
 }
 
+func (h *MainHelper) PreloadMigrations() {
+	preloadMigrations(*h.Settings.DriverName, h.SQLStore)
+}
+
 func (h *MainHelper) Close() error {
 	if h.SQLStore != nil {
 		h.SQLStore.Close()
@@ -240,6 +294,10 @@ func (h *MainHelper) Close() error {
 		os.RemoveAll(h.testResourcePath)
 	}
 
+	if h.storePool != nil {
+		h.storePool.Close()
+	}
+
 	if r := recover(); r != nil {
 		log.Fatalln(r)
 	}
@@ -321,3 +379,9 @@ func (h *MainHelper) execOnEachReplica(query string, args ...any) error {
 	}
 	return nil
 }
+
+func (h *MainHelper) Parallel(t *testing.T) {
+	if h.Options.RunParallel {
+		t.Parallel()
+	}
+}