[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

server/channels/store/sqlstore/channel_store_test.go

@@ -25,6 +25,10 @@ func TestSearchChannelStore(t *testing.T) {
 }
 
 func TestChannelSearchQuerySQLInjection(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	for _, st := range storeTypes {
 		t.Run(st.Name, func(t *testing.T) {
 			s := &SqlChannelStore{

server/channels/store/sqlstore/context_test.go

@@ -12,6 +12,10 @@ import (
 )
 
 func TestContextMaster(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	ctx := context.Background()
 
 	m := WithMaster(ctx)
@@ -19,6 +23,10 @@ func TestContextMaster(t *testing.T) {
 }
 
 func TestRequestContextWithMaster(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	t.Run("set and get", func(t *testing.T) {
 		var rctx request.CTX = request.TestContext(t)
 

server/channels/store/sqlstore/init_test.go

@@ -7,8 +7,11 @@ import (
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 )
 
-func InitTest(logger mlog.LoggerIFace) {
-	initStores(logger)
+var enableFullyParallelTests bool
+
+func InitTest(logger mlog.LoggerIFace, parallelism int) {
+	enableFullyParallelTests = parallelism > 1
+	initStores(logger, parallelism)
 }
 
 func TearDownTest() {

server/channels/store/sqlstore/main_test.go

@@ -4,8 +4,12 @@
 package sqlstore_test
 
 import (
+	"flag"
+	"os"
+	"strconv"
 	"testing"
 
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
 	"github.com/mattermost/mattermost/server/v8/channels/testlib"
 )
@@ -13,10 +17,21 @@ import (
 var mainHelper *testlib.MainHelper
 
 func TestMain(m *testing.M) {
+	var parallelism int
+	if f := flag.Lookup("test.parallel"); f != nil {
+		parallelism, _ = strconv.Atoi(f.Value.String())
+	}
+	runParallel := os.Getenv("ENABLE_FULLY_PARALLEL_TESTS") == "true" && parallelism > 1
+	if runParallel {
+		mlog.Info("Fully parallel tests enabled", mlog.Int("parallelism", parallelism))
+	} else {
+		parallelism = 1
+	}
+
 	mainHelper = testlib.NewMainHelperWithOptions(nil)
 	defer mainHelper.Close()
 
-	sqlstore.InitTest(mainHelper.Logger)
+	sqlstore.InitTest(mainHelper.Logger, parallelism)
 
 	mainHelper.Main(m)
 	sqlstore.TearDownTest()

server/channels/store/sqlstore/store_test.go

@@ -41,7 +41,32 @@ type storeType struct {
 	Store       store.Store
 }
 
-var storeTypes []*storeType
+var (
+	storeTypes []*storeType
+	storePools []*TestPool
+)
+
+func getStoresFromPools(t *testing.T) []*storeType {
+	stores := make([]*storeType, len(storePools))
+	for i, pool := range storePools {
+		entry := pool.Get(t)
+		if entry == nil {
+			panic("failed to get store from pool")
+		}
+
+		entry.Store.DropAllTables()
+		entry.Store.MarkSystemRanUnitTests()
+
+		stores[i] = &storeType{
+			Name:        *entry.Settings.DriverName,
+			SqlSettings: entry.Settings,
+			SqlStore:    entry.Store,
+			Store:       entry.Store,
+		}
+	}
+
+	return stores
+}
 
 func newStoreType(name, driver string) *storeType {
 	return &storeType{
@@ -57,7 +82,14 @@ func StoreTest(t *testing.T, f func(*testing.T, request.CTX, store.Store)) {
 			panic(err)
 		}
 	}()
-	for _, st := range storeTypes {
+
+	stores := storeTypes
+	if enableFullyParallelTests {
+		t.Parallel()
+		stores = getStoresFromPools(t)
+	}
+
+	for _, st := range stores {
 		st := st
 		rctx := request.TestContext(t)
 
@@ -65,6 +97,7 @@ func StoreTest(t *testing.T, f func(*testing.T, request.CTX, store.Store)) {
 			if testing.Short() {
 				t.SkipNow()
 			}
+
 			f(t, rctx, st.Store)
 		})
 	}
@@ -78,13 +111,21 @@ func StoreTestWithSearchTestEngine(t *testing.T, f func(*testing.T, store.Store,
 		}
 	}()
 
-	for _, st := range storeTypes {
+	stores := storeTypes
+	if enableFullyParallelTests {
+		t.Parallel()
+		stores = getStoresFromPools(t)
+	}
+
+	for _, st := range stores {
 		st := st
 		searchTestEngine := &searchtest.SearchTestEngine{
 			Driver: *st.SqlSettings.DriverName,
 		}
 
-		t.Run(st.Name, func(t *testing.T) { f(t, st.Store, searchTestEngine) })
+		t.Run(st.Name, func(t *testing.T) {
+			f(t, st.Store, searchTestEngine)
+		})
 	}
 }
 
@@ -95,7 +136,14 @@ func StoreTestWithSqlStore(t *testing.T, f func(*testing.T, request.CTX, store.S
 			panic(err)
 		}
 	}()
-	for _, st := range storeTypes {
+
+	stores := storeTypes
+	if enableFullyParallelTests {
+		t.Parallel()
+		stores = getStoresFromPools(t)
+	}
+
+	for _, st := range stores {
 		st := st
 		rctx := request.TestContext(t)
 
@@ -108,7 +156,7 @@ func StoreTestWithSqlStore(t *testing.T, f func(*testing.T, request.CTX, store.S
 	}
 }
 
-func initStores(logger mlog.LoggerIFace) {
+func initStores(logger mlog.LoggerIFace, parallelism int) {
 	if testing.Short() {
 		return
 	}
@@ -120,12 +168,31 @@ func initStores(logger mlog.LoggerIFace) {
 			storeTypes = append(storeTypes, newStoreType("MySQL", model.DatabaseDriverMysql))
 		case "postgres":
 			storeTypes = append(storeTypes, newStoreType("PostgreSQL", model.DatabaseDriverPostgres))
+			if enableFullyParallelTests {
+				pgStorePool, err := NewTestPool(logger, model.DatabaseDriverPostgres, parallelism)
+				if err != nil {
+					panic(err)
+				}
+				storePools = append(storePools, pgStorePool)
+			}
 		}
 	} else {
 		storeTypes = append(storeTypes,
 			newStoreType("MySQL", model.DatabaseDriverMysql),
 			newStoreType("PostgreSQL", model.DatabaseDriverPostgres),
 		)
+
+		if enableFullyParallelTests {
+			pgStorePool, err := NewTestPool(logger, model.DatabaseDriverPostgres, parallelism)
+			if err != nil {
+				panic(err)
+			}
+			msStorePool, err := NewTestPool(logger, model.DatabaseDriverMysql, parallelism)
+			if err != nil {
+				panic(err)
+			}
+			storePools = append(storePools, pgStorePool, msStorePool)
+		}
 	}
 
 	defer func() {
@@ -177,7 +244,19 @@ func tearDownStores() {
 				wg.Done()
 			}()
 		}
+
+		var wgPool sync.WaitGroup
+		wgPool.Add(len(storePools))
+		for _, pool := range storePools {
+			pool := pool
+			go func() {
+				defer wgPool.Done()
+				pool.Close()
+			}()
+		}
+
 		wg.Wait()
+		wgPool.Wait()
 	})
 }
 
@@ -185,6 +264,10 @@ func tearDownStores() {
 // before the fix in MM-28397.
 // Keeping it here to help avoiding future regressions.
 func TestStoreLicenseRace(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	logger := mlog.CreateTestLogger(t)
 
 	settings, err := makeSqlSettings(model.DatabaseDriverPostgres)
@@ -422,6 +505,10 @@ func TestGetReplica(t *testing.T) {
 }
 
 func TestGetDbVersion(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	logger := mlog.CreateTestLogger(t)
 
 	testDrivers := []string{
@@ -449,6 +536,10 @@ func TestGetDbVersion(t *testing.T) {
 }
 
 func TestEnsureMinimumDBVersion(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	tests := []struct {
 		driver string
 		ver    string
@@ -530,6 +621,10 @@ func TestEnsureMinimumDBVersion(t *testing.T) {
 }
 
 func TestIsBinaryParamEnabled(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	tests := []struct {
 		store    SqlStore
 		expected bool
@@ -677,6 +772,10 @@ func TestGetAllConns(t *testing.T) {
 }
 
 func TestIsDuplicate(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testErrors := map[error]bool{
 		&pq.Error{Code: "42P06"}:                          false,
 		&pq.Error{Code: PGDupTableErrorCode}:              true,
@@ -696,6 +795,10 @@ func TestIsDuplicate(t *testing.T) {
 }
 
 func TestVersionString(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	versions := []struct {
 		input  int
 		driver string
@@ -735,6 +838,10 @@ func TestVersionString(t *testing.T) {
 }
 
 func TestReplicaLagQuery(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testDrivers := []string{
 		model.DatabaseDriverPostgres,
 		model.DatabaseDriverMysql,
@@ -796,6 +903,10 @@ func TestReplicaLagQuery(t *testing.T) {
 }
 
 func TestInvalidReplicaLagDataSource(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	logger := mlog.CreateConsoleTestLogger(t)
 
 	testDrivers := []string{
@@ -910,6 +1021,10 @@ func TestMySQLReadTimeout(t *testing.T) {
 }
 
 func TestGetDBSchemaVersion(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testDrivers := []string{
 		model.DatabaseDriverPostgres,
 		model.DatabaseDriverMysql,
@@ -950,6 +1065,10 @@ func TestGetDBSchemaVersion(t *testing.T) {
 }
 
 func TestGetLocalSchemaVersion(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testDrivers := []string{
 		model.DatabaseDriverPostgres,
 		model.DatabaseDriverMysql,
@@ -978,6 +1097,10 @@ func TestGetLocalSchemaVersion(t *testing.T) {
 }
 
 func TestGetAppliedMigrations(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testDrivers := []string{
 		model.DatabaseDriverPostgres,
 		model.DatabaseDriverMysql,
@@ -1026,6 +1149,10 @@ func TestGetAppliedMigrations(t *testing.T) {
 }
 
 func TestSkipMigrationsOption(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	testDrivers := []string{
 		model.DatabaseDriverPostgres,
 		model.DatabaseDriverMysql,

server/channels/store/sqlstore/testpool.go

@@ -0,0 +1,114 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sqlstore
+
+import (
+	"sync"
+	"testing"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+)
+
+// TestPool is used to facilitate the efficient (and safe) use of test stores in parallel tests (e.g. in api4, app, sqlstore).
+type TestPool struct {
+	entries map[string]*TestPoolEntry
+	mut     sync.Mutex
+	logger  mlog.LoggerIFace
+}
+
+type TestPoolEntry struct {
+	Store    *SqlStore
+	Settings *model.SqlSettings
+}
+
+func NewTestPool(logger mlog.LoggerIFace, driverName string, poolSize int) (*TestPool, error) {
+	logger.Info("Creating test store pool", mlog.Int("poolSize", poolSize))
+
+	entries := make(map[string]*TestPoolEntry, poolSize)
+
+	var mut sync.Mutex
+	var eg errgroup.Group
+	for i := 0; i < poolSize; i++ {
+		eg.Go(func() error {
+			settings := storetest.MakeSqlSettings(driverName, false)
+			sqlStore, err := New(*settings, logger, nil)
+			if err != nil {
+				return err
+			}
+
+			mut.Lock()
+			logger.Info("Initializing test store in pool", mlog.String("datasource", *settings.DataSource))
+			entries[*settings.DataSource] = &TestPoolEntry{
+				Store:    sqlStore,
+				Settings: settings,
+			}
+			mut.Unlock()
+
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return &TestPool{
+		entries: entries,
+		logger:  logger,
+	}, nil
+}
+
+func (p *TestPool) Get(t testing.TB) *TestPoolEntry {
+	p.mut.Lock()
+	defer p.mut.Unlock()
+
+	p.logger.Info("Getting from test store pool", mlog.Int("poolSize", len(p.entries)))
+
+	var poolEntry *TestPoolEntry
+	for _, entry := range p.entries {
+		poolEntry = entry
+		delete(p.entries, *entry.Settings.DataSource)
+		break
+	}
+
+	// No more stores available in the pool
+	if poolEntry == nil {
+		return nil
+	}
+
+	p.logger.Info("Got store from pool", mlog.String("datasource", *poolEntry.Settings.DataSource), mlog.Int("poolSize", len(p.entries)))
+
+	dataSource := *poolEntry.Settings.DataSource
+
+	// Return store to pool on test cleanup
+	t.Cleanup(func() {
+		p.mut.Lock()
+		defer p.mut.Unlock()
+		p.logger.Info("Returning to test store pool", mlog.String("datasource", dataSource), mlog.Int("poolSize", len(p.entries)))
+		p.entries[dataSource] = poolEntry
+	})
+
+	return poolEntry
+}
+
+func (p *TestPool) Close() {
+	p.mut.Lock()
+	defer p.mut.Unlock()
+
+	var wg sync.WaitGroup
+	wg.Add(len(p.entries))
+	for _, entry := range p.entries {
+		entry := entry
+		go func() {
+			defer wg.Done()
+			entry.Store.Close()
+			storetest.CleanupSqlSettings(entry.Settings)
+		}()
+	}
+	wg.Wait()
+}

server/channels/store/sqlstore/utils_test.go

@@ -14,6 +14,10 @@ import (
 )
 
 func TestMapStringsToQueryParams(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	t.Run("one item", func(t *testing.T) {
 		input := []string{"apple"}
 
@@ -37,8 +41,10 @@ func TestMapStringsToQueryParams(t *testing.T) {
 	})
 }
 
-var keys string
-var params map[string]any
+var (
+	keys   string
+	params map[string]any
+)
 
 func BenchmarkMapStringsToQueryParams(b *testing.B) {
 	b.Run("one item", func(b *testing.B) {
@@ -56,6 +62,10 @@ func BenchmarkMapStringsToQueryParams(b *testing.B) {
 }
 
 func TestSanitizeSearchTerm(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	term := "test"
 	result := sanitizeSearchTerm(term, "\\")
 	require.Equal(t, result, term)
@@ -82,6 +92,10 @@ func TestSanitizeSearchTerm(t *testing.T) {
 }
 
 func TestRemoveNonAlphaNumericUnquotedTerms(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	const (
 		sep           = " "
 		chineseHello  = "你好"
@@ -109,6 +123,10 @@ func TestRemoveNonAlphaNumericUnquotedTerms(t *testing.T) {
 }
 
 func TestMySQLJSONArgs(t *testing.T) {
+	if enableFullyParallelTests {
+		t.Parallel()
+	}
+
 	tests := []struct {
 		props     map[string]string
 		args      []any