Enable products for tests (#22757)

* Enable products for channels tests

* increase unit test timeout; check IsConfigReadOnly

* make app-layers

* Avoid loading boards tempaltes between tests to improve speed

* Fix delete query to be compatible with both databases

* Avoid preserving the templates for boards store tests

* Run all tests in one command

* Revert "Run all tests in one command"

This reverts commit 0330f7cd8f96b47776770e8f80ba6ba18d98b8d1.

* concurrent pkg group tests in CI

* Revert "Revert "Run all tests in one command""

This reverts commit 73892fec772575ff054a99ba9e00a7b57ca74164.

* Revert "concurrent pkg group tests in CI"

This reverts commit 550fb6cdd4a7f14d9632f2626bc66f389173b5d9.

* try testing 3 subsets of packages concurrently to improve time taken

* Revert "try testing 3 subsets of packages concurrently to improve time taken"

This reverts commit 97475f3c4eafa8bbe047097e0841220884f68cdc.

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: wiggin77 <wiggin77@warpmail.net>

server/channels/testlib/helper.go

@@ -41,6 +41,11 @@ type HelperOptions struct {
 }
 
 func NewMainHelper() *MainHelper {
+	// Ignore any globally defined datasource if a test dsn defined
+	if os.Getenv("TEST_DATABASE_MYSQL_DSN") != "" || os.Getenv("TEST_DATABASE_POSTGRESQL_DSN") != "" {
+		os.Unsetenv("MM_SQLSETTINGS_DATASOURCE")
+	}
+
 	return NewMainHelperWithOptions(&HelperOptions{
 		EnableStore:     true,
 		EnableResources: true,
@@ -48,6 +53,11 @@ func NewMainHelper() *MainHelper {
 }
 
 func NewMainHelperWithOptions(options *HelperOptions) *MainHelper {
+	// Ignore any globally defined datasource if a test dsn defined
+	if os.Getenv("TEST_DATABASE_MYSQL_DSN") != "" || os.Getenv("TEST_DATABASE_POSTGRESQL_DSN") != "" {
+		os.Unsetenv("MM_SQLSETTINGS_DATASOURCE")
+	}
+
 	var mainHelper MainHelper
 	flag.Parse()
 
@@ -153,6 +163,7 @@ func (h *MainHelper) setupResources() {
 func (h *MainHelper) PreloadMigrations() {
 	var buf []byte
 	var err error
+
 	basePath := os.Getenv("MM_SERVER_PATH")
 	if basePath == "" {
 		basePath = "mattermost-server/server"
@@ -177,6 +188,62 @@ func (h *MainHelper) PreloadMigrations() {
 	if err != nil {
 		panic(errors.Wrap(err, "Error preloading migrations. Check if you have &multiStatements=true in your DSN if you are using MySQL. Or perhaps the schema changed? If yes, then update the warmup files accordingly"))
 	}
+
+	h.PreloadBoardsMigrationsIfNeeded()
+}
+
+// PreloadBoardsMigrationsIfNeeded loads boards migrations if the
+// focalboard_schema_migrations table exists already.
+// Besides this, the same compatibility and breaking conditions that
+// PreloadMigrations has apply here.
+//
+// Re-generate the files with:
+// pg_dump -a -h localhost -U mmuser -d <> --no-comments --inserts -t focalboard_system_settings
+// mysqldump -u root -p <> --no-create-info --extended-insert=FALSE focalboard_system_settings
+func (h *MainHelper) PreloadBoardsMigrationsIfNeeded() {
+	tableSchemaFn := "current_schema()"
+	if *h.Settings.DriverName == model.DatabaseDriverMysql {
+		tableSchemaFn = "DATABASE()"
+	}
+
+	basePath := os.Getenv("MM_SERVER_PATH")
+	if basePath == "" {
+		basePath = "mattermost-server/server"
+	}
+	relPath := "channels/testlib/testdata"
+
+	handle := h.SQLStore.GetMasterX()
+	var boardsTableCount int
+	gErr := handle.Get(&boardsTableCount, `
+      SELECT COUNT(*)
+        FROM INFORMATION_SCHEMA.TABLES
+       WHERE TABLE_SCHEMA = `+tableSchemaFn+`
+         AND TABLE_NAME = 'focalboard_schema_migrations'`)
+	if gErr != nil {
+		panic(errors.Wrap(gErr, "Error preloading migrations. Cannot query INFORMATION_SCHEMA table to check for focalboard_schema_migrations table"))
+	}
+
+	var buf []byte
+	var err error
+	if boardsTableCount != 0 {
+		switch *h.Settings.DriverName {
+		case model.DatabaseDriverPostgres:
+			boardsFinalPath := filepath.Join(basePath, relPath, "boards_postgres_migration_warmup.sql")
+			buf, err = os.ReadFile(boardsFinalPath)
+			if err != nil {
+				panic(fmt.Errorf("cannot read file: %v", err))
+			}
+		case model.DatabaseDriverMysql:
+			boardsFinalPath := filepath.Join(basePath, relPath, "boards_mysql_migration_warmup.sql")
+			buf, err = os.ReadFile(boardsFinalPath)
+			if err != nil {
+				panic(fmt.Errorf("cannot read file: %v", err))
+			}
+		}
+		if _, err := handle.Exec(string(buf)); err != nil {
+			panic(errors.Wrap(err, "Error preloading boards migrations. Check if you have &multiStatements=true in your DSN if you are using MySQL. Or perhaps the schema changed? If yes, then update the warmup files accordingly"))
+		}
+	}
 }
 
 func (h *MainHelper) Close() error {

server/channels/testlib/store.go

@@ -101,6 +101,9 @@ func GetMockStoreForSetupFunctions() *mocks.Store {
 	oAuthStore := mocks.OAuthStore{}
 	groupStore := mocks.GroupStore{}
 
+	pluginStore := mocks.PluginStore{}
+	pluginStore.On("List", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil)
+
 	mockStore.On("System").Return(&systemStore)
 	mockStore.On("User").Return(&userStore)
 	mockStore.On("Post").Return(&postStore)
@@ -116,6 +119,7 @@ func GetMockStoreForSetupFunctions() *mocks.Store {
 	mockStore.On("OAuth").Return(&oAuthStore)
 	mockStore.On("Group").Return(&groupStore)
 	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+	mockStore.On("Plugin").Return(&pluginStore)
 
 	return &mockStore
 }

server/channels/testlib/testdata/boards_mysql_migration_warmup.sql

@@ -0,0 +1,41 @@
+-- MySQL dump 10.13  Distrib 5.7.12, for Linux (x86_64)
+--
+-- Host: localhost    Database: mattermost_test
+-- ------------------------------------------------------
+-- Server version       5.7.12
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Dumping data for table `focalboard_system_settings`
+--
+
+LOCK TABLES `focalboard_system_settings` WRITE;
+/*!40000 ALTER TABLE `focalboard_system_settings` DISABLE KEYS */;
+INSERT INTO `focalboard_system_settings` VALUES ('CategoryUuidIdMigrationComplete','true');
+INSERT INTO `focalboard_system_settings` VALUES ('DeDuplicateCategoryBoardTableComplete','true');
+INSERT INTO `focalboard_system_settings` VALUES ('DeletedMembershipBoardsMigrationComplete','true');
+INSERT INTO `focalboard_system_settings` VALUES ('TeamLessBoardsMigrationComplete','true');
+INSERT INTO `focalboard_system_settings` VALUES ('UniqueIDsMigrationComplete','true');
+/*!40000 ALTER TABLE `focalboard_system_settings` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2023-03-31 11:37:35

server/channels/testlib/testdata/boards_postgres_migration_warmup.sql

@@ -0,0 +1,26 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 10.20 (Debian 10.20-1.pgdg90+1)
+-- Dumped by pg_dump version 14.7
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+
+--
+-- Data for Name: focalboard_system_settings; Type: TABLE DATA; Schema: public; Owner: mmuser
+--
+
+INSERT INTO public.focalboard_system_settings VALUES ('UniqueIDsMigrationComplete', 'true');
+INSERT INTO public.focalboard_system_settings VALUES ('TeamLessBoardsMigrationComplete', 'true');
+INSERT INTO public.focalboard_system_settings VALUES ('DeletedMembershipBoardsMigrationComplete', 'true');
+INSERT INTO public.focalboard_system_settings VALUES ('CategoryUuidIdMigrationComplete', 'true');
+INSERT INTO public.focalboard_system_settings VALUES ('DeDuplicateCategoryBoardTableComplete', 'true');
+
+
+--
+-- PostgreSQL database dump complete
+--