Dockerized build updated tests (#9943)

* testlib: introduce and leverage

This doesn't yet factor out the individual test helpers: many packages
still rely on `api4` directly to do this, but now wire up the test store
setup through this package. `app` and `store`, in particular, don't use
`testlib` because of circular dependencies at the moment.

* cmd: command_test.go: use api4 testlib

* cmd: plugin_test.go: remove dependence on test-config.json

* cmd: config_test.go use configured database settings

* ensure test-(te|ee) exit with status code

* test-server: run all tests, deprecating test-te/test-ee

* cmd/mattermost/commands: fix unit tests

Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously.

* wrap run/check into a test helper

It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown.

* additional TestConfigGet granularity

* customized config path to avoid default location

* be explicit if the storetest initialization fails

* generate safe coverprofile names in the presence of subtests

* additional TestConfigShow granularity

* fix permission_test.go typo

* fix webhook tests

* actually flag.Parse() to skip database setup on os.Execed tests

* fix recent regression in #9962, not caught by unit tests

Makefile

@@ -71,7 +71,6 @@ TESTFLAGSEE ?= -short
 
 # Packages lists
 TE_PACKAGES=$(shell go list ./...)
-TE_PACKAGES_COMMA=$(shell echo $(TE_PACKAGES) | tr ' ' ',')
 
 # Plugins Packages
 PLUGIN_PACKAGES=mattermost-plugin-zoom mattermost-plugin-jira
@@ -83,15 +82,16 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
 	IGNORE:=$(shell cp $(BUILD_ENTERPRISE_DIR)/imports/imports.go imports/)
 	IGNORE:=$(shell rm -f enterprise)
 	IGNORE:=$(shell ln -s $(BUILD_ENTERPRISE_DIR) enterprise)
+else
+	IGNORE:=$(shell rm -f imports/imports.go)
 endif
 
 EE_PACKAGES=$(shell go list ./enterprise/...)
-EE_PACKAGES_COMMA=$(shell echo $(EE_PACKAGES) | tr ' ' ',')
 
 ifeq ($(BUILD_ENTERPRISE_READY),true)
-ALL_PACKAGES_COMMA=$(TE_PACKAGES_COMMA),$(EE_PACKAGES_COMMA)
+ALL_PACKAGES=$(TE_PACKAGES) $(EE_PACKAGES)
 else
-ALL_PACKAGES_COMMA=$(TE_PACKAGES_COMMA)
+ALL_PACKAGES=$(TE_PACKAGES)
 endif
 
 
@@ -354,33 +354,6 @@ do-cover-file: ## Creates the test coverage report file.
 go-junit-report:
 	go get -u github.com/jstemmer/go-junit-report
 
-test-te: start-docker go-junit-report do-cover-file ## Runs tests in the team edition.
-	@echo Testing TE
-	@echo "Packages to test: "$(TE_PACKAGES)
-	find . -name 'cprofile*.out' -exec sh -c 'rm "{}"' \;
-	$(GO) test $(GOFLAGS) -run=$(TESTS) $(TESTFLAGS) -p 1 -v -timeout=2000s -covermode=count -coverpkg=$(ALL_PACKAGES_COMMA) -exec $(ROOT)/scripts/test-xprog.sh $(TE_PACKAGES) | tee output-test-te
-	cat output-test-te | $(GOPATH)/bin/go-junit-report > report-te.xml && rm output-test-te
-	find . -name 'cprofile*.out' -exec sh -c 'tail -n +2 {} >> cover.out ; rm "{}"' \;
-
-test-ee: start-docker go-junit-report do-cover-file ## Runs tests in the enterprise edition.
-	@echo Testing EE
-
-	rm -f enterprise/config/*.crt
-	rm -f enterprise/config/*.key
-
-ifeq ($(BUILD_ENTERPRISE_READY),true)
-	@echo Testing EE
-	@echo "Packages to test: "$(EE_PACKAGES)
-	find . -name 'cprofile*.out' -exec sh -c 'rm "{}"' \;
-	$(GO) test $(GOFLAGS) -run=$(TESTS) $(TESTFLAGSEE) -p 1 -v -timeout=2000s -covermode=count -coverpkg=$(ALL_PACKAGES_COMMA) -exec $(ROOT)/scripts/test-xprog.sh $(EE_PACKAGES) 2>&1 | tee output-test-ee
-	cat output-test-ee | $(GOPATH)/bin/go-junit-report > report-ee.xml && rm output-test-ee
-	find . -name 'cprofile*.out' -exec sh -c 'tail -n +2 {} >> cover.out ; rm "{}"' \;
-	rm -f enterprise/config/*.crt
-	rm -f enterprise/config/*.key
-else
-	@echo Skipping EE Tests
-endif
-
 test-compile:
 	@echo COMPILE TESTS
 
@@ -388,8 +361,13 @@ test-compile:
 		$(GO) test $(GOFLAGS) -c $$package; \
 	done
 
-test-server: test-te test-ee ## Runs tests.
+test-server: start-docker go-junit-report do-cover-file ## Runs tests.
-	find . -type d -name data -not -path './vendor/*' | xargs rm -rf
+ifeq ($(BUILD_ENTERPRISE_READY),true)
+	@echo Running all tests
+else
+	@echo Running only TE tests
+endif
+	./scripts/test.sh "$(GO)" "$(GOFLAGS)" "$(ALL_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)"
 
 internal-test-web-client: ## Runs web client tests.
 	$(GO) run $(GOFLAGS) $(PLATFORM_FILES) test web_client_tests

api4/api_test.go

@@ -1,52 +0,0 @@
-// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api4
-
-import (
-	"flag"
-	"os"
-	"testing"
-
-	"github.com/mattermost/mattermost-server/mlog"
-	"github.com/mattermost/mattermost-server/store/storetest"
-	"github.com/mattermost/mattermost-server/utils"
-)
-
-func TestMain(m *testing.M) {
-	flag.Parse()
-
-	// Setup a global logger to catch tests logging outside of app context
-	// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
-	mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
-		EnableConsole: true,
-		ConsoleJson:   true,
-		ConsoleLevel:  "error",
-		EnableFile:    false,
-	}))
-
-	utils.TranslationsPreInit()
-
-	// In the case where a dev just wants to run a single test, it's faster to just use the default
-	// store.
-	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
-		mlog.Info("-test.run used, not creating temporary containers")
-		os.Exit(m.Run())
-	}
-
-	status := 0
-
-	container, settings, err := storetest.NewMySQLContainer()
-	if err != nil {
-		panic(err)
-	}
-
-	UseTestStore(container, settings)
-
-	defer func() {
-		StopTestStore()
-		os.Exit(status)
-	}()
-
-	status = m.Run()
-}

api4/apitestlib.go

@@ -21,8 +21,6 @@ import (
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
-	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/mattermost/mattermost-server/web"
 	"github.com/mattermost/mattermost-server/wsapi"
@@ -52,33 +50,18 @@ type TestHelper struct {
 	tempWorkspace     string
 }
 
-type persistentTestStore struct {
+// testStore tracks the active test store.
-	store.Store
+// This is a bridge between the new testlib ownership of the test store and the existing usage
-}
+// of the api4 test helper by many packages. In the future, this test helper would ideally belong
+// to the testlib altogether.
+var testStore store.Store
 
-func (*persistentTestStore) Close() {}
+func UseTestStore(store store.Store) {
-
+	testStore = store
-var testStoreContainer *storetest.RunningContainer
-var testStore *persistentTestStore
-
-// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
-// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
-func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
-	testStoreContainer = container
-	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
-}
-
-func StopTestStore() {
-	if testStoreContainer != nil {
-		testStoreContainer.Stop()
-		testStoreContainer = nil
-	}
 }
 
 func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHelper {
-	if testStore != nil {
+	testStore.DropAllTables()
-		testStore.DropAllTables()
-	}
 
 	permConfig, err := os.Open(utils.FindConfigFile("config.json"))
 	if err != nil {
@@ -96,9 +79,7 @@ func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHel
 	}
 
 	options := []app.Option{app.ConfigFile(tempConfig.Name()), app.DisableConfigWatch}
-	if testStore != nil {
+	options = append(options, app.StoreOverride(testStore))
-		options = append(options, app.StoreOverride(testStore))
-	}
 
 	s, err := app.NewServer(options...)
 	if err != nil {
@@ -117,9 +98,7 @@ func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHel
 		cfg.EmailSettings.SendEmailNotifications = true
 	})
 	prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
-	if testStore != nil {
+	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
-		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
-	}
 	if updateConfig != nil {
 		th.App.UpdateConfig(updateConfig)
 	}
@@ -205,7 +184,6 @@ func (me *TestHelper) TearDown() {
 	utils.EnableDebugLogForTest()
 
 	if err := recover(); err != nil {
-		StopTestStore()
 		panic(err)
 	}
 }

api4/main_test.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api4
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+	UseTestStore(mainHelper.Store)
+
+	mainHelper.Main(m)
+}

api4/user.go

@@ -603,7 +603,7 @@ func autocompleteUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	var autocomplete *model.UserAutocomplete
+	var autocomplete model.UserAutocomplete
 
 	if len(channelId) > 0 {
 		// Applying the provided teamId here is useful for DMs and GMs which don't belong

app/app_test.go

@@ -4,57 +4,14 @@
 package app
 
 import (
-	"flag"
 	"fmt"
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store/storetest"
-	"github.com/mattermost/mattermost-server/utils"
 )
 
-func TestMain(m *testing.M) {
-	flag.Parse()
-
-	// Setup a global logger to catch tests logging outside of app context
-	// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
-	mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
-		EnableConsole: true,
-		ConsoleJson:   true,
-		ConsoleLevel:  "error",
-		EnableFile:    false,
-	}))
-
-	utils.TranslationsPreInit()
-
-	// In the case where a dev just wants to run a single test, it's faster to just use the default
-	// store.
-	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
-		mlog.Info("-test.run used, not creating temporary containers")
-		os.Exit(m.Run())
-	}
-
-	status := 0
-
-	container, settings, err := storetest.NewMySQLContainer()
-	if err != nil {
-		panic(err)
-	}
-
-	UseTestStore(container, settings)
-
-	defer func() {
-		StopTestStore()
-		os.Exit(status)
-	}()
-
-	status = m.Run()
-}
-
 /* Temporarily comment out until MM-11108
 func TestAppRace(t *testing.T) {
 	for i := 0; i < 10; i++ {
@@ -88,7 +45,7 @@ func TestDoAdvancedPermissionsMigration(t *testing.T) {
 	th := Setup()
 	defer th.TearDown()
 
-	if testStoreSqlSupplier == nil {
+	if mainHelper.SqlSupplier == nil {
 		t.Skip("This test requires a TestStore to be run.")
 	}
 
@@ -467,7 +424,7 @@ func TestDoEmojisPermissionsMigration(t *testing.T) {
 	th := Setup()
 	defer th.TearDown()
 
-	if testStoreSqlSupplier == nil {
+	if mainHelper.SqlSupplier == nil {
 		t.Skip("This test requires a TestStore to be run.")
 	}
 

app/helper_test.go

@@ -13,12 +13,8 @@ import (
 
 	"testing"
 
-	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
-	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/mattermost/mattermost-server/utils/testutils"
 )
@@ -40,37 +36,8 @@ type TestHelper struct {
 	MockedHTTPService *testutils.MockedHTTPService
 }
 
-type persistentTestStore struct {
-	store.Store
-}
-
-func (*persistentTestStore) Close() {}
-
-var testStoreContainer *storetest.RunningContainer
-var testStore *persistentTestStore
-var testStoreSqlSupplier *sqlstore.SqlSupplier
-var testClusterInterface *FakeClusterInterface
-
-// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
-// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
-func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
-	testClusterInterface = &FakeClusterInterface{}
-	testStoreContainer = container
-	testStoreSqlSupplier = sqlstore.NewSqlSupplier(*settings, nil)
-	testStore = &persistentTestStore{store.NewLayeredStore(testStoreSqlSupplier, nil, testClusterInterface)}
-}
-
-func StopTestStore() {
-	if testStoreContainer != nil {
-		testStoreContainer.Stop()
-		testStoreContainer = nil
-	}
-}
-
 func setupTestHelper(enterprise bool) *TestHelper {
-	if testStore != nil {
+	mainHelper.Store.DropAllTables()
-		testStore.DropAllTables()
-	}
 
 	permConfig, err := os.Open(utils.FindConfigFile("config.json"))
 	if err != nil {
@@ -88,9 +55,7 @@ func setupTestHelper(enterprise bool) *TestHelper {
 	}
 
 	options := []Option{ConfigFile(tempConfig.Name()), DisableConfigWatch}
-	if testStore != nil {
+	options = append(options, StoreOverride(mainHelper.Store))
-		options = append(options, StoreOverride(testStore))
-	}
 
 	s, err := NewServer(options...)
 	if err != nil {
@@ -106,9 +71,7 @@ func setupTestHelper(enterprise bool) *TestHelper {
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.MaxUsersPerTeam = 50 })
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.RateLimitSettings.Enable = false })
 	prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
-	if testStore != nil {
+	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
-		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
-	}
 	serverErr := th.App.StartServer()
 	if serverErr != nil {
 		panic(serverErr)
@@ -446,7 +409,6 @@ func (me *TestHelper) TearDown() {
 	me.ShutdownApp()
 	os.Remove(me.tempConfigPath)
 	if err := recover(); err != nil {
-		StopTestStore()
 		panic(err)
 	}
 	if me.tempWorkspace != "" {
@@ -455,25 +417,25 @@ func (me *TestHelper) TearDown() {
 }
 
 func (me *TestHelper) ResetRoleMigration() {
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {
 		panic(err)
 	}
 
-	testClusterInterface.sendClearRoleCacheMessage()
+	mainHelper.ClusterInterface.SendClearRoleCacheMessage()
 
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
 		panic(err)
 	}
 }
 
 func (me *TestHelper) ResetEmojisMigration() {
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("UPDATE Roles SET Permissions=REPLACE(Permissions, ', manage_emojis', '') WHERE builtin=True"); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("UPDATE Roles SET Permissions=REPLACE(Permissions, ', manage_emojis', '') WHERE builtin=True"); err != nil {
 		panic(err)
 	}
 
-	testClusterInterface.sendClearRoleCacheMessage()
+	mainHelper.ClusterInterface.SendClearRoleCacheMessage()
 
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": EMOJIS_PERMISSIONS_MIGRATION_KEY}); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": EMOJIS_PERMISSIONS_MIGRATION_KEY}); err != nil {
 		panic(err)
 	}
 }
@@ -533,36 +495,3 @@ func (me *TestHelper) SetupPluginAPI() *PluginAPI {
 
 	return NewPluginAPI(me.App, manifest)
 }
-
-type FakeClusterInterface struct {
-	clusterMessageHandler einterfaces.ClusterMessageHandler
-}
-
-func (me *FakeClusterInterface) StartInterNodeCommunication() {}
-func (me *FakeClusterInterface) StopInterNodeCommunication()  {}
-func (me *FakeClusterInterface) RegisterClusterMessageHandler(event string, crm einterfaces.ClusterMessageHandler) {
-	me.clusterMessageHandler = crm
-}
-func (me *FakeClusterInterface) GetClusterId() string                             { return "" }
-func (me *FakeClusterInterface) IsLeader() bool                                   { return false }
-func (me *FakeClusterInterface) GetMyClusterInfo() *model.ClusterInfo             { return nil }
-func (me *FakeClusterInterface) GetClusterInfos() []*model.ClusterInfo            { return nil }
-func (me *FakeClusterInterface) SendClusterMessage(cluster *model.ClusterMessage) {}
-func (me *FakeClusterInterface) NotifyMsg(buf []byte)                             {}
-func (me *FakeClusterInterface) GetClusterStats() ([]*model.ClusterStats, *model.AppError) {
-	return nil, nil
-}
-func (me *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppError) {
-	return []string{}, nil
-}
-func (me *FakeClusterInterface) GetPluginStatuses() (model.PluginStatuses, *model.AppError) {
-	return nil, nil
-}
-func (me *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
-	return nil
-}
-func (me *FakeClusterInterface) sendClearRoleCacheMessage() {
-	me.clusterMessageHandler(&model.ClusterMessage{
-		Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES,
-	})
-}

app/main_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

build/legacy.mk

@@ -0,0 +1,5 @@
+# test-te used to just run the team edition tests, but now runs whatever is available
+test-te: test-server
+
+# test-ee used to just run the enterprise edition tests, but now runs whatever is available
+test-ee: test-server

cmd/mattermost/commands/channel_test.go

@@ -8,50 +8,49 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestJoinChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	channel := th.CreatePublicChannel()
 
-	CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+	th.CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
 
 	// Joining twice should succeed
-	CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+	th.CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
 
 	// should fail because channel does not exist
-	require.Error(t, RunCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name+"asdf", th.BasicUser2.Email))
+	require.Error(t, th.RunCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name+"asdf", th.BasicUser2.Email))
 }
 
 func TestRemoveChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	channel := th.CreatePublicChannel()
 
-	CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+	th.CheckCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
 
 	// should fail because channel does not exist
-	require.Error(t, RunCommand(t, "channel", "remove", th.BasicTeam.Name+":doesnotexist", th.BasicUser2.Email))
+	require.Error(t, th.RunCommand(t, "channel", "remove", th.BasicTeam.Name+":doesnotexist", th.BasicUser2.Email))
 
 	time.Sleep(time.Second)
 
-	CheckCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+	th.CheckCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
 
 	time.Sleep(time.Second)
 
 	// Leaving twice should succeed
-	CheckCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+	th.CheckCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
 }
 
 func TestMoveChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	team1 := th.BasicTeam
@@ -68,22 +67,22 @@ func TestMoveChannel(t *testing.T) {
 	origin := team1.Name + ":" + channel.Name
 	dest := team2.Name
 
-	CheckCommand(t, "channel", "add", origin, adminEmail)
+	th.CheckCommand(t, "channel", "add", origin, adminEmail)
 
 	// should fail with nil because errors are logged instead of returned when a channel does not exist
-	CheckCommand(t, "channel", "move", dest, team1.Name+":doesnotexist", "--username", adminUsername)
+	th.CheckCommand(t, "channel", "move", dest, team1.Name+":doesnotexist", "--username", adminUsername)
 
-	CheckCommand(t, "channel", "move", dest, origin, "--username", adminUsername)
+	th.CheckCommand(t, "channel", "move", dest, origin, "--username", adminUsername)
 }
 
 func TestListChannels(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	channel := th.CreatePublicChannel()
 	th.Client.Must(th.Client.DeleteChannel(channel.Id))
 
-	output := CheckCommand(t, "channel", "list", th.BasicTeam.Name)
+	output := th.CheckCommand(t, "channel", "list", th.BasicTeam.Name)
 
 	if !strings.Contains(string(output), "town-square") {
 		t.Fatal("should have channels")
@@ -95,37 +94,37 @@ func TestListChannels(t *testing.T) {
 }
 
 func TestRestoreChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	channel := th.CreatePublicChannel()
 	th.Client.Must(th.Client.DeleteChannel(channel.Id))
 
-	CheckCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+	th.CheckCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
 
 	// restoring twice should succeed
-	CheckCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+	th.CheckCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
 }
 
 func TestCreateChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 
-	CheckCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--name", name)
+	th.CheckCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--name", name)
 
 	name = name + "-private"
-	CheckCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--private", "--name", name)
+	th.CheckCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--private", "--name", name)
 }
 
 func TestRenameChannel(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	channel := th.CreatePublicChannel()
-	CheckCommand(t, "channel", "rename", th.BasicTeam.Name+":"+channel.Name, "newchannelname10", "--display_name", "New Display Name")
+	th.CheckCommand(t, "channel", "rename", th.BasicTeam.Name+":"+channel.Name, "newchannelname10", "--display_name", "New Display Name")
 
 	// Get the channel from the DB
 	updatedChannel, _ := th.App.GetChannel(channel.Id)

cmd/mattermost/commands/cmdtestlib.go

@@ -4,8 +4,11 @@
 package commands
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,33 +16,159 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/api4"
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/testlib"
 )
 
 var coverprofileCounters map[string]int = make(map[string]int)
 
-func execArgs(t *testing.T, args []string) []string {
+var mainHelper *testlib.MainHelper
-	ret := []string{"-test.run", "ExecCommand"}
+
+type testHelper struct {
+	*api4.TestHelper
+
+	config            *model.Config
+	tempDir           string
+	configFilePath    string
+	disableAutoConfig bool
+}
+
+// Setup creates an instance of testHelper.
+func Setup() *testHelper {
+	dir, err := ioutil.TempDir("", "testHelper")
+	if err != nil {
+		panic("failed to create temporary directory: " + err.Error())
+	}
+
+	api4TestHelper := api4.Setup()
+
+	testHelper := &testHelper{
+		TestHelper:     api4TestHelper,
+		tempDir:        dir,
+		configFilePath: filepath.Join(dir, "config-helper.json"),
+	}
+
+	config := &model.Config{}
+	config.SetDefaults()
+	testHelper.SetConfig(config)
+
+	return testHelper
+}
+
+// InitBasic simply proxies to api4.InitBasic, while still returning a testHelper.
+func (h *testHelper) InitBasic() *testHelper {
+	h.TestHelper.InitBasic()
+	return h
+}
+
+// TemporaryDirectory returns the temporary directory created for user by the test helper.
+func (h *testHelper) TemporaryDirectory() string {
+	return h.tempDir
+}
+
+// Config returns the configuration passed to a running command.
+func (h *testHelper) Config() *model.Config {
+	return h.config.Clone()
+}
+
+// ConfigPath returns the path to the temporary config file passed to a running command.
+func (h *testHelper) ConfigPath() string {
+	return h.configFilePath
+}
+
+// SetConfig replaces the configuration passed to a running command.
+func (h *testHelper) SetConfig(config *model.Config) {
+	config.SqlSettings = *mainHelper.Settings
+	h.config = config
+
+	if err := ioutil.WriteFile(h.configFilePath, []byte(config.ToJson()), 0600); err != nil {
+		panic("failed to write file " + h.configFilePath + ": " + err.Error())
+	}
+}
+
+// SetAutoConfig configures whether the --config flag is automatically passed to a running command.
+func (h *testHelper) SetAutoConfig(autoConfig bool) {
+	h.disableAutoConfig = !autoConfig
+}
+
+// TearDown cleans up temporary files and assets created during the life of the test helper.
+func (h *testHelper) TearDown() {
+	h.TestHelper.TearDown()
+	os.RemoveAll(h.tempDir)
+}
+
+func (h *testHelper) execArgs(t *testing.T, args []string) []string {
+	ret := []string{"-test.v", "-test.run", "ExecCommand"}
 	if coverprofile := flag.Lookup("test.coverprofile").Value.String(); coverprofile != "" {
 		dir := filepath.Dir(coverprofile)
 		base := filepath.Base(coverprofile)
 		baseParts := strings.SplitN(base, ".", 2)
-		coverprofileCounters[t.Name()] = coverprofileCounters[t.Name()] + 1
+		name := strings.Replace(t.Name(), "/", "_", -1)
-		baseParts[0] = fmt.Sprintf("%v-%v-%v", baseParts[0], t.Name(), coverprofileCounters[t.Name()])
+		coverprofileCounters[name] = coverprofileCounters[name] + 1
+		baseParts[0] = fmt.Sprintf("%v-%v-%v", baseParts[0], name, coverprofileCounters[name])
 		ret = append(ret, "-test.coverprofile", filepath.Join(dir, strings.Join(baseParts, ".")))
 	}
-	return append(append(ret, "--", "--disableconfigwatch"), args...)
+
+	ret = append(ret, "--", "--disableconfigwatch")
+
+	// Unless the test passes a `--config` of its own, create a temporary one from the default
+	// configuration with the current test database applied.
+	hasConfig := h.disableAutoConfig
+	for _, arg := range args {
+		if arg == "--config" {
+			hasConfig = true
+			break
+		}
+	}
+
+	if !hasConfig {
+		ret = append(ret, "--config", h.configFilePath)
+	}
+
+	ret = append(ret, args...)
+
+	return ret
 }
 
-func CheckCommand(t *testing.T, args ...string) string {
+// CheckCommand invokes the test binary, returning the output modified for assertion testing.
+func (h *testHelper) CheckCommand(t *testing.T, args ...string) string {
 	path, err := os.Executable()
 	require.NoError(t, err)
-	output, err := exec.Command(path, execArgs(t, args)...).CombinedOutput()
+	output, err := exec.Command(path, h.execArgs(t, args)...).CombinedOutput()
 	require.NoError(t, err, string(output))
 	return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(string(output)), "PASS"))
 }
 
-func RunCommand(t *testing.T, args ...string) error {
+// RunCommand invokes the test binary, returning only any error.
+func (h *testHelper) RunCommand(t *testing.T, args ...string) error {
 	path, err := os.Executable()
 	require.NoError(t, err)
-	return exec.Command(path, execArgs(t, args)...).Run()
+	return exec.Command(path, h.execArgs(t, args)...).Run()
+}
+
+// RunCommandWithOutput is a variant of RunCommand that returns the unmodified output and any error.
+func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string, error) {
+	path, err := os.Executable()
+	require.NoError(t, err)
+
+	cmd := exec.Command(path, h.execArgs(t, args)...)
+
+	var buf bytes.Buffer
+	reader, writer := io.Pipe()
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	done := make(chan bool)
+	go func() {
+		io.Copy(&buf, reader)
+		close(done)
+	}()
+
+	err = cmd.Run()
+	writer.Close()
+	<-done
+
+	return buf.String(), err
 }

cmd/mattermost/commands/command_test.go

@@ -4,18 +4,21 @@
 package commands
 
 import (
-	"os"
-	"os/exec"
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
+	"github.com/mattermost/mattermost-server/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestCreateCommand(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
+
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	th.SetConfig(config)
+
 	team := th.BasicTeam
 	adminUser := th.TeamAdminUser
 	user := th.BasicUser
@@ -107,13 +110,9 @@ func TestCreateCommand(t *testing.T) {
 		},
 	}
 
-	path, err := os.Executable()
-	require.NoError(t, err)
-
 	for _, testCase := range testCases {
 		t.Run(testCase.Description, func(t *testing.T) {
-
+			actual, _ := th.RunCommandWithOutput(t, testCase.Args...)
-			actual, _ := exec.Command(path, execArgs(t, testCase.Args)...).CombinedOutput()
 
 			cmds, _ := th.SystemAdminClient.ListCommands(team.Id, true)
 
@@ -132,9 +131,11 @@ func TestCreateCommand(t *testing.T) {
 	}
 }
 
-/* Commenting it out because race condition
 func TestDeleteCommand(t *testing.T) {
-	th := app.Setup().InitBasic()
+	// Skipped due to v5.6 RC build issues.
+	t.Skip()
+
+	th := Setup().InitBasic()
 	defer th.TearDown()
 	url := "http://localhost:8000/test-command"
 	team := th.BasicTeam
@@ -158,14 +159,13 @@ func TestDeleteCommand(t *testing.T) {
 		require.Nil(t, err)
 		assert.Equal(t, len(commands), 1)
 
-		CheckCommand(t, "command", "delete", command.Id)
+		th.CheckCommand(t, "command", "delete", command.Id)
 		commands, err = th.App.ListTeamCommands(team.Id)
 		require.Nil(t, err)
 		assert.Equal(t, len(commands), 0)
 	})
 
 	t.Run("not existing command", func(t *testing.T) {
-		assert.Error(t, RunCommand(t, "command", "delete", "invalid"))
+		assert.Error(t, th.RunCommand(t, "command", "delete", "invalid"))
 	})
 }
-*/

cmd/mattermost/commands/config_flag_test.go

@@ -4,6 +4,7 @@
 package commands
 
 import (
+	"encoding/json"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -11,21 +12,13 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"encoding/json"
-
 	"github.com/mattermost/mattermost-server/utils"
 )
 
 func TestConfigFlag(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
+	th := Setup()
-	require.NoError(t, err)
+	defer th.TearDown()
-	defer os.RemoveAll(dir)
+	dir := th.TemporaryDirectory()
-
-	utils.TranslationsPreInit()
-	config, _, _, err := utils.LoadConfig("config.json")
-	require.Nil(t, err)
-	configPath := filepath.Join(dir, "foo.json")
-	require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600))
 
 	timezones := utils.LoadTimezones("timezones.json")
 	tzConfigPath := filepath.Join(dir, "timezones.json")
@@ -41,8 +34,15 @@ func TestConfigFlag(t *testing.T) {
 	defer os.Chdir(prevDir)
 	os.Chdir(dir)
 
-	require.Error(t, RunCommand(t, "version"))
+	t.Run("version without a config file should fail", func(t *testing.T) {
-	CheckCommand(t, "--config", "foo.json", "version")
+		th.SetAutoConfig(false)
-	CheckCommand(t, "--config", "./foo.json", "version")
+		defer th.SetAutoConfig(true)
-	CheckCommand(t, "--config", configPath, "version")
+		require.Error(t, th.RunCommand(t, "version"))
+	})
+
+	t.Run("version with varying paths to the config file", func(t *testing.T) {
+		th.CheckCommand(t, "--config", filepath.Base(th.ConfigPath()), "version")
+		th.CheckCommand(t, "--config", "./"+filepath.Base(th.ConfigPath()), "version")
+		th.CheckCommand(t, "version")
+	})
 }

cmd/mattermost/commands/config_test.go

@@ -4,16 +4,12 @@
 package commands
 
 import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/model"
 )
@@ -72,88 +68,90 @@ type TestNewTeamSettings struct {
 }
 
 func TestConfigValidate(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
+	th := Setup()
-	require.NoError(t, err)
+	defer th.TearDown()
-	defer os.RemoveAll(dir)
 
-	path := filepath.Join(dir, "config.json")
+	assert.Error(t, th.RunCommand(t, "--config", "foo.json", "config", "validate"))
-	config := &model.Config{}
+	th.CheckCommand(t, "config", "validate")
-	config.SetDefaults()
-	require.NoError(t, ioutil.WriteFile(path, []byte(config.ToJson()), 0600))
-
-	assert.Error(t, RunCommand(t, "--config", "foo.json", "config", "validate"))
-	CheckCommand(t, "--config", path, "config", "validate")
 }
 
 func TestConfigGet(t *testing.T) {
-	// Error when no arguments are given
+	th := Setup()
-	assert.Error(t, RunCommand(t, "config", "get"))
+	defer th.TearDown()
 
-	// Error when more than one config settings are given
+	t.Run("Error when no arguments are given", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "config", "get", "abc", "def"))
+		assert.Error(t, th.RunCommand(t, "config", "get"))
+	})
 
-	// Error when a config setting which is not in the config.json is given
+	t.Run("Error when more than one config settings are given", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "config", "get", "abc"))
+		assert.Error(t, th.RunCommand(t, "config", "get", "abc", "def"))
+	})
 
-	// No Error when a config setting which is  in the config.json is given
+	t.Run("Error when a config setting which is not in the config.json is given", func(t *testing.T) {
-	CheckCommand(t, "config", "get", "MessageExportSettings")
+		assert.Error(t, th.RunCommand(t, "config", "get", "abc"))
-	CheckCommand(t, "config", "get", "MessageExportSettings.GlobalRelaySettings")
+	})
-	CheckCommand(t, "config", "get", "MessageExportSettings.GlobalRelaySettings.CustomerType")
 
-	// check output
+	t.Run("No Error when a config setting which is in the config.json is given", func(t *testing.T) {
-	output := CheckCommand(t, "config", "get", "MessageExportSettings")
+		th.CheckCommand(t, "config", "get", "MessageExportSettings")
+		th.CheckCommand(t, "config", "get", "MessageExportSettings.GlobalRelaySettings")
+		th.CheckCommand(t, "config", "get", "MessageExportSettings.GlobalRelaySettings.CustomerType")
+	})
 
-	assert.Contains(t, string(output), "EnableExport")
+	t.Run("check output", func(t *testing.T) {
-	assert.Contains(t, string(output), "ExportFormat")
+		output := th.CheckCommand(t, "config", "get", "MessageExportSettings")
-	assert.Contains(t, string(output), "DailyRunTime")
+
-	assert.Contains(t, string(output), "ExportFromTimestamp")
+		assert.Contains(t, string(output), "EnableExport")
+		assert.Contains(t, string(output), "ExportFormat")
+		assert.Contains(t, string(output), "DailyRunTime")
+		assert.Contains(t, string(output), "ExportFromTimestamp")
+	})
 }
 
 func TestConfigSet(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
+	th := Setup()
-	require.NoError(t, err)
+	defer th.TearDown()
-	defer os.RemoveAll(dir)
 
-	path := filepath.Join(dir, "config.json")
+	t.Run("Error when no arguments are given", func(t *testing.T) {
-	config := &model.Config{}
+		assert.Error(t, th.RunCommand(t, "config", "set"))
-	config.SetDefaults()
+	})
-	require.NoError(t, ioutil.WriteFile(path, []byte(config.ToJson()), 0600))
 
-	// Error when no arguments are given
+	t.Run("Error when only one argument is given", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "--config", path, "config", "set"))
+		assert.Error(t, th.RunCommand(t, "config", "set", "test"))
+	})
 
-	// Error when only one argument is given
+	t.Run("Error when the wrong key is set", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "--config", path, "config", "set", "test"))
+		assert.Error(t, th.RunCommand(t, "config", "set", "invalid-key", "value"))
+		assert.Error(t, th.RunCommand(t, "config", "get", "invalid-key"))
+	})
 
-	// Error when the wrong key is set
+	t.Run("Error when the wrong value is set", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "--config", path, "config", "set", "invalid-key", "value"))
+		assert.Error(t, th.RunCommand(t, "config", "set", "EmailSettings.ConnectionSecurity", "invalid"))
-	assert.Error(t, RunCommand(t, "--config", path, "config", "get", "invalid-key"))
+		output := th.CheckCommand(t, "config", "get", "EmailSettings.ConnectionSecurity")
+		assert.NotContains(t, string(output), "invalid")
+	})
 
-	// Error when the wrong value is set
+	t.Run("Error when the wrong locale is set", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "--config", path, "config", "set", "EmailSettings.ConnectionSecurity", "invalid"))
+		th.CheckCommand(t, "config", "set", "LocalizationSettings.DefaultServerLocale", "es")
-	output := CheckCommand(t, "--config", path, "config", "get", "EmailSettings.ConnectionSecurity")
+		assert.Error(t, th.RunCommand(t, "config", "set", "LocalizationSettings.DefaultServerLocale", "invalid"))
-	assert.NotContains(t, string(output), "invalid")
+		output := th.CheckCommand(t, "config", "get", "LocalizationSettings.DefaultServerLocale")
+		assert.NotContains(t, string(output), "invalid")
+		assert.NotContains(t, string(output), "\"en\"")
+	})
 
-	// Error when the wrong locale is set
+	t.Run("Success when a valid value is set", func(t *testing.T) {
-	CheckCommand(t, "--config", path, "config", "set", "LocalizationSettings.DefaultServerLocale", "es")
+		assert.NoError(t, th.RunCommand(t, "config", "set", "EmailSettings.ConnectionSecurity", "TLS"))
-	assert.Error(t, RunCommand(t, "--config", path, "config", "set", "LocalizationSettings.DefaultServerLocale", "invalid"))
+		output := th.CheckCommand(t, "config", "get", "EmailSettings.ConnectionSecurity")
-	output = CheckCommand(t, "--config", path, "config", "get", "LocalizationSettings.DefaultServerLocale")
+		assert.Contains(t, string(output), "TLS")
-	assert.NotContains(t, string(output), "invalid")
+	})
-	assert.NotContains(t, string(output), "\"en\"")
 
-	// Success when a valid value is set
+	t.Run("Success when a valid locale is set", func(t *testing.T) {
-	CheckCommand(t, "--config", path, "config", "set", "EmailSettings.ConnectionSecurity", "TLS")
+		assert.NoError(t, th.RunCommand(t, "config", "set", "LocalizationSettings.DefaultServerLocale", "es"))
-	output = CheckCommand(t, "--config", path, "config", "get", "EmailSettings.ConnectionSecurity")
+		output := th.CheckCommand(t, "config", "get", "LocalizationSettings.DefaultServerLocale")
-	assert.Contains(t, string(output), "TLS")
+		assert.Contains(t, string(output), "\"es\"")
-
+	})
-	// Success when a valid locale is set
-	CheckCommand(t, "--config", path, "config", "set", "LocalizationSettings.DefaultServerLocale", "es")
-	output = CheckCommand(t, "--config", path, "config", "get", "LocalizationSettings.DefaultServerLocale")
-	assert.Contains(t, string(output), "\"es\"")
 }
 
 func TestStructToMap(t *testing.T) {
-
 	cases := []struct {
 		Name     string
 		Input    interface{}
@@ -324,7 +322,6 @@ func TestConfigToMap(t *testing.T) {
 }
 
 func TestPrintMap(t *testing.T) {
-
 	inputCases := []interface{}{
 		map[string]interface{}{
 			"CustomerType": "A9",
@@ -387,7 +384,6 @@ func TestPrintMap(t *testing.T) {
 }
 
 func TestPrintConfigValues(t *testing.T) {
-
 	outputs := []string{
 		"Siteurl: \"abc\"\nWebsocketurl: \"def\"\nLicensedfieldlocation: \"ghi\"\n",
 		"Sitename: \"abc\"\nMaxuserperteam: \"1\"\n",
@@ -464,39 +460,42 @@ func TestPrintConfigValues(t *testing.T) {
 }
 
 func TestConfigShow(t *testing.T) {
+	th := Setup()
+	defer th.TearDown()
 
-	// error
+	t.Run("error with unknown subcommand", func(t *testing.T) {
-	assert.Error(t, RunCommand(t, "config", "show", "abc"))
+		assert.Error(t, th.RunCommand(t, "config", "show", "abc"))
+	})
 
-	// no error
+	t.Run("successfully dumping config", func(t *testing.T) {
-	CheckCommand(t, "config", "show")
+		output := th.CheckCommand(t, "config", "show")
-
+		assert.Contains(t, string(output), "SqlSettings")
-	// check the output
+		assert.Contains(t, string(output), "MessageExportSettings")
-	output := CheckCommand(t, "config", "show")
+		assert.Contains(t, string(output), "AnnouncementSettings")
-	assert.Contains(t, string(output), "SqlSettings")
+	})
-	assert.Contains(t, string(output), "MessageExportSettings")
-	assert.Contains(t, string(output), "AnnouncementSettings")
 }
 
 func TestSetConfig(t *testing.T) {
+	th := Setup()
+	defer th.TearDown()
+
 	// Error when no argument is given
-	assert.Error(t, RunCommand(t, "config", "set"))
+	assert.Error(t, th.RunCommand(t, "config", "set"))
 
 	// No Error when more than one argument is given
-	CheckCommand(t, "config", "set", "ThemeSettings.AllowedThemes", "hello", "World")
+	th.CheckCommand(t, "config", "set", "ThemeSettings.AllowedThemes", "hello", "World")
 
 	// No Error when two arguments are given
-	CheckCommand(t, "config", "set", "ThemeSettings.AllowedThemes", "hello")
+	th.CheckCommand(t, "config", "set", "ThemeSettings.AllowedThemes", "hello")
 
 	// Error when only one argument is given
-	assert.Error(t, RunCommand(t, "config", "set", "ThemeSettings.AllowedThemes"))
+	assert.Error(t, th.RunCommand(t, "config", "set", "ThemeSettings.AllowedThemes"))
 
 	// Error when config settings not in the config file are given
-	assert.Error(t, RunCommand(t, "config", "set", "Abc"))
+	assert.Error(t, th.RunCommand(t, "config", "set", "Abc"))
 }
 
 func TestUpdateMap(t *testing.T) {
-
 	// create a config to make changes
 	config := TestNewConfig{
 		TestNewServiceSettings{
@@ -577,7 +576,6 @@ func TestUpdateMap(t *testing.T) {
 }
 
 func contains(configMap map[string]interface{}, v interface{}, configSettings []string) bool {
-
 	res := configMap[configSettings[0]]
 
 	value := reflect.ValueOf(res)

cmd/mattermost/commands/export_test.go

@@ -4,15 +4,11 @@
 package commands
 
 import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/utils"
 )
 
 // There are no tests that actually run the Message Export job, because it can take a long time to complete depending
@@ -20,47 +16,49 @@ import (
 // fails fast if invalid flags are supplied
 
 func TestMessageExportNotEnabled(t *testing.T) {
-	configPath := writeTempConfig(t, false)
+	th := Setup()
-	defer os.RemoveAll(filepath.Dir(configPath))
+	defer th.TearDown()
+
+	config := th.Config()
+	config.MessageExportSettings.EnableExport = model.NewBool(false)
+	th.SetConfig(config)
 
 	// should fail fast because the feature isn't enabled
-	require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule"))
+	require.Error(t, th.RunCommand(t, "export", "schedule"))
 }
 
 func TestMessageExportInvalidFormat(t *testing.T) {
-	configPath := writeTempConfig(t, true)
+	th := Setup()
-	defer os.RemoveAll(filepath.Dir(configPath))
+	defer th.TearDown()
+
+	config := th.Config()
+	config.MessageExportSettings.EnableExport = model.NewBool(true)
+	th.SetConfig(config)
 
 	// should fail fast because format isn't supported
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule"))
+	require.Error(t, th.RunCommand(t, "--format", "not_actiance", "export", "schedule"))
 }
 
 func TestMessageExportNegativeExportFrom(t *testing.T) {
-	configPath := writeTempConfig(t, true)
+	th := Setup()
-	defer os.RemoveAll(filepath.Dir(configPath))
+	defer th.TearDown()
+
+	config := th.Config()
+	config.MessageExportSettings.EnableExport = model.NewBool(true)
+	th.SetConfig(config)
 
 	// should fail fast because export from must be a valid timestamp
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule"))
+	require.Error(t, th.RunCommand(t, "--format", "actiance", "--exportFrom", "-1", "export", "schedule"))
 }
 
 func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
-	configPath := writeTempConfig(t, true)
+	th := Setup()
-	defer os.RemoveAll(filepath.Dir(configPath))
+	defer th.TearDown()
+
+	config := th.Config()
+	config.MessageExportSettings.EnableExport = model.NewBool(true)
+	th.SetConfig(config)
 
 	// should fail fast because timeout seconds must be a positive int
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule"))
+	require.Error(t, th.RunCommand(t, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule"))
-}
-
-func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string {
-	dir, err := ioutil.TempDir("", "")
-	require.NoError(t, err)
-
-	utils.TranslationsPreInit()
-	config, _, _, appErr := utils.LoadConfig("config.json")
-	require.Nil(t, appErr)
-	config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled)
-	configPath := filepath.Join(dir, "foo.json")
-	require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600))
-
-	return configPath
 }

cmd/mattermost/commands/main_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package commands
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/api4"
+	"github.com/mattermost/mattermost-server/testlib"
+)
+
+func TestMain(m *testing.M) {
+	// Command tests are run by re-invoking the test binary in question, so avoid creating
+	// another container when we detect same.
+	flag.Parse()
+	if filter := flag.Lookup("test.run").Value.String(); filter == "ExecCommand" {
+		status := m.Run()
+		os.Exit(status)
+		return
+	}
+
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+	api4.UseTestStore(mainHelper.Store)
+
+	mainHelper.Main(m)
+}

cmd/mattermost/commands/permissions_test.go

@@ -4,37 +4,26 @@
 package commands
 
 import (
-	"os"
-	"os/exec"
-	"strings"
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/mattermost/mattermost-server/utils"
 )
 
 func TestPermissionsExport_rejectsUnlicensed(t *testing.T) {
-	permissionsLicenseRequiredTest(t, "export")
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	actual, _ := th.RunCommandWithOutput(t, "permissions", "export")
+	assert.Contains(t, actual, utils.T("cli.license.critical"))
 }
 
 func TestPermissionsImport_rejectsUnlicensed(t *testing.T) {
-	permissionsLicenseRequiredTest(t, "import")
+	th := Setup().InitBasic()
-}
-
-func permissionsLicenseRequiredTest(t *testing.T, subcommand string) {
-	th := api4.Setup().InitBasic()
 	defer th.TearDown()
 
-	path, err := os.Executable()
+	actual, _ := th.RunCommandWithOutput(t, "permissions", "import")
-	if err != nil {
-		t.Fail()
-	}
-	args := []string{"-test.run", "ExecCommand", "--", "--disableconfigwatch", "permissions", subcommand}
-	output, _ := exec.Command(path, args...).CombinedOutput()
 
-	actual := string(output)
+	assert.Contains(t, actual, utils.T("cli.license.critical"))
-	expected := utils.T("cli.license.critical")
-	if !strings.Contains(actual, expected) {
-		t.Errorf("Expected '%v' but got '%v'.", expected, actual)
-	}
 }

cmd/mattermost/commands/plugin_test.go

@@ -5,38 +5,43 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestPlugin(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	config := th.Config()
+	*config.PluginSettings.EnableUploads = true
+	*config.PluginSettings.Directory = "./test-plugins"
+	*config.PluginSettings.ClientDirectory = "./test-client-plugins"
+	th.SetConfig(config)
+
 	os.MkdirAll("./test-plugins", os.ModePerm)
 	os.MkdirAll("./test-client-plugins", os.ModePerm)
 
-	th := api4.Setup().InitBasic()
-	defer th.TearDown()
-
 	path, _ := utils.FindDir("tests")
 
 	os.Chdir(filepath.Join("..", "..", ".."))
 
-	CheckCommand(t, "--config", filepath.Join(path, "test-config.json"), "plugin", "add", filepath.Join(path, "testplugin.tar.gz"))
+	th.CheckCommand(t, "plugin", "add", filepath.Join(path, "testplugin.tar.gz"))
 
-	CheckCommand(t, "--config", filepath.Join(path, "test-config.json"), "plugin", "enable", "testplugin")
+	th.CheckCommand(t, "plugin", "enable", "testplugin")
-	cfg, _, _, err := utils.LoadConfig(filepath.Join(path, "test-config.json"))
+	cfg, _, _, err := utils.LoadConfig(th.ConfigPath())
 	require.Nil(t, err)
 	assert.Equal(t, cfg.PluginSettings.PluginStates["testplugin"].Enable, true)
 
-	CheckCommand(t, "--config", filepath.Join(path, "test-config.json"), "plugin", "disable", "testplugin")
+	th.CheckCommand(t, "plugin", "disable", "testplugin")
-	cfg, _, _, err = utils.LoadConfig(filepath.Join(path, "test-config.json"))
+	cfg, _, _, err = utils.LoadConfig(th.ConfigPath())
 	require.Nil(t, err)
 	assert.Equal(t, cfg.PluginSettings.PluginStates["testplugin"].Enable, false)
 
-	CheckCommand(t, "--config", filepath.Join(path, "test-config.json"), "plugin", "list")
+	th.CheckCommand(t, "plugin", "list")
 
-	CheckCommand(t, "--config", filepath.Join(path, "test-config.json"), "plugin", "delete", "testplugin")
+	th.CheckCommand(t, "plugin", "delete", "testplugin")
 
 	os.Chdir(filepath.Join("cmd", "mattermost", "commands"))
 }

cmd/mattermost/commands/roles_test.go

@@ -6,15 +6,14 @@ package commands
 import (
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/model"
 )
 
 func TestAssignRole(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
-	CheckCommand(t, "roles", "system_admin", th.BasicUser.Email)
+	th.CheckCommand(t, "roles", "system_admin", th.BasicUser.Email)
 
 	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
 		t.Fatal(result.Err)
@@ -25,7 +24,7 @@ func TestAssignRole(t *testing.T) {
 		}
 	}
 
-	CheckCommand(t, "roles", "member", th.BasicUser.Email)
+	th.CheckCommand(t, "roles", "member", th.BasicUser.Email)
 
 	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
 		t.Fatal(result.Err)

cmd/mattermost/commands/sampledata_test.go

@@ -6,20 +6,19 @@ package commands
 import (
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/stretchr/testify/require"
 )
 
 func TestSampledataBadParameters(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	// should fail because you need at least 1 worker
-	require.Error(t, RunCommand(t, "sampledata", "--workers", "0"))
+	require.Error(t, th.RunCommand(t, "sampledata", "--workers", "0"))
 
 	// should fail because you have more team memberships than teams
-	require.Error(t, RunCommand(t, "sampledata", "--teams", "10", "--teams-memberships", "11"))
+	require.Error(t, th.RunCommand(t, "sampledata", "--teams", "10", "--teams-memberships", "11"))
 
 	// should fail because you have more channel memberships than channels per team
-	require.Error(t, RunCommand(t, "sampledata", "--channels-per-team", "10", "--channel-memberships", "11"))
+	require.Error(t, th.RunCommand(t, "sampledata", "--channels-per-team", "10", "--channel-memberships", "11"))
 }

cmd/mattermost/commands/team_test.go

@@ -7,19 +7,18 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/model"
 )
 
 func TestCreateTeam(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
 	found := th.SystemAdminClient.Must(th.SystemAdminClient.TeamExists(name, "")).(bool)
 
@@ -29,10 +28,10 @@ func TestCreateTeam(t *testing.T) {
 }
 
 func TestJoinTeam(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
-	CheckCommand(t, "team", "add", th.BasicTeam.Name, th.BasicUser.Email)
+	th.CheckCommand(t, "team", "add", th.BasicTeam.Name, th.BasicUser.Email)
 
 	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetUsersInTeam(th.BasicTeam.Id, 0, 1000, "")).([]*model.User)
 
@@ -51,10 +50,10 @@ func TestJoinTeam(t *testing.T) {
 }
 
 func TestLeaveTeam(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
-	CheckCommand(t, "team", "remove", th.BasicTeam.Name, th.BasicUser.Email)
+	th.CheckCommand(t, "team", "remove", th.BasicTeam.Name, th.BasicUser.Email)
 
 	profiles := th.Client.Must(th.Client.GetUsersInTeam(th.BasicTeam.Id, 0, 1000, "")).([]*model.User)
 
@@ -80,16 +79,16 @@ func TestLeaveTeam(t *testing.T) {
 }
 
 func TestListTeams(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	output := CheckCommand(t, "team", "list", th.BasicTeam.Name, th.BasicUser.Email)
+	output := th.CheckCommand(t, "team", "list", th.BasicTeam.Name, th.BasicUser.Email)
 
 	if !strings.Contains(string(output), name) {
 		t.Fatal("should have the created team")
@@ -97,18 +96,18 @@ func TestListTeams(t *testing.T) {
 }
 
 func TestListArchivedTeams(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	CheckCommand(t, "team", "archive", name)
+	th.CheckCommand(t, "team", "archive", name)
 
-	output := CheckCommand(t, "team", "list", th.BasicTeam.Name, th.BasicUser.Email)
+	output := th.CheckCommand(t, "team", "list", th.BasicTeam.Name, th.BasicUser.Email)
 
 	if !strings.Contains(string(output), name+" (archived)") {
 		t.Fatal("should have archived team")
@@ -116,16 +115,16 @@ func TestListArchivedTeams(t *testing.T) {
 }
 
 func TestSearchTeamsByName(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	output := CheckCommand(t, "team", "search", name)
+	output := th.CheckCommand(t, "team", "search", name)
 
 	if !strings.Contains(string(output), name) {
 		t.Fatal("should have the created team")
@@ -133,16 +132,16 @@ func TestSearchTeamsByName(t *testing.T) {
 }
 
 func TestSearchTeamsByDisplayName(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	output := CheckCommand(t, "team", "search", displayName)
+	output := th.CheckCommand(t, "team", "search", displayName)
 
 	if !strings.Contains(string(output), name) {
 		t.Fatal("should have the created team")
@@ -150,18 +149,18 @@ func TestSearchTeamsByDisplayName(t *testing.T) {
 }
 
 func TestSearchArchivedTeamsByName(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	CheckCommand(t, "team", "archive", name)
+	th.CheckCommand(t, "team", "archive", name)
 
-	output := CheckCommand(t, "team", "search", name)
+	output := th.CheckCommand(t, "team", "search", name)
 
 	if !strings.Contains(string(output), "(archived)") {
 		t.Fatal("should have archived team")
@@ -169,18 +168,18 @@ func TestSearchArchivedTeamsByName(t *testing.T) {
 }
 
 func TestArchiveTeams(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	name := "name" + id
 	displayName := "Name " + id
 
-	CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+	th.CheckCommand(t, "team", "create", "--name", name, "--display_name", displayName)
 
-	CheckCommand(t, "team", "archive", name)
+	th.CheckCommand(t, "team", "archive", name)
 
-	output := CheckCommand(t, "team", "list")
+	output := th.CheckCommand(t, "team", "list")
 
 	if !strings.Contains(string(output), name+" (archived)") {
 		t.Fatal("should have archived team")

cmd/mattermost/commands/user_test.go

@@ -6,22 +6,21 @@ package commands
 import (
 	"testing"
 
-	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/stretchr/testify/require"
 )
 
 func TestCreateUserWithTeam(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	id := model.NewId()
 	email := "success+" + id + "@simulator.amazonses.com"
 	username := "name" + id
 
-	CheckCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+	th.CheckCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
 
-	CheckCommand(t, "team", "add", th.BasicTeam.Id, email)
+	th.CheckCommand(t, "team", "add", th.BasicTeam.Id, email)
 
 	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetUsersInTeam(th.BasicTeam.Id, 0, 1000, "")).([]*model.User)
 
@@ -40,14 +39,14 @@ func TestCreateUserWithTeam(t *testing.T) {
 }
 
 func TestCreateUserWithoutTeam(t *testing.T) {
-	th := api4.Setup()
+	th := Setup()
 	defer th.TearDown()
 
 	id := model.NewId()
 	email := "success+" + id + "@simulator.amazonses.com"
 	username := "name" + id
 
-	CheckCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+	th.CheckCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
 
 	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
 		t.Fatal(result.Err)
@@ -58,10 +57,10 @@ func TestCreateUserWithoutTeam(t *testing.T) {
 }
 
 func TestResetPassword(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
-	CheckCommand(t, "user", "password", th.BasicUser.Email, "password2")
+	th.CheckCommand(t, "user", "password", th.BasicUser.Email, "password2")
 
 	th.Client.Logout()
 	th.BasicUser.Password = "password2"
@@ -69,23 +68,23 @@ func TestResetPassword(t *testing.T) {
 }
 
 func TestMakeUserActiveAndInactive(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	// first inactivate the user
-	CheckCommand(t, "user", "deactivate", th.BasicUser.Email)
+	th.CheckCommand(t, "user", "deactivate", th.BasicUser.Email)
 
 	// activate the inactive user
-	CheckCommand(t, "user", "activate", th.BasicUser.Email)
+	th.CheckCommand(t, "user", "activate", th.BasicUser.Email)
 }
 
 func TestChangeUserEmail(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
 	newEmail := model.NewId() + "@mattermost-test.com"
 
-	CheckCommand(t, "user", "email", th.BasicUser.Username, newEmail)
+	th.CheckCommand(t, "user", "email", th.BasicUser.Username, newEmail)
 	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err == nil {
 		t.Fatal("should've updated to the new email")
 	}
@@ -99,21 +98,21 @@ func TestChangeUserEmail(t *testing.T) {
 	}
 
 	// should fail because using an invalid email
-	require.Error(t, RunCommand(t, "user", "email", th.BasicUser.Username, "wrong$email.com"))
+	require.Error(t, th.RunCommand(t, "user", "email", th.BasicUser.Username, "wrong$email.com"))
 
 	// should fail because missing one parameter
-	require.Error(t, RunCommand(t, "user", "email", th.BasicUser.Username))
+	require.Error(t, th.RunCommand(t, "user", "email", th.BasicUser.Username))
 
 	// should fail because missing both parameters
-	require.Error(t, RunCommand(t, "user", "email"))
+	require.Error(t, th.RunCommand(t, "user", "email"))
 
 	// should fail because have more than 2  parameters
-	require.Error(t, RunCommand(t, "user", "email", th.BasicUser.Username, "new@email.com", "extra!"))
+	require.Error(t, th.RunCommand(t, "user", "email", th.BasicUser.Username, "new@email.com", "extra!"))
 
 	// should fail because user not found
-	require.Error(t, RunCommand(t, "user", "email", "invalidUser", newEmail))
+	require.Error(t, th.RunCommand(t, "user", "email", "invalidUser", newEmail))
 
 	// should fail because email already in use
-	require.Error(t, RunCommand(t, "user", "email", th.BasicUser.Username, th.BasicUser2.Email))
+	require.Error(t, th.RunCommand(t, "user", "email", th.BasicUser.Username, th.BasicUser2.Email))
 
 }

cmd/mattermost/commands/version_test.go

@@ -8,5 +8,8 @@ import (
 )
 
 func TestVersion(t *testing.T) {
-	CheckCommand(t, "version")
+	th := Setup()
+	defer th.TearDown()
+
+	th.CheckCommand(t, "version")
 }

cmd/mattermost/commands/webhook_test.go

@@ -15,10 +15,18 @@ import (
 )
 
 func TestListWebhooks(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 	adminClient := th.SystemAdminClient
 
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	config.ServiceSettings.EnableIncomingWebhooks = true
+	config.ServiceSettings.EnableOutgoingWebhooks = true
+	config.ServiceSettings.EnablePostUsernameOverride = true
+	config.ServiceSettings.EnablePostIconOverride = true
+	th.SetConfig(config)
+
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableIncomingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableOutgoingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnablePostUsernameOverride = true })
@@ -41,7 +49,7 @@ func TestListWebhooks(t *testing.T) {
 	_, resp = adminClient.CreateOutgoingWebhook(outHook)
 	api4.CheckNoError(t, resp)
 
-	output := CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
+	output := th.CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
 
 	if !strings.Contains(string(output), dispName) {
 		t.Fatal("should have incoming webhooks")
@@ -54,9 +62,17 @@ func TestListWebhooks(t *testing.T) {
 }
 
 func TestCreateIncomingWebhook(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	config.ServiceSettings.EnableIncomingWebhooks = true
+	config.ServiceSettings.EnableOutgoingWebhooks = true
+	config.ServiceSettings.EnablePostUsernameOverride = true
+	config.ServiceSettings.EnablePostIconOverride = true
+	th.SetConfig(config)
+
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableIncomingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableOutgoingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnablePostUsernameOverride = true })
@@ -70,16 +86,16 @@ func TestCreateIncomingWebhook(t *testing.T) {
 	th.RemovePermissionFromRole(model.PERMISSION_MANAGE_WEBHOOKS.Id, model.TEAM_USER_ROLE_ID)
 
 	// should fail because you need to specify valid channel
-	require.Error(t, RunCommand(t, "webhook", "create-incoming"))
+	require.Error(t, th.RunCommand(t, "webhook", "create-incoming"))
-	require.Error(t, RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicTeam.Name+":doesnotexist"))
+	require.Error(t, th.RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicTeam.Name+":doesnotexist"))
 
 	// should fail because you need to specify valid user
-	require.Error(t, RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id))
+	require.Error(t, th.RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id))
-	require.Error(t, RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id, "--user", "doesnotexist"))
+	require.Error(t, th.RunCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id, "--user", "doesnotexist"))
 
 	description := "myhookinc"
 	displayName := "myhookinc"
-	CheckCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id, "--user", th.BasicUser.Email, "--description", description, "--display-name", displayName)
+	th.CheckCommand(t, "webhook", "create-incoming", "--channel", th.BasicChannel.Id, "--user", th.BasicUser.Email, "--description", description, "--display-name", displayName)
 
 	webhooks, err := th.App.GetIncomingWebhooksPage(0, 1000)
 	if err != nil {
@@ -98,9 +114,17 @@ func TestCreateIncomingWebhook(t *testing.T) {
 }
 
 func TestModifyIncomingWebhook(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	config.ServiceSettings.EnableIncomingWebhooks = true
+	config.ServiceSettings.EnableOutgoingWebhooks = true
+	config.ServiceSettings.EnablePostUsernameOverride = true
+	config.ServiceSettings.EnablePostIconOverride = true
+	th.SetConfig(config)
+
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableIncomingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableOutgoingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnablePostUsernameOverride = true })
@@ -131,9 +155,9 @@ func TestModifyIncomingWebhook(t *testing.T) {
 	}()
 
 	// should fail because you need to specify valid incoming webhook
-	require.Error(t, RunCommand(t, "webhook", "modify-incoming", "doesnotexist"))
+	require.Error(t, th.RunCommand(t, "webhook", "modify-incoming", "doesnotexist"))
 	// should fail because you need to specify valid channel
-	require.Error(t, RunCommand(t, "webhook", "modify-incoming", oldHook.Id, "--channel", th.BasicTeam.Name+":doesnotexist"))
+	require.Error(t, th.RunCommand(t, "webhook", "modify-incoming", oldHook.Id, "--channel", th.BasicTeam.Name+":doesnotexist"))
 
 	modifiedDescription := "myhookincdesc2"
 	modifiedDisplayName := "myhookincname2"
@@ -141,7 +165,7 @@ func TestModifyIncomingWebhook(t *testing.T) {
 	modifiedChannelLocked := true
 	modifiedChannelId := th.BasicChannel2.Id
 
-	CheckCommand(t, "webhook", "modify-incoming", oldHook.Id, "--channel", modifiedChannelId, "--description", modifiedDescription, "--display-name", modifiedDisplayName, "--icon", modifiedIconUrl, "--lock-to-channel", strconv.FormatBool(modifiedChannelLocked))
+	th.CheckCommand(t, "webhook", "modify-incoming", oldHook.Id, "--channel", modifiedChannelId, "--description", modifiedDescription, "--display-name", modifiedDisplayName, "--icon", modifiedIconUrl, "--lock-to-channel", strconv.FormatBool(modifiedChannelLocked))
 
 	modifiedHook, err := th.App.GetIncomingWebhook(oldHook.Id)
 	if err != nil {
@@ -153,9 +177,17 @@ func TestModifyIncomingWebhook(t *testing.T) {
 }
 
 func TestCreateOutgoingWebhook(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	config.ServiceSettings.EnableIncomingWebhooks = true
+	config.ServiceSettings.EnableOutgoingWebhooks = true
+	config.ServiceSettings.EnablePostUsernameOverride = true
+	config.ServiceSettings.EnablePostIconOverride = true
+	th.SetConfig(config)
+
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableIncomingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableOutgoingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnablePostUsernameOverride = true })
@@ -178,24 +210,24 @@ func TestCreateOutgoingWebhook(t *testing.T) {
 	callbackURL2 := "http://localhost:8000/my-webhook-handler2"
 
 	// should fail because team is not specified
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
 
 	// should fail because user is not specified
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2))
 
 	// should fail because display name is not specified
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--team", team, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--team", team, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
 
 	// should fail because trigger words are not specified
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
 
 	// should fail because callback URLs are not specified
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--user", user))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--team", team, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--user", user))
 
 	// should fail because outgoing webhooks cannot be made for private channels
-	require.Error(t, RunCommand(t, "webhook", "create-outgoing", "--team", team, "--channel", th.BasicPrivateChannel.Id, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
+	require.Error(t, th.RunCommand(t, "webhook", "create-outgoing", "--team", team, "--channel", th.BasicPrivateChannel.Id, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user))
 
-	CheckCommand(t, "webhook", "create-outgoing", "--team", team, "--channel", th.BasicChannel.Id, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user)
+	th.CheckCommand(t, "webhook", "create-outgoing", "--team", team, "--channel", th.BasicChannel.Id, "--display-name", displayName, "--trigger-word", triggerWord1, "--trigger-word", triggerWord2, "--url", callbackURL1, "--url", callbackURL2, "--user", user)
 
 	webhooks, err := th.App.GetOutgoingWebhooksPage(0, 1000)
 	if err != nil {
@@ -214,10 +246,18 @@ func TestCreateOutgoingWebhook(t *testing.T) {
 }
 
 func TestDeleteWebhooks(t *testing.T) {
-	th := api4.Setup().InitBasic()
+	th := Setup().InitBasic()
 	defer th.TearDown()
 	adminClient := th.SystemAdminClient
 
+	config := th.Config()
+	*config.ServiceSettings.EnableCommands = true
+	config.ServiceSettings.EnableIncomingWebhooks = true
+	config.ServiceSettings.EnableOutgoingWebhooks = true
+	config.ServiceSettings.EnablePostUsernameOverride = true
+	config.ServiceSettings.EnablePostIconOverride = true
+	th.SetConfig(config)
+
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableIncomingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnableOutgoingWebhooks = true })
 	th.App.UpdateConfig(func(cfg *model.Config) { cfg.ServiceSettings.EnablePostUsernameOverride = true })
@@ -240,7 +280,7 @@ func TestDeleteWebhooks(t *testing.T) {
 	outgoingHook, resp := adminClient.CreateOutgoingWebhook(outHookStruct)
 	api4.CheckNoError(t, resp)
 
-	hooksBeforeDeletion := CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
+	hooksBeforeDeletion := th.CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
 
 	if !strings.Contains(string(hooksBeforeDeletion), dispName) {
 		t.Fatal("Should have incoming webhooks")
@@ -250,10 +290,10 @@ func TestDeleteWebhooks(t *testing.T) {
 		t.Fatal("Should have outgoing webhooks")
 	}
 
-	CheckCommand(t, "webhook", "delete", incomingHook.Id)
+	th.CheckCommand(t, "webhook", "delete", incomingHook.Id)
-	CheckCommand(t, "webhook", "delete", outgoingHook.Id)
+	th.CheckCommand(t, "webhook", "delete", outgoingHook.Id)
 
-	hooksAfterDeletion := CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
+	hooksAfterDeletion := th.CheckCommand(t, "webhook", "list", th.BasicTeam.Name)
 
 	if strings.Contains(string(hooksAfterDeletion), dispName) {
 		t.Fatal("Should not have incoming webhooks")

migrations/helper_test.go

@@ -10,12 +10,8 @@ import (
 	"time"
 
 	"github.com/mattermost/mattermost-server/app"
-	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
-	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 )
 
@@ -34,37 +30,8 @@ type TestHelper struct {
 	tempWorkspace  string
 }
 
-type persistentTestStore struct {
-	store.Store
-}
-
-func (*persistentTestStore) Close() {}
-
-var testStoreContainer *storetest.RunningContainer
-var testStore *persistentTestStore
-var testStoreSqlSupplier *sqlstore.SqlSupplier
-var testClusterInterface *FakeClusterInterface
-
-// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
-// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
-func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
-	testClusterInterface = &FakeClusterInterface{}
-	testStoreContainer = container
-	testStoreSqlSupplier = sqlstore.NewSqlSupplier(*settings, nil)
-	testStore = &persistentTestStore{store.NewLayeredStore(testStoreSqlSupplier, nil, testClusterInterface)}
-}
-
-func StopTestStore() {
-	if testStoreContainer != nil {
-		testStoreContainer.Stop()
-		testStoreContainer = nil
-	}
-}
-
 func setupTestHelper(enterprise bool) *TestHelper {
-	if testStore != nil {
+	mainHelper.Store.DropAllTables()
-		testStore.DropAllTables()
-	}
 
 	permConfig, err := os.Open(utils.FindConfigFile("config.json"))
 	if err != nil {
@@ -82,9 +49,7 @@ func setupTestHelper(enterprise bool) *TestHelper {
 	}
 
 	options := []app.Option{app.ConfigFile(tempConfig.Name()), app.DisableConfigWatch}
-	if testStore != nil {
+	options = append(options, app.StoreOverride(mainHelper.Store))
-		options = append(options, app.StoreOverride(testStore))
-	}
 
 	s, err := app.NewServer(options...)
 	if err != nil {
@@ -100,9 +65,8 @@ func setupTestHelper(enterprise bool) *TestHelper {
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.MaxUsersPerTeam = 50 })
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.RateLimitSettings.Enable = false })
 	prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
-	if testStore != nil {
+	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
-		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
+
-	}
 	serverErr := th.App.StartServer()
 	if serverErr != nil {
 		panic(serverErr)
@@ -295,7 +259,6 @@ func (me *TestHelper) TearDown() {
 	me.Server.Shutdown()
 	os.Remove(me.tempConfigPath)
 	if err := recover(); err != nil {
-		StopTestStore()
 		panic(err)
 	}
 	if me.tempWorkspace != "" {
@@ -304,13 +267,13 @@ func (me *TestHelper) TearDown() {
 }
 
 func (me *TestHelper) ResetRoleMigration() {
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {
 		panic(err)
 	}
 
-	testClusterInterface.sendClearRoleCacheMessage()
+	mainHelper.ClusterInterface.SendClearRoleCacheMessage()
 
-	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": app.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
+	if _, err := mainHelper.SqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": app.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
 		panic(err)
 	}
 }
@@ -330,36 +293,3 @@ func (me *TestHelper) DeleteAllJobsByTypeAndMigrationKey(jobType string, migrati
 		}
 	}
 }
-
-type FakeClusterInterface struct {
-	clusterMessageHandler einterfaces.ClusterMessageHandler
-}
-
-func (me *FakeClusterInterface) StartInterNodeCommunication() {}
-func (me *FakeClusterInterface) StopInterNodeCommunication()  {}
-func (me *FakeClusterInterface) RegisterClusterMessageHandler(event string, crm einterfaces.ClusterMessageHandler) {
-	me.clusterMessageHandler = crm
-}
-func (me *FakeClusterInterface) GetClusterId() string                             { return "" }
-func (me *FakeClusterInterface) IsLeader() bool                                   { return false }
-func (me *FakeClusterInterface) GetMyClusterInfo() *model.ClusterInfo             { return nil }
-func (me *FakeClusterInterface) GetClusterInfos() []*model.ClusterInfo            { return nil }
-func (me *FakeClusterInterface) SendClusterMessage(cluster *model.ClusterMessage) {}
-func (me *FakeClusterInterface) NotifyMsg(buf []byte)                             {}
-func (me *FakeClusterInterface) GetClusterStats() ([]*model.ClusterStats, *model.AppError) {
-	return nil, nil
-}
-func (me *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppError) {
-	return []string{}, nil
-}
-func (me *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
-	return nil
-}
-func (me *FakeClusterInterface) sendClearRoleCacheMessage() {
-	me.clusterMessageHandler(&model.ClusterMessage{
-		Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES,
-	})
-}
-func (me *FakeClusterInterface) GetPluginStatuses() (model.PluginStatuses, *model.AppError) {
-	return nil, nil
-}

migrations/main_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

migrations/migrations_test.go

@@ -4,56 +4,13 @@
 package migrations
 
 import (
-	"flag"
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store/storetest"
-	"github.com/mattermost/mattermost-server/utils"
 )
 
-func TestMain(m *testing.M) {
-	flag.Parse()
-
-	// Setup a global logger to catch tests logging outside of app context
-	// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
-	mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
-		EnableConsole: true,
-		ConsoleJson:   true,
-		ConsoleLevel:  "error",
-		EnableFile:    false,
-	}))
-
-	utils.TranslationsPreInit()
-
-	// In the case where a dev just wants to run a single test, it's faster to just use the default
-	// store.
-	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
-		mlog.Info("-test.run used, not creating temporary containers")
-		os.Exit(m.Run())
-	}
-
-	status := 0
-
-	container, settings, err := storetest.NewMySQLContainer()
-	if err != nil {
-		panic(err)
-	}
-
-	UseTestStore(container, settings)
-
-	defer func() {
-		StopTestStore()
-		os.Exit(status)
-	}()
-
-	status = m.Run()
-}
-
 func TestGetMigrationState(t *testing.T) {
 	th := Setup()
 	defer th.TearDown()

scripts/test.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+GO=$1
+GOFLAGS=$2
+PACKAGES=$3
+TESTS=$4
+TESTFLAGS=$5
+
+PACKAGES_COMMA=$(echo $PACKAGES | tr ' ' ',')
+
+echo "Packages to test: $PACKAGES"
+find . -name 'cprofile*.out' -exec sh -c 'rm "{}"' \;
+find . -type d -name data -not -path './vendor/*' | xargs rm -rf
+
+$GO test $GOFLAGS -run=$TESTS $TESTFLAGS -p 1 -v -timeout=2000s -covermode=count -coverpkg=$PACKAGES_COMMA -exec $DIR/test-xprog.sh $PACKAGES 2>&1 | tee output
+EXIT_STATUS=$?
+
+cat output | $GOPATH/bin/go-junit-report > report.xml
+rm output
+find . -name 'cprofile*.out' -exec sh -c 'tail -n +2 {} >> cover.out ; rm "{}"' \;
+rm -f config/*.crt
+rm -f config/*.key
+
+exit $EXIT_STATUS

testlib/cluster.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package testlib
+
+import (
+	"github.com/mattermost/mattermost-server/einterfaces"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+type FakeClusterInterface struct {
+	clusterMessageHandler einterfaces.ClusterMessageHandler
+}
+
+func (c *FakeClusterInterface) StartInterNodeCommunication() {}
+
+func (c *FakeClusterInterface) StopInterNodeCommunication() {}
+
+func (c *FakeClusterInterface) RegisterClusterMessageHandler(event string, crm einterfaces.ClusterMessageHandler) {
+	c.clusterMessageHandler = crm
+}
+
+func (c *FakeClusterInterface) GetClusterId() string { return "" }
+
+func (c *FakeClusterInterface) IsLeader() bool { return false }
+
+func (c *FakeClusterInterface) GetMyClusterInfo() *model.ClusterInfo { return nil }
+
+func (c *FakeClusterInterface) GetClusterInfos() []*model.ClusterInfo { return nil }
+
+func (c *FakeClusterInterface) SendClusterMessage(cluster *model.ClusterMessage) {}
+
+func (c *FakeClusterInterface) NotifyMsg(buf []byte) {}
+
+func (c *FakeClusterInterface) GetClusterStats() ([]*model.ClusterStats, *model.AppError) {
+	return nil, nil
+}
+
+func (c *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppError) {
+	return []string{}, nil
+}
+
+func (c *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
+	return nil
+}
+
+func (c *FakeClusterInterface) SendClearRoleCacheMessage() {
+	c.clusterMessageHandler(&model.ClusterMessage{
+		Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES,
+	})
+}
+
+func (c *FakeClusterInterface) GetPluginStatuses() (model.PluginStatuses, *model.AppError) {
+	return nil, nil
+}

testlib/doc.go

@@ -0,0 +1,5 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+// testlib exposes helper methods for running unit tests against a containerized test store.
+package testlib

testlib/helper.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package testlib
+
+import (
+	"os"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/mlog"
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/store"
+	"github.com/mattermost/mattermost-server/store/sqlstore"
+	"github.com/mattermost/mattermost-server/store/storetest"
+	"github.com/mattermost/mattermost-server/utils"
+)
+
+type MainHelper struct {
+	Settings         *model.SqlSettings
+	Store            store.Store
+	SqlSupplier      *sqlstore.SqlSupplier
+	ClusterInterface *FakeClusterInterface
+
+	container *storetest.RunningContainer
+	status    int
+}
+
+func NewMainHelper() *MainHelper {
+	// Setup a global logger to catch tests logging outside of app context
+	// The global logger will be stomped by apps initalizing but that's fine for testing.
+	// Ideally this won't happen.
+	mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
+		EnableConsole: true,
+		ConsoleJson:   true,
+		ConsoleLevel:  "error",
+		EnableFile:    false,
+	}))
+
+	utils.TranslationsPreInit()
+
+	container, settings, err := storetest.NewMySQLContainer()
+	if err != nil {
+		panic("failed to start mysql container: " + err.Error())
+	}
+
+	testClusterInterface := &FakeClusterInterface{}
+	testStoreSqlSupplier := sqlstore.NewSqlSupplier(*settings, nil)
+	testStore := &TestStore{store.NewLayeredStore(testStoreSqlSupplier, nil, testClusterInterface)}
+
+	return &MainHelper{
+		Settings:         settings,
+		Store:            testStore,
+		SqlSupplier:      testStoreSqlSupplier,
+		ClusterInterface: testClusterInterface,
+		container:        container,
+	}
+}
+
+func (h *MainHelper) Main(m *testing.M) {
+	h.status = m.Run()
+}
+
+func (h *MainHelper) Close() error {
+	h.container.Stop()
+	h.container = nil
+
+	os.Exit(h.status)
+
+	return nil
+}

testlib/store.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package testlib
+
+import (
+	"github.com/mattermost/mattermost-server/store"
+)
+
+type TestStore struct {
+	store.Store
+}
+
+func (*TestStore) Close() {
+}

tests/test-config.json

@@ -1,401 +0,0 @@
-{
-    "ServiceSettings": {
-        "SiteURL": "",
-        "WebsocketURL": "",
-        "LicenseFileLocation": "",
-        "ListenAddress": ":8065",
-        "ConnectionSecurity": "",
-        "TLSCertFile": "",
-        "TLSKeyFile": "",
-        "UseLetsEncrypt": false,
-        "LetsEncryptCertificateCacheFile": "./config/letsencrypt.cache",
-        "Forward80To443": false,
-        "ReadTimeout": 300,
-        "WriteTimeout": 300,
-        "MaximumLoginAttempts": 10,
-        "GoroutineHealthThreshold": -1,
-        "GoogleDeveloperKey": "",
-        "EnableOAuthServiceProvider": false,
-        "EnableIncomingWebhooks": true,
-        "EnableOutgoingWebhooks": true,
-        "EnableCommands": true,
-        "EnableOnlyAdminIntegrations": true,
-        "EnablePostUsernameOverride": false,
-        "EnablePostIconOverride": false,
-        "EnableLinkPreviews": false,
-        "EnableTesting": false,
-        "EnableDeveloper": false,
-        "EnableSecurityFixAlert": true,
-        "EnableInsecureOutgoingConnections": false,
-        "AllowedUntrustedInternalConnections": "",
-        "EnableMultifactorAuthentication": false,
-        "EnforceMultifactorAuthentication": false,
-        "EnableUserAccessTokens": false,
-        "AllowCorsFrom": "",
-        "AllowCookiesForSubdomains": false,
-        "SessionLengthWebInDays": 30,
-        "SessionLengthMobileInDays": 30,
-        "SessionLengthSSOInDays": 30,
-        "SessionCacheInMinutes": 10,
-        "SessionIdleTimeoutInMinutes": 0,
-        "WebsocketSecurePort": 443,
-        "WebsocketPort": 80,
-        "WebserverMode": "gzip",
-        "EnableCustomEmoji": false,
-        "EnableEmojiPicker": true,
-        "RestrictCustomEmojiCreation": "all",
-        "RestrictPostDelete": "all",
-        "AllowEditPost": "always",
-        "PostEditTimeLimit": -1,
-        "TimeBetweenUserTypingUpdatesMilliseconds": 5000,
-        "EnablePostSearch": true,
-        "EnableUserTypingMessages": true,
-        "EnableChannelViewedMessages": true,
-        "EnableUserStatuses": true,
-        "ExperimentalEnableAuthenticationTransfer": true,
-        "ClusterLogTimeoutMilliseconds": 2000,
-        "CloseUnusedDirectMessages": false,
-        "EnablePreviewFeatures": true,
-        "EnableTutorial": true,
-        "ExperimentalEnableDefaultChannelLeaveJoinMessages": true,
-        "ExperimentalGroupUnreadChannels": "disabled",
-        "ImageProxyType": "",
-        "ImageProxyURL": "",
-        "ImageProxyOptions": "",
-        "EnableAPITeamDeletion": false,
-        "ExperimentalEnableHardenedMode": false
-    },
-    "TeamSettings": {
-        "SiteName": "Mattermost",
-        "MaxUsersPerTeam": 50,
-        "EnableTeamCreation": true,
-        "EnableUserCreation": true,
-        "EnableOpenServer": false,
-        "EnableUserDeactivation": false,
-        "RestrictCreationToDomains": "",
-        "EnableCustomBrand": false,
-        "CustomBrandText": "",
-        "CustomDescriptionText": "",
-        "RestrictDirectMessage": "any",
-        "RestrictTeamInvite": "all",
-        "RestrictPublicChannelManagement": "all",
-        "RestrictPrivateChannelManagement": "all",
-        "RestrictPublicChannelCreation": "all",
-        "RestrictPrivateChannelCreation": "all",
-        "RestrictPublicChannelDeletion": "all",
-        "RestrictPrivateChannelDeletion": "all",
-        "RestrictPrivateChannelManageMembers": "all",
-        "EnableXToLeaveChannelsFromLHS": false,
-        "UserStatusAwayTimeout": 300,
-        "MaxChannelsPerTeam": 2000,
-        "MaxNotificationsPerChannel": 1000,
-        "EnableConfirmNotificationsToChannel": true,
-        "TeammateNameDisplay": "username",
-        "ExperimentalEnableAutomaticReplies": false,
-        "ExperimentalHideTownSquareinLHS": false,
-        "ExperimentalTownSquareIsReadOnly": false,
-        "ExperimentalPrimaryTeam": ""
-    },
-    "ClientRequirements": {
-        "AndroidLatestVersion": "",
-        "AndroidMinVersion": "",
-        "DesktopLatestVersion": "",
-        "DesktopMinVersion": "",
-        "IosLatestVersion": "",
-        "IosMinVersion": ""
-    },
-    "SqlSettings": {
-        "DriverName": "mysql",
-        "DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s",
-        "DataSourceReplicas": [],
-        "DataSourceSearchReplicas": [],
-        "MaxIdleConns": 20,
-        "MaxOpenConns": 300,
-        "Trace": false,
-        "AtRestEncryptKey": "jdh9iergmse3w9mt53snasugmmi9r6it",
-        "QueryTimeout": 30
-    },
-    "LogSettings": {
-        "EnableConsole": true,
-        "ConsoleLevel": "DEBUG",
-        "ConsoleJson": true,
-        "EnableFile": true,
-        "FileLevel": "INFO",
-        "FileJson": true,
-        "FileLocation": "",
-        "EnableWebhookDebugging": true,
-        "EnableDiagnostics": true
-    },
-    "PasswordSettings": {
-        "MinimumLength": 5,
-        "Lowercase": false,
-        "Number": false,
-        "Uppercase": false,
-        "Symbol": false
-    },
-    "FileSettings": {
-        "EnableFileAttachments": true,
-        "EnableMobileUpload": true,
-        "EnableMobileDownload": true,
-        "MaxFileSize": 52428800,
-        "DriverName": "local",
-        "Directory": "./data/",
-        "EnablePublicLink": false,
-        "PublicLinkSalt": "3xh7ztscuezjp1jkdjybtejrtw59xjt1",
-        "InitialFont": "luximbi.ttf",
-        "AmazonS3AccessKeyId": "",
-        "AmazonS3SecretAccessKey": "",
-        "AmazonS3Bucket": "",
-        "AmazonS3Region": "",
-        "AmazonS3Endpoint": "s3.amazonaws.com",
-        "AmazonS3SSL": true,
-        "AmazonS3SignV2": false,
-        "AmazonS3SSE": false,
-        "AmazonS3Trace": false
-    },
-    "EmailSettings": {
-        "EnableSignUpWithEmail": true,
-        "EnableSignInWithEmail": true,
-        "EnableSignInWithUsername": true,
-        "SendEmailNotifications": true,
-        "UseChannelInEmailNotifications": false,
-        "RequireEmailVerification": false,
-        "FeedbackName": "",
-        "FeedbackEmail": "test@example.com",
-        "FeedbackOrganization": "",
-        "EnableSMTPAuth": false,
-        "SMTPUsername": "",
-        "SMTPPassword": "",
-        "SMTPServer": "dockerhost",
-        "SMTPPort": "2500",
-        "ConnectionSecurity": "",
-        "InviteSalt": "n3mceqsek4j5ichs5hw9sudwx3cfbtqa",
-        "SendPushNotifications": false,
-        "PushNotificationServer": "",
-        "PushNotificationContents": "generic",
-        "EnableEmailBatching": false,
-        "EmailBatchingBufferSize": 256,
-        "EmailBatchingInterval": 30,
-        "EnablePreviewModeBanner": true,
-        "SkipServerCertificateVerification": false,
-        "EmailNotificationContentsType": "full",
-        "LoginButtonColor": "",
-        "LoginButtonBorderColor": "",
-        "LoginButtonTextColor": ""
-    },
-    "RateLimitSettings": {
-        "Enable": false,
-        "PerSec": 10,
-        "MaxBurst": 100,
-        "MemoryStoreSize": 10000,
-        "VaryByRemoteAddr": true,
-        "VaryByUser": false,
-        "VaryByHeader": ""
-    },
-    "PrivacySettings": {
-        "ShowEmailAddress": true,
-        "ShowFullName": true
-    },
-    "SupportSettings": {
-        "TermsOfServiceLink": "https://about.mattermost.com/default-terms/",
-        "PrivacyPolicyLink": "https://about.mattermost.com/default-privacy-policy/",
-        "AboutLink": "https://about.mattermost.com/default-about/",
-        "HelpLink": "https://about.mattermost.com/default-help/",
-        "ReportAProblemLink": "https://about.mattermost.com/default-report-a-problem/",
-        "SupportEmail": "feedback@mattermost.com"
-    },
-    "AnnouncementSettings": {
-        "EnableBanner": false,
-        "BannerText": "",
-        "BannerColor": "#f2a93b",
-        "BannerTextColor": "#333333",
-        "AllowBannerDismissal": true
-    },
-    "ThemeSettings": {
-        "EnableThemeSelection": true,
-        "DefaultTheme": "default",
-        "AllowCustomThemes": true,
-        "AllowedThemes": []
-    },
-    "GitLabSettings": {
-        "Enable": false,
-        "Secret": "",
-        "Id": "",
-        "Scope": "",
-        "AuthEndpoint": "",
-        "TokenEndpoint": "",
-        "UserApiEndpoint": ""
-    },
-    "GoogleSettings": {
-        "Enable": false,
-        "Secret": "",
-        "Id": "",
-        "Scope": "profile email",
-        "AuthEndpoint": "https://accounts.google.com/o/oauth2/v2/auth",
-        "TokenEndpoint": "https://www.googleapis.com/oauth2/v4/token",
-        "UserApiEndpoint": "https://www.googleapis.com/plus/v1/people/me"
-    },
-    "Office365Settings": {
-        "Enable": false,
-        "Secret": "",
-        "Id": "",
-        "Scope": "User.Read",
-        "AuthEndpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
-        "TokenEndpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
-        "UserApiEndpoint": "https://graph.microsoft.com/v1.0/me"
-    },
-    "LdapSettings": {
-        "Enable": false,
-        "EnableSync": false,
-        "LdapServer": "",
-        "LdapPort": 389,
-        "ConnectionSecurity": "",
-        "BaseDN": "",
-        "BindUsername": "",
-        "BindPassword": "",
-        "UserFilter": "",
-        "FirstNameAttribute": "",
-        "LastNameAttribute": "",
-        "EmailAttribute": "",
-        "UsernameAttribute": "",
-        "NicknameAttribute": "",
-        "IdAttribute": "",
-        "PositionAttribute": "",
-        "LoginIdAttribute": "",
-        "SyncIntervalMinutes": 60,
-        "SkipCertificateVerification": false,
-        "QueryTimeout": 60,
-        "MaxPageSize": 0,
-        "LoginFieldName": "",
-        "LoginButtonColor": "",
-        "LoginButtonBorderColor": "",
-        "LoginButtonTextColor": ""
-    },
-    "ComplianceSettings": {
-        "Enable": false,
-        "Directory": "./data/",
-        "EnableDaily": false
-    },
-    "LocalizationSettings": {
-        "DefaultServerLocale": "en",
-        "DefaultClientLocale": "en",
-        "AvailableLocales": ""
-    },
-    "SamlSettings": {
-        "Enable": false,
-        "EnableSyncWithLdap": false,
-        "Verify": true,
-        "Encrypt": true,
-        "IdpUrl": "",
-        "IdpDescriptorUrl": "",
-        "AssertionConsumerServiceURL": "",
-        "ScopingIDPProviderId": "",
-        "ScopingIDPName": "",
-        "IdpCertificateFile": "",
-        "PublicCertificateFile": "",
-        "PrivateKeyFile": "",
-        "FirstNameAttribute": "",
-        "LastNameAttribute": "",
-        "EmailAttribute": "",
-        "UsernameAttribute": "",
-        "NicknameAttribute": "",
-        "LocaleAttribute": "",
-        "PositionAttribute": "",
-        "LoginButtonText": "With SAML",
-        "LoginButtonColor": "",
-        "LoginButtonBorderColor": "",
-        "LoginButtonTextColor": ""
-    },
-    "NativeAppSettings": {
-        "AppDownloadLink": "https://about.mattermost.com/downloads/",
-        "AndroidAppDownloadLink": "https://about.mattermost.com/mattermost-android-app/",
-        "IosAppDownloadLink": "https://about.mattermost.com/mattermost-ios-app/"
-    },
-    "ClusterSettings": {
-        "Enable": false,
-        "ClusterName": "",
-        "OverrideHostname": "",
-        "UseIpAddress": true,
-        "UseExperimentalGossip": false,
-        "ReadOnlyConfig": true,
-        "GossipPort": 8074,
-        "StreamingPort": 8075,
-        "MaxIdleConns": 100,
-        "MaxIdleConnsPerHost": 128,
-        "IdleConnTimeoutMilliseconds": 90000
-    },
-    "MetricsSettings": {
-        "Enable": false,
-        "BlockProfileRate": 0,
-        "ListenAddress": ":8067"
-    },
-    "ExperimentalSettings": {
-        "ClientSideCertEnable": false,
-        "ClientSideCertCheck": "secondary"
-    },
-    "AnalyticsSettings": {
-        "MaxUsersForStatistics": 2500
-    },
-    "ElasticsearchSettings": {
-        "ConnectionUrl": "http://dockerhost:9200",
-        "Username": "elastic",
-        "Password": "changeme",
-        "EnableIndexing": false,
-        "EnableSearching": false,
-        "Sniff": true,
-        "PostIndexReplicas": 1,
-        "PostIndexShards": 1,
-        "AggregatePostsAfterDays": 365,
-        "PostsAggregatorJobStartTime": "03:00",
-        "IndexPrefix": "",
-        "LiveIndexingBatchSize": 1,
-        "BulkIndexingTimeWindowSeconds": 3600,
-        "RequestTimeoutSeconds": 30
-    },
-    "DataRetentionSettings": {
-        "EnableMessageDeletion": false,
-        "EnableFileDeletion": false,
-        "MessageRetentionDays": 365,
-        "FileRetentionDays": 365,
-        "DeletionJobStartTime": "02:00"
-    },
-    "MessageExportSettings": {
-        "EnableExport": false,
-        "ExportFormat": "actiance",
-        "DailyRunTime": "01:00",
-        "ExportFromTimestamp": 0,
-        "BatchSize": 10000,
-        "GlobalRelaySettings": {
-            "CustomerType": "A9",
-            "SmtpUsername": "",
-            "SmtpPassword": "",
-            "EmailAddress": ""
-        }
-    },
-    "JobSettings": {
-        "RunJobs": true,
-        "RunScheduler": true
-    },
-    "PluginSettings": {
-        "Enable": true,
-        "EnableUploads": true,
-        "Directory": "./test-plugins",
-        "ClientDirectory": "./test-client-plugins",
-        "Plugins": {},
-        "PluginStates": {
-            "jira": {
-                "Enable": true
-            },
-            "testplugin": {
-                "Enable": false
-            }
-        }
-    },
-    "DisplaySettings": {
-        "CustomUrlSchemes": [],
-        "ExperimentalTimezone": false
-    },
-    "TimezoneSettings": {
-        "SupportedTimezonesPath": "timezones.json"
-    }
-}

web/handlers_test.go

@@ -18,7 +18,7 @@ func handlerForHTTPErrors(c *Context, w http.ResponseWriter, r *http.Request) {
 }
 
 func TestHandlerServeHTTPErrors(t *testing.T) {
-	s, err := app.NewServer(app.StoreOverride(testStore), app.DisableConfigWatch)
+	s, err := app.NewServer(app.StoreOverride(mainHelper.Store), app.DisableConfigWatch)
 	defer s.Shutdown()
 
 	web := New(s, s.AppOptions, s.Router)
@@ -61,7 +61,7 @@ func handlerForHTTPSecureTransport(c *Context, w http.ResponseWriter, r *http.Re
 }
 
 func TestHandlerServeHTTPSecureTransport(t *testing.T) {
-	s, err := app.NewServer(app.StoreOverride(testStore), app.DisableConfigWatch)
+	s, err := app.NewServer(app.StoreOverride(mainHelper.Store), app.DisableConfigWatch)
 	defer s.Shutdown()
 
 	a := s.FakeApp()

web/main_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package web
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

web/web_test.go

@@ -5,37 +5,15 @@ package web
 
 import (
 	"fmt"
-	"os"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/app"
-	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
-	"github.com/mattermost/mattermost-server/store/storetest"
-	"github.com/mattermost/mattermost-server/utils"
 )
 
 var ApiClient *model.Client4
 var URL string
 
-type persistentTestStore struct {
-	store.Store
-}
-
-func (*persistentTestStore) Close() {}
-
-var testStoreContainer *storetest.RunningContainer
-var testStore *persistentTestStore
-
-func StopTestStore() {
-	if testStoreContainer != nil {
-		testStoreContainer.Stop()
-		testStoreContainer = nil
-	}
-}
-
 type TestHelper struct {
 	App    *app.App
 	Server *app.Server
@@ -48,7 +26,9 @@ type TestHelper struct {
 }
 
 func Setup() *TestHelper {
-	s, err := app.NewServer(app.StoreOverride(testStore), app.DisableConfigWatch)
+	mainHelper.Store.DropAllTables()
+
+	s, err := app.NewServer(app.StoreOverride(mainHelper.Store), app.DisableConfigWatch)
 	if err != nil {
 		panic(err)
 	}
@@ -103,7 +83,6 @@ func (th *TestHelper) InitBasic() *TestHelper {
 func (th *TestHelper) TearDown() {
 	th.Server.Shutdown()
 	if err := recover(); err != nil {
-		StopTestStore()
 		panic(err)
 	}
 }
@@ -125,37 +104,6 @@ func TestStatic(t *testing.T) {
 }
 */
 
-func TestMain(m *testing.M) {
-	// Setup a global logger to catch tests logging outside of app context
-	// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
-	mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
-		EnableConsole: true,
-		ConsoleJson:   true,
-		ConsoleLevel:  "error",
-		EnableFile:    false,
-	}))
-
-	utils.TranslationsPreInit()
-
-	status := 0
-
-	container, settings, err := storetest.NewPostgreSQLContainer()
-	if err != nil {
-		panic(err)
-	}
-
-	testStoreContainer = container
-	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
-
-	defer func() {
-		StopTestStore()
-		os.Exit(status)
-	}()
-
-	status = m.Run()
-
-}
-
 func TestCheckClientCompatability(t *testing.T) {
 	//Browser Name, UA String, expected result (if the browser should fail the test false and if it should pass the true)
 	type uaTest struct {