Dockerized build updated tests (#9943)

* testlib: introduce and leverage

This doesn't yet factor out the individual test helpers: many packages
still rely on `api4` directly to do this, but now wire up the test store
setup through this package. `app` and `store`, in particular, don't use
`testlib` because of circular dependencies at the moment.

* cmd: command_test.go: use api4 testlib

* cmd: plugin_test.go: remove dependence on test-config.json

* cmd: config_test.go use configured database settings

* ensure test-(te|ee) exit with status code

* test-server: run all tests, deprecating test-te/test-ee

* cmd/mattermost/commands: fix unit tests

Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously.

* wrap run/check into a test helper

It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown.

* additional TestConfigGet granularity

* customized config path to avoid default location

* be explicit if the storetest initialization fails

* generate safe coverprofile names in the presence of subtests

* additional TestConfigShow granularity

* fix permission_test.go typo

* fix webhook tests

* actually flag.Parse() to skip database setup on os.Execed tests

* fix recent regression in #9962, not caught by unit tests

cmd/mattermost/commands/cmdtestlib.go

@@ -4,8 +4,11 @@
 package commands
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,33 +16,159 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/api4"
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/testlib"
 )
 
 var coverprofileCounters map[string]int = make(map[string]int)
 
-func execArgs(t *testing.T, args []string) []string {
-	ret := []string{"-test.run", "ExecCommand"}
+var mainHelper *testlib.MainHelper
+
+type testHelper struct {
+	*api4.TestHelper
+
+	config            *model.Config
+	tempDir           string
+	configFilePath    string
+	disableAutoConfig bool
+}
+
+// Setup creates an instance of testHelper.
+func Setup() *testHelper {
+	dir, err := ioutil.TempDir("", "testHelper")
+	if err != nil {
+		panic("failed to create temporary directory: " + err.Error())
+	}
+
+	api4TestHelper := api4.Setup()
+
+	testHelper := &testHelper{
+		TestHelper:     api4TestHelper,
+		tempDir:        dir,
+		configFilePath: filepath.Join(dir, "config-helper.json"),
+	}
+
+	config := &model.Config{}
+	config.SetDefaults()
+	testHelper.SetConfig(config)
+
+	return testHelper
+}
+
+// InitBasic simply proxies to api4.InitBasic, while still returning a testHelper.
+func (h *testHelper) InitBasic() *testHelper {
+	h.TestHelper.InitBasic()
+	return h
+}
+
+// TemporaryDirectory returns the temporary directory created for user by the test helper.
+func (h *testHelper) TemporaryDirectory() string {
+	return h.tempDir
+}
+
+// Config returns the configuration passed to a running command.
+func (h *testHelper) Config() *model.Config {
+	return h.config.Clone()
+}
+
+// ConfigPath returns the path to the temporary config file passed to a running command.
+func (h *testHelper) ConfigPath() string {
+	return h.configFilePath
+}
+
+// SetConfig replaces the configuration passed to a running command.
+func (h *testHelper) SetConfig(config *model.Config) {
+	config.SqlSettings = *mainHelper.Settings
+	h.config = config
+
+	if err := ioutil.WriteFile(h.configFilePath, []byte(config.ToJson()), 0600); err != nil {
+		panic("failed to write file " + h.configFilePath + ": " + err.Error())
+	}
+}
+
+// SetAutoConfig configures whether the --config flag is automatically passed to a running command.
+func (h *testHelper) SetAutoConfig(autoConfig bool) {
+	h.disableAutoConfig = !autoConfig
+}
+
+// TearDown cleans up temporary files and assets created during the life of the test helper.
+func (h *testHelper) TearDown() {
+	h.TestHelper.TearDown()
+	os.RemoveAll(h.tempDir)
+}
+
+func (h *testHelper) execArgs(t *testing.T, args []string) []string {
+	ret := []string{"-test.v", "-test.run", "ExecCommand"}
 	if coverprofile := flag.Lookup("test.coverprofile").Value.String(); coverprofile != "" {
 		dir := filepath.Dir(coverprofile)
 		base := filepath.Base(coverprofile)
 		baseParts := strings.SplitN(base, ".", 2)
-		coverprofileCounters[t.Name()] = coverprofileCounters[t.Name()] + 1
-		baseParts[0] = fmt.Sprintf("%v-%v-%v", baseParts[0], t.Name(), coverprofileCounters[t.Name()])
+		name := strings.Replace(t.Name(), "/", "_", -1)
+		coverprofileCounters[name] = coverprofileCounters[name] + 1
+		baseParts[0] = fmt.Sprintf("%v-%v-%v", baseParts[0], name, coverprofileCounters[name])
 		ret = append(ret, "-test.coverprofile", filepath.Join(dir, strings.Join(baseParts, ".")))
 	}
-	return append(append(ret, "--", "--disableconfigwatch"), args...)
+
+	ret = append(ret, "--", "--disableconfigwatch")
+
+	// Unless the test passes a `--config` of its own, create a temporary one from the default
+	// configuration with the current test database applied.
+	hasConfig := h.disableAutoConfig
+	for _, arg := range args {
+		if arg == "--config" {
+			hasConfig = true
+			break
+		}
+	}
+
+	if !hasConfig {
+		ret = append(ret, "--config", h.configFilePath)
+	}
+
+	ret = append(ret, args...)
+
+	return ret
 }
 
-func CheckCommand(t *testing.T, args ...string) string {
+// CheckCommand invokes the test binary, returning the output modified for assertion testing.
+func (h *testHelper) CheckCommand(t *testing.T, args ...string) string {
 	path, err := os.Executable()
 	require.NoError(t, err)
-	output, err := exec.Command(path, execArgs(t, args)...).CombinedOutput()
+	output, err := exec.Command(path, h.execArgs(t, args)...).CombinedOutput()
 	require.NoError(t, err, string(output))
 	return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(string(output)), "PASS"))
 }
 
-func RunCommand(t *testing.T, args ...string) error {
+// RunCommand invokes the test binary, returning only any error.
+func (h *testHelper) RunCommand(t *testing.T, args ...string) error {
 	path, err := os.Executable()
 	require.NoError(t, err)
-	return exec.Command(path, execArgs(t, args)...).Run()
+	return exec.Command(path, h.execArgs(t, args)...).Run()
+}
+
+// RunCommandWithOutput is a variant of RunCommand that returns the unmodified output and any error.
+func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string, error) {
+	path, err := os.Executable()
+	require.NoError(t, err)
+
+	cmd := exec.Command(path, h.execArgs(t, args)...)
+
+	var buf bytes.Buffer
+	reader, writer := io.Pipe()
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	done := make(chan bool)
+	go func() {
+		io.Copy(&buf, reader)
+		close(done)
+	}()
+
+	err = cmd.Run()
+	writer.Close()
+	<-done
+
+	return buf.String(), err
 }