unittest using externally managed database (#9400)

* MM-12083: unittest using externally managed database * cherry-pick Makefile changes from @cpanato * Jenkins changes (#9915) * add docker compose * udpate * when using minio dont need to set the region * update * add wait for it script * using old minio * add new jenkins file * update makefile * add dockerfile * rename the docker-compose proj to avoid colision (#9917) * rename the docker-compose proj to avoid colision * enable debug * enable debug to double checkt the branchs and fix docker-compose name (#9919) * add ee hash to check (#9920) * fix name (#9921) * update jenkins file to push from branch and prs * if a new push comes in stop the running build * split mysql and postgres variables * add script to run jenkins-like env in local dev env * update docker-compose project name to use uuid to make it more randon * fix DCNAME definition * update elasticsearch docker image * revert test * tidy up stages, and wait for mysql differently * update docker image and add check for postgres * checking if is ready * update docker compose to have a wait for deps * add readme and rename dockerfile * fix -unittest setup * using mm docker image * restore parallel unit tests at the package level Spin up a dedicated database for each package under test to avoid races in accessing the same tables. Simplify the interface for configuring the test database to just a DSN instead of multiple exports for each field. * try to work around root mysql access in CI * update local-test-env.sh too * MYSQL_ROOT_HOST: % * fix missing quotes * setting some memory limits for mysql * revert memory docker compose does not support * fix env name for postgres * expose errors in app/export_test.go * fix test label, better error checking on teardown * increase query timeout for tests * fix export_test * update local dev script * add configurable mysql root passwd
2018-12-10 11:58:09 -05:00
--- a/store/storetest/settings.go
+++ b/store/storetest/settings.go
@@ -0,0 +1,233 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package storetest
+
+import (
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+
+	"database/sql"
+
+	"github.com/go-sql-driver/mysql"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/model"
+)
+
+const (
+	defaultMysqlDSN      = "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s"
+	defaultPostgresqlDSN = "postgres://mmuser:mostest@dockerhost:5432/mattermost_test?sslmode=disable&connect_timeout=10"
+	defaultMysqlRootPWD  = "passwd"
+)
+
+func getEnv(name, defaultValue string) string {
+	if value := os.Getenv(name); value != "" {
+		return value
+	} else {
+		return defaultValue
+	}
+}
+
+func log(message string) {
+	verbose := false
+	if verboseFlag := flag.Lookup("test.v"); verboseFlag != nil {
+		verbose = verboseFlag.Value.String() == "true"
+	}
+	if verboseFlag := flag.Lookup("v"); verboseFlag != nil {
+		verbose = verboseFlag.Value.String() == "true"
+	}
+
+	if verbose {
+		fmt.Println(message)
+	}
+}
+
+// MySQLSettings returns the database settings to connect to the MySQL unittesting database.
+// The database name is generated randomly and must be created before use.
+func MySQLSettings() *model.SqlSettings {
+	dsn := getEnv("TEST_DATABASE_MYSQL_DSN", defaultMysqlDSN)
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	cfg.DBName = "db" + model.NewId()
+
+	return databaseSettings("mysql", cfg.FormatDSN())
+}
+
+// PostgresSQLSettings returns the database settings to connect to the PostgreSQL unittesting database.
+// The database name is generated randomly and must be created before use.
+func PostgreSQLSettings() *model.SqlSettings {
+	dsn := getEnv("TEST_DATABASE_POSTGRESQL_DSN", defaultPostgresqlDSN)
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	// Generate a random database name
+	dsnUrl.Path = "db" + model.NewId()
+
+	return databaseSettings("postgres", dsnUrl.String())
+}
+
+func mySQLRootDSN(dsn string) string {
+	rootPwd := getEnv("TEST_DATABASE_MYSQL_ROOT_PASSWD", defaultMysqlRootPWD)
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	cfg.User = "root"
+	cfg.Passwd = rootPwd
+	cfg.DBName = "mysql"
+
+	return cfg.FormatDSN()
+}
+
+func postgreSQLRootDSN(dsn string) string {
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	// // Assume the unittesting database has the same password.
+	// password := ""
+	// if dsnUrl.User != nil {
+	// 	password, _ = dsnUrl.User.Password()
+	// }
+
+	// dsnUrl.User = url.UserPassword("", password)
+	dsnUrl.Path = "postgres"
+
+	return dsnUrl.String()
+}
+
+func mySQLDSNDatabase(dsn string) string {
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	return cfg.DBName
+}
+
+func postgreSQLDSNDatabase(dsn string) string {
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	return path.Base(dsnUrl.Path)
+}
+
+func databaseSettings(driver, dataSource string) *model.SqlSettings {
+	settings := &model.SqlSettings{
+		DriverName:                  &driver,
+		DataSource:                  &dataSource,
+		DataSourceReplicas:          []string{},
+		DataSourceSearchReplicas:    []string{},
+		MaxIdleConns:                new(int),
+		ConnMaxLifetimeMilliseconds: new(int),
+		MaxOpenConns:                new(int),
+		Trace:                       false,
+		AtRestEncryptKey:            model.NewRandomString(32),
+		QueryTimeout:                new(int),
+	}
+	*settings.MaxIdleConns = 10
+	*settings.ConnMaxLifetimeMilliseconds = 3600000
+	*settings.MaxOpenConns = 100
+	*settings.QueryTimeout = 60
+
+	return settings
+}
+
+// execAsRoot executes the given sql as root against the testing database
+func execAsRoot(settings *model.SqlSettings, sqlCommand string) error {
+	var dsn string
+	var driver = *settings.DriverName
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		dsn = mySQLRootDSN(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		dsn = postgreSQLRootDSN(*settings.DataSource)
+	default:
+		return fmt.Errorf("unsupported driver %s", driver)
+	}
+
+	db, err := sql.Open(driver, dsn)
+	if err != nil {
+		return errors.Wrapf(err, "failed to connect to %s database as root", driver)
+	}
+	defer db.Close()
+	if _, err = db.Exec(sqlCommand); err != nil {
+		return errors.Wrapf(err, "failed to execute `%s` against %s database as root", sqlCommand, driver)
+	}
+
+	return nil
+}
+
+// MakeSqlSettings creates a randomly named database and returns the corresponding sql settings
+func MakeSqlSettings(driver string) *model.SqlSettings {
+	var settings *model.SqlSettings
+	var dbName string
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		settings = MySQLSettings()
+		dbName = mySQLDSNDatabase(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		settings = PostgreSQLSettings()
+		dbName = postgreSQLDSNDatabase(*settings.DataSource)
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	if err := execAsRoot(settings, "CREATE DATABASE "+dbName); err != nil {
+		panic("failed to create temporary database " + dbName + ": " + err.Error())
+	}
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON "+dbName+".* TO 'mmuser'"); err != nil {
+			panic("failed to grant mmuser permission to " + dbName + ":" + err.Error())
+		}
+	case model.DATABASE_DRIVER_POSTGRES:
+		if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON DATABASE \""+dbName+"\" TO mmuser"); err != nil {
+			panic("failed to grant mmuser permission to " + dbName + ":" + err.Error())
+		}
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	log("Created temporary database " + dbName)
+
+	return settings
+}
+
+func CleanupSqlSettings(settings *model.SqlSettings) {
+	var driver = *settings.DriverName
+	var dbName string
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		dbName = mySQLDSNDatabase(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		dbName = postgreSQLDSNDatabase(*settings.DataSource)
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	if err := execAsRoot(settings, "DROP DATABASE "+dbName); err != nil {
+		panic("failed to drop temporary database " + dbName + ": " + err.Error())
+	}
+
+	log("Dropped temporary database " + dbName)
+}