unittest using externally managed database (#9400)

* MM-12083: unittest using externally managed database * cherry-pick Makefile changes from @cpanato * Jenkins changes (#9915) * add docker compose * udpate * when using minio dont need to set the region * update * add wait for it script * using old minio * add new jenkins file * update makefile * add dockerfile * rename the docker-compose proj to avoid colision (#9917) * rename the docker-compose proj to avoid colision * enable debug * enable debug to double checkt the branchs and fix docker-compose name (#9919) * add ee hash to check (#9920) * fix name (#9921) * update jenkins file to push from branch and prs * if a new push comes in stop the running build * split mysql and postgres variables * add script to run jenkins-like env in local dev env * update docker-compose project name to use uuid to make it more randon * fix DCNAME definition * update elasticsearch docker image * revert test * tidy up stages, and wait for mysql differently * update docker image and add check for postgres * checking if is ready * update docker compose to have a wait for deps * add readme and rename dockerfile * fix -unittest setup * using mm docker image * restore parallel unit tests at the package level Spin up a dedicated database for each package under test to avoid races in accessing the same tables. Simplify the interface for configuring the test database to just a DSN instead of multiple exports for each field. * try to work around root mysql access in CI * update local-test-env.sh too * MYSQL_ROOT_HOST: % * fix missing quotes * setting some memory limits for mysql * revert memory docker compose does not support * fix env name for postgres * expose errors in app/export_test.go * fix test label, better error checking on teardown * increase query timeout for tests * fix export_test * update local dev script * add configurable mysql root passwd
2018-12-10 11:58:09 -05:00
--- a/store/sqlstore/store_test.go
+++ b/store/sqlstore/store_test.go
@@ -15,23 +15,15 @@ import (
 	"github.com/mattermost/mattermost-server/utils"
 )

-var storeTypes = []*struct {
+type storeType struct {
 	Name        string
-	Func        func() (*storetest.RunningContainer, *model.SqlSettings, error)
-	Container   *storetest.RunningContainer
+	SqlSettings *model.SqlSettings
 	SqlSupplier *SqlSupplier
 	Store       store.Store
-}{
-	{
-		Name: "MySQL",
-		Func: storetest.NewMySQLContainer,
-	},
-	{
-		Name: "PostgreSQL",
-		Func: storetest.NewPostgreSQLContainer,
-	},
 }

+var storeTypes []*storeType
+
 func StoreTest(t *testing.T, f func(*testing.T, store.Store)) {
 	defer func() {
 		if err := recover(); err != nil {
@@ -59,6 +51,15 @@ func StoreTestWithSqlSupplier(t *testing.T, f func(*testing.T, store.Store, stor
 }

 func initStores() {
+	storeTypes = append(storeTypes, &storeType{
+		Name:        "MySQL",
+		SqlSettings: storetest.MakeSqlSettings(model.DATABASE_DRIVER_MYSQL),
+	})
+	storeTypes = append(storeTypes, &storeType{
+		Name:        "PostgreSQL",
+		SqlSettings: storetest.MakeSqlSettings(model.DATABASE_DRIVER_POSTGRES),
+	})
+
 	defer func() {
 		if err := recover(); err != nil {
 			tearDownStores()
@@ -66,29 +67,18 @@ func initStores() {
 		}
 	}()
 	var wg sync.WaitGroup
-	errCh := make(chan error, len(storeTypes))
-	wg.Add(len(storeTypes))
 	for _, st := range storeTypes {
 		st := st
+		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			container, settings, err := st.Func()
-			if err != nil {
-				errCh <- err
-				return
-			}
-			st.Container = container
-			st.SqlSupplier = NewSqlSupplier(*settings, nil)
+			st.SqlSupplier = NewSqlSupplier(*st.SqlSettings, nil)
 			st.Store = store.NewLayeredStore(st.SqlSupplier, nil, nil)
+			st.Store.DropAllTables()
 			st.Store.MarkSystemRanUnitTests()
 		}()
 	}
 	wg.Wait()
-	select {
-	case err := <-errCh:
-		panic(err)
-	default:
-	}
 }

 var tearDownStoresOnce sync.Once
@@ -103,9 +93,6 @@ func tearDownStores() {
 				if st.Store != nil {
 					st.Store.Close()
 				}
-				if st.Container != nil {
-					st.Container.Stop()
-				}
 				wg.Done()
 			}()
 		}
--- a/store/storetest/docker.go
+++ b/store/storetest/docker.go
@@ -1,143 +0,0 @@
-// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package storetest
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"os/exec"
-	"strings"
-	"time"
-
-	"github.com/mattermost/mattermost-server/mlog"
-	"github.com/mattermost/mattermost-server/model"
-)
-
-type Container struct {
-	Id              string
-	NetworkSettings struct {
-		Ports map[string][]struct {
-			HostPort string
-		}
-	}
-}
-
-type RunningContainer struct {
-	Container
-}
-
-func (c *RunningContainer) Stop() error {
-	mlog.Info(fmt.Sprintf("Removing container: %v", c.Id))
-	return exec.Command("docker", "rm", "-f", c.Id).Run()
-}
-
-func NewMySQLContainer() (*RunningContainer, *model.SqlSettings, error) {
-	container, err := runContainer([]string{
-		"-e", "MYSQL_ROOT_PASSWORD=mostest",
-		"-e", "MYSQL_USER=mmuser",
-		"-e", "MYSQL_PASSWORD=mostest",
-		"-e", "MYSQL_DATABASE=mattermost_test",
-		"--tmpfs", "/var/lib/mysql",
-		"mysql:5.7",
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-	mlog.Info("Waiting for mysql connectivity")
-	port := container.NetworkSettings.Ports["3306/tcp"][0].HostPort
-	if err := waitForPort(port); err != nil {
-		container.Stop()
-		return nil, nil, err
-	}
-	return container, databaseSettings("mysql", "mmuser:mostest@tcp(127.0.0.1:"+port+")/mattermost_test?charset=utf8mb4,utf8"), nil
-}
-
-func NewPostgreSQLContainer() (*RunningContainer, *model.SqlSettings, error) {
-	container, err := runContainer([]string{
-		"-e", "POSTGRES_USER=mmuser",
-		"-e", "POSTGRES_PASSWORD=mostest",
-		"--tmpfs", "/var/lib/postgresql/data",
-		"postgres:9.4",
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-	mlog.Info("Waiting for postgres connectivity")
-	port := container.NetworkSettings.Ports["5432/tcp"][0].HostPort
-	if err := waitForPort(port); err != nil {
-		container.Stop()
-		return nil, nil, err
-	}
-	return container, databaseSettings("postgres", "postgres://mmuser:mostest@127.0.0.1:"+port+"?sslmode=disable"), nil
-}
-
-func databaseSettings(driver, dataSource string) *model.SqlSettings {
-	settings := &model.SqlSettings{
-		DriverName:                  &driver,
-		DataSource:                  &dataSource,
-		DataSourceReplicas:          []string{},
-		DataSourceSearchReplicas:    []string{},
-		MaxIdleConns:                new(int),
-		ConnMaxLifetimeMilliseconds: new(int),
-		MaxOpenConns:                new(int),
-		Trace:                       false,
-		AtRestEncryptKey:            model.NewRandomString(32),
-		QueryTimeout:                new(int),
-	}
-	*settings.MaxIdleConns = 10
-	*settings.ConnMaxLifetimeMilliseconds = 3600000
-	*settings.MaxOpenConns = 100
-	*settings.QueryTimeout = 10
-	return settings
-}
-
-func runContainer(args []string) (*RunningContainer, error) {
-	name := "mattermost-storetest-" + model.NewId()
-	dockerArgs := append([]string{"run", "-d", "-P", "--name", name}, args...)
-	out, err := exec.Command("docker", dockerArgs...).Output()
-	if err != nil {
-		return nil, err
-	}
-	id := strings.TrimSpace(string(out))
-	out, err = exec.Command("docker", "inspect", id).Output()
-	if err != nil {
-		exec.Command("docker", "rm", "-f", id).Run()
-		return nil, err
-	}
-	var containers []Container
-	if err := json.Unmarshal(out, &containers); err != nil {
-		exec.Command("docker", "rm", "-f", id).Run()
-		return nil, err
-	}
-	mlog.Info(fmt.Sprintf("Running container: %v", id))
-	return &RunningContainer{containers[0]}, nil
-}
-
-func waitForPort(port string) error {
-	deadline := time.Now().Add(time.Minute * 10)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, time.Minute)
-		if err != nil {
-			return err
-		}
-		if err = conn.SetReadDeadline(time.Now().Add(time.Millisecond * 500)); err != nil {
-			return err
-		}
-		_, err = conn.Read(make([]byte, 1))
-		conn.Close()
-		if err == nil {
-			return nil
-		}
-		if e, ok := err.(net.Error); ok && e.Timeout() {
-			return nil
-		}
-		if err != io.EOF {
-			return err
-		}
-		time.Sleep(time.Millisecond * 200)
-	}
-	return fmt.Errorf("timeout waiting for port %v", port)
-}
--- a/store/storetest/settings.go
+++ b/store/storetest/settings.go
@@ -0,0 +1,233 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package storetest
+
+import (
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+
+	"database/sql"
+
+	"github.com/go-sql-driver/mysql"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/model"
+)
+
+const (
+	defaultMysqlDSN      = "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s"
+	defaultPostgresqlDSN = "postgres://mmuser:mostest@dockerhost:5432/mattermost_test?sslmode=disable&connect_timeout=10"
+	defaultMysqlRootPWD  = "passwd"
+)
+
+func getEnv(name, defaultValue string) string {
+	if value := os.Getenv(name); value != "" {
+		return value
+	} else {
+		return defaultValue
+	}
+}
+
+func log(message string) {
+	verbose := false
+	if verboseFlag := flag.Lookup("test.v"); verboseFlag != nil {
+		verbose = verboseFlag.Value.String() == "true"
+	}
+	if verboseFlag := flag.Lookup("v"); verboseFlag != nil {
+		verbose = verboseFlag.Value.String() == "true"
+	}
+
+	if verbose {
+		fmt.Println(message)
+	}
+}
+
+// MySQLSettings returns the database settings to connect to the MySQL unittesting database.
+// The database name is generated randomly and must be created before use.
+func MySQLSettings() *model.SqlSettings {
+	dsn := getEnv("TEST_DATABASE_MYSQL_DSN", defaultMysqlDSN)
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	cfg.DBName = "db" + model.NewId()
+
+	return databaseSettings("mysql", cfg.FormatDSN())
+}
+
+// PostgresSQLSettings returns the database settings to connect to the PostgreSQL unittesting database.
+// The database name is generated randomly and must be created before use.
+func PostgreSQLSettings() *model.SqlSettings {
+	dsn := getEnv("TEST_DATABASE_POSTGRESQL_DSN", defaultPostgresqlDSN)
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	// Generate a random database name
+	dsnUrl.Path = "db" + model.NewId()
+
+	return databaseSettings("postgres", dsnUrl.String())
+}
+
+func mySQLRootDSN(dsn string) string {
+	rootPwd := getEnv("TEST_DATABASE_MYSQL_ROOT_PASSWD", defaultMysqlRootPWD)
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	cfg.User = "root"
+	cfg.Passwd = rootPwd
+	cfg.DBName = "mysql"
+
+	return cfg.FormatDSN()
+}
+
+func postgreSQLRootDSN(dsn string) string {
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	// // Assume the unittesting database has the same password.
+	// password := ""
+	// if dsnUrl.User != nil {
+	// 	password, _ = dsnUrl.User.Password()
+	// }
+
+	// dsnUrl.User = url.UserPassword("", password)
+	dsnUrl.Path = "postgres"
+
+	return dsnUrl.String()
+}
+
+func mySQLDSNDatabase(dsn string) string {
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	return cfg.DBName
+}
+
+func postgreSQLDSNDatabase(dsn string) string {
+	dsnUrl, err := url.Parse(dsn)
+	if err != nil {
+		panic("failed to parse dsn " + dsn + ": " + err.Error())
+	}
+
+	return path.Base(dsnUrl.Path)
+}
+
+func databaseSettings(driver, dataSource string) *model.SqlSettings {
+	settings := &model.SqlSettings{
+		DriverName:                  &driver,
+		DataSource:                  &dataSource,
+		DataSourceReplicas:          []string{},
+		DataSourceSearchReplicas:    []string{},
+		MaxIdleConns:                new(int),
+		ConnMaxLifetimeMilliseconds: new(int),
+		MaxOpenConns:                new(int),
+		Trace:                       false,
+		AtRestEncryptKey:            model.NewRandomString(32),
+		QueryTimeout:                new(int),
+	}
+	*settings.MaxIdleConns = 10
+	*settings.ConnMaxLifetimeMilliseconds = 3600000
+	*settings.MaxOpenConns = 100
+	*settings.QueryTimeout = 60
+
+	return settings
+}
+
+// execAsRoot executes the given sql as root against the testing database
+func execAsRoot(settings *model.SqlSettings, sqlCommand string) error {
+	var dsn string
+	var driver = *settings.DriverName
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		dsn = mySQLRootDSN(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		dsn = postgreSQLRootDSN(*settings.DataSource)
+	default:
+		return fmt.Errorf("unsupported driver %s", driver)
+	}
+
+	db, err := sql.Open(driver, dsn)
+	if err != nil {
+		return errors.Wrapf(err, "failed to connect to %s database as root", driver)
+	}
+	defer db.Close()
+	if _, err = db.Exec(sqlCommand); err != nil {
+		return errors.Wrapf(err, "failed to execute `%s` against %s database as root", sqlCommand, driver)
+	}
+
+	return nil
+}
+
+// MakeSqlSettings creates a randomly named database and returns the corresponding sql settings
+func MakeSqlSettings(driver string) *model.SqlSettings {
+	var settings *model.SqlSettings
+	var dbName string
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		settings = MySQLSettings()
+		dbName = mySQLDSNDatabase(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		settings = PostgreSQLSettings()
+		dbName = postgreSQLDSNDatabase(*settings.DataSource)
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	if err := execAsRoot(settings, "CREATE DATABASE "+dbName); err != nil {
+		panic("failed to create temporary database " + dbName + ": " + err.Error())
+	}
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON "+dbName+".* TO 'mmuser'"); err != nil {
+			panic("failed to grant mmuser permission to " + dbName + ":" + err.Error())
+		}
+	case model.DATABASE_DRIVER_POSTGRES:
+		if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON DATABASE \""+dbName+"\" TO mmuser"); err != nil {
+			panic("failed to grant mmuser permission to " + dbName + ":" + err.Error())
+		}
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	log("Created temporary database " + dbName)
+
+	return settings
+}
+
+func CleanupSqlSettings(settings *model.SqlSettings) {
+	var driver = *settings.DriverName
+	var dbName string
+
+	switch driver {
+	case model.DATABASE_DRIVER_MYSQL:
+		dbName = mySQLDSNDatabase(*settings.DataSource)
+	case model.DATABASE_DRIVER_POSTGRES:
+		dbName = postgreSQLDSNDatabase(*settings.DataSource)
+	default:
+		panic("unsupported driver " + driver)
+	}
+
+	if err := execAsRoot(settings, "DROP DATABASE "+dbName); err != nil {
+		panic("failed to drop temporary database " + dbName + ": " + err.Error())
+	}
+
+	log("Dropped temporary database " + dbName)
+}
--- a/store/storetest/user_store.go
+++ b/store/storetest/user_store.go
@@ -77,7 +77,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) {
 	if err := (<-ss.User().Save(&u1)).Err; err != nil {
 		t.Fatal("couldn't save user", err)
 	}
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam))

@@ -108,7 +108,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) {
 		if err := (<-ss.User().Save(&u1)).Err; err != nil {
 			t.Fatal("couldn't save item", err)
 		}
-		defer ss.User().PermanentDelete(u1.Id)
+		defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 		store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam))
 	}
@@ -119,7 +119,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) {
 	if err := (<-ss.User().Save(&u1)).Err; err != nil {
 		t.Fatal("couldn't save item", err)
 	}
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	if err := (<-ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam)).Err; err == nil {
 		t.Fatal("should be the limit")
@@ -130,14 +130,14 @@ func testUserStoreUpdate(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	u2.AuthService = "ldap"
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))

 	time.Sleep(100 * time.Millisecond)
@@ -166,7 +166,7 @@ func testUserStoreUpdate(t *testing.T, ss store.Store) {
 	oldEmail := u3.Email
 	u3.AuthService = "gitlab"
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u3.Id}, -1))

 	u3.Email = MakeEmail()
@@ -198,7 +198,7 @@ func testUserStoreUpdateUpdateAt(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	time.Sleep(10 * time.Millisecond)
@@ -221,7 +221,7 @@ func testUserStoreUpdateFailedPasswordAttempts(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	if err := (<-ss.User().UpdateFailedPasswordAttempts(u1.Id, 3)).Err; err != nil {
@@ -242,7 +242,7 @@ func testUserStoreGet(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	if r1 := <-ss.User().Get(u1.Id); r1.Err != nil {
@@ -262,7 +262,7 @@ func testUserCount(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	if result := <-ss.User().GetTotalUsersCount(); result.Err != nil {
@@ -278,13 +278,13 @@ func testGetAllUsingAuthService(t *testing.T, ss store.Store) {
 	u1.Email = MakeEmail()
 	u1.AuthService = "someservice"
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	u2.AuthService = "someservice"
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if r1 := <-ss.User().GetAllUsingAuthService(u1.AuthService); r1.Err != nil {
 		t.Fatal(r1.Err)
@@ -300,12 +300,12 @@ func testUserStoreGetAllProfiles(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if r1 := <-ss.User().GetAllProfiles(0, 100); r1.Err != nil {
 		t.Fatal(r1.Err)
@@ -344,7 +344,7 @@ func testUserStoreGetAllProfiles(t *testing.T, ss store.Store) {
 	u3 := &model.User{}
 	u3.Email = MakeEmail()
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	if r2 := <-ss.User().GetEtagForAllProfiles(); r2.Err != nil {
 		t.Fatal(r2.Err)
@@ -361,13 +361,13 @@ func testUserStoreGetProfiles(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	if r1 := <-ss.User().GetProfiles(teamId, 0, 100); r1.Err != nil {
@@ -408,7 +408,7 @@ func testUserStoreGetProfiles(t *testing.T, ss store.Store) {
 	u3 := &model.User{}
 	u3.Email = MakeEmail()
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u3.Id}, -1))

 	if r2 := <-ss.User().GetEtagForProfiles(teamId); r2.Err != nil {
@@ -426,13 +426,13 @@ func testUserStoreGetProfilesInChannel(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	c1 := model.Channel{}
@@ -504,13 +504,13 @@ func testUserStoreGetProfilesInChannelByStatus(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	c1 := model.Channel{}
@@ -586,13 +586,13 @@ func testUserStoreGetProfilesWithoutTeam(t *testing.T, ss store.Store) {
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{}
 	u2.Username = "a000000001" + model.NewId()
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if r1 := <-ss.User().GetProfilesWithoutTeam(0, 100); r1.Err != nil {
 		t.Fatal(r1.Err)
@@ -623,13 +623,13 @@ func testUserStoreGetAllProfilesInChannel(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	c1 := model.Channel{}
@@ -713,13 +713,13 @@ func testUserStoreGetProfilesNotInChannel(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	c1 := model.Channel{}
@@ -808,13 +808,13 @@ func testUserStoreGetProfilesByIds(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	if r1 := <-ss.User().GetProfileByIds([]string{u1.Id}, false); r1.Err != nil {
@@ -953,14 +953,14 @@ func testUserStoreGetProfilesByUsernames(t *testing.T, ss store.Store) {
 	u1.Email = MakeEmail()
 	u1.Username = "username1" + model.NewId()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	u2.Username = "username2" + model.NewId()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	if r1 := <-ss.User().GetProfilesByUsernames([]string{u1.Username, u2.Username}, teamId); r1.Err != nil {
@@ -999,7 +999,7 @@ func testUserStoreGetProfilesByUsernames(t *testing.T, ss store.Store) {
 	u3.Email = MakeEmail()
 	u3.Username = "username3" + model.NewId()
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: team2Id, UserId: u3.Id}, -1))

 	if r1 := <-ss.User().GetProfilesByUsernames([]string{u1.Username, u3.Username}, ""); r1.Err != nil {
@@ -1040,13 +1040,13 @@ func testUserStoreGetSystemAdminProfiles(t *testing.T, ss store.Store) {
 	u1.Email = MakeEmail()
 	u1.Roles = model.SYSTEM_USER_ROLE_ID + " " + model.SYSTEM_ADMIN_ROLE_ID
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	if r1 := <-ss.User().GetSystemAdminProfiles(); r1.Err != nil {
@@ -1065,7 +1065,7 @@ func testUserStoreGetByEmail(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamid, UserId: u1.Id}, -1))

 	if err := (<-ss.User().GetByEmail(u1.Email)).Err; err != nil {
@@ -1087,7 +1087,7 @@ func testUserStoreGetByAuthData(t *testing.T, ss store.Store) {
 	u1.AuthData = &auth
 	u1.AuthService = "service"
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	if err := (<-ss.User().GetByAuth(u1.AuthData, u1.AuthService)).Err; err != nil {
@@ -1107,7 +1107,7 @@ func testUserStoreGetByUsername(t *testing.T, ss store.Store) {
 	u1.Email = MakeEmail()
 	u1.Username = model.NewId()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	if err := (<-ss.User().GetByUsername(u1.Username)).Err; err != nil {
@@ -1129,7 +1129,7 @@ func testUserStoreGetForLogin(t *testing.T, ss store.Store) {
 		AuthData:    &auth,
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	auth2 := model.NewId()

@@ -1140,7 +1140,7 @@ func testUserStoreGetForLogin(t *testing.T, ss store.Store) {
 		AuthData:    &auth2,
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if result := <-ss.User().GetForLogin(u1.Username, true, true); result.Err != nil {
 		t.Fatal("Should have gotten user by username", result.Err)
@@ -1170,7 +1170,7 @@ func testUserStoreUpdatePassword(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	hashedPassword := model.HashPassword("newpwd")
@@ -1193,7 +1193,7 @@ func testUserStoreDelete(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))

 	if err := (<-ss.User().PermanentDelete(u1.Id)).Err; err != nil {
@@ -1207,7 +1207,7 @@ func testUserStoreUpdateAuthData(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	service := "someservice"
@@ -1252,14 +1252,14 @@ func testUserUnreadCount(t *testing.T, ss store.Store) {
 	u1.Username = "user1" + model.NewId()
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	u2.Username = "user2" + model.NewId()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1))

 	if err := (<-ss.Channel().Save(&c1, -1)).Err; err != nil {
@@ -1330,7 +1330,7 @@ func testUserStoreUpdateMfaSecret(t *testing.T, ss store.Store) {
 	u1 := model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(&u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	time.Sleep(100 * time.Millisecond)

@@ -1348,7 +1348,7 @@ func testUserStoreUpdateMfaActive(t *testing.T, ss store.Store) {
 	u1 := model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(&u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	time.Sleep(100 * time.Millisecond)

@@ -1370,7 +1370,7 @@ func testUserStoreGetRecentlyActiveUsersForTeam(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Status().SaveOrUpdate(&model.Status{UserId: u1.Id, Status: model.STATUS_ONLINE, Manual: false, LastActivityAt: model.GetMillis(), ActiveChannel: ""}))
 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1))
@@ -1384,7 +1384,7 @@ func testUserStoreGetNewUsersForTeam(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Status().SaveOrUpdate(&model.Status{UserId: u1.Id, Status: model.STATUS_ONLINE, Manual: false, LastActivityAt: model.GetMillis(), ActiveChannel: ""}))
 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1))
@@ -1419,14 +1419,14 @@ func testUserStoreSearch(t *testing.T, ss store.Store) {
 		Email:     "harold" + model.NewId() + "@simulator.amazonses.com",
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{
 		Username: "jim-bobby" + model.NewId(),
 		Email:    MakeEmail(),
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	u3 := &model.User{
 		Username: "jimbo3" + model.NewId(),
@@ -1434,7 +1434,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) {
 		DeleteAt: 1,
 	}
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	u5 := &model.User{
 		Username:  "yu" + model.NewId(),
@@ -1444,7 +1444,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) {
 		Email:     MakeEmail(),
 	}
 	store.Must(ss.User().Save(u5))
-	defer ss.User().PermanentDelete(u5.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u5.Id)) }()

 	u6 := &model.User{
 		Username:  "underscore" + model.NewId(),
@@ -1454,7 +1454,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) {
 		Email:     MakeEmail(),
 	}
 	store.Must(ss.User().Save(u6))
-	defer ss.User().PermanentDelete(u6.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u6.Id)) }()

 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1))
@@ -1716,14 +1716,14 @@ func testUserStoreSearchNotInChannel(t *testing.T, ss store.Store) {
 		Email:     "harold" + model.NewId() + "@simulator.amazonses.com",
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{
 		Username: "jim2-bobby" + model.NewId(),
 		Email:    MakeEmail(),
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	u3 := &model.User{
 		Username: "jimbo3" + model.NewId(),
@@ -1731,7 +1731,7 @@ func testUserStoreSearchNotInChannel(t *testing.T, ss store.Store) {
 		DeleteAt: 1,
 	}
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1))
@@ -1923,14 +1923,14 @@ func testUserStoreSearchInChannel(t *testing.T, ss store.Store) {
 		Email:     "harold" + model.NewId() + "@simulator.amazonses.com",
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{
 		Username: "jim-bobby" + model.NewId(),
 		Email:    MakeEmail(),
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	u3 := &model.User{
 		Username: "jimbo3" + model.NewId(),
@@ -1938,7 +1938,7 @@ func testUserStoreSearchInChannel(t *testing.T, ss store.Store) {
 		DeleteAt: 1,
 	}
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1))
@@ -2069,14 +2069,14 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 		Email:     "harold" + model.NewId() + "@simulator.amazonses.com",
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{
 		Username: "jim-bobby" + model.NewId(),
 		Email:    MakeEmail(),
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	u3 := &model.User{
 		Username: "jimbo3" + model.NewId(),
@@ -2084,7 +2084,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 		DeleteAt: 1,
 	}
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	u4 := &model.User{
 		Username: "simon" + model.NewId(),
@@ -2092,7 +2092,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 		DeleteAt: 0,
 	}
 	store.Must(ss.User().Save(u4))
-	defer ss.User().PermanentDelete(u4.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u4.Id)) }()

 	u5 := &model.User{
 		Username:  "yu" + model.NewId(),
@@ -2102,7 +2102,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 		Email:     MakeEmail(),
 	}
 	store.Must(ss.User().Save(u5))
-	defer ss.User().PermanentDelete(u5.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u5.Id)) }()

 	u6 := &model.User{
 		Username:  "underscore" + model.NewId(),
@@ -2112,7 +2112,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 		Email:     MakeEmail(),
 	}
 	store.Must(ss.User().Save(u6))
-	defer ss.User().PermanentDelete(u6.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u6.Id)) }()

 	teamId1 := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId1, UserId: u1.Id}, -1))
@@ -2186,7 +2186,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) {
 			[]*model.User{},
 		},
 		{
-			"search jimb, team1",
+			"search jimb, team2",
 			teamId2,
 			"jimb",
 			&model.UserSearchOptions{
@@ -2241,14 +2241,14 @@ func testUserStoreSearchWithoutTeam(t *testing.T, ss store.Store) {
 		Email:     "harold" + model.NewId() + "@simulator.amazonses.com",
 	}
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	u2 := &model.User{
 		Username: "jim2-bobby" + model.NewId(),
 		Email:    MakeEmail(),
 	}
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	u3 := &model.User{
 		Username: "jimbo3" + model.NewId(),
@@ -2256,7 +2256,7 @@ func testUserStoreSearchWithoutTeam(t *testing.T, ss store.Store) {
 		DeleteAt: 1,
 	}
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()

 	tid := model.NewId()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u3.Id}, -1))
@@ -2329,7 +2329,7 @@ func testUserStoreAnalyticsGetInactiveUsersCount(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	var count int64

@@ -2343,7 +2343,7 @@ func testUserStoreAnalyticsGetInactiveUsersCount(t *testing.T, ss store.Store) {
 	u2.Email = MakeEmail()
 	u2.DeleteAt = model.GetMillis()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if result := <-ss.User().AnalyticsGetInactiveUsersCount(); result.Err != nil {
 		t.Fatal(result.Err)
@@ -2375,12 +2375,12 @@ func testUserStoreAnalyticsGetSystemAdminCount(t *testing.T, ss store.Store) {
 	if err := (<-ss.User().Save(&u1)).Err; err != nil {
 		t.Fatal("couldn't save user", err)
 	}
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	if err := (<-ss.User().Save(&u2)).Err; err != nil {
 		t.Fatal("couldn't save user", err)
 	}
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()

 	if result := <-ss.User().AnalyticsGetSystemAdminCount(); result.Err != nil {
 		t.Fatal(result.Err)
@@ -2398,14 +2398,14 @@ func testUserStoreGetProfilesNotInTeam(t *testing.T, ss store.Store) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	store.Must(ss.User().Save(u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1))
 	store.Must(ss.User().UpdateUpdateAt(u1.Id))

 	u2 := &model.User{}
 	u2.Email = MakeEmail()
 	store.Must(ss.User().Save(u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.User().UpdateUpdateAt(u2.Id))

 	var initialUsersNotInTeam int
@@ -2512,7 +2512,7 @@ func testUserStoreGetProfilesNotInTeam(t *testing.T, ss store.Store) {
 	u3 := &model.User{}
 	u3.Email = MakeEmail()
 	store.Must(ss.User().Save(u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()
 	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u3.Id}, -1))
 	store.Must(ss.User().UpdateUpdateAt(u3.Id))

@@ -2550,13 +2550,13 @@ func testUserStoreClearAllCustomRoleAssignments(t *testing.T, ss store.Store) {
 	}

 	store.Must(ss.User().Save(&u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()
 	store.Must(ss.User().Save(&u2))
-	defer ss.User().PermanentDelete(u2.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }()
 	store.Must(ss.User().Save(&u3))
-	defer ss.User().PermanentDelete(u3.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }()
 	store.Must(ss.User().Save(&u4))
-	defer ss.User().PermanentDelete(u4.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u4.Id)) }()

 	require.Nil(t, (<-ss.User().ClearAllCustomRoleAssignments()).Err)

@@ -2584,7 +2584,7 @@ func testUserStoreGetAllAfter(t *testing.T, ss store.Store) {
 		Roles:    "system_user system_admin system_post_all",
 	}
 	store.Must(ss.User().Save(&u1))
-	defer ss.User().PermanentDelete(u1.Id)
+	defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }()

 	r1 := <-ss.User().GetAllAfter(10000, strings.Repeat("0", 26))
 	require.Nil(t, r1.Err)