MM-11262: database config store (#10281)

* vendor github.com/jmoiron/sqlx

* MM-11262: introduce a database store

* revert unnecessary fmt.Errorf

* simplify unit test helper methods

* remote TODO re: retry

* relocate initializeConfigurationsTable for clarity

* factor out a commonStore

* acquire database config lock on close for safety

* add missing header

* fix lock comment

Gopkg.lock

@@ -375,6 +375,17 @@
   pruneopts = "UT"
   revision = "1d33003b386959af197ba96475f198c114627b5e"
 
+[[projects]]
+  digest = "1:6c41d4f998a03b6604227ccad36edaed6126c397e5d78709ef4814a1145a6757"
+  name = "github.com/jmoiron/sqlx"
+  packages = [
+    ".",
+    "reflectx",
+  ]
+  pruneopts = "UT"
+  revision = "d161d7a76b5661016ad0b085869f77fd410f3e6a"
+  version = "v1.2.0"
+
 [[projects]]
   digest = "1:0a69a1c0db3591fcefb47f115b224592c8dfa4368b7ba9fae509d5e16cdc95c8"
   name = "github.com/konsorten/go-windows-terminal-sequences"
@@ -1108,6 +1119,7 @@
     "github.com/hashicorp/memberlist",
     "github.com/icrowley/fake",
     "github.com/jaytaylor/html2text",
+    "github.com/jmoiron/sqlx",
     "github.com/lib/pq",
     "github.com/mattermost/gorp",
     "github.com/mattermost/rsc/qr",

api4/apitestlib.go

@@ -79,7 +79,7 @@ func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHel
 		panic(err)
 	}
 
-	options := []app.Option{app.ConfigFile(tempConfig.Name(), false)}
+	options := []app.Option{app.Config(tempConfig.Name(), false)}
 	options = append(options, app.StoreOverride(testStore))
 
 	s, err := app.NewServer(options...)

app/helper_test.go

@@ -51,7 +51,7 @@ func setupTestHelper(enterprise bool, tb testing.TB) *TestHelper {
 		panic(err)
 	}
 
-	options := []Option{ConfigFile(tempConfig.Name(), false)}
+	options := []Option{Config(tempConfig.Name(), false)}
 	options = append(options, StoreOverride(mainHelper.Store))
 	options = append(options, SetLogger(mlog.NewTestingLogger(tb)))
 

app/options.go

@@ -38,11 +38,12 @@ func StoreOverride(override interface{}) Option {
 	}
 }
 
-func ConfigFile(file string, watch bool) Option {
+// Config applies the given config dsn, whether a path to config.json or a database connection string.
+func Config(dsn string, watch bool) Option {
 	return func(s *Server) error {
-		configStore, err := config.NewFileStore(file, watch)
+		configStore, err := config.NewStore(dsn, watch)
 		if err != nil {
-			return errors.Wrap(err, "failed to apply ConfigFile option")
+			return errors.Wrap(err, "failed to apply Config option")
 		}
 
 		s.configStore = configStore
@@ -50,6 +51,7 @@ func ConfigFile(file string, watch bool) Option {
 	}
 }
 
+// ConfigStore applies the given config store, typically to replace the traditional sources with a memory store for testing.
 func ConfigStore(configStore config.Store) Option {
 	return func(s *Server) error {
 		s.configStore = configStore

cmd/mattermost/commands/config.go

@@ -138,19 +138,31 @@ func configSubpathCmdF(command *cobra.Command, args []string) error {
 	return nil
 }
 
+func getConfigStore(command *cobra.Command) (config.Store, error) {
+	if err := utils.TranslationsPreInit(); err != nil {
+		return nil, errors.Wrap(err, "failed to initialize i18n")
+	}
+
+	configDSN, err := command.Flags().GetString("config")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse --config flag")
+	}
+
+	configStore, err := config.NewStore(configDSN, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize config store")
+	}
+
+	return configStore, nil
+}
+
 func configGetCmdF(command *cobra.Command, args []string) error {
-	app, err := InitDBCommandContextCobra(command)
+	configStore, err := getConfigStore(command)
 	if err != nil {
 		return err
 	}
-	defer app.Shutdown()
 
-	// create the model for config
-	// Note: app.Config() returns a pointer, make appropriate changes
-	config := app.Config()
-
-	// get the print config setting and any error if there is
-	out, err := printConfigValues(configToMap(*config), strings.Split(args[0], "."), args[0])
+	out, err := printConfigValues(configToMap(*configStore.Get()), strings.Split(args[0], "."), args[0])
 	if err != nil {
 		return err
 	}
@@ -161,18 +173,17 @@ func configGetCmdF(command *cobra.Command, args []string) error {
 }
 
 func configShowCmdF(command *cobra.Command, args []string) error {
-	app, err := InitDBCommandContextCobra(command)
+	configStore, err := getConfigStore(command)
 	if err != nil {
 		return err
 	}
-	defer app.Shutdown()
 
 	err = cobra.NoArgs(command, args)
 	if err != nil {
 		return err
 	}
 
-	fmt.Printf("%s", prettyPrintStruct(*app.Config()))
+	fmt.Printf("%s", prettyPrintStruct(*configStore.Get()))
 	return nil
 }
 
@@ -199,26 +210,24 @@ func printConfigValues(configMap map[string]interface{}, configSetting []string,
 }
 
 func configSetCmdF(command *cobra.Command, args []string) error {
-	app, err := InitDBCommandContextCobra(command)
+	configStore, err := getConfigStore(command)
 	if err != nil {
 		return err
 	}
 
-	defer app.Shutdown()
-
 	// args[0] -> holds the config setting that we want to change
 	// args[1:] -> the new value of the config setting
 	configSetting := args[0]
 	newVal := args[1:]
 
 	// create the function to update config
-	oldConfig := app.Config()
-	newConfig := app.Config()
-	f := updateConfigValue(configSetting, newVal, oldConfig, newConfig)
+	oldConfig := configStore.Get()
+	newConfig := configStore.Get()
 
-	app.UpdateConfig(f)
-	if err := newConfig.IsValid(); err != nil {
-		return err
+	f := updateConfigValue(configSetting, newVal, oldConfig, newConfig)
+	f(newConfig)
+	if _, err := configStore.Set(newConfig); err != nil {
+		return errors.Wrap(err, "failed to set config")
 	}
 
 	// UpdateConfig above would have already fixed these invalid locales, but we check again
@@ -228,8 +237,7 @@ func configSetCmdF(command *cobra.Command, args []string) error {
 		return errors.New("Invalid locale configuration")
 	}
 
-	// make the changes persist
-	app.PersistConfig()
+	configStore.Save()
 
 	return nil
 }

cmd/mattermost/commands/init.go

@@ -30,13 +30,13 @@ func InitDBCommandContextCobra(command *cobra.Command) (*app.App, error) {
 	return a, nil
 }
 
-func InitDBCommandContext(configFileLocation string) (*app.App, error) {
+func InitDBCommandContext(configDSN string) (*app.App, error) {
 	if err := utils.TranslationsPreInit(); err != nil {
 		return nil, err
 	}
 	model.AppErrorInit(utils.T)
 
-	s, err := app.NewServer(app.ConfigFile(configFileLocation, true))
+	s, err := app.NewServer(app.Config(configDSN, false))
 	if err != nil {
 		return nil, err
 	}

cmd/mattermost/commands/server.go

@@ -43,9 +43,9 @@ func serverCmdF(command *cobra.Command, args []string) error {
 	return runServer(config, disableConfigWatch, usedPlatform, interruptChan)
 }
 
-func runServer(configFileLocation string, disableConfigWatch bool, usedPlatform bool, interruptChan chan os.Signal) error {
+func runServer(configDSN string, disableConfigWatch bool, usedPlatform bool, interruptChan chan os.Signal) error {
 	options := []app.Option{
-		app.ConfigFile(configFileLocation, !disableConfigWatch),
+		app.Config(configDSN, !disableConfigWatch),
 		app.RunJobs,
 		app.JoinCluster,
 		app.StartElasticsearch,

config/common.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package config
+
+import (
+	"io"
+	"sync"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/pkg/errors"
+)
+
+// commonStore enables code sharing between different backing implementations
+type commonStore struct {
+	emitter
+
+	configLock           sync.RWMutex
+	config               *model.Config
+	environmentOverrides map[string]interface{}
+}
+
+// Get fetches the current, cached configuration.
+func (cs *commonStore) Get() *model.Config {
+	cs.configLock.RLock()
+	defer cs.configLock.RUnlock()
+
+	return cs.config
+}
+
+// GetEnvironmentOverrides fetches the configuration fields overridden by environment variables.
+func (cs *commonStore) GetEnvironmentOverrides() map[string]interface{} {
+	cs.configLock.RLock()
+	defer cs.configLock.RUnlock()
+
+	return cs.environmentOverrides
+}
+
+// set replaces the current configuration in its entirety, without updating the backing store.
+//
+// This function assumes no lock has been acquired, as it acquires a write lock itself.
+func (cs *commonStore) set(newCfg *model.Config, isValid func(*model.Config) error) (*model.Config, error) {
+	cs.configLock.Lock()
+	var unlockOnce sync.Once
+	defer unlockOnce.Do(cs.configLock.Unlock)
+
+	oldCfg := cs.config
+
+	// TODO: disallow attempting to save a directly modified config (comparing pointers). This
+	// wouldn't be an exhaustive check, given the use of pointers throughout the data
+	// structure, but might prevent common mistakes. Requires upstream changes first.
+	// if newCfg == oldCfg {
+	// 	return nil, errors.New("old configuration modified instead of cloning")
+	// }
+
+	newCfg = newCfg.Clone()
+	newCfg.SetDefaults()
+
+	// Sometimes the config is received with "fake" data in sensitive fielcs. Apply the real
+	// data from the existing config as necessary.
+	desanitize(oldCfg, newCfg)
+
+	if err := newCfg.IsValid(); err != nil {
+		return nil, errors.Wrap(err, "new configuration is invalid")
+	}
+
+	// Allow backing-store specific checks.
+	if isValid != nil {
+		if err := isValid(newCfg); err != nil {
+			return nil, err
+		}
+	}
+
+	// Ideally, Set would persist automatically and abstract this completely away from the
+	// client. Doing so requires a few upstream changes first, so for now an explicit Save()
+	// remains required.
+	// if err := cs.persist(newCfg); err != nil {
+	// 	return nil, errors.Wrap(err, "failed to persist")
+	// }
+
+	cs.config = newCfg
+
+	unlockOnce.Do(cs.configLock.Unlock)
+
+	// Notify listeners synchronously. Ideally, this would be asynchronous, but existing code
+	// assumes this and there would be increased complexity to avoid racing updates.
+	cs.invokeConfigListeners(oldCfg, newCfg)
+
+	return oldCfg, nil
+}
+
+// load updates the current configuration from the given io.ReadCloser.
+//
+// This function assumes no lock has been acquired, as it acquires a write lock itself.
+func (cs *commonStore) load(f io.ReadCloser, needsSave bool, persist func(*model.Config) error) error {
+	allowEnvironmentOverrides := true
+	loadedCfg, environmentOverrides, err := unmarshalConfig(f, allowEnvironmentOverrides)
+	if err != nil {
+		return errors.Wrapf(err, "failed to unmarshal config")
+	}
+
+	// SetDefaults generates various keys and salts if not previously configured. Determine if
+	// such a change will be made before invoking. This method will not effect the save: that
+	// remains the responsibility of the caller.
+	needsSave = needsSave || loadedCfg.SqlSettings.AtRestEncryptKey == nil || len(*loadedCfg.SqlSettings.AtRestEncryptKey) == 0
+	needsSave = needsSave || loadedCfg.FileSettings.PublicLinkSalt == nil || len(*loadedCfg.FileSettings.PublicLinkSalt) == 0
+	needsSave = needsSave || loadedCfg.EmailSettings.InviteSalt == nil || len(*loadedCfg.EmailSettings.InviteSalt) == 0
+
+	loadedCfg.SetDefaults()
+
+	if err := loadedCfg.IsValid(); err != nil {
+		return errors.Wrap(err, "invalid config")
+	}
+
+	if changed := fixConfig(loadedCfg); changed {
+		needsSave = true
+	}
+
+	cs.configLock.Lock()
+	var unlockOnce sync.Once
+	defer unlockOnce.Do(cs.configLock.Unlock)
+
+	if needsSave {
+		if err = persist(loadedCfg); err != nil {
+			return errors.Wrap(err, "failed to persist required changes after load")
+		}
+	}
+
+	oldCfg := cs.config
+	cs.config = loadedCfg
+	cs.environmentOverrides = environmentOverrides
+
+	unlockOnce.Do(cs.configLock.Unlock)
+
+	// Notify listeners synchronously. Ideally, this would be asynchronous, but existing code
+	// assumes this and there would be increased complexity to avoid racing updates.
+	cs.invokeConfigListeners(oldCfg, loadedCfg)
+
+	return nil
+}

config/common_test.go

@@ -0,0 +1,17 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func prepareExpectedConfig(t *testing.T, expectedCfg *model.Config) *model.Config {
+	// These fields require special initialization for our tests.
+	expectedCfg = expectedCfg.Clone()
+	expectedCfg.MessageExportSettings.GlobalRelaySettings = &model.GlobalRelayMessageExportSettings{}
+	expectedCfg.PluginSettings.Plugins = make(map[string]map[string]interface{})
+	expectedCfg.PluginSettings.PluginStates = make(map[string]*model.PluginState)
+
+	return expectedCfg
+}

config/database.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package config
+
+import (
+	"bytes"
+	"database/sql"
+	"io/ioutil"
+	"net/url"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/mlog"
+	"github.com/mattermost/mattermost-server/model"
+
+	// Load the MySQL driver
+	_ "github.com/go-sql-driver/mysql"
+	// Load the Postgres driver
+	_ "github.com/lib/pq"
+)
+
+// DatabaseStore is a config store backed by a database.
+type DatabaseStore struct {
+	commonStore
+
+	originalDsn    string
+	driverName     string
+	dataSourceName string
+	db             *sqlx.DB
+}
+
+// NewDatabaseStore creates a new instance of a config store backed by the given database.
+func NewDatabaseStore(dsn string) (ds *DatabaseStore, err error) {
+	driverName, dataSourceName, err := parseDSN(dsn)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid DSN")
+	}
+
+	db, err := sqlx.Open(driverName, dataSourceName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to connect to %s database", driverName)
+	}
+
+	ds = &DatabaseStore{
+		driverName:     driverName,
+		originalDsn:    dsn,
+		dataSourceName: dataSourceName,
+		db:             db,
+	}
+	if err = initializeConfigurationsTable(ds.db); err != nil {
+		return nil, errors.Wrap(err, "failed to initialize")
+	}
+
+	if err = ds.Load(); err != nil {
+		return nil, errors.Wrap(err, "failed to load")
+	}
+
+	return ds, nil
+}
+
+// initializeConfigurationsTable ensures the requisite tables in place to form the backing store.
+func initializeConfigurationsTable(db *sqlx.DB) error {
+	_, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS Configurations (
+		    Id VARCHAR(26) PRIMARY KEY,
+		    Value TEXT NOT NULL,
+		    CreateAt BIGINT NOT NULL,
+		    Active BOOLEAN NULL UNIQUE
+		)
+	`)
+	if err != nil {
+		return errors.Wrap(err, "failed to create Configurations table")
+	}
+
+	return nil
+}
+
+// parseDSN splits up a connection string into a driver name and data source name.
+//
+// For example:
+//	mysql://mmuser:mostest@dockerhost:5432/mattermost_test
+// returns
+//	driverName = mysql
+//	dataSourceName = mmuser:mostest@dockerhost:5432/mattermost_test
+//
+// By contrast, a Postgres DSN is returned unmodified.
+func parseDSN(dsn string) (string, string, error) {
+	// Treat the DSN as the URL that it is.
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return "", "", errors.Wrap(err, "failed to parse DSN as URL")
+	}
+
+	scheme := u.Scheme
+	switch scheme {
+	case "mysql":
+		// Strip off the mysql:// for the dsn with which to connect.
+		u.Scheme = ""
+		dsn = strings.TrimPrefix(u.String(), "//")
+
+	case "postgres":
+		// No changes required
+
+	default:
+		return "", "", errors.Wrapf(err, "unsupported scheme %s", scheme)
+	}
+
+	return scheme, dsn, nil
+}
+
+// Set replaces the current configuration in its entirety, without updating the backing store.
+func (ds *DatabaseStore) Set(newCfg *model.Config) (*model.Config, error) {
+	return ds.commonStore.set(newCfg, nil)
+}
+
+// persist writes the configuration to the configured database.
+func (ds *DatabaseStore) persist(cfg *model.Config) error {
+	b, err := marshalConfig(cfg)
+	if err != nil {
+		return errors.Wrap(err, "failed to serialize")
+	}
+
+	id := model.NewId()
+	value := string(b)
+	createAt := model.GetMillis()
+
+	tx, err := ds.db.Beginx()
+	if err != nil {
+		return errors.Wrap(err, "failed to begin transaction")
+	}
+	defer func() {
+		// Rollback after Commit just returns sql.ErrTxDone.
+		if err := tx.Rollback(); err != nil && err != sql.ErrTxDone {
+			mlog.Error("Failed to rollback configuration transaction", mlog.Err(err))
+		}
+	}()
+
+	params := map[string]interface{}{
+		"id":        id,
+		"value":     value,
+		"create_at": createAt,
+		"key":       "ConfigurationId",
+	}
+
+	if _, err := tx.Exec("UPDATE Configurations SET Active = NULL WHERE Active"); err != nil {
+		return errors.Wrap(err, "failed to deactivate current configuration")
+	}
+
+	if _, err := tx.NamedExec("INSERT INTO Configurations (Id, Value, CreateAt, Active) VALUES (:id, :value, :create_at, TRUE)", params); err != nil {
+		return errors.Wrap(err, "failed to record new configuration")
+	}
+
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "failed to commit transaction")
+	}
+
+	return nil
+}
+
+// Load updates the current configuration from the backing store.
+func (ds *DatabaseStore) Load() (err error) {
+	var needsSave bool
+	var configurationData []byte
+
+	row := ds.db.QueryRow("SELECT Value FROM Configurations WHERE Active")
+	if err = row.Scan(&configurationData); err != nil && err != sql.ErrNoRows {
+		return errors.Wrap(err, "failed to query active configuration")
+	}
+
+	// Initialize from the default config if no active configuration could be found.
+	if len(configurationData) == 0 {
+		needsSave = true
+
+		defaultCfg := model.Config{}
+		defaultCfg.SetDefaults()
+
+		// Assume the database storing the config is also to be used for the application.
+		// This can be overridden using environment variables on first start if necessary,
+		// or changed from the system console afterwards.
+		*defaultCfg.SqlSettings.DriverName = ds.driverName
+		*defaultCfg.SqlSettings.DataSource = ds.dataSourceName
+
+		configurationData, err = marshalConfig(&defaultCfg)
+		if err != nil {
+			return errors.Wrap(err, "failed to serialize default config")
+		}
+	}
+
+	return ds.commonStore.load(ioutil.NopCloser(bytes.NewReader(configurationData)), needsSave, ds.persist)
+}
+
+// Save writes the current configuration to the backing store.
+func (ds *DatabaseStore) Save() error {
+	ds.configLock.RLock()
+	defer ds.configLock.RUnlock()
+
+	return ds.persist(ds.config)
+}
+
+// String returns the path to the database backing the config, masking the password.
+func (ds *DatabaseStore) String() string {
+	u, _ := url.Parse(ds.originalDsn)
+
+	// Strip out the password to avoid leaking in logs.
+	u.User = url.User(u.User.Username())
+
+	return u.String()
+}
+
+// Close cleans up resources associated with the store.
+func (ds *DatabaseStore) Close() error {
+	ds.configLock.Lock()
+	defer ds.configLock.Unlock()
+
+	return ds.db.Close()
+}

config/database_test.go

@@ -0,0 +1,470 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package config_test
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/config"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func setupConfigDatabase(t *testing.T, cfg *model.Config) (string, func()) {
+	t.Helper()
+	os.Clearenv()
+	truncateTables(t)
+
+	cfgData, err := config.MarshalConfig(cfg)
+	require.NoError(t, err)
+
+	db := sqlx.NewDb(mainHelper.SqlSupplier.GetMaster().Db, *mainHelper.Settings.DriverName)
+	err = config.InitializeConfigurationsTable(db)
+	require.NoError(t, err)
+
+	id := model.NewId()
+	_, err = db.NamedExec("INSERT INTO Configurations (Id, Value, CreateAt, Active) VALUES(:Id, :Value, :CreateAt, TRUE)", map[string]interface{}{
+		"Id":       id,
+		"Value":    cfgData,
+		"CreateAt": model.GetMillis(),
+	})
+	require.NoError(t, err)
+
+	return id, func() {
+		truncateTables(t)
+	}
+}
+
+// getActualDatabaseConfig returns the active configuration in the database without relying on a config store.
+func getActualDatabaseConfig(t *testing.T) *model.Config {
+	t.Helper()
+
+	var actualCfgData []byte
+	db := sqlx.NewDb(mainHelper.SqlSupplier.GetMaster().Db, *mainHelper.Settings.DriverName)
+	err := db.Get(&actualCfgData, "SELECT Value FROM Configurations WHERE Active")
+	require.NoError(t, err)
+
+	actualCfg, _, err := config.UnmarshalConfig(bytes.NewReader(actualCfgData), false)
+	require.Nil(t, err)
+
+	return actualCfg
+}
+
+// assertDatabaseEqualsConfig verifies the active in-database configuration equals the given config.
+func assertDatabaseEqualsConfig(t *testing.T, expectedCfg *model.Config) {
+	t.Helper()
+
+	expectedCfg = prepareExpectedConfig(t, expectedCfg)
+	actualCfg := getActualDatabaseConfig(t)
+	assert.Equal(t, expectedCfg, actualCfg)
+}
+
+// assertDatabaseNotEqualsConfig verifies the in-database configuration does not equal the given config.
+func assertDatabaseNotEqualsConfig(t *testing.T, expectedCfg *model.Config) {
+	t.Helper()
+
+	expectedCfg = prepareExpectedConfig(t, expectedCfg)
+	actualCfg := getActualDatabaseConfig(t)
+	assert.NotEqual(t, expectedCfg, actualCfg)
+}
+
+func TestDatabaseStoreNew(t *testing.T) {
+	t.Run("no existing configuration - initialization required", func(t *testing.T) {
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		assert.Equal(t, model.SERVICE_SETTINGS_DEFAULT_SITE_URL, *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("existing config, initialization required", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, testConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		assert.Equal(t, "http://TestStoreNew", *ds.Get().ServiceSettings.SiteURL)
+		assertDatabaseNotEqualsConfig(t, testConfig)
+	})
+
+	t.Run("already minimally configured", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, minimalConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		assert.Equal(t, "http://minimal", *ds.Get().ServiceSettings.SiteURL)
+		assertDatabaseEqualsConfig(t, minimalConfig)
+	})
+
+	t.Run("invalid url", func(t *testing.T) {
+		_, err := config.NewDatabaseStore("")
+		require.Error(t, err)
+	})
+
+	t.Run("unsupported scheme", func(t *testing.T) {
+		_, err := config.NewDatabaseStore("invalid")
+		require.Error(t, err)
+	})
+}
+
+func TestDatabaseStoreGet(t *testing.T) {
+	_, tearDown := setupConfigDatabase(t, testConfig)
+	defer tearDown()
+
+	ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+	require.NoError(t, err)
+	defer ds.Close()
+
+	cfg := ds.Get()
+	assert.Equal(t, "http://TestStoreNew", *cfg.ServiceSettings.SiteURL)
+
+	cfg2 := ds.Get()
+	assert.Equal(t, "http://TestStoreNew", *cfg.ServiceSettings.SiteURL)
+
+	assert.True(t, cfg == cfg2, "Get() returned different configuration instances")
+
+	newCfg := &model.Config{}
+	oldCfg, err := ds.Set(newCfg)
+	require.NoError(t, err)
+
+	assert.True(t, oldCfg == cfg, "returned config after set() changed original")
+	assert.False(t, newCfg == cfg, "returned config should have been different from original")
+}
+
+func TestDatabaseStoreGetEnivironmentOverrides(t *testing.T) {
+	_, tearDown := setupConfigDatabase(t, testConfig)
+	defer tearDown()
+
+	ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+	require.NoError(t, err)
+	defer ds.Close()
+
+	assert.Equal(t, "http://TestStoreNew", *ds.Get().ServiceSettings.SiteURL)
+	assert.Empty(t, ds.GetEnvironmentOverrides())
+
+	os.Setenv("MM_SERVICESETTINGS_SITEURL", "http://override")
+
+	ds, err = config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+	require.NoError(t, err)
+	defer ds.Close()
+
+	assert.Equal(t, "http://override", *ds.Get().ServiceSettings.SiteURL)
+	assert.Equal(t, map[string]interface{}{"ServiceSettings": map[string]interface{}{"SiteURL": true}}, ds.GetEnvironmentOverrides())
+}
+
+func TestDatabaseStoreSet(t *testing.T) {
+	t.Run("set same pointer value", func(t *testing.T) {
+		t.Skip("not yet implemented")
+
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		_, err = ds.Set(ds.Get())
+		if assert.Error(t, err) {
+			assert.EqualError(t, err, "old configuration modified instead of cloning")
+		}
+	})
+
+	t.Run("defaults required", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, minimalConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		oldCfg := ds.Get()
+
+		newCfg := &model.Config{}
+
+		retCfg, err := ds.Set(newCfg)
+		require.NoError(t, err)
+		assert.Equal(t, oldCfg, retCfg)
+
+		assert.Equal(t, model.SERVICE_SETTINGS_DEFAULT_SITE_URL, *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("desanitization required", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, ldapConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		oldCfg := ds.Get()
+
+		newCfg := &model.Config{}
+		newCfg.LdapSettings.BindPassword = sToP(model.FAKE_SETTING)
+
+		retCfg, err := ds.Set(newCfg)
+		require.NoError(t, err)
+		assert.Equal(t, oldCfg, retCfg)
+
+		assert.Equal(t, "password", *ds.Get().LdapSettings.BindPassword)
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		newCfg := &model.Config{}
+		newCfg.ServiceSettings.SiteURL = sToP("invalid")
+
+		_, err = ds.Set(newCfg)
+		if assert.Error(t, err) {
+			assert.EqualError(t, err, "new configuration is invalid: Config.IsValid: model.config.is_valid.site_url.app_error, ")
+		}
+
+		assert.Equal(t, model.SERVICE_SETTINGS_DEFAULT_SITE_URL, *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("read-only ignored", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, readOnlyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		newCfg := &model.Config{
+			ServiceSettings: model.ServiceSettings{
+				SiteURL: sToP("http://new"),
+			},
+		}
+
+		_, err = ds.Set(newCfg)
+		require.NoError(t, err)
+
+		assert.Equal(t, "http://new", *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("persist failed", func(t *testing.T) {
+		t.Skip("skipping persistence test inside Set")
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		db := sqlx.NewDb(mainHelper.SqlSupplier.GetMaster().Db, *mainHelper.Settings.DriverName)
+		_, err = db.Exec("DROP TABLE Configurations")
+		require.NoError(t, err)
+
+		newCfg := &model.Config{}
+
+		_, err = ds.Set(newCfg)
+		if assert.Error(t, err) {
+			assert.True(t, strings.HasPrefix(err.Error(), "failed to persist: failed to write to database"))
+		}
+
+		assert.Equal(t, model.SERVICE_SETTINGS_DEFAULT_SITE_URL, *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("listeners notified", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		oldCfg := ds.Get()
+
+		called := make(chan bool, 1)
+		callback := func(oldfg, newCfg *model.Config) {
+			called <- true
+		}
+		ds.AddListener(callback)
+
+		newCfg := &model.Config{}
+
+		retCfg, err := ds.Set(newCfg)
+		require.NoError(t, err)
+		assert.Equal(t, oldCfg, retCfg)
+
+		select {
+		case <-called:
+		case <-time.After(5 * time.Second):
+			t.Fatal("callback should have been called when config written")
+		}
+	})
+}
+
+func TestDatabaseStoreLoad(t *testing.T) {
+	t.Run("active configuration no longer exists", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		truncateTables(t)
+
+		err = ds.Load()
+		require.NoError(t, err)
+		assertDatabaseNotEqualsConfig(t, emptyConfig)
+	})
+
+	t.Run("honour environment", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, minimalConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		os.Setenv("MM_SERVICESETTINGS_SITEURL", "http://override")
+
+		err = ds.Load()
+		require.NoError(t, err)
+		assert.Equal(t, "http://override", *ds.Get().ServiceSettings.SiteURL)
+		assert.Equal(t, map[string]interface{}{"ServiceSettings": map[string]interface{}{"SiteURL": true}}, ds.GetEnvironmentOverrides())
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		cfgData, err := config.MarshalConfig(invalidConfig)
+		require.NoError(t, err)
+
+		db := sqlx.NewDb(mainHelper.SqlSupplier.GetMaster().Db, *mainHelper.Settings.DriverName)
+		truncateTables(t)
+		id := model.NewId()
+		_, err = db.NamedExec("INSERT INTO Configurations (Id, Value, CreateAt, Active) VALUES(:Id, :Value, :CreateAt, TRUE)", map[string]interface{}{
+			"Id":       id,
+			"Value":    cfgData,
+			"CreateAt": model.GetMillis(),
+		})
+		require.NoError(t, err)
+
+		err = ds.Load()
+		if assert.Error(t, err) {
+			assert.EqualError(t, err, "invalid config: Config.IsValid: model.config.is_valid.site_url.app_error, ")
+		}
+	})
+
+	t.Run("fixes required", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, fixesRequiredConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		err = ds.Load()
+		require.NoError(t, err)
+		assertDatabaseNotEqualsConfig(t, fixesRequiredConfig)
+		assert.Equal(t, "http://trailingslash", *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("listeners notifed", func(t *testing.T) {
+		_, tearDown := setupConfigDatabase(t, emptyConfig)
+		defer tearDown()
+
+		ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+		require.NoError(t, err)
+		defer ds.Close()
+
+		called := make(chan bool, 1)
+		callback := func(oldfg, newCfg *model.Config) {
+			called <- true
+		}
+		ds.AddListener(callback)
+
+		err = ds.Load()
+		require.NoError(t, err)
+
+		select {
+		case <-called:
+		case <-time.After(5 * time.Second):
+			t.Fatal("callback should have been called when config loaded")
+		}
+	})
+}
+
+func TestDatabaseStoreSave(t *testing.T) {
+	_, tearDown := setupConfigDatabase(t, minimalConfig)
+	defer tearDown()
+
+	ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+	require.NoError(t, err)
+	defer ds.Close()
+
+	newCfg := &model.Config{
+		ServiceSettings: model.ServiceSettings{
+			SiteURL: sToP("http://new"),
+		},
+	}
+
+	t.Run("set without save", func(t *testing.T) {
+		_, err = ds.Set(newCfg)
+		require.NoError(t, err)
+
+		err = ds.Load()
+		require.NoError(t, err)
+
+		assert.Equal(t, "http://minimal", *ds.Get().ServiceSettings.SiteURL)
+	})
+
+	t.Run("set with save", func(t *testing.T) {
+		_, err = ds.Set(newCfg)
+		require.NoError(t, err)
+
+		err = ds.Save()
+		require.NoError(t, err)
+
+		err = ds.Load()
+		require.NoError(t, err)
+
+		assert.Equal(t, "http://new", *ds.Get().ServiceSettings.SiteURL)
+	})
+}
+
+func TestDatabaseStoreString(t *testing.T) {
+	_, tearDown := setupConfigDatabase(t, emptyConfig)
+	defer tearDown()
+
+	ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource))
+	require.NoError(t, err)
+	defer ds.Close()
+
+	actualStringURL, err := url.Parse(ds.String())
+	require.NoError(t, err)
+
+	assert.Equal(t, *mainHelper.Settings.DriverName, actualStringURL.Scheme)
+	actualUsername := actualStringURL.User.Username()
+	actualPassword, _ := actualStringURL.User.Password()
+	assert.NotEmpty(t, actualUsername)
+	assert.Empty(t, actualPassword, "should mask password")
+}

config/export_test.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"io"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+// MarshalConfig exposes the internal marshalConfig to tests only.
+func MarshalConfig(cfg *model.Config) ([]byte, error) {
+	return marshalConfig(cfg)
+}
+
+// UnmarshalConfig exposes the internal unmarshalConfig to tests only.
+func UnmarshalConfig(r io.Reader, allowEnvironmentOverrides bool) (*model.Config, map[string]interface{}, error) {
+	return unmarshalConfig(r, allowEnvironmentOverrides)
+}
+
+// InitializeConfigurationsTable exposes the internal initializeConfigurationsTable to test only.
+func InitializeConfigurationsTable(db *sqlx.DB) error {
+	return initializeConfigurationsTable(db)
+}

config/file.go

@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"sync"
 
 	"github.com/pkg/errors"
 
@@ -25,14 +24,11 @@ var (
 
 // FileStore is a config store backed by a file such as config/config.json.
 type FileStore struct {
-	emitter
+	commonStore
 
-	config               *model.Config
-	environmentOverrides map[string]interface{}
-	configLock           sync.RWMutex
-	path                 string
-	watch                bool
-	watcher              *watcher
+	path    string
+	watch   bool
+	watcher *watcher
 }
 
 // NewFileStore creates a new instance of a config store backed by the given file path.
@@ -90,68 +86,15 @@ func resolveConfigFilePath(path string) (string, error) {
 	return "", fmt.Errorf("failed to find config file %s", path)
 }
 
-// Get fetches the current, cached configuration.
-func (fs *FileStore) Get() *model.Config {
-	fs.configLock.RLock()
-	defer fs.configLock.RUnlock()
-
-	return fs.config
-}
-
-// GetEnvironmentOverrides fetches the configuration fields overridden by environment variables.
-func (fs *FileStore) GetEnvironmentOverrides() map[string]interface{} {
-	fs.configLock.RLock()
-	defer fs.configLock.RUnlock()
-
-	return fs.environmentOverrides
-}
-
 // Set replaces the current configuration in its entirety, without updating the backing store.
 func (fs *FileStore) Set(newCfg *model.Config) (*model.Config, error) {
-	fs.configLock.Lock()
-	var unlockOnce sync.Once
-	defer unlockOnce.Do(fs.configLock.Unlock)
+	return fs.commonStore.set(newCfg, func(cfg *model.Config) error {
+		if *fs.config.ClusterSettings.Enable && *fs.config.ClusterSettings.ReadOnlyConfig {
+			return ErrReadOnlyConfiguration
+		}
 
-	oldCfg := fs.config
-
-	// TODO: disallow attempting to save a directly modified config (comparing pointers). This
-	// wouldn't be an exhaustive check, given the use of pointers throughout the data
-	// structure, but might prevent common mistakes. Requires upstream changes first.
-	// if newCfg == oldCfg {
-	// 	return nil, errors.New("old configuration modified instead of cloning")
-	// }
-
-	newCfg = newCfg.Clone()
-	newCfg.SetDefaults()
-
-	// Sometimes the config is received with "fake" data in sensitive fields. Apply the real
-	// data from the existing config as necessary.
-	desanitize(oldCfg, newCfg)
-
-	if err := newCfg.IsValid(); err != nil {
-		return nil, errors.Wrap(err, "new configuration is invalid")
-	}
-
-	if *oldCfg.ClusterSettings.Enable && *oldCfg.ClusterSettings.ReadOnlyConfig {
-		return nil, ErrReadOnlyConfiguration
-	}
-
-	// Ideally, Set would persist automatically and abstract this completely away from the
-	// client. Doing so requires a few upstream changes first, so for now an explicit Save()
-	// remains required.
-	// if err := fs.persist(newCfg); err != nil {
-	// 	return nil, errors.Wrap(err, "failed to persist")
-	// }
-
-	fs.config = newCfg
-
-	unlockOnce.Do(fs.configLock.Unlock)
-
-	// Notify listeners synchronously. Ideally, this would be asynchronous, but existing code
-	// assumes this and there would be increased complexity to avoid racing updates.
-	fs.invokeConfigListeners(oldCfg, newCfg)
-
-	return oldCfg, nil
+		return nil
+	})
 }
 
 // persist writes the configuration to the configured file.
@@ -206,50 +149,7 @@ func (fs *FileStore) Load() (err error) {
 		}
 	}()
 
-	allowEnvironmentOverrides := true
-	loadedCfg, environmentOverrides, err := unmarshalConfig(f, allowEnvironmentOverrides)
-	if err != nil {
-		return errors.Wrapf(err, "failed to unmarshal config from %s", fs.path)
-	}
-
-	// SetDefaults generates various keys and salts if not previously configured. Determine if
-	// such a change will be made before invoking. This method will not effect the save: that
-	// remains the responsibility of the caller.
-	needsSave = needsSave || loadedCfg.SqlSettings.AtRestEncryptKey == nil || len(*loadedCfg.SqlSettings.AtRestEncryptKey) == 0
-	needsSave = needsSave || loadedCfg.FileSettings.PublicLinkSalt == nil || len(*loadedCfg.FileSettings.PublicLinkSalt) == 0
-	needsSave = needsSave || loadedCfg.EmailSettings.InviteSalt == nil || len(*loadedCfg.EmailSettings.InviteSalt) == 0
-
-	loadedCfg.SetDefaults()
-
-	if err := loadedCfg.IsValid(); err != nil {
-		return errors.Wrap(err, "invalid config")
-	}
-
-	if changed := fixConfig(loadedCfg); changed {
-		needsSave = true
-	}
-
-	fs.configLock.Lock()
-	var unlockOnce sync.Once
-	defer unlockOnce.Do(fs.configLock.Unlock)
-
-	if needsSave {
-		if err = fs.persist(loadedCfg); err != nil {
-			return errors.Wrap(err, "failed to persist required changes after load")
-		}
-	}
-
-	oldCfg := fs.config
-	fs.config = loadedCfg
-	fs.environmentOverrides = environmentOverrides
-
-	unlockOnce.Do(fs.configLock.Unlock)
-
-	// Notify listeners synchronously. Ideally, this would be asynchronous, but existing code
-	// assumes this and there would be increased complexity to avoid racing updates.
-	fs.invokeConfigListeners(oldCfg, loadedCfg)
-
-	return nil
+	return fs.commonStore.load(f, needsSave, fs.persist)
 }
 
 // Save writes the current configuration to the backing store.

config/file_test.go

@@ -104,36 +104,32 @@ func setupConfigFile(t *testing.T, cfg *model.Config) (string, func()) {
 	}
 }
 
-// assertFileEqualsConfig verifies the on disk contents of the given path equal the given config.
-func assertFileEqualsConfig(t *testing.T, expectedCfg *model.Config, path string) {
+// getActualFileConfig returns the configuration present in the given file without relying a config store.
+func getActualFileConfig(t *testing.T, path string) *model.Config {
+	t.Helper()
+
 	f, err := os.Open(path)
 	require.Nil(t, err)
-
-	// These fields require special initialization for our tests.
-	expectedCfg = expectedCfg.Clone()
-	expectedCfg.MessageExportSettings.GlobalRelaySettings = &model.GlobalRelayMessageExportSettings{}
-	expectedCfg.PluginSettings.Plugins = make(map[string]map[string]interface{})
-	expectedCfg.PluginSettings.PluginStates = make(map[string]*model.PluginState)
+	defer f.Close()
 
 	actualCfg, _, err := config.UnmarshalConfig(f, false)
 	require.Nil(t, err)
 
+	return actualCfg
+}
+
+// assertFileEqualsConfig verifies the on disk contents of the given path equal the given config.
+func assertFileEqualsConfig(t *testing.T, expectedCfg *model.Config, path string) {
+	expectedCfg = prepareExpectedConfig(t, expectedCfg)
+	actualCfg := getActualFileConfig(t, path)
+
 	assert.Equal(t, expectedCfg, actualCfg)
 }
 
 // assertFileNotEqualsConfig verifies the on disk contents of the given path does not equal the given config.
 func assertFileNotEqualsConfig(t *testing.T, expectedCfg *model.Config, path string) {
-	f, err := os.Open(path)
-	require.Nil(t, err)
-
-	// These fields require special initialization for our tests.
-	expectedCfg = expectedCfg.Clone()
-	expectedCfg.MessageExportSettings.GlobalRelaySettings = &model.GlobalRelayMessageExportSettings{}
-	expectedCfg.PluginSettings.Plugins = make(map[string]map[string]interface{})
-	expectedCfg.PluginSettings.PluginStates = make(map[string]*model.PluginState)
-
-	actualCfg, _, err := config.UnmarshalConfig(f, false)
-	require.Nil(t, err)
+	expectedCfg = prepareExpectedConfig(t, expectedCfg)
+	actualCfg := getActualFileConfig(t, path)
 
 	assert.NotEqual(t, expectedCfg, actualCfg)
 }

config/main_test.go

@@ -1,17 +1,45 @@
-package config
+package config_test
 
 import (
-	"io"
+	"testing"
 
+	"github.com/go-sql-driver/mysql"
 	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/testlib"
+	"github.com/stretchr/testify/require"
 )
 
-// MarshalConfig exposes the internal marshalConfig to tests only.
-func MarshalConfig(cfg *model.Config) ([]byte, error) {
-	return marshalConfig(cfg)
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
 }
 
-// UnmarshalConfig exposes the internal unmarshalConfig to tests only.
-func UnmarshalConfig(r io.Reader, allowEnvironmentOverrides bool) (*model.Config, map[string]interface{}, error) {
-	return unmarshalConfig(r, allowEnvironmentOverrides)
+// truncateTables clears tables used by the config package for reuse in other tests
+func truncateTables(t *testing.T) {
+	t.Helper()
+
+	switch *mainHelper.Settings.DriverName {
+	case model.DATABASE_DRIVER_MYSQL:
+		_, err := mainHelper.SqlSupplier.GetMaster().Db.Exec("TRUNCATE TABLE Configurations")
+		if err != nil {
+			if driverErr, ok := err.(*mysql.MySQLError); ok {
+				// Ignore if the Configurations table does not exist.
+				if driverErr.Number == 1146 {
+					return
+				}
+			}
+		}
+		require.NoError(t, err)
+
+	case model.DATABASE_DRIVER_POSTGRES:
+		_, err := mainHelper.SqlSupplier.GetMaster().Db.Exec("TRUNCATE TABLE Configurations")
+		require.NoError(t, err)
+
+	default:
+		t.Fatalf("unsupported driver name: %s", *mainHelper.Settings.DriverName)
+	}
 }

config/store.go

@@ -4,6 +4,8 @@
 package config
 
 import (
+	"strings"
+
 	"github.com/mattermost/mattermost-server/model"
 )
 
@@ -39,3 +41,12 @@ type Store interface {
 	// Close cleans up resources associated with the store.
 	Close() error
 }
+
+// NewStore creates a database or file store given a data source name by which to connect.
+func NewStore(dsn string, watch bool) (Store, error) {
+	if strings.HasPrefix(dsn, "mysql://") || strings.HasPrefix(dsn, "postgres://") {
+		return NewDatabaseStore(dsn)
+	}
+
+	return NewFileStore(dsn, watch)
+}

config/store_test.go

@@ -0,0 +1,35 @@
+package config_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewStore(t *testing.T) {
+	t.Run("database dsn", func(t *testing.T) {
+		ds, err := config.NewStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource), false)
+		require.NoError(t, err)
+		ds.Close()
+	})
+
+	t.Run("database dsn, watch ignored", func(t *testing.T) {
+		ds, err := config.NewStore(fmt.Sprintf("%s://%s", *mainHelper.Settings.DriverName, *mainHelper.Settings.DataSource), true)
+		require.NoError(t, err)
+		ds.Close()
+	})
+
+	t.Run("file dsn", func(t *testing.T) {
+		fs, err := config.NewStore("config.json", false)
+		require.NoError(t, err)
+		fs.Close()
+	})
+
+	t.Run("file dsn, watch", func(t *testing.T) {
+		fs, err := config.NewStore("config.json", true)
+		require.NoError(t, err)
+		fs.Close()
+	})
+}

migrations/helper_test.go

@@ -49,7 +49,7 @@ func setupTestHelper(enterprise bool) *TestHelper {
 		panic(err)
 	}
 
-	options := []app.Option{app.ConfigFile(tempConfig.Name(), false)}
+	options := []app.Option{app.Config(tempConfig.Name(), false)}
 	options = append(options, app.StoreOverride(mainHelper.Store))
 
 	s, err := app.NewServer(options...)

vendor/github.com/jmoiron/sqlx/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+tags
+environ

vendor/github.com/jmoiron/sqlx/.travis.yml

@@ -0,0 +1,27 @@
+# vim: ft=yaml sw=2 ts=2
+
+language: go
+
+# enable database services
+services:
+  - mysql
+  - postgresql
+
+# create test database
+before_install:
+  - mysql -e 'CREATE DATABASE IF NOT EXISTS sqlxtest;'
+  - psql -c 'create database sqlxtest;' -U postgres
+  - go get github.com/mattn/goveralls
+  - export SQLX_MYSQL_DSN="travis:@/sqlxtest?parseTime=true"
+  - export SQLX_POSTGRES_DSN="postgres://postgres:@localhost/sqlxtest?sslmode=disable"
+  - export SQLX_SQLITE_DSN="$HOME/sqlxtest.db"
+
+# go versions to test
+go:
+  - "1.8"
+  - "1.9"
+  - "1.10.x"
+
+# run tests w/ coverage
+script:
+  - travis_retry $GOPATH/bin/goveralls -service=travis-ci

vendor/github.com/jmoiron/sqlx/LICENSE

@@ -0,0 +1,23 @@
+ Copyright (c) 2013, Jason Moiron
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+

vendor/github.com/jmoiron/sqlx/README.md

@@ -0,0 +1,187 @@
+# sqlx
+
+[![Build Status](https://travis-ci.org/jmoiron/sqlx.svg?branch=master)](https://travis-ci.org/jmoiron/sqlx) [![Coverage Status](https://coveralls.io/repos/github/jmoiron/sqlx/badge.svg?branch=master)](https://coveralls.io/github/jmoiron/sqlx?branch=master) [![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/jmoiron/sqlx) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/jmoiron/sqlx/master/LICENSE)
+
+sqlx is a library which provides a set of extensions on go's standard
+`database/sql` library.  The sqlx versions of `sql.DB`, `sql.TX`, `sql.Stmt`,
+et al. all leave the underlying interfaces untouched, so that their interfaces
+are a superset on the standard ones.  This makes it relatively painless to
+integrate existing codebases using database/sql with sqlx.
+
+Major additional concepts are:
+
+* Marshal rows into structs (with embedded struct support), maps, and slices
+* Named parameter support including prepared statements
+* `Get` and `Select` to go quickly from query to struct/slice
+
+In addition to the [godoc API documentation](http://godoc.org/github.com/jmoiron/sqlx),
+there is also some [standard documentation](http://jmoiron.github.io/sqlx/) that
+explains how to use `database/sql` along with sqlx.
+
+## Recent Changes
+
+* The [introduction](https://github.com/jmoiron/sqlx/pull/387) of `sql.ColumnType` sets the required minimum Go version to 1.8.
+
+* sqlx/types.JsonText has been renamed to JSONText to follow Go naming conventions.
+
+This breaks backwards compatibility, but it's in a way that is trivially fixable
+(`s/JsonText/JSONText/g`).  The `types` package is both experimental and not in
+active development currently.
+
+* Using Go 1.6 and below with `types.JSONText` and `types.GzippedText` can be _potentially unsafe_, **especially** when used with common auto-scan sqlx idioms like `Select` and `Get`. See [golang bug #13905](https://github.com/golang/go/issues/13905).
+
+### Backwards Compatibility
+
+There is no Go1-like promise of absolute stability, but I take the issue seriously
+and will maintain the library in a compatible state unless vital bugs prevent me 
+from doing so.  Since [#59](https://github.com/jmoiron/sqlx/issues/59) and 
+[#60](https://github.com/jmoiron/sqlx/issues/60) necessitated breaking behavior, 
+a wider API cleanup was done at the time of fixing.  It's possible this will happen
+in future;  if it does, a git tag will be provided for users requiring the old
+behavior to continue to use it until such a time as they can migrate.
+
+## install
+
+    go get github.com/jmoiron/sqlx
+
+## issues
+
+Row headers can be ambiguous (`SELECT 1 AS a, 2 AS a`), and the result of
+`Columns()` does not fully qualify column names in queries like:
+
+```sql
+SELECT a.id, a.name, b.id, b.name FROM foos AS a JOIN foos AS b ON a.parent = b.id;
+```
+
+making a struct or map destination ambiguous.  Use `AS` in your queries
+to give columns distinct names, `rows.Scan` to scan them manually, or 
+`SliceScan` to get a slice of results.
+
+## usage
+
+Below is an example which shows some common use cases for sqlx.  Check 
+[sqlx_test.go](https://github.com/jmoiron/sqlx/blob/master/sqlx_test.go) for more
+usage.
+
+
+```go
+package main
+
+import (
+    "database/sql"
+    "fmt"
+    "log"
+    
+    _ "github.com/lib/pq"
+    "github.com/jmoiron/sqlx"
+)
+
+var schema = `
+CREATE TABLE person (
+    first_name text,
+    last_name text,
+    email text
+);
+
+CREATE TABLE place (
+    country text,
+    city text NULL,
+    telcode integer
+)`
+
+type Person struct {
+    FirstName string `db:"first_name"`
+    LastName  string `db:"last_name"`
+    Email     string
+}
+
+type Place struct {
+    Country string
+    City    sql.NullString
+    TelCode int
+}
+
+func main() {
+    // this Pings the database trying to connect, panics on error
+    // use sqlx.Open() for sql.Open() semantics
+    db, err := sqlx.Connect("postgres", "user=foo dbname=bar sslmode=disable")
+    if err != nil {
+        log.Fatalln(err)
+    }
+
+    // exec the schema or fail; multi-statement Exec behavior varies between
+    // database drivers;  pq will exec them all, sqlite3 won't, ymmv
+    db.MustExec(schema)
+    
+    tx := db.MustBegin()
+    tx.MustExec("INSERT INTO person (first_name, last_name, email) VALUES ($1, $2, $3)", "Jason", "Moiron", "jmoiron@jmoiron.net")
+    tx.MustExec("INSERT INTO person (first_name, last_name, email) VALUES ($1, $2, $3)", "John", "Doe", "johndoeDNE@gmail.net")
+    tx.MustExec("INSERT INTO place (country, city, telcode) VALUES ($1, $2, $3)", "United States", "New York", "1")
+    tx.MustExec("INSERT INTO place (country, telcode) VALUES ($1, $2)", "Hong Kong", "852")
+    tx.MustExec("INSERT INTO place (country, telcode) VALUES ($1, $2)", "Singapore", "65")
+    // Named queries can use structs, so if you have an existing struct (i.e. person := &Person{}) that you have populated, you can pass it in as &person
+    tx.NamedExec("INSERT INTO person (first_name, last_name, email) VALUES (:first_name, :last_name, :email)", &Person{"Jane", "Citizen", "jane.citzen@example.com"})
+    tx.Commit()
+
+    // Query the database, storing results in a []Person (wrapped in []interface{})
+    people := []Person{}
+    db.Select(&people, "SELECT * FROM person ORDER BY first_name ASC")
+    jason, john := people[0], people[1]
+
+    fmt.Printf("%#v\n%#v", jason, john)
+    // Person{FirstName:"Jason", LastName:"Moiron", Email:"jmoiron@jmoiron.net"}
+    // Person{FirstName:"John", LastName:"Doe", Email:"johndoeDNE@gmail.net"}
+
+    // You can also get a single result, a la QueryRow
+    jason = Person{}
+    err = db.Get(&jason, "SELECT * FROM person WHERE first_name=$1", "Jason")
+    fmt.Printf("%#v\n", jason)
+    // Person{FirstName:"Jason", LastName:"Moiron", Email:"jmoiron@jmoiron.net"}
+
+    // if you have null fields and use SELECT *, you must use sql.Null* in your struct
+    places := []Place{}
+    err = db.Select(&places, "SELECT * FROM place ORDER BY telcode ASC")
+    if err != nil {
+        fmt.Println(err)
+        return
+    }
+    usa, singsing, honkers := places[0], places[1], places[2]
+    
+    fmt.Printf("%#v\n%#v\n%#v\n", usa, singsing, honkers)
+    // Place{Country:"United States", City:sql.NullString{String:"New York", Valid:true}, TelCode:1}
+    // Place{Country:"Singapore", City:sql.NullString{String:"", Valid:false}, TelCode:65}
+    // Place{Country:"Hong Kong", City:sql.NullString{String:"", Valid:false}, TelCode:852}
+
+    // Loop through rows using only one struct
+    place := Place{}
+    rows, err := db.Queryx("SELECT * FROM place")
+    for rows.Next() {
+        err := rows.StructScan(&place)
+        if err != nil {
+            log.Fatalln(err)
+        } 
+        fmt.Printf("%#v\n", place)
+    }
+    // Place{Country:"United States", City:sql.NullString{String:"New York", Valid:true}, TelCode:1}
+    // Place{Country:"Hong Kong", City:sql.NullString{String:"", Valid:false}, TelCode:852}
+    // Place{Country:"Singapore", City:sql.NullString{String:"", Valid:false}, TelCode:65}
+
+    // Named queries, using `:name` as the bindvar.  Automatic bindvar support
+    // which takes into account the dbtype based on the driverName on sqlx.Open/Connect
+    _, err = db.NamedExec(`INSERT INTO person (first_name,last_name,email) VALUES (:first,:last,:email)`, 
+        map[string]interface{}{
+            "first": "Bin",
+            "last": "Smuth",
+            "email": "bensmith@allblacks.nz",
+    })
+
+    // Selects Mr. Smith from the database
+    rows, err = db.NamedQuery(`SELECT * FROM person WHERE first_name=:fn`, map[string]interface{}{"fn": "Bin"})
+
+    // Named queries can also use structs.  Their bind names follow the same rules
+    // as the name -> db mapping, so struct fields are lowercased and the `db` tag
+    // is taken into consideration.
+    rows, err = db.NamedQuery(`SELECT * FROM person WHERE first_name=:first_name`, jason)
+}
+```
+

vendor/github.com/jmoiron/sqlx/bind.go

@@ -0,0 +1,217 @@
+package sqlx
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"errors"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/jmoiron/sqlx/reflectx"
+)
+
+// Bindvar types supported by Rebind, BindMap and BindStruct.
+const (
+	UNKNOWN = iota
+	QUESTION
+	DOLLAR
+	NAMED
+	AT
+)
+
+// BindType returns the bindtype for a given database given a drivername.
+func BindType(driverName string) int {
+	switch driverName {
+	case "postgres", "pgx", "pq-timeouts", "cloudsqlpostgres":
+		return DOLLAR
+	case "mysql":
+		return QUESTION
+	case "sqlite3":
+		return QUESTION
+	case "oci8", "ora", "goracle":
+		return NAMED
+	case "sqlserver":
+		return AT
+	}
+	return UNKNOWN
+}
+
+// FIXME: this should be able to be tolerant of escaped ?'s in queries without
+// losing much speed, and should be to avoid confusion.
+
+// Rebind a query from the default bindtype (QUESTION) to the target bindtype.
+func Rebind(bindType int, query string) string {
+	switch bindType {
+	case QUESTION, UNKNOWN:
+		return query
+	}
+
+	// Add space enough for 10 params before we have to allocate
+	rqb := make([]byte, 0, len(query)+10)
+
+	var i, j int
+
+	for i = strings.Index(query, "?"); i != -1; i = strings.Index(query, "?") {
+		rqb = append(rqb, query[:i]...)
+
+		switch bindType {
+		case DOLLAR:
+			rqb = append(rqb, '$')
+		case NAMED:
+			rqb = append(rqb, ':', 'a', 'r', 'g')
+		case AT:
+			rqb = append(rqb, '@', 'p')
+		}
+
+		j++
+		rqb = strconv.AppendInt(rqb, int64(j), 10)
+
+		query = query[i+1:]
+	}
+
+	return string(append(rqb, query...))
+}
+
+// Experimental implementation of Rebind which uses a bytes.Buffer.  The code is
+// much simpler and should be more resistant to odd unicode, but it is twice as
+// slow.  Kept here for benchmarking purposes and to possibly replace Rebind if
+// problems arise with its somewhat naive handling of unicode.
+func rebindBuff(bindType int, query string) string {
+	if bindType != DOLLAR {
+		return query
+	}
+
+	b := make([]byte, 0, len(query))
+	rqb := bytes.NewBuffer(b)
+	j := 1
+	for _, r := range query {
+		if r == '?' {
+			rqb.WriteRune('$')
+			rqb.WriteString(strconv.Itoa(j))
+			j++
+		} else {
+			rqb.WriteRune(r)
+		}
+	}
+
+	return rqb.String()
+}
+
+// In expands slice values in args, returning the modified query string
+// and a new arg list that can be executed by a database. The `query` should
+// use the `?` bindVar.  The return value uses the `?` bindVar.
+func In(query string, args ...interface{}) (string, []interface{}, error) {
+	// argMeta stores reflect.Value and length for slices and
+	// the value itself for non-slice arguments
+	type argMeta struct {
+		v      reflect.Value
+		i      interface{}
+		length int
+	}
+
+	var flatArgsCount int
+	var anySlices bool
+
+	meta := make([]argMeta, len(args))
+
+	for i, arg := range args {
+		if a, ok := arg.(driver.Valuer); ok {
+			arg, _ = a.Value()
+		}
+		v := reflect.ValueOf(arg)
+		t := reflectx.Deref(v.Type())
+
+		// []byte is a driver.Value type so it should not be expanded
+		if t.Kind() == reflect.Slice && t != reflect.TypeOf([]byte{}) {
+			meta[i].length = v.Len()
+			meta[i].v = v
+
+			anySlices = true
+			flatArgsCount += meta[i].length
+
+			if meta[i].length == 0 {
+				return "", nil, errors.New("empty slice passed to 'in' query")
+			}
+		} else {
+			meta[i].i = arg
+			flatArgsCount++
+		}
+	}
+
+	// don't do any parsing if there aren't any slices;  note that this means
+	// some errors that we might have caught below will not be returned.
+	if !anySlices {
+		return query, args, nil
+	}
+
+	newArgs := make([]interface{}, 0, flatArgsCount)
+	buf := make([]byte, 0, len(query)+len(", ?")*flatArgsCount)
+
+	var arg, offset int
+
+	for i := strings.IndexByte(query[offset:], '?'); i != -1; i = strings.IndexByte(query[offset:], '?') {
+		if arg >= len(meta) {
+			// if an argument wasn't passed, lets return an error;  this is
+			// not actually how database/sql Exec/Query works, but since we are
+			// creating an argument list programmatically, we want to be able
+			// to catch these programmer errors earlier.
+			return "", nil, errors.New("number of bindVars exceeds arguments")
+		}
+
+		argMeta := meta[arg]
+		arg++
+
+		// not a slice, continue.
+		// our questionmark will either be written before the next expansion
+		// of a slice or after the loop when writing the rest of the query
+		if argMeta.length == 0 {
+			offset = offset + i + 1
+			newArgs = append(newArgs, argMeta.i)
+			continue
+		}
+
+		// write everything up to and including our ? character
+		buf = append(buf, query[:offset+i+1]...)
+
+		for si := 1; si < argMeta.length; si++ {
+			buf = append(buf, ", ?"...)
+		}
+
+		newArgs = appendReflectSlice(newArgs, argMeta.v, argMeta.length)
+
+		// slice the query and reset the offset. this avoids some bookkeeping for
+		// the write after the loop
+		query = query[offset+i+1:]
+		offset = 0
+	}
+
+	buf = append(buf, query...)
+
+	if arg < len(meta) {
+		return "", nil, errors.New("number of bindVars less than number arguments")
+	}
+
+	return string(buf), newArgs, nil
+}
+
+func appendReflectSlice(args []interface{}, v reflect.Value, vlen int) []interface{} {
+	switch val := v.Interface().(type) {
+	case []interface{}:
+		args = append(args, val...)
+	case []int:
+		for i := range val {
+			args = append(args, val[i])
+		}
+	case []string:
+		for i := range val {
+			args = append(args, val[i])
+		}
+	default:
+		for si := 0; si < vlen; si++ {
+			args = append(args, v.Index(si).Interface())
+		}
+	}
+
+	return args
+}

vendor/github.com/jmoiron/sqlx/doc.go

@@ -0,0 +1,12 @@
+// Package sqlx provides general purpose extensions to database/sql.
+//
+// It is intended to seamlessly wrap database/sql and provide convenience
+// methods which are useful in the development of database driven applications.
+// None of the underlying database/sql methods are changed.  Instead all extended
+// behavior is implemented through new methods defined on wrapper types.
+//
+// Additions include scanning into structs, named query support, rebinding
+// queries for different drivers, convenient shorthands for common error handling
+// and more.
+//
+package sqlx

vendor/github.com/jmoiron/sqlx/go.mod

@@ -0,0 +1,7 @@
+module github.com/jmoiron/sqlx
+
+require (
+	github.com/go-sql-driver/mysql v1.4.0
+	github.com/lib/pq v1.0.0
+	github.com/mattn/go-sqlite3 v1.9.0
+)

vendor/github.com/jmoiron/sqlx/go.sum

@@ -0,0 +1,6 @@
+github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=

vendor/github.com/jmoiron/sqlx/named.go

@@ -0,0 +1,356 @@
+package sqlx
+
+// Named Query Support
+//
+//  * BindMap - bind query bindvars to map/struct args
+//	* NamedExec, NamedQuery - named query w/ struct or map
+//  * NamedStmt - a pre-compiled named query which is a prepared statement
+//
+// Internal Interfaces:
+//
+//  * compileNamedQuery - rebind a named query, returning a query and list of names
+//  * bindArgs, bindMapArgs, bindAnyArgs - given a list of names, return an arglist
+//
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"unicode"
+
+	"github.com/jmoiron/sqlx/reflectx"
+)
+
+// NamedStmt is a prepared statement that executes named queries.  Prepare it
+// how you would execute a NamedQuery, but pass in a struct or map when executing.
+type NamedStmt struct {
+	Params      []string
+	QueryString string
+	Stmt        *Stmt
+}
+
+// Close closes the named statement.
+func (n *NamedStmt) Close() error {
+	return n.Stmt.Close()
+}
+
+// Exec executes a named statement using the struct passed.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) Exec(arg interface{}) (sql.Result, error) {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return *new(sql.Result), err
+	}
+	return n.Stmt.Exec(args...)
+}
+
+// Query executes a named statement using the struct argument, returning rows.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) Query(arg interface{}) (*sql.Rows, error) {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return nil, err
+	}
+	return n.Stmt.Query(args...)
+}
+
+// QueryRow executes a named statement against the database.  Because sqlx cannot
+// create a *sql.Row with an error condition pre-set for binding errors, sqlx
+// returns a *sqlx.Row instead.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryRow(arg interface{}) *Row {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return &Row{err: err}
+	}
+	return n.Stmt.QueryRowx(args...)
+}
+
+// MustExec execs a NamedStmt, panicing on error
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) MustExec(arg interface{}) sql.Result {
+	res, err := n.Exec(arg)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+// Queryx using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) Queryx(arg interface{}) (*Rows, error) {
+	r, err := n.Query(arg)
+	if err != nil {
+		return nil, err
+	}
+	return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err
+}
+
+// QueryRowx this NamedStmt.  Because of limitations with QueryRow, this is
+// an alias for QueryRow.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryRowx(arg interface{}) *Row {
+	return n.QueryRow(arg)
+}
+
+// Select using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) Select(dest interface{}, arg interface{}) error {
+	rows, err := n.Queryx(arg)
+	if err != nil {
+		return err
+	}
+	// if something happens here, we want to make sure the rows are Closed
+	defer rows.Close()
+	return scanAll(rows, dest, false)
+}
+
+// Get using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) Get(dest interface{}, arg interface{}) error {
+	r := n.QueryRowx(arg)
+	return r.scanAny(dest, false)
+}
+
+// Unsafe creates an unsafe version of the NamedStmt
+func (n *NamedStmt) Unsafe() *NamedStmt {
+	r := &NamedStmt{Params: n.Params, Stmt: n.Stmt, QueryString: n.QueryString}
+	r.Stmt.unsafe = true
+	return r
+}
+
+// A union interface of preparer and binder, required to be able to prepare
+// named statements (as the bindtype must be determined).
+type namedPreparer interface {
+	Preparer
+	binder
+}
+
+func prepareNamed(p namedPreparer, query string) (*NamedStmt, error) {
+	bindType := BindType(p.DriverName())
+	q, args, err := compileNamedQuery([]byte(query), bindType)
+	if err != nil {
+		return nil, err
+	}
+	stmt, err := Preparex(p, q)
+	if err != nil {
+		return nil, err
+	}
+	return &NamedStmt{
+		QueryString: q,
+		Params:      args,
+		Stmt:        stmt,
+	}, nil
+}
+
+func bindAnyArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
+	if maparg, ok := arg.(map[string]interface{}); ok {
+		return bindMapArgs(names, maparg)
+	}
+	return bindArgs(names, arg, m)
+}
+
+// private interface to generate a list of interfaces from a given struct
+// type, given a list of names to pull out of the struct.  Used by public
+// BindStruct interface.
+func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
+	arglist := make([]interface{}, 0, len(names))
+
+	// grab the indirected value of arg
+	v := reflect.ValueOf(arg)
+	for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; {
+		v = v.Elem()
+	}
+
+	err := m.TraversalsByNameFunc(v.Type(), names, func(i int, t []int) error {
+		if len(t) == 0 {
+			return fmt.Errorf("could not find name %s in %#v", names[i], arg)
+		}
+
+		val := reflectx.FieldByIndexesReadOnly(v, t)
+		arglist = append(arglist, val.Interface())
+
+		return nil
+	})
+
+	return arglist, err
+}
+
+// like bindArgs, but for maps.
+func bindMapArgs(names []string, arg map[string]interface{}) ([]interface{}, error) {
+	arglist := make([]interface{}, 0, len(names))
+
+	for _, name := range names {
+		val, ok := arg[name]
+		if !ok {
+			return arglist, fmt.Errorf("could not find name %s in %#v", name, arg)
+		}
+		arglist = append(arglist, val)
+	}
+	return arglist, nil
+}
+
+// bindStruct binds a named parameter query with fields from a struct argument.
+// The rules for binding field names to parameter names follow the same
+// conventions as for StructScan, including obeying the `db` struct tags.
+func bindStruct(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
+	bound, names, err := compileNamedQuery([]byte(query), bindType)
+	if err != nil {
+		return "", []interface{}{}, err
+	}
+
+	arglist, err := bindArgs(names, arg, m)
+	if err != nil {
+		return "", []interface{}{}, err
+	}
+
+	return bound, arglist, nil
+}
+
+// bindMap binds a named parameter query with a map of arguments.
+func bindMap(bindType int, query string, args map[string]interface{}) (string, []interface{}, error) {
+	bound, names, err := compileNamedQuery([]byte(query), bindType)
+	if err != nil {
+		return "", []interface{}{}, err
+	}
+
+	arglist, err := bindMapArgs(names, args)
+	return bound, arglist, err
+}
+
+// -- Compilation of Named Queries
+
+// Allow digits and letters in bind params;  additionally runes are
+// checked against underscores, meaning that bind params can have be
+// alphanumeric with underscores.  Mind the difference between unicode
+// digits and numbers, where '5' is a digit but '五' is not.
+var allowedBindRunes = []*unicode.RangeTable{unicode.Letter, unicode.Digit}
+
+// FIXME: this function isn't safe for unicode named params, as a failing test
+// can testify.  This is not a regression but a failure of the original code
+// as well.  It should be modified to range over runes in a string rather than
+// bytes, even though this is less convenient and slower.  Hopefully the
+// addition of the prepared NamedStmt (which will only do this once) will make
+// up for the slightly slower ad-hoc NamedExec/NamedQuery.
+
+// compile a NamedQuery into an unbound query (using the '?' bindvar) and
+// a list of names.
+func compileNamedQuery(qs []byte, bindType int) (query string, names []string, err error) {
+	names = make([]string, 0, 10)
+	rebound := make([]byte, 0, len(qs))
+
+	inName := false
+	last := len(qs) - 1
+	currentVar := 1
+	name := make([]byte, 0, 10)
+
+	for i, b := range qs {
+		// a ':' while we're in a name is an error
+		if b == ':' {
+			// if this is the second ':' in a '::' escape sequence, append a ':'
+			if inName && i > 0 && qs[i-1] == ':' {
+				rebound = append(rebound, ':')
+				inName = false
+				continue
+			} else if inName {
+				err = errors.New("unexpected `:` while reading named param at " + strconv.Itoa(i))
+				return query, names, err
+			}
+			inName = true
+			name = []byte{}
+		} else if inName && i > 0 && b == '=' {
+			rebound = append(rebound, ':', '=')
+			inName = false
+			continue
+			// if we're in a name, and this is an allowed character, continue
+		} else if inName && (unicode.IsOneOf(allowedBindRunes, rune(b)) || b == '_' || b == '.') && i != last {
+			// append the byte to the name if we are in a name and not on the last byte
+			name = append(name, b)
+			// if we're in a name and it's not an allowed character, the name is done
+		} else if inName {
+			inName = false
+			// if this is the final byte of the string and it is part of the name, then
+			// make sure to add it to the name
+			if i == last && unicode.IsOneOf(allowedBindRunes, rune(b)) {
+				name = append(name, b)
+			}
+			// add the string representation to the names list
+			names = append(names, string(name))
+			// add a proper bindvar for the bindType
+			switch bindType {
+			// oracle only supports named type bind vars even for positional
+			case NAMED:
+				rebound = append(rebound, ':')
+				rebound = append(rebound, name...)
+			case QUESTION, UNKNOWN:
+				rebound = append(rebound, '?')
+			case DOLLAR:
+				rebound = append(rebound, '$')
+				for _, b := range strconv.Itoa(currentVar) {
+					rebound = append(rebound, byte(b))
+				}
+				currentVar++
+			case AT:
+				rebound = append(rebound, '@', 'p')
+				for _, b := range strconv.Itoa(currentVar) {
+					rebound = append(rebound, byte(b))
+				}
+				currentVar++
+			}
+			// add this byte to string unless it was not part of the name
+			if i != last {
+				rebound = append(rebound, b)
+			} else if !unicode.IsOneOf(allowedBindRunes, rune(b)) {
+				rebound = append(rebound, b)
+			}
+		} else {
+			// this is a normal byte and should just go onto the rebound query
+			rebound = append(rebound, b)
+		}
+	}
+
+	return string(rebound), names, err
+}
+
+// BindNamed binds a struct or a map to a query with named parameters.
+// DEPRECATED: use sqlx.Named` instead of this, it may be removed in future.
+func BindNamed(bindType int, query string, arg interface{}) (string, []interface{}, error) {
+	return bindNamedMapper(bindType, query, arg, mapper())
+}
+
+// Named takes a query using named parameters and an argument and
+// returns a new query with a list of args that can be executed by
+// a database.  The return value uses the `?` bindvar.
+func Named(query string, arg interface{}) (string, []interface{}, error) {
+	return bindNamedMapper(QUESTION, query, arg, mapper())
+}
+
+func bindNamedMapper(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
+	if maparg, ok := arg.(map[string]interface{}); ok {
+		return bindMap(bindType, query, maparg)
+	}
+	return bindStruct(bindType, query, arg, m)
+}
+
+// NamedQuery binds a named query and then runs Query on the result using the
+// provided Ext (sqlx.Tx, sqlx.Db).  It works with both structs and with
+// map[string]interface{} types.
+func NamedQuery(e Ext, query string, arg interface{}) (*Rows, error) {
+	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
+	if err != nil {
+		return nil, err
+	}
+	return e.Queryx(q, args...)
+}
+
+// NamedExec uses BindStruct to get a query executable by the driver and
+// then runs Exec on the result.  Returns an error from the binding
+// or the query excution itself.
+func NamedExec(e Ext, query string, arg interface{}) (sql.Result, error) {
+	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
+	if err != nil {
+		return nil, err
+	}
+	return e.Exec(q, args...)
+}

vendor/github.com/jmoiron/sqlx/named_context.go

@@ -0,0 +1,132 @@
+// +build go1.8
+
+package sqlx
+
+import (
+	"context"
+	"database/sql"
+)
+
+// A union interface of contextPreparer and binder, required to be able to
+// prepare named statements with context (as the bindtype must be determined).
+type namedPreparerContext interface {
+	PreparerContext
+	binder
+}
+
+func prepareNamedContext(ctx context.Context, p namedPreparerContext, query string) (*NamedStmt, error) {
+	bindType := BindType(p.DriverName())
+	q, args, err := compileNamedQuery([]byte(query), bindType)
+	if err != nil {
+		return nil, err
+	}
+	stmt, err := PreparexContext(ctx, p, q)
+	if err != nil {
+		return nil, err
+	}
+	return &NamedStmt{
+		QueryString: q,
+		Params:      args,
+		Stmt:        stmt,
+	}, nil
+}
+
+// ExecContext executes a named statement using the struct passed.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) ExecContext(ctx context.Context, arg interface{}) (sql.Result, error) {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return *new(sql.Result), err
+	}
+	return n.Stmt.ExecContext(ctx, args...)
+}
+
+// QueryContext executes a named statement using the struct argument, returning rows.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryContext(ctx context.Context, arg interface{}) (*sql.Rows, error) {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return nil, err
+	}
+	return n.Stmt.QueryContext(ctx, args...)
+}
+
+// QueryRowContext executes a named statement against the database.  Because sqlx cannot
+// create a *sql.Row with an error condition pre-set for binding errors, sqlx
+// returns a *sqlx.Row instead.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryRowContext(ctx context.Context, arg interface{}) *Row {
+	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
+	if err != nil {
+		return &Row{err: err}
+	}
+	return n.Stmt.QueryRowxContext(ctx, args...)
+}
+
+// MustExecContext execs a NamedStmt, panicing on error
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) MustExecContext(ctx context.Context, arg interface{}) sql.Result {
+	res, err := n.ExecContext(ctx, arg)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+// QueryxContext using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryxContext(ctx context.Context, arg interface{}) (*Rows, error) {
+	r, err := n.QueryContext(ctx, arg)
+	if err != nil {
+		return nil, err
+	}
+	return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err
+}
+
+// QueryRowxContext this NamedStmt.  Because of limitations with QueryRow, this is
+// an alias for QueryRow.
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) QueryRowxContext(ctx context.Context, arg interface{}) *Row {
+	return n.QueryRowContext(ctx, arg)
+}
+
+// SelectContext using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) SelectContext(ctx context.Context, dest interface{}, arg interface{}) error {
+	rows, err := n.QueryxContext(ctx, arg)
+	if err != nil {
+		return err
+	}
+	// if something happens here, we want to make sure the rows are Closed
+	defer rows.Close()
+	return scanAll(rows, dest, false)
+}
+
+// GetContext using this NamedStmt
+// Any named placeholder parameters are replaced with fields from arg.
+func (n *NamedStmt) GetContext(ctx context.Context, dest interface{}, arg interface{}) error {
+	r := n.QueryRowxContext(ctx, arg)
+	return r.scanAny(dest, false)
+}
+
+// NamedQueryContext binds a named query and then runs Query on the result using the
+// provided Ext (sqlx.Tx, sqlx.Db).  It works with both structs and with
+// map[string]interface{} types.
+func NamedQueryContext(ctx context.Context, e ExtContext, query string, arg interface{}) (*Rows, error) {
+	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
+	if err != nil {
+		return nil, err
+	}
+	return e.QueryxContext(ctx, q, args...)
+}
+
+// NamedExecContext uses BindStruct to get a query executable by the driver and
+// then runs Exec on the result.  Returns an error from the binding
+// or the query excution itself.
+func NamedExecContext(ctx context.Context, e ExtContext, query string, arg interface{}) (sql.Result, error) {
+	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
+	if err != nil {
+		return nil, err
+	}
+	return e.ExecContext(ctx, q, args...)
+}

vendor/github.com/jmoiron/sqlx/reflectx/README.md

@@ -0,0 +1,17 @@
+# reflectx
+
+The sqlx package has special reflect needs.  In particular, it needs to:
+
+* be able to map a name to a field
+* understand embedded structs
+* understand mapping names to fields by a particular tag
+* user specified name -> field mapping functions
+
+These behaviors mimic the behaviors by the standard library marshallers and also the
+behavior of standard Go accessors.
+
+The first two are amply taken care of by `Reflect.Value.FieldByName`, and the third is
+addressed by `Reflect.Value.FieldByNameFunc`, but these don't quite understand struct
+tags in the ways that are vital to most marshallers, and they are slow.
+
+This reflectx package extends reflect to achieve these goals.

vendor/github.com/jmoiron/sqlx/reflectx/reflect.go

@@ -0,0 +1,441 @@
+// Package reflectx implements extensions to the standard reflect lib suitable
+// for implementing marshalling and unmarshalling packages.  The main Mapper type
+// allows for Go-compatible named attribute access, including accessing embedded
+// struct attributes and the ability to use  functions and struct tags to
+// customize field names.
+//
+package reflectx
+
+import (
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// A FieldInfo is metadata for a struct field.
+type FieldInfo struct {
+	Index    []int
+	Path     string
+	Field    reflect.StructField
+	Zero     reflect.Value
+	Name     string
+	Options  map[string]string
+	Embedded bool
+	Children []*FieldInfo
+	Parent   *FieldInfo
+}
+
+// A StructMap is an index of field metadata for a struct.
+type StructMap struct {
+	Tree  *FieldInfo
+	Index []*FieldInfo
+	Paths map[string]*FieldInfo
+	Names map[string]*FieldInfo
+}
+
+// GetByPath returns a *FieldInfo for a given string path.
+func (f StructMap) GetByPath(path string) *FieldInfo {
+	return f.Paths[path]
+}
+
+// GetByTraversal returns a *FieldInfo for a given integer path.  It is
+// analogous to reflect.FieldByIndex, but using the cached traversal
+// rather than re-executing the reflect machinery each time.
+func (f StructMap) GetByTraversal(index []int) *FieldInfo {
+	if len(index) == 0 {
+		return nil
+	}
+
+	tree := f.Tree
+	for _, i := range index {
+		if i >= len(tree.Children) || tree.Children[i] == nil {
+			return nil
+		}
+		tree = tree.Children[i]
+	}
+	return tree
+}
+
+// Mapper is a general purpose mapper of names to struct fields.  A Mapper
+// behaves like most marshallers in the standard library, obeying a field tag
+// for name mapping but also providing a basic transform function.
+type Mapper struct {
+	cache      map[reflect.Type]*StructMap
+	tagName    string
+	tagMapFunc func(string) string
+	mapFunc    func(string) string
+	mutex      sync.Mutex
+}
+
+// NewMapper returns a new mapper using the tagName as its struct field tag.
+// If tagName is the empty string, it is ignored.
+func NewMapper(tagName string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+	}
+}
+
+// NewMapperTagFunc returns a new mapper which contains a mapper for field names
+// AND a mapper for tag values.  This is useful for tags like json which can
+// have values like "name,omitempty".
+func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper {
+	return &Mapper{
+		cache:      make(map[reflect.Type]*StructMap),
+		tagName:    tagName,
+		mapFunc:    mapFunc,
+		tagMapFunc: tagMapFunc,
+	}
+}
+
+// NewMapperFunc returns a new mapper which optionally obeys a field tag and
+// a struct field name mapper func given by f.  Tags will take precedence, but
+// for any other field, the mapped name will be f(field.Name)
+func NewMapperFunc(tagName string, f func(string) string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+		mapFunc: f,
+	}
+}
+
+// TypeMap returns a mapping of field strings to int slices representing
+// the traversal down the struct to reach the field.
+func (m *Mapper) TypeMap(t reflect.Type) *StructMap {
+	m.mutex.Lock()
+	mapping, ok := m.cache[t]
+	if !ok {
+		mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc)
+		m.cache[t] = mapping
+	}
+	m.mutex.Unlock()
+	return mapping
+}
+
+// FieldMap returns the mapper's mapping of field names to reflect values.  Panics
+// if v's Kind is not Struct, or v is not Indirectable to a struct kind.
+func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	r := map[string]reflect.Value{}
+	tm := m.TypeMap(v.Type())
+	for tagName, fi := range tm.Names {
+		r[tagName] = FieldByIndexes(v, fi.Index)
+	}
+	return r
+}
+
+// FieldByName returns a field by its mapped name as a reflect.Value.
+// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind.
+// Returns zero Value if the name is not found.
+func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	fi, ok := tm.Names[name]
+	if !ok {
+		return v
+	}
+	return FieldByIndexes(v, fi.Index)
+}
+
+// FieldsByName returns a slice of values corresponding to the slice of names
+// for the value.  Panics if v's Kind is not Struct or v is not Indirectable
+// to a struct Kind.  Returns zero Value for each name not found.
+func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	vals := make([]reflect.Value, 0, len(names))
+	for _, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			vals = append(vals, *new(reflect.Value))
+		} else {
+			vals = append(vals, FieldByIndexes(v, fi.Index))
+		}
+	}
+	return vals
+}
+
+// TraversalsByName returns a slice of int slices which represent the struct
+// traversals for each mapped name.  Panics if t is not a struct or Indirectable
+// to a struct.  Returns empty int slice for each name not found.
+func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int {
+	r := make([][]int, 0, len(names))
+	m.TraversalsByNameFunc(t, names, func(_ int, i []int) error {
+		if i == nil {
+			r = append(r, []int{})
+		} else {
+			r = append(r, i)
+		}
+
+		return nil
+	})
+	return r
+}
+
+// TraversalsByNameFunc traverses the mapped names and calls fn with the index of
+// each name and the struct traversal represented by that name. Panics if t is not
+// a struct or Indirectable to a struct. Returns the first error returned by fn or nil.
+func (m *Mapper) TraversalsByNameFunc(t reflect.Type, names []string, fn func(int, []int) error) error {
+	t = Deref(t)
+	mustBe(t, reflect.Struct)
+	tm := m.TypeMap(t)
+	for i, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			if err := fn(i, nil); err != nil {
+				return err
+			}
+		} else {
+			if err := fn(i, fi.Index); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// FieldByIndexes returns a value for the field given by the struct traversal
+// for the given value.
+func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+		// if this is a pointer and it's nil, allocate a new value and set it
+		if v.Kind() == reflect.Ptr && v.IsNil() {
+			alloc := reflect.New(Deref(v.Type()))
+			v.Set(alloc)
+		}
+		if v.Kind() == reflect.Map && v.IsNil() {
+			v.Set(reflect.MakeMap(v.Type()))
+		}
+	}
+	return v
+}
+
+// FieldByIndexesReadOnly returns a value for a particular struct traversal,
+// but is not concerned with allocating nil pointers because the value is
+// going to be used for reading and not setting.
+func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+	}
+	return v
+}
+
+// Deref is Indirect for reflect.Types
+func Deref(t reflect.Type) reflect.Type {
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return t
+}
+
+// -- helpers & utilities --
+
+type kinder interface {
+	Kind() reflect.Kind
+}
+
+// mustBe checks a value against a kind, panicing with a reflect.ValueError
+// if the kind isn't that which is required.
+func mustBe(v kinder, expected reflect.Kind) {
+	if k := v.Kind(); k != expected {
+		panic(&reflect.ValueError{Method: methodName(), Kind: k})
+	}
+}
+
+// methodName returns the caller of the function calling methodName
+func methodName() string {
+	pc, _, _, _ := runtime.Caller(2)
+	f := runtime.FuncForPC(pc)
+	if f == nil {
+		return "unknown method"
+	}
+	return f.Name()
+}
+
+type typeQueue struct {
+	t  reflect.Type
+	fi *FieldInfo
+	pp string // Parent path
+}
+
+// A copying append that creates a new slice each time.
+func apnd(is []int, i int) []int {
+	x := make([]int, len(is)+1)
+	for p, n := range is {
+		x[p] = n
+	}
+	x[len(x)-1] = i
+	return x
+}
+
+type mapf func(string) string
+
+// parseName parses the tag and the target name for the given field using
+// the tagName (eg 'json' for `json:"foo"` tags), mapFunc for mapping the
+// field's name to a target name, and tagMapFunc for mapping the tag to
+// a target name.
+func parseName(field reflect.StructField, tagName string, mapFunc, tagMapFunc mapf) (tag, fieldName string) {
+	// first, set the fieldName to the field's name
+	fieldName = field.Name
+	// if a mapFunc is set, use that to override the fieldName
+	if mapFunc != nil {
+		fieldName = mapFunc(fieldName)
+	}
+
+	// if there's no tag to look for, return the field name
+	if tagName == "" {
+		return "", fieldName
+	}
+
+	// if this tag is not set using the normal convention in the tag,
+	// then return the fieldname..  this check is done because according
+	// to the reflect documentation:
+	//    If the tag does not have the conventional format,
+	//    the value returned by Get is unspecified.
+	// which doesn't sound great.
+	if !strings.Contains(string(field.Tag), tagName+":") {
+		return "", fieldName
+	}
+
+	// at this point we're fairly sure that we have a tag, so lets pull it out
+	tag = field.Tag.Get(tagName)
+
+	// if we have a mapper function, call it on the whole tag
+	// XXX: this is a change from the old version, which pulled out the name
+	// before the tagMapFunc could be run, but I think this is the right way
+	if tagMapFunc != nil {
+		tag = tagMapFunc(tag)
+	}
+
+	// finally, split the options from the name
+	parts := strings.Split(tag, ",")
+	fieldName = parts[0]
+
+	return tag, fieldName
+}
+
+// parseOptions parses options out of a tag string, skipping the name
+func parseOptions(tag string) map[string]string {
+	parts := strings.Split(tag, ",")
+	options := make(map[string]string, len(parts))
+	if len(parts) > 1 {
+		for _, opt := range parts[1:] {
+			// short circuit potentially expensive split op
+			if strings.Contains(opt, "=") {
+				kv := strings.Split(opt, "=")
+				options[kv[0]] = kv[1]
+				continue
+			}
+			options[opt] = ""
+		}
+	}
+	return options
+}
+
+// getMapping returns a mapping for the t type, using the tagName, mapFunc and
+// tagMapFunc to determine the canonical names of fields.
+func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc mapf) *StructMap {
+	m := []*FieldInfo{}
+
+	root := &FieldInfo{}
+	queue := []typeQueue{}
+	queue = append(queue, typeQueue{Deref(t), root, ""})
+
+QueueLoop:
+	for len(queue) != 0 {
+		// pop the first item off of the queue
+		tq := queue[0]
+		queue = queue[1:]
+
+		// ignore recursive field
+		for p := tq.fi.Parent; p != nil; p = p.Parent {
+			if tq.fi.Field.Type == p.Field.Type {
+				continue QueueLoop
+			}
+		}
+
+		nChildren := 0
+		if tq.t.Kind() == reflect.Struct {
+			nChildren = tq.t.NumField()
+		}
+		tq.fi.Children = make([]*FieldInfo, nChildren)
+
+		// iterate through all of its fields
+		for fieldPos := 0; fieldPos < nChildren; fieldPos++ {
+
+			f := tq.t.Field(fieldPos)
+
+			// parse the tag and the target name using the mapping options for this field
+			tag, name := parseName(f, tagName, mapFunc, tagMapFunc)
+
+			// if the name is "-", disabled via a tag, skip it
+			if name == "-" {
+				continue
+			}
+
+			fi := FieldInfo{
+				Field:   f,
+				Name:    name,
+				Zero:    reflect.New(f.Type).Elem(),
+				Options: parseOptions(tag),
+			}
+
+			// if the path is empty this path is just the name
+			if tq.pp == "" {
+				fi.Path = fi.Name
+			} else {
+				fi.Path = tq.pp + "." + fi.Name
+			}
+
+			// skip unexported fields
+			if len(f.PkgPath) != 0 && !f.Anonymous {
+				continue
+			}
+
+			// bfs search of anonymous embedded structs
+			if f.Anonymous {
+				pp := tq.pp
+				if tag != "" {
+					pp = fi.Path
+				}
+
+				fi.Embedded = true
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				nChildren := 0
+				ft := Deref(f.Type)
+				if ft.Kind() == reflect.Struct {
+					nChildren = ft.NumField()
+				}
+				fi.Children = make([]*FieldInfo, nChildren)
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, pp})
+			} else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) {
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				fi.Children = make([]*FieldInfo, Deref(f.Type).NumField())
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path})
+			}
+
+			fi.Index = apnd(tq.fi.Index, fieldPos)
+			fi.Parent = tq.fi
+			tq.fi.Children[fieldPos] = &fi
+			m = append(m, &fi)
+		}
+	}
+
+	flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}}
+	for _, fi := range flds.Index {
+		flds.Paths[fi.Path] = fi
+		if fi.Name != "" && !fi.Embedded {
+			flds.Names[fi.Path] = fi
+		}
+	}
+
+	return flds
+}

vendor/github.com/jmoiron/sqlx/sqlx_context.go

@@ -0,0 +1,346 @@
+// +build go1.8
+
+package sqlx
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+)
+
+// ConnectContext to a database and verify with a ping.
+func ConnectContext(ctx context.Context, driverName, dataSourceName string) (*DB, error) {
+	db, err := Open(driverName, dataSourceName)
+	if err != nil {
+		return db, err
+	}
+	err = db.PingContext(ctx)
+	return db, err
+}
+
+// QueryerContext is an interface used by GetContext and SelectContext
+type QueryerContext interface {
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error)
+	QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row
+}
+
+// PreparerContext is an interface used by PreparexContext.
+type PreparerContext interface {
+	PrepareContext(ctx context.Context, query string) (*sql.Stmt, error)
+}
+
+// ExecerContext is an interface used by MustExecContext and LoadFileContext
+type ExecerContext interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+}
+
+// ExtContext is a union interface which can bind, query, and exec, with Context
+// used by NamedQueryContext and NamedExecContext.
+type ExtContext interface {
+	binder
+	QueryerContext
+	ExecerContext
+}
+
+// SelectContext executes a query using the provided Queryer, and StructScans
+// each row into dest, which must be a slice.  If the slice elements are
+// scannable, then the result set must have only one column.  Otherwise,
+// StructScan is used. The *sql.Rows are closed automatically.
+// Any placeholder parameters are replaced with supplied args.
+func SelectContext(ctx context.Context, q QueryerContext, dest interface{}, query string, args ...interface{}) error {
+	rows, err := q.QueryxContext(ctx, query, args...)
+	if err != nil {
+		return err
+	}
+	// if something happens here, we want to make sure the rows are Closed
+	defer rows.Close()
+	return scanAll(rows, dest, false)
+}
+
+// PreparexContext prepares a statement.
+//
+// The provided context is used for the preparation of the statement, not for
+// the execution of the statement.
+func PreparexContext(ctx context.Context, p PreparerContext, query string) (*Stmt, error) {
+	s, err := p.PrepareContext(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	return &Stmt{Stmt: s, unsafe: isUnsafe(p), Mapper: mapperFor(p)}, err
+}
+
+// GetContext does a QueryRow using the provided Queryer, and scans the
+// resulting row to dest.  If dest is scannable, the result must only have one
+// column. Otherwise, StructScan is used.  Get will return sql.ErrNoRows like
+// row.Scan would. Any placeholder parameters are replaced with supplied args.
+// An error is returned if the result set is empty.
+func GetContext(ctx context.Context, q QueryerContext, dest interface{}, query string, args ...interface{}) error {
+	r := q.QueryRowxContext(ctx, query, args...)
+	return r.scanAny(dest, false)
+}
+
+// LoadFileContext exec's every statement in a file (as a single call to Exec).
+// LoadFileContext may return a nil *sql.Result if errors are encountered
+// locating or reading the file at path.  LoadFile reads the entire file into
+// memory, so it is not suitable for loading large data dumps, but can be useful
+// for initializing schemas or loading indexes.
+//
+// FIXME: this does not really work with multi-statement files for mattn/go-sqlite3
+// or the go-mysql-driver/mysql drivers;  pq seems to be an exception here.  Detecting
+// this by requiring something with DriverName() and then attempting to split the
+// queries will be difficult to get right, and its current driver-specific behavior
+// is deemed at least not complex in its incorrectness.
+func LoadFileContext(ctx context.Context, e ExecerContext, path string) (*sql.Result, error) {
+	realpath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+	contents, err := ioutil.ReadFile(realpath)
+	if err != nil {
+		return nil, err
+	}
+	res, err := e.ExecContext(ctx, string(contents))
+	return &res, err
+}
+
+// MustExecContext execs the query using e and panics if there was an error.
+// Any placeholder parameters are replaced with supplied args.
+func MustExecContext(ctx context.Context, e ExecerContext, query string, args ...interface{}) sql.Result {
+	res, err := e.ExecContext(ctx, query, args...)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+// PrepareNamedContext returns an sqlx.NamedStmt
+func (db *DB) PrepareNamedContext(ctx context.Context, query string) (*NamedStmt, error) {
+	return prepareNamedContext(ctx, db, query)
+}
+
+// NamedQueryContext using this DB.
+// Any named placeholder parameters are replaced with fields from arg.
+func (db *DB) NamedQueryContext(ctx context.Context, query string, arg interface{}) (*Rows, error) {
+	return NamedQueryContext(ctx, db, query, arg)
+}
+
+// NamedExecContext using this DB.
+// Any named placeholder parameters are replaced with fields from arg.
+func (db *DB) NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) {
+	return NamedExecContext(ctx, db, query, arg)
+}
+
+// SelectContext using this DB.
+// Any placeholder parameters are replaced with supplied args.
+func (db *DB) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return SelectContext(ctx, db, dest, query, args...)
+}
+
+// GetContext using this DB.
+// Any placeholder parameters are replaced with supplied args.
+// An error is returned if the result set is empty.
+func (db *DB) GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return GetContext(ctx, db, dest, query, args...)
+}
+
+// PreparexContext returns an sqlx.Stmt instead of a sql.Stmt.
+//
+// The provided context is used for the preparation of the statement, not for
+// the execution of the statement.
+func (db *DB) PreparexContext(ctx context.Context, query string) (*Stmt, error) {
+	return PreparexContext(ctx, db, query)
+}
+
+// QueryxContext queries the database and returns an *sqlx.Rows.
+// Any placeholder parameters are replaced with supplied args.
+func (db *DB) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) {
+	r, err := db.DB.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	return &Rows{Rows: r, unsafe: db.unsafe, Mapper: db.Mapper}, err
+}
+
+// QueryRowxContext queries the database and returns an *sqlx.Row.
+// Any placeholder parameters are replaced with supplied args.
+func (db *DB) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row {
+	rows, err := db.DB.QueryContext(ctx, query, args...)
+	return &Row{rows: rows, err: err, unsafe: db.unsafe, Mapper: db.Mapper}
+}
+
+// MustBeginTx starts a transaction, and panics on error.  Returns an *sqlx.Tx instead
+// of an *sql.Tx.
+//
+// The provided context is used until the transaction is committed or rolled
+// back. If the context is canceled, the sql package will roll back the
+// transaction. Tx.Commit will return an error if the context provided to
+// MustBeginContext is canceled.
+func (db *DB) MustBeginTx(ctx context.Context, opts *sql.TxOptions) *Tx {
+	tx, err := db.BeginTxx(ctx, opts)
+	if err != nil {
+		panic(err)
+	}
+	return tx
+}
+
+// MustExecContext (panic) runs MustExec using this database.
+// Any placeholder parameters are replaced with supplied args.
+func (db *DB) MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result {
+	return MustExecContext(ctx, db, query, args...)
+}
+
+// BeginTxx begins a transaction and returns an *sqlx.Tx instead of an
+// *sql.Tx.
+//
+// The provided context is used until the transaction is committed or rolled
+// back. If the context is canceled, the sql package will roll back the
+// transaction. Tx.Commit will return an error if the context provided to
+// BeginxContext is canceled.
+func (db *DB) BeginTxx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) {
+	tx, err := db.DB.BeginTx(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &Tx{Tx: tx, driverName: db.driverName, unsafe: db.unsafe, Mapper: db.Mapper}, err
+}
+
+// StmtxContext returns a version of the prepared statement which runs within a
+// transaction. Provided stmt can be either *sql.Stmt or *sqlx.Stmt.
+func (tx *Tx) StmtxContext(ctx context.Context, stmt interface{}) *Stmt {
+	var s *sql.Stmt
+	switch v := stmt.(type) {
+	case Stmt:
+		s = v.Stmt
+	case *Stmt:
+		s = v.Stmt
+	case *sql.Stmt:
+		s = v
+	default:
+		panic(fmt.Sprintf("non-statement type %v passed to Stmtx", reflect.ValueOf(stmt).Type()))
+	}
+	return &Stmt{Stmt: tx.StmtContext(ctx, s), Mapper: tx.Mapper}
+}
+
+// NamedStmtContext returns a version of the prepared statement which runs
+// within a transaction.
+func (tx *Tx) NamedStmtContext(ctx context.Context, stmt *NamedStmt) *NamedStmt {
+	return &NamedStmt{
+		QueryString: stmt.QueryString,
+		Params:      stmt.Params,
+		Stmt:        tx.StmtxContext(ctx, stmt.Stmt),
+	}
+}
+
+// PreparexContext returns an sqlx.Stmt instead of a sql.Stmt.
+//
+// The provided context is used for the preparation of the statement, not for
+// the execution of the statement.
+func (tx *Tx) PreparexContext(ctx context.Context, query string) (*Stmt, error) {
+	return PreparexContext(ctx, tx, query)
+}
+
+// PrepareNamedContext returns an sqlx.NamedStmt
+func (tx *Tx) PrepareNamedContext(ctx context.Context, query string) (*NamedStmt, error) {
+	return prepareNamedContext(ctx, tx, query)
+}
+
+// MustExecContext runs MustExecContext within a transaction.
+// Any placeholder parameters are replaced with supplied args.
+func (tx *Tx) MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result {
+	return MustExecContext(ctx, tx, query, args...)
+}
+
+// QueryxContext within a transaction and context.
+// Any placeholder parameters are replaced with supplied args.
+func (tx *Tx) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) {
+	r, err := tx.Tx.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	return &Rows{Rows: r, unsafe: tx.unsafe, Mapper: tx.Mapper}, err
+}
+
+// SelectContext within a transaction and context.
+// Any placeholder parameters are replaced with supplied args.
+func (tx *Tx) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return SelectContext(ctx, tx, dest, query, args...)
+}
+
+// GetContext within a transaction and context.
+// Any placeholder parameters are replaced with supplied args.
+// An error is returned if the result set is empty.
+func (tx *Tx) GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	return GetContext(ctx, tx, dest, query, args...)
+}
+
+// QueryRowxContext within a transaction and context.
+// Any placeholder parameters are replaced with supplied args.
+func (tx *Tx) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row {
+	rows, err := tx.Tx.QueryContext(ctx, query, args...)
+	return &Row{rows: rows, err: err, unsafe: tx.unsafe, Mapper: tx.Mapper}
+}
+
+// NamedExecContext using this Tx.
+// Any named placeholder parameters are replaced with fields from arg.
+func (tx *Tx) NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) {
+	return NamedExecContext(ctx, tx, query, arg)
+}
+
+// SelectContext using the prepared statement.
+// Any placeholder parameters are replaced with supplied args.
+func (s *Stmt) SelectContext(ctx context.Context, dest interface{}, args ...interface{}) error {
+	return SelectContext(ctx, &qStmt{s}, dest, "", args...)
+}
+
+// GetContext using the prepared statement.
+// Any placeholder parameters are replaced with supplied args.
+// An error is returned if the result set is empty.
+func (s *Stmt) GetContext(ctx context.Context, dest interface{}, args ...interface{}) error {
+	return GetContext(ctx, &qStmt{s}, dest, "", args...)
+}
+
+// MustExecContext (panic) using this statement.  Note that the query portion of
+// the error output will be blank, as Stmt does not expose its query.
+// Any placeholder parameters are replaced with supplied args.
+func (s *Stmt) MustExecContext(ctx context.Context, args ...interface{}) sql.Result {
+	return MustExecContext(ctx, &qStmt{s}, "", args...)
+}
+
+// QueryRowxContext using this statement.
+// Any placeholder parameters are replaced with supplied args.
+func (s *Stmt) QueryRowxContext(ctx context.Context, args ...interface{}) *Row {
+	qs := &qStmt{s}
+	return qs.QueryRowxContext(ctx, "", args...)
+}
+
+// QueryxContext using this statement.
+// Any placeholder parameters are replaced with supplied args.
+func (s *Stmt) QueryxContext(ctx context.Context, args ...interface{}) (*Rows, error) {
+	qs := &qStmt{s}
+	return qs.QueryxContext(ctx, "", args...)
+}
+
+func (q *qStmt) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
+	return q.Stmt.QueryContext(ctx, args...)
+}
+
+func (q *qStmt) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) {
+	r, err := q.Stmt.QueryContext(ctx, args...)
+	if err != nil {
+		return nil, err
+	}
+	return &Rows{Rows: r, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper}, err
+}
+
+func (q *qStmt) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row {
+	rows, err := q.Stmt.QueryContext(ctx, args...)
+	return &Row{rows: rows, err: err, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper}
+}
+
+func (q *qStmt) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
+	return q.Stmt.ExecContext(ctx, args...)
+}

web/web_test.go

@@ -47,7 +47,7 @@ func Setup() *TestHelper {
 		panic(err)
 	}
 
-	options := []app.Option{app.ConfigFile(tempConfig.Name(), false)}
+	options := []app.Option{app.Config(tempConfig.Name(), false)}
 	options = append(options, app.StoreOverride(mainHelper.Store))
 
 	s, err := app.NewServer(options...)