Moved some common SQL function to public utils as they are used in plugins (#26412)
* Moved some common SQL function tu public utls as they are used in plugins * goimported file * Added tests * Created sub-package * MOved SetupConnection to public sql utils
Этот коммит содержится в:
Harshil Sharma
коммит произвёл
GitHub
GitHub
родитель
f34445a6f4
Коммит
4fda7e6f34
126
server/public/utils/sql/sql_utils.go
Обычный файл
126
server/public/utils/sql/sql_utils.go
Обычный файл
@@ -0,0 +1,126 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
dbsql "database/sql"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
DBPingTimeoutSecs = 10
|
||||
|
||||
replicaLagPrefix = "replica-lag"
|
||||
)
|
||||
|
||||
// ResetReadTimeout removes the timeout constraint from the MySQL dsn.
|
||||
func ResetReadTimeout(dataSource string) (string, error) {
|
||||
config, err := mysql.ParseDSN(dataSource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
config.ReadTimeout = 0
|
||||
return config.FormatDSN(), nil
|
||||
}
|
||||
|
||||
// AppendMultipleStatementsFlag attached dsn parameters to MySQL dsn in order to make migrations work.
|
||||
func AppendMultipleStatementsFlag(dataSource string) (string, error) {
|
||||
config, err := mysql.ParseDSN(dataSource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if config.Params == nil {
|
||||
config.Params = map[string]string{}
|
||||
}
|
||||
|
||||
config.Params["multiStatements"] = "true"
|
||||
return config.FormatDSN(), nil
|
||||
}
|
||||
|
||||
// SetupConnection sets up the connection to the database and pings it to make sure it's alive.
|
||||
// It also applies any database configuration settings that are required.
|
||||
func SetupConnection(logger mlog.LoggerIFace, connType string, dataSource string, settings *model.SqlSettings, attempts int) (*dbsql.DB, error) {
|
||||
db, err := dbsql.Open(*settings.DriverName, dataSource)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open SQL connection")
|
||||
}
|
||||
|
||||
// At this point, we have passed sql.Open, so we deliberately ignore any errors.
|
||||
sanitized, _ := SanitizeDataSource(*settings.DriverName, dataSource)
|
||||
|
||||
logger = logger.With(
|
||||
mlog.String("database", connType),
|
||||
mlog.String("dataSource", sanitized),
|
||||
)
|
||||
|
||||
for i := 0; i < attempts; i++ {
|
||||
logger.Info("Pinging SQL")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), DBPingTimeoutSecs*time.Second)
|
||||
defer cancel()
|
||||
err = db.PingContext(ctx)
|
||||
if err != nil {
|
||||
if i == attempts-1 {
|
||||
return nil, err
|
||||
}
|
||||
logger.Error("Failed to ping DB", mlog.Int("retrying in seconds", DBPingTimeoutSecs), mlog.Err(err))
|
||||
time.Sleep(DBPingTimeoutSecs * time.Second)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if strings.HasPrefix(connType, replicaLagPrefix) {
|
||||
// If this is a replica lag connection, we just open one connection.
|
||||
//
|
||||
// Arguably, if the query doesn't require a special credential, it does take up
|
||||
// one extra connection from the replica DB. But falling back to the replica
|
||||
// data source when the replica lag data source is null implies an ordering constraint
|
||||
// which makes things brittle and is not a good design.
|
||||
// If connections are an overhead, it is advised to use a connection pool.
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
} else {
|
||||
db.SetMaxIdleConns(*settings.MaxIdleConns)
|
||||
db.SetMaxOpenConns(*settings.MaxOpenConns)
|
||||
}
|
||||
db.SetConnMaxLifetime(time.Duration(*settings.ConnMaxLifetimeMilliseconds) * time.Millisecond)
|
||||
db.SetConnMaxIdleTime(time.Duration(*settings.ConnMaxIdleTimeMilliseconds) * time.Millisecond)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func SanitizeDataSource(driverName, dataSource string) (string, error) {
|
||||
switch driverName {
|
||||
case model.DatabaseDriverPostgres:
|
||||
u, err := url.Parse(dataSource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.User = url.UserPassword("****", "****")
|
||||
params := u.Query()
|
||||
params.Del("user")
|
||||
params.Del("password")
|
||||
u.RawQuery = params.Encode()
|
||||
return u.String(), nil
|
||||
case model.DatabaseDriverMysql:
|
||||
cfg, err := mysql.ParseDSN(dataSource)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfg.User = "****"
|
||||
cfg.Passwd = "****"
|
||||
return cfg.FormatDSN(), nil
|
||||
default:
|
||||
return "", errors.New("invalid drivername. Not postgres or mysql.")
|
||||
}
|
||||
}
|
||||
109
server/public/utils/sql/sql_utils_test.go
Обычный файл
109
server/public/utils/sql/sql_utils_test.go
Обычный файл
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAppendMultipleStatementsFlag(t *testing.T) {
|
||||
testCases := []struct {
|
||||
Scenario string
|
||||
DSN string
|
||||
ExpectedDSN string
|
||||
}{
|
||||
{
|
||||
"Should append multiStatements param to the DSN path with existing params",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost?writeTimeout=30s",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost?writeTimeout=30s&multiStatements=true",
|
||||
},
|
||||
{
|
||||
"Should append multiStatements param to the DSN path with no existing params",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost?multiStatements=true",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.Scenario, func(t *testing.T) {
|
||||
res, err := AppendMultipleStatementsFlag(tc.DSN)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.ExpectedDSN, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetReadTimeout(t *testing.T) {
|
||||
testCases := []struct {
|
||||
Scenario string
|
||||
DSN string
|
||||
ExpectedDSN string
|
||||
}{
|
||||
{
|
||||
"Should re move read timeout param from the DSN",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost?readTimeout=30s",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost",
|
||||
},
|
||||
{
|
||||
"Should change nothing as there is no read timeout param specified",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost",
|
||||
"user:rand?&ompasswith@character@unix(/var/run/mysqld/mysqld.sock)/mattermost",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.Scenario, func(t *testing.T) {
|
||||
res, err := ResetReadTimeout(tc.DSN)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.ExpectedDSN, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDataSource(t *testing.T) {
|
||||
t.Run(model.DatabaseDriverPostgres, func(t *testing.T) {
|
||||
testCases := []struct {
|
||||
Original string
|
||||
Sanitized string
|
||||
}{
|
||||
{
|
||||
"postgres://mmuser:mostest@localhost/dummy?sslmode=disable",
|
||||
"postgres://%2A%2A%2A%2A:%2A%2A%2A%2A@localhost/dummy?sslmode=disable",
|
||||
},
|
||||
{
|
||||
"postgres://localhost/dummy?sslmode=disable&user=mmuser&password=mostest",
|
||||
"postgres://%2A%2A%2A%2A:%2A%2A%2A%2A@localhost/dummy?sslmode=disable",
|
||||
},
|
||||
}
|
||||
driver := model.DatabaseDriverPostgres
|
||||
for _, tc := range testCases {
|
||||
out, err := SanitizeDataSource(driver, tc.Original)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.Sanitized, out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(model.DatabaseDriverMysql, func(t *testing.T) {
|
||||
testCases := []struct {
|
||||
Original string
|
||||
Sanitized string
|
||||
}{
|
||||
{
|
||||
"mmuser:mostest@tcp(localhost:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s",
|
||||
"****:****@tcp(localhost:3306)/mattermost_test?readTimeout=30s&writeTimeout=30s&charset=utf8mb4%2Cutf8",
|
||||
},
|
||||
}
|
||||
driver := model.DatabaseDriverMysql
|
||||
for _, tc := range testCases {
|
||||
out, err := SanitizeDataSource(driver, tc.Original)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.Sanitized, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
Ссылка в новой задаче
Block a user