mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

store/sqlstore cleanup and postgres tests (#7595)

Просмотр исходного кода 
* sqlstore cleanup / postgres tests

* remove stopped containers

* cmd/platform compile fix

* remove test-postgres target from makefile
Этот коммит содержится в:
Chris
2017-10-09 10:16:14 -07:00
коммит произвёл Joram Wilander
родитель 70e5f00241
Коммит 0f66b6e726
22 изменённых файлов: 334 добавлений и 119 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

16
Makefile
Просмотреть файл

@@ -343,22 +343,6 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
rm -f config/*.key
endif
test-postgres:
@echo Testing Postgres
@sed -i'' -e 's|"DriverName": "mysql"|"DriverName": "postgres"|g' config/config.json
@sed -i'' -e 's|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|g' config/config.json
$(GO) test $(GOFLAGS) -run=$(TESTS) -test.v -test.timeout=2000s -covermode=count -coverprofile=cprofile.out -coverpkg=$(ALL_PACKAGES_COMMA) github.com/mattermost/mattermost-server/store/sqlstore || exit 1; \
if [ -f cprofile.out ]; then \
tail -n +2 cprofile.out >> cover.out; \
rm cprofile.out; \
fi; \
@sed -i'' -e 's|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|g' config/config.json
@sed -i'' -e 's|"DriverName": "postgres"|"DriverName": "mysql"|g' config/config.json
@rm config/config.json-e
test-server: test-te test-ee
internal-test-web-client:

2
app/admin.go
Просмотреть файл

@@ -187,7 +187,7 @@ func (a *App) RecycleDatabaseConnection() {
oldStore := a.Srv.Store
l4g.Warn(utils.T("api.admin.recycle_db_start.warn"))
a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster)
a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster)
a.Jobs.Store = a.Srv.Store

4
app/post.go
Просмотреть файл

@@ -613,6 +613,10 @@ func (a *App) SearchPostsInTeam(terms string, userId string, teamId string, isOr
return postList, nil
} else {
if !*utils.Cfg.ServiceSettings.EnablePostSearch {
return nil, model.NewAppError("SearchPostsInTeam", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
}
channels := []store.StoreChannel{}
for _, params := range paramsList {

2
app/server.go
Просмотреть файл

@@ -85,7 +85,7 @@ func (a *App) NewServer() {
}
func (a *App) InitStores() {
a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster)
a.Srv.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster)
}
type VaryBy struct{}

6
app/user.go
Просмотреть файл

@@ -438,7 +438,7 @@ func (a *App) GetUsersPage(page int, perPage int, asAdmin bool) ([]*model.User,
}
func (a *App) GetUsersEtag() string {
return (<-a.Srv.Store.User().GetEtagForAllProfiles()).Data.(string)
return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForAllProfiles()).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
}
func (a *App) GetUsersInTeam(teamId string, offset int, limit int) ([]*model.User, *model.AppError) {
@@ -492,11 +492,11 @@ func (a *App) GetUsersNotInTeamPage(teamId string, page int, perPage int, asAdmi
}
func (a *App) GetUsersInTeamEtag(teamId string) string {
return (<-a.Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string)
return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
}
func (a *App) GetUsersNotInTeamEtag(teamId string) string {
return (<-a.Srv.Store.User().GetEtagForProfilesNotInTeam(teamId)).Data.(string)
return fmt.Sprintf("%v.%v.%v", (<-a.Srv.Store.User().GetEtagForProfilesNotInTeam(teamId)).Data.(string), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
}
func (a *App) GetUsersInChannel(channelId string, offset int, limit int) ([]*model.User, *model.AppError) {

3
cmd/platform/jobserver.go
Просмотреть файл

@@ -10,6 +10,7 @@ import (
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/store/sqlstore"
"github.com/mattermost/mattermost-server/utils"
"github.com/spf13/cobra"
)
@@ -36,7 +37,7 @@ func jobserverCmdF(cmd *cobra.Command, args []string) {
}
defer l4g.Close()
a.Jobs.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Metrics), a.Metrics, a.Cluster)
a.Jobs.Store = store.NewLayeredStore(sqlstore.NewSqlSupplier(utils.Cfg.SqlSettings, a.Metrics), a.Metrics, a.Cluster)
defer a.Jobs.Store.Close()
a.Jobs.LoadLicense()

3
store/sqlstore/audit_store.go
Просмотреть файл

@@ -8,7 +8,6 @@ import (
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/utils"
)
type SqlAuditStore struct {
@@ -83,7 +82,7 @@ func (s SqlAuditStore) PermanentDeleteByUser(userId string) store.StoreChannel {
func (s SqlAuditStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var query string
if *utils.Cfg.SqlSettings.DriverName == "postgres" {
if s.DriverName() == "postgres" {
query = "DELETE from Audits WHERE Id = any (array (SELECT Id FROM Audits WHERE CreateAt < :EndTime LIMIT :Limit))"
} else {
query = "DELETE from Audits WHERE CreateAt < :EndTime LIMIT :Limit"

8
store/sqlstore/channel_store.go
Просмотреть файл

@@ -1042,7 +1042,7 @@ func (s SqlChannelStore) UpdateLastViewedAt(channelIds []string, userId string)
var updateQuery string
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
updateQuery = `UPDATE
ChannelMembers
SET
@@ -1053,7 +1053,7 @@ func (s SqlChannelStore) UpdateLastViewedAt(channelIds []string, userId string)
WHERE
UserId = :UserId
AND (` + updateIdQuery + `)`
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
updateQuery = `UPDATE
ChannelMembers
SET
@@ -1253,7 +1253,7 @@ func (s SqlChannelStore) performSearch(searchQuery string, term string, paramete
if term == "" {
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", "", 1)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
splitTerm := strings.Fields(term)
for i, t := range strings.Fields(term) {
if i == len(splitTerm)-1 {
@@ -1267,7 +1267,7 @@ func (s SqlChannelStore) performSearch(searchQuery string, term string, paramete
searchClause := fmt.Sprintf("AND (%s) @@ to_tsquery('simple', :Term)", "Name || ' ' || DisplayName")
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
splitTerm := strings.Fields(term)
for i, t := range strings.Fields(term) {
splitTerm[i] = "+" + t + "*"

9
store/sqlstore/command_store.go
Просмотреть файл

@@ -92,7 +92,14 @@ func (s SqlCommandStore) GetByTrigger(teamId string, trigger string) store.Store
return store.Do(func(result *store.StoreResult) {
var command model.Command
if err := s.GetReplica().SelectOne(&command, "SELECT * FROM Commands WHERE TeamId = :TeamId AND `Trigger` = :Trigger AND DeleteAt = 0", map[string]interface{}{"TeamId": teamId, "Trigger": trigger}); err != nil {
var query string
if s.DriverName() == "mysql" {
query = "SELECT * FROM Commands WHERE TeamId = :TeamId AND `Trigger` = :Trigger AND DeleteAt = 0"
} else {
query = "SELECT * FROM Commands WHERE TeamId = :TeamId AND \"trigger\" = :Trigger AND DeleteAt = 0"
}
if err := s.GetReplica().SelectOne(&command, query, map[string]interface{}{"TeamId": teamId, "Trigger": trigger}); err != nil {
result.Err = model.NewAppError("SqlCommandStore.GetByTrigger", "store.sql_command.get_by_trigger.app_error", nil, "teamId="+teamId+", trigger="+trigger+", err="+err.Error(), http.StatusInternalServerError)
}

2
store/sqlstore/file_info_store.go
Просмотреть файл

@@ -219,7 +219,7 @@ func (fs SqlFileInfoStore) PermanentDelete(fileId string) store.StoreChannel {
func (s SqlFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var query string
if *utils.Cfg.SqlSettings.DriverName == "postgres" {
if s.DriverName() == "postgres" {
query = "DELETE from FileInfo WHERE Id = any (array (SELECT Id FROM FileInfo WHERE CreateAt < :EndTime LIMIT :Limit))"
} else {
query = "DELETE from FileInfo WHERE CreateAt < :EndTime LIMIT :Limit"

5
store/sqlstore/oauth_store.go
Просмотреть файл

@@ -10,7 +10,6 @@ import (
"github.com/mattermost/gorp"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/utils"
)
type SqlOAuthStore struct {
@@ -336,9 +335,9 @@ func (as SqlOAuthStore) deleteOAuthAppSessions(transaction *gorp.Transaction, cl
result := store.StoreResult{}
query := ""
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if as.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = "DELETE FROM Sessions s USING OAuthAccessData o WHERE o.Token = s.Token AND o.ClientId = :Id"
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if as.DriverName() == model.DATABASE_DRIVER_MYSQL {
query = "DELETE s.* FROM Sessions s INNER JOIN OAuthAccessData o ON o.Token = s.Token WHERE o.ClientId = :Id"
}

18
store/sqlstore/post_store.go
Просмотреть файл

@@ -713,14 +713,6 @@ var specialSearchChar = []string{
func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
if !*utils.Cfg.ServiceSettings.EnablePostSearch {
list := model.NewPostList()
result.Data = list
result.Err = model.NewAppError("SqlPostStore.Search", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v params=%v", teamId, userId, params.ToJson()), http.StatusNotImplemented)
return
}
queryParams := map[string]interface{}{
"TeamId": teamId,
"UserId": userId,
@@ -833,7 +825,7 @@ func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchP
if terms == "" {
// we've already confirmed that we have a channel or user to search for
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", "", 1)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
// Parse text for wildcards
if wildcard, err := regexp.Compile("\\*($| )"); err == nil {
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
@@ -847,7 +839,7 @@ func (s SqlPostStore) Search(teamId string, userId string, params *model.SearchP
searchClause := fmt.Sprintf("AND %s @@ to_tsquery(:Terms)", searchType)
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType)
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
@@ -912,7 +904,7 @@ func (s SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) store.Sto
ORDER BY Name DESC
LIMIT 30`
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query =
`SELECT
TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, COUNT(DISTINCT Posts.UserId) AS Value
@@ -966,7 +958,7 @@ func (s SqlPostStore) AnalyticsPostCountsByDay(teamId string) store.StoreChannel
ORDER BY Name DESC
LIMIT 30`
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query =
`SELECT
TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value
@@ -1110,7 +1102,7 @@ func (s SqlPostStore) GetPostsBatchForIndexing(startTime int64, limit int) store
func (s SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var query string
if *utils.Cfg.SqlSettings.DriverName == "postgres" {
if s.DriverName() == "postgres" {
query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < :EndTime LIMIT :Limit))"
} else {
query = "DELETE from Posts WHERE CreateAt < :EndTime LIMIT :Limit"

4
store/sqlstore/preference_store.go
Просмотреть файл

@@ -101,7 +101,7 @@ func (s SqlPreferenceStore) save(transaction *gorp.Transaction, preference *mode
"Value": preference.Value,
}
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
if _, err := transaction.Exec(
`INSERT INTO
Preferences
@@ -112,7 +112,7 @@ func (s SqlPreferenceStore) save(transaction *gorp.Transaction, preference *mode
Value = :Value`, params); err != nil {
result.Err = model.NewAppError("SqlPreferenceStore.save", "store.sql_preference.save.updating.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
// postgres has no way to upsert values until version 9.5 and trying inserting and then updating causes transactions to abort
count, err := transaction.SelectInt(
`SELECT

1
store/sqlstore/store.go
Просмотреть файл

@@ -40,6 +40,7 @@ import (
}*/
type SqlStore interface {
DriverName() string
GetCurrentSchemaVersion() string
GetMaster() *gorp.DbMap
GetSearchReplica() *gorp.DbMap

106
store/sqlstore/store_test.go
Просмотреть файл

@@ -4,22 +4,110 @@
package sqlstore
import (
"flag"
"os"
"sync"
"testing"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/store/storetest"
"github.com/mattermost/mattermost-server/utils"
)
var sqlStore store.Store
var storeTypes = []*struct {
Name string
Func func() (*storetest.RunningContainer, *model.SqlSettings, error)
Container *storetest.RunningContainer
Store store.Store
}{
{
Name: "MySQL",
Func: storetest.NewMySQLContainer,
},
{
Name: "PostgreSQL",
Func: storetest.NewPostgreSQLContainer,
},
}
func StoreTest(t *testing.T, f func(*testing.T, store.Store)) {
if sqlStore == nil {
utils.TranslationsPreInit()
utils.LoadConfig("config.json")
utils.InitTranslations(utils.Cfg.LocalizationSettings)
sqlStore = store.NewLayeredStore(NewSqlSupplier(nil), nil, nil)
sqlStore.MarkSystemRanUnitTests()
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
for _, st := range storeTypes {
st := st
t.Run(st.Name, func(t *testing.T) { f(t, st.Store) })
}
f(t, sqlStore)
}
func initStores() {
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
var wg sync.WaitGroup
errCh := make(chan error, len(storeTypes))
wg.Add(len(storeTypes))
for _, st := range storeTypes {
st := st
go func() {
defer wg.Done()
container, settings, err := st.Func()
if err != nil {
errCh <- err
return
}
st.Container = container
st.Store = store.NewLayeredStore(NewSqlSupplier(*settings, nil), nil, nil)
st.Store.MarkSystemRanUnitTests()
}()
}
wg.Wait()
select {
case err := <-errCh:
panic(err)
default:
}
}
var tearDownStoresOnce sync.Once
func tearDownStores() {
tearDownStoresOnce.Do(func() {
var wg sync.WaitGroup
wg.Add(len(storeTypes))
for _, st := range storeTypes {
st := st
go func() {
st.Store.Close()
st.Container.Stop()
wg.Done()
}()
}
wg.Wait()
})
}
func TestMain(m *testing.M) {
flag.Parse()
utils.TranslationsPreInit()
utils.LoadConfig("config.json")
utils.InitTranslations(utils.Cfg.LocalizationSettings)
status := 0
initStores()
defer func() {
tearDownStores()
os.Exit(status)
}()
status = m.Run()
}

86
store/sqlstore/supplier.go
Просмотреть файл

@@ -96,12 +96,14 @@ type SqlSupplier struct {
replicas []*gorp.DbMap
searchReplicas []*gorp.DbMap
oldStores SqlSupplierOldStores
settings *model.SqlSettings
}
func NewSqlSupplier(metrics einterfaces.MetricsInterface) *SqlSupplier {
func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlSupplier {
supplier := &SqlSupplier{
rrCounter: 0,
srCounter: 0,
settings: &settings,
}
supplier.initConnection()
@@ -173,8 +175,8 @@ func (s *SqlSupplier) Next() store.LayeredStoreSupplier {
return s.next
}
func setupConnection(con_type string, driver string, dataSource string, maxIdle int, maxOpen int, trace bool) *gorp.DbMap {
db, err := dbsql.Open(driver, dataSource)
func setupConnection(con_type string, dataSource string, settings *model.SqlSettings) *gorp.DbMap {
db, err := dbsql.Open(*settings.DriverName, dataSource)
if err != nil {
l4g.Critical(utils.T("store.sql.open_conn.critical"), err)
time.Sleep(time.Second)
@@ -200,19 +202,19 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle
}
}
db.SetMaxIdleConns(maxIdle)
db.SetMaxOpenConns(maxOpen)
db.SetMaxIdleConns(*settings.MaxIdleConns)
db.SetMaxOpenConns(*settings.MaxOpenConns)
db.SetConnMaxLifetime(time.Duration(MAX_DB_CONN_LIFETIME) * time.Minute)
var dbmap *gorp.DbMap
connectionTimeout := time.Duration(*utils.Cfg.SqlSettings.QueryTimeout) * time.Second
connectionTimeout := time.Duration(*settings.QueryTimeout) * time.Second
if driver == "sqlite3" {
if *settings.DriverName == "sqlite3" {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.SqliteDialect{}, QueryTimeout: connectionTimeout}
} else if driver == model.DATABASE_DRIVER_MYSQL {
} else if *settings.DriverName == model.DATABASE_DRIVER_MYSQL {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8MB4"}, QueryTimeout: connectionTimeout}
} else if driver == model.DATABASE_DRIVER_POSTGRES {
} else if *settings.DriverName == model.DATABASE_DRIVER_POSTGRES {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.PostgresDialect{}, QueryTimeout: connectionTimeout}
} else {
l4g.Critical(utils.T("store.sql.dialect_driver.critical"))
@@ -220,7 +222,7 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle
os.Exit(EXIT_NO_DRIVER)
}
if trace {
if settings.Trace {
dbmap.TraceOn("", sqltrace.New(os.Stdout, "sql-trace:", sqltrace.Lmicroseconds))
}
@@ -228,34 +230,32 @@ func setupConnection(con_type string, driver string, dataSource string, maxIdle
}
func (s *SqlSupplier) initConnection() {
s.master = setupConnection("master", *utils.Cfg.SqlSettings.DriverName,
*utils.Cfg.SqlSettings.DataSource, *utils.Cfg.SqlSettings.MaxIdleConns,
*utils.Cfg.SqlSettings.MaxOpenConns, utils.Cfg.SqlSettings.Trace)
s.master = setupConnection("master", *s.settings.DataSource, s.settings)
if len(utils.Cfg.SqlSettings.DataSourceReplicas) == 0 {
if len(s.settings.DataSourceReplicas) == 0 {
s.replicas = make([]*gorp.DbMap, 1)
s.replicas[0] = s.master
} else {
s.replicas = make([]*gorp.DbMap, len(utils.Cfg.SqlSettings.DataSourceReplicas))
for i, replica := range utils.Cfg.SqlSettings.DataSourceReplicas {
s.replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), *utils.Cfg.SqlSettings.DriverName, replica,
*utils.Cfg.SqlSettings.MaxIdleConns, *utils.Cfg.SqlSettings.MaxOpenConns,
utils.Cfg.SqlSettings.Trace)
s.replicas = make([]*gorp.DbMap, len(s.settings.DataSourceReplicas))
for i, replica := range s.settings.DataSourceReplicas {
s.replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), replica, s.settings)
}
}
if len(utils.Cfg.SqlSettings.DataSourceSearchReplicas) == 0 {
if len(s.settings.DataSourceSearchReplicas) == 0 {
s.searchReplicas = s.replicas
} else {
s.searchReplicas = make([]*gorp.DbMap, len(utils.Cfg.SqlSettings.DataSourceSearchReplicas))
for i, replica := range utils.Cfg.SqlSettings.DataSourceSearchReplicas {
s.searchReplicas[i] = setupConnection(fmt.Sprintf("search-replica-%v", i), *utils.Cfg.SqlSettings.DriverName, replica,
*utils.Cfg.SqlSettings.MaxIdleConns, *utils.Cfg.SqlSettings.MaxOpenConns,
utils.Cfg.SqlSettings.Trace)
s.searchReplicas = make([]*gorp.DbMap, len(s.settings.DataSourceSearchReplicas))
for i, replica := range s.settings.DataSourceSearchReplicas {
s.searchReplicas[i] = setupConnection(fmt.Sprintf("search-replica-%v", i), replica, s.settings)
}
}
}
func (ss *SqlSupplier) DriverName() string {
return *ss.settings.DriverName
}
func (ss *SqlSupplier) GetCurrentSchemaVersion() string {
version, _ := ss.GetMaster().SelectStr("SELECT Value FROM Systems WHERE Name='Version'")
return version
@@ -281,7 +281,7 @@ func (ss *SqlSupplier) TotalMasterDbConnections() int {
func (ss *SqlSupplier) TotalReadDbConnections() int {
if len(utils.Cfg.SqlSettings.DataSourceReplicas) == 0 {
if len(ss.settings.DataSourceReplicas) == 0 {
return 0
}
@@ -294,7 +294,7 @@ func (ss *SqlSupplier) TotalReadDbConnections() int {
}
func (ss *SqlSupplier) TotalSearchDbConnections() int {
if len(utils.Cfg.SqlSettings.DataSourceSearchReplicas) == 0 {
if len(ss.settings.DataSourceSearchReplicas) == 0 {
return 0
}
@@ -318,7 +318,7 @@ func (ss *SqlSupplier) MarkSystemRanUnitTests() {
}
func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
count, err := ss.GetMaster().SelectInt(
`SELECT count(relname) FROM pg_class WHERE relname=$1`,
strings.ToLower(tableName),
@@ -332,7 +332,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
return count > 0
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
count, err := ss.GetMaster().SelectInt(
`SELECT
@@ -363,7 +363,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
}
func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool {
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
count, err := ss.GetMaster().SelectInt(
`SELECT COUNT(0)
FROM pg_attribute
@@ -386,7 +386,7 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool
return count > 0
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
count, err := ss.GetMaster().SelectInt(
`SELECT
@@ -423,7 +423,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri
return false
}
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + postgresColType + " DEFAULT '" + defaultValue + "'")
if err != nil {
l4g.Critical(utils.T("store.sql.create_column.critical"), err)
@@ -433,7 +433,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri
return true
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + mySqlColType + " DEFAULT '" + defaultValue + "'")
if err != nil {
l4g.Critical(utils.T("store.sql.create_column.critical"), err)
@@ -488,9 +488,9 @@ func (ss *SqlSupplier) RenameColumnIfExists(tableName string, oldColumnName stri
}
var err error
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " CHANGE " + oldColumnName + " " + newColumnName + " " + colType)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " RENAME COLUMN " + oldColumnName + " TO " + newColumnName)
}
@@ -510,9 +510,9 @@ func (ss *SqlSupplier) GetMaxLengthOfColumnIfExists(tableName string, columnName
var result string
var err error
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
result, err = ss.GetMaster().SelectStr("SELECT CHARACTER_MAXIMUM_LENGTH FROM information_schema.columns WHERE table_name = '" + tableName + "' AND COLUMN_NAME = '" + columnName + "'")
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
result, err = ss.GetMaster().SelectStr("SELECT character_maximum_length FROM information_schema.columns WHERE table_name = '" + strings.ToLower(tableName) + "' AND column_name = '" + strings.ToLower(columnName) + "'")
}
@@ -531,9 +531,9 @@ func (ss *SqlSupplier) AlterColumnTypeIfExists(tableName string, columnName stri
}
var err error
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " MODIFY " + columnName + " " + mySqlColType)
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
} else if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType)
}
@@ -565,7 +565,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
uniqueStr = "UNIQUE "
}
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
_, errExists := ss.GetMaster().SelectStr("SELECT $1::regclass", indexName)
// It should fail if the index does not exist
if errExists == nil {
@@ -587,7 +587,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
time.Sleep(time.Second)
os.Exit(EXIT_CREATE_INDEX_POSTGRES)
}
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName)
if err != nil {
@@ -622,7 +622,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) bool {
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
_, err := ss.GetMaster().SelectStr("SELECT $1::regclass", indexName)
// It should fail if the index does not exist
if err != nil {
@@ -637,7 +637,7 @@ func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) b
}
return true
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName)
if err != nil {

4
store/sqlstore/supplier_reactions.go
Просмотреть файл

@@ -147,8 +147,8 @@ func (s *SqlSupplier) ReactionPermanentDeleteBatch(ctx context.Context, endTime
result := store.NewSupplierResult()
var query string
if *utils.Cfg.SqlSettings.DriverName == "postgres" {
query = "DELETE from Reactions WHERE Id = any (array (SELECT Id FROM Reactions WHERE CreateAt < :EndTime LIMIT :Limit))"
if s.DriverName() == "postgres" {
query = "DELETE from Reactions WHERE CreateAt = any (array (SELECT CreateAt FROM Reactions WHERE CreateAt < :EndTime LIMIT :Limit))"
} else {
query = "DELETE from Reactions WHERE CreateAt < :EndTime LIMIT :Limit"
}

4
store/sqlstore/team_store.go
Просмотреть файл

@@ -266,7 +266,7 @@ func (s SqlTeamStore) GetAllTeamListing() store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
query := "SELECT * FROM Teams WHERE AllowOpenInvite = 1"
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = "SELECT * FROM Teams WHERE AllowOpenInvite = true"
}
@@ -289,7 +289,7 @@ func (s SqlTeamStore) GetAllTeamPageListing(offset int, limit int) store.StoreCh
return store.Do(func(result *store.StoreResult) {
query := "SELECT * FROM Teams WHERE AllowOpenInvite = 1 LIMIT :Limit OFFSET :Offset"
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = "SELECT * FROM Teams WHERE AllowOpenInvite = true LIMIT :Limit OFFSET :Offset"
}

4
store/sqlstore/upgrade.go
Просмотреть файл

@@ -135,11 +135,11 @@ func UpgradeDatabaseToVersion33(sqlStore SqlStore) {
}
// increase size of Value column of Preferences table to match the size of the ThemeProps column
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if sqlStore.DriverName() == model.DATABASE_DRIVER_POSTGRES {
if _, err := transaction.Exec("ALTER TABLE Preferences ALTER COLUMN Value TYPE varchar(2000)"); err != nil {
themeMigrationFailed(err)
}
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if sqlStore.DriverName() == model.DATABASE_DRIVER_MYSQL {
if _, err := transaction.Exec("ALTER TABLE Preferences MODIFY Value text"); err != nil {
themeMigrationFailed(err)
}

9
store/sqlstore/user_access_token_store.go
Просмотреть файл

@@ -10,7 +10,6 @@ import (
"github.com/mattermost/gorp"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/utils"
)
type SqlUserAccessTokenStore struct {
@@ -80,9 +79,9 @@ func (s SqlUserAccessTokenStore) deleteSessionsAndTokensById(transaction *gorp.T
result := store.StoreResult{}
query := ""
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = "DELETE FROM Sessions s USING UserAccessTokens o WHERE o.Token = s.Token AND o.Id = :Id"
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
query = "DELETE s.* FROM Sessions s INNER JOIN UserAccessTokens o ON o.Token = s.Token WHERE o.Id = :Id"
}
@@ -132,9 +131,9 @@ func (s SqlUserAccessTokenStore) deleteSessionsandTokensByUser(transaction *gorp
result := store.StoreResult{}
query := ""
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
query = "DELETE FROM Sessions s USING UserAccessTokens o WHERE o.Token = s.Token AND o.UserId = :UserId"
} else if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
query = "DELETE s.* FROM Sessions s INNER JOIN UserAccessTokens o ON o.Token = s.Token WHERE o.UserId = :UserId"
}

16
store/sqlstore/user_store.go
Просмотреть файл

@@ -78,7 +78,7 @@ func (us SqlUserStore) CreateIndexesIfNotExists() {
us.CreateIndexIfNotExists("idx_users_create_at", "Users", "CreateAt")
us.CreateIndexIfNotExists("idx_users_delete_at", "Users", "DeleteAt")
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if us.DriverName() == model.DATABASE_DRIVER_POSTGRES {
us.CreateIndexIfNotExists("idx_users_email_lower", "Users", "lower(Email)")
us.CreateIndexIfNotExists("idx_users_username_lower", "Users", "lower(Username)")
us.CreateIndexIfNotExists("idx_users_nickname_lower", "Users", "lower(Nickname)")
@@ -321,9 +321,9 @@ func (s SqlUserStore) GetEtagForAllProfiles() store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
updateAt, err := s.GetReplica().SelectInt("SELECT UpdateAt FROM Users ORDER BY UpdateAt DESC LIMIT 1")
if err != nil {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis())
} else {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt)
}
})
}
@@ -349,9 +349,9 @@ func (s SqlUserStore) GetEtagForProfiles(teamId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
updateAt, err := s.GetReplica().SelectInt("SELECT UpdateAt FROM Users, TeamMembers WHERE TeamMembers.TeamId = :TeamId AND Users.Id = TeamMembers.UserId ORDER BY UpdateAt DESC LIMIT 1", map[string]interface{}{"TeamId": teamId})
if err != nil {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis())
} else {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt)
}
})
}
@@ -1066,7 +1066,7 @@ func (us SqlUserStore) performSearch(searchQuery string, term string, options ma
for i, term := range splitTerms {
fields := []string{}
for _, field := range splitFields {
if *utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
if us.DriverName() == model.DATABASE_DRIVER_POSTGRES {
fields = append(fields, fmt.Sprintf("lower(%s) LIKE lower(%s) escape '*' ", field, fmt.Sprintf(":Term%d", i)))
} else {
fields = append(fields, fmt.Sprintf("%s LIKE %s escape '*' ", field, fmt.Sprintf(":Term%d", i)))
@@ -1159,9 +1159,9 @@ func (us SqlUserStore) GetEtagForProfilesNotInTeam(teamId string) store.StoreCha
`, map[string]interface{}{"TeamId": teamId})
if err != nil {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, model.GetMillis())
} else {
result.Data = fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, updateAt, utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
result.Data = fmt.Sprintf("%v.%v", model.CurrentVersion, updateAt)
}
})
}

141
store/storetest/docker.go Обычный файл
Просмотреть файл

@@ -0,0 +1,141 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package storetest
import (
"encoding/json"
"fmt"
"io"
"net"
"os/exec"
"strings"
"time"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/model"
)
type Container struct {
Id string
NetworkSettings struct {
Ports map[string][]struct {
HostPort string
}
}
}
type RunningContainer struct {
Container
}
func (c *RunningContainer) Stop() error {
l4g.Info("removing container: %v", c.Id)
return exec.Command("docker", "rm", "-f", c.Id).Run()
}
func NewMySQLContainer() (*RunningContainer, *model.SqlSettings, error) {
container, err := runContainer([]string{
"-e", "MYSQL_ROOT_PASSWORD=mostest",
"-e", "MYSQL_USER=mmuser",
"-e", "MYSQL_PASSWORD=mostest",
"-e", "MYSQL_DATABASE=mattermost_test",
"--tmpfs", "/var/lib/mysql",
"mysql:5.7",
})
if err != nil {
return nil, nil, err
}
l4g.Info("Waiting for mysql connectivity")
port := container.NetworkSettings.Ports["3306/tcp"][0].HostPort
if err := waitForPort(port); err != nil {
container.Stop()
return nil, nil, err
}
return container, databaseSettings("mysql", "mmuser:mostest@tcp(127.0.0.1:"+port+")/mattermost_test?charset=utf8mb4,utf8"), nil
}
func NewPostgreSQLContainer() (*RunningContainer, *model.SqlSettings, error) {
container, err := runContainer([]string{
"-e", "POSTGRES_USER=mmuser",
"-e", "POSTGRES_PASSWORD=mostest",
"--tmpfs", "/var/lib/postgresql/data",
"postgres:9.4",
})
if err != nil {
return nil, nil, err
}
l4g.Info("Waiting for postgres connectivity")
port := container.NetworkSettings.Ports["5432/tcp"][0].HostPort
if err := waitForPort(port); err != nil {
container.Stop()
return nil, nil, err
}
return container, databaseSettings("postgres", "postgres://mmuser:mostest@127.0.0.1:"+port+"?sslmode=disable"), nil
}
func databaseSettings(driver, dataSource string) *model.SqlSettings {
settings := &model.SqlSettings{
DriverName: &driver,
DataSource: &dataSource,
DataSourceReplicas: []string{},
DataSourceSearchReplicas: []string{},
MaxIdleConns: new(int),
MaxOpenConns: new(int),
Trace: false,
AtRestEncryptKey: model.NewRandomString(32),
QueryTimeout: new(int),
}
*settings.MaxIdleConns = 10
*settings.MaxOpenConns = 100
*settings.QueryTimeout = 10
return settings
}
func runContainer(args []string) (*RunningContainer, error) {
name := "mattermost-storetest-" + model.NewId()
dockerArgs := append([]string{"run", "-d", "-P", "--name", name}, args...)
out, err := exec.Command("docker", dockerArgs...).Output()
if err != nil {
return nil, err
}
id := strings.TrimSpace(string(out))
out, err = exec.Command("docker", "inspect", id).Output()
if err != nil {
exec.Command("docker", "rm", "-f", id).Run()
return nil, err
}
var containers []Container
if err := json.Unmarshal(out, &containers); err != nil {
exec.Command("docker", "rm", "-f", id).Run()
return nil, err
}
l4g.Info("running container: %v", id)
return &RunningContainer{containers[0]}, nil
}
func waitForPort(port string) error {
for i := 0; i < 120; i++ {
conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, time.Minute)
if err != nil {
return err
}
if err = conn.SetReadDeadline(time.Now().Add(time.Millisecond * 500)); err != nil {
return err
}
_, err = conn.Read(make([]byte, 1))
conn.Close()
if err == nil {
return nil
}
if e, ok := err.(net.Error); ok && e.Timeout() {
return nil
}
if err != io.EOF {
return err
}
time.Sleep(time.Millisecond * 200)
}
return fmt.Errorf("timeout waiting for port %v", port)
}