mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Fixing behavior of Replicas and SearchReplicas in canary environments (#14576)

Просмотр исходного кода 
* Fixing behavior of Replicas and SearchReplicas in canary environments

* Trying to fix tests

* Revert "Trying to fix tests"

This reverts commit 3531da961844f5cb8557efcd3b570b06f362c6df.

* Revert "Fixing behavior of Replicas and SearchReplicas in canary environments"

This reverts commit 0c05901c843e4ccd60c8320fb4e0123b1bacf430.

* Revert "Disable read/search db replicas in TE/E0 (#14400)"

This reverts commit ef5ac519d9.

* Making the store aware of the license

* Readding the unit tests

* Fixing sqlstor supplier tests

* Adding mutex to ensure license write consistency and fixing tests

* Fixing tests

* Fixing tests

* Shuting down server properly during tests

* Trying to fix tests

* Trying to fix the tests

* Skipping flaky tests

Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
2020-05-27 15:42:48 +02:00
коммит произвёл GitHub
родитель 553af3a694
Коммит ae328153d5
10 изменённых файлов: 123 добавлений и 64 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
app/post_test.go
Просмотреть файл

@@ -560,6 +560,7 @@ func TestImageProxy(t *testing.T) {
}
func TestMaxPostSize(t *testing.T) {
t.Skip("TODO: fix flaky test")
t.Parallel()
testCases := []struct {

20
app/server.go
Просмотреть файл

@@ -52,8 +52,8 @@ import (
var MaxNotificationsPerChannelDefault int64 = 1000000
type Server struct {
Store store.Store
sqlStore *sqlstore.SqlSupplier
Store store.Store
WebSocketRouter *WebSocketRouter
// RootRouter is the starting point for all HTTP requests to the server.
@@ -309,24 +309,6 @@ func NewServer(options ...Option) (*Server, error) {
license := s.License()
if license == nil && len(s.Config().SqlSettings.DataSourceReplicas) > 0 {
mlog.Warn("Read replicas functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license.")
s.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceReplicas = []string{}
})
s.Store.Close()
s.Store = s.newStore()
}
if license == nil && len(s.Config().SqlSettings.DataSourceSearchReplicas) > 0 {
mlog.Warn("Search replicas functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license.")
s.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceSearchReplicas = []string{}
})
s.Store.Close()
s.Store = s.newStore()
}
if license == nil {
s.UpdateConfig(func(cfg *model.Config) {
cfg.TeamSettings.MaxNotificationsPerChannel = &MaxNotificationsPerChannelDefault

5
app/server_app_adapters.go
Просмотреть файл

@@ -77,6 +77,11 @@ func (s *Server) RunOldAppInitialization() error {
searchStore.UpdateConfig(cfg)
})
s.sqlStore.UpdateLicense(s.License())
s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
s.sqlStore.UpdateLicense(newLicense)
})
return store.NewTimerLayer(
searchStore,
s.Metrics,

57
app/server_test.go
Просмотреть файл

@@ -23,6 +23,7 @@ import (
"github.com/mattermost/mattermost-server/v5/config"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store/storetest"
"github.com/mattermost/mattermost-server/v5/utils/fileutils"
"github.com/stretchr/testify/require"
)
@@ -42,61 +43,73 @@ func TestStartServerSuccess(t *testing.T) {
}
func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {
t.Skip("TODO: fix flaky test")
cfg := model.Config{}
cfg.SetDefaults()
driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
if driverName == "" {
driverName = model.DATABASE_DRIVER_POSTGRES
}
dsn := ""
if driverName == model.DATABASE_DRIVER_POSTGRES {
dsn = os.Getenv("TEST_DATABASE_POSTGRESQL_DSN")
} else {
dsn = os.Getenv("TEST_DATABASE_MYSQL_DSN")
}
cfg.SqlSettings = *storetest.MakeSqlSettings(driverName)
if dsn != "" {
cfg.SqlSettings.DataSource = &dsn
}
cfg.SqlSettings.DataSourceReplicas = []string{*cfg.SqlSettings.DataSource}
cfg.SqlSettings.DataSourceSearchReplicas = []string{*cfg.SqlSettings.DataSource}
t.Run("Read Replicas with no License", func(t *testing.T) {
s, err := NewServer(func(server *Server) error {
configStore, _ := config.NewFileStore("config.json", true)
configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
server.configStore = configStore
server.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceReplicas = []string{*cfg.SqlSettings.DataSource}
})
return nil
})
require.NoError(t, err)
require.Equal(t, s.sqlStore.GetMaster(), s.sqlStore.GetReplica())
require.Len(t, s.Config().SqlSettings.DataSourceReplicas, 0)
defer s.Shutdown()
require.Same(t, s.sqlStore.GetMaster(), s.sqlStore.GetReplica())
require.Len(t, s.Config().SqlSettings.DataSourceReplicas, 1)
})
t.Run("Read Replicas With License", func(t *testing.T) {
s, err := NewServer(func(server *Server) error {
configStore, _ := config.NewFileStore("config.json", true)
configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
server.configStore = configStore
server.licenseValue.Store(model.NewTestLicense())
server.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceReplicas = []string{*cfg.SqlSettings.DataSource}
})
return nil
})
require.NoError(t, err)
require.NotEqual(t, s.sqlStore.GetMaster(), s.sqlStore.GetReplica())
defer s.Shutdown()
require.NotSame(t, s.sqlStore.GetMaster(), s.sqlStore.GetReplica())
require.Len(t, s.Config().SqlSettings.DataSourceReplicas, 1)
})
t.Run("Search Replicas with no License", func(t *testing.T) {
s, err := NewServer(func(server *Server) error {
configStore, _ := config.NewFileStore("config.json", true)
configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
server.configStore = configStore
server.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceSearchReplicas = []string{*cfg.SqlSettings.DataSource}
})
return nil
})
require.NoError(t, err)
require.Equal(t, s.sqlStore.GetMaster(), s.sqlStore.GetSearchReplica())
require.Len(t, s.Config().SqlSettings.DataSourceSearchReplicas, 0)
defer s.Shutdown()
require.Same(t, s.sqlStore.GetMaster(), s.sqlStore.GetSearchReplica())
require.Len(t, s.Config().SqlSettings.DataSourceSearchReplicas, 1)
})
t.Run("Search Replicas With License", func(t *testing.T) {
s, err := NewServer(func(server *Server) error {
configStore, _ := config.NewFileStore("config.json", true)
configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
server.configStore = configStore
server.licenseValue.Store(model.NewTestLicense())
server.UpdateConfig(func(cfg *model.Config) {
cfg.SqlSettings.DataSourceSearchReplicas = []string{*cfg.SqlSettings.DataSource}
})
return nil
})
require.NoError(t, err)
require.NotEqual(t, s.sqlStore.GetMaster(), s.sqlStore.GetSearchReplica())
defer s.Shutdown()
require.NotSame(t, s.sqlStore.GetMaster(), s.sqlStore.GetSearchReplica())
require.Len(t, s.Config().SqlSettings.DataSourceSearchReplicas, 1)
})
}

25
config/utils.go
Просмотреть файл

@@ -35,21 +35,6 @@ func desanitize(actual, target *model.Config) {
if *target.SqlSettings.DataSource == model.FAKE_SETTING {
*target.SqlSettings.DataSource = *actual.SqlSettings.DataSource
}
if len(target.SqlSettings.DataSourceReplicas) == 1 && target.SqlSettings.DataSourceReplicas[0] == model.FAKE_SETTING {
target.SqlSettings.DataSourceReplicas = make([]string, len(actual.SqlSettings.DataSourceReplicas))
for i := range target.SqlSettings.DataSourceReplicas {
target.SqlSettings.DataSourceReplicas[i] = actual.SqlSettings.DataSourceReplicas[i]
}
}
if len(target.SqlSettings.DataSourceSearchReplicas) == 1 && target.SqlSettings.DataSourceSearchReplicas[0] == model.FAKE_SETTING {
target.SqlSettings.DataSourceSearchReplicas = make([]string, len(actual.SqlSettings.DataSourceSearchReplicas))
for i := range target.SqlSettings.DataSourceSearchReplicas {
target.SqlSettings.DataSourceSearchReplicas[i] = actual.SqlSettings.DataSourceSearchReplicas[i]
}
}
if *target.SqlSettings.AtRestEncryptKey == model.FAKE_SETTING {
target.SqlSettings.AtRestEncryptKey = actual.SqlSettings.AtRestEncryptKey
}
@@ -57,6 +42,16 @@ func desanitize(actual, target *model.Config) {
if *target.ElasticsearchSettings.Password == model.FAKE_SETTING {
*target.ElasticsearchSettings.Password = *actual.ElasticsearchSettings.Password
}
target.SqlSettings.DataSourceReplicas = make([]string, len(actual.SqlSettings.DataSourceReplicas))
for i := range target.SqlSettings.DataSourceReplicas {
target.SqlSettings.DataSourceReplicas[i] = actual.SqlSettings.DataSourceReplicas[i]
}
target.SqlSettings.DataSourceSearchReplicas = make([]string, len(actual.SqlSettings.DataSourceSearchReplicas))
for i := range target.SqlSettings.DataSourceSearchReplicas {
target.SqlSettings.DataSourceSearchReplicas[i] = actual.SqlSettings.DataSourceSearchReplicas[i]
}
}
// fixConfig patches invalid or missing data in the configuration, returning true if changed.

4
config/utils_test.go
Просмотреть файл

@@ -50,8 +50,8 @@ func TestDesanitize(t *testing.T) {
target.SqlSettings.DataSource = sToP(model.FAKE_SETTING)
target.SqlSettings.AtRestEncryptKey = sToP(model.FAKE_SETTING)
target.ElasticsearchSettings.Password = sToP(model.FAKE_SETTING)
target.SqlSettings.DataSourceReplicas = []string{model.FAKE_SETTING}
target.SqlSettings.DataSourceSearchReplicas = []string{model.FAKE_SETTING}
target.SqlSettings.DataSourceReplicas = append(target.SqlSettings.DataSourceReplicas, "old_replica0")
target.SqlSettings.DataSourceSearchReplicas = append(target.SqlSettings.DataSourceReplicas, "old_search_replica0")
actualClone := actual.Clone()
desanitize(actual, target)

2
model/config.go
Просмотреть файл

@@ -3429,8 +3429,6 @@ func (o *Config) Sanitize() {
}
*o.SqlSettings.DataSource = FAKE_SETTING
o.SqlSettings.DataSourceReplicas = []string{FAKE_SETTING}
o.SqlSettings.DataSourceSearchReplicas = []string{FAKE_SETTING}
*o.SqlSettings.AtRestEncryptKey = FAKE_SETTING
*o.ElasticsearchSettings.Password = FAKE_SETTING

2
model/config_test.go
Просмотреть файл

@@ -1239,8 +1239,6 @@ func TestConfigSanitize(t *testing.T) {
assert.Equal(t, FAKE_SETTING, *c.EmailSettings.SMTPPassword)
assert.Equal(t, FAKE_SETTING, *c.GitLabSettings.Secret)
assert.Equal(t, FAKE_SETTING, *c.SqlSettings.DataSource)
assert.Equal(t, []string{FAKE_SETTING}, c.SqlSettings.DataSourceReplicas)
assert.Equal(t, []string{FAKE_SETTING}, c.SqlSettings.DataSourceSearchReplicas)
assert.Equal(t, FAKE_SETTING, *c.SqlSettings.AtRestEncryptKey)
assert.Equal(t, FAKE_SETTING, *c.ElasticsearchSettings.Password)
assert.Equal(t, FAKE_SETTING, c.SqlSettings.DataSourceReplicas[0])

15
store/sqlstore/supplier.go
Просмотреть файл

@@ -11,6 +11,7 @@ import (
"fmt"
"os"
"strings"
"sync"
"sync/atomic"
"time"
@@ -113,6 +114,8 @@ type SqlSupplier struct {
settings *model.SqlSettings
lockedToMaster bool
context context.Context
license *model.License
licenseMutex sync.Mutex
}
type TraceOnAdapter struct{}
@@ -327,6 +330,10 @@ func (ss *SqlSupplier) GetMaster() *gorp.DbMap {
}
func (ss *SqlSupplier) GetSearchReplica() *gorp.DbMap {
if ss.license == nil {
return ss.GetMaster()
}
if len(ss.settings.DataSourceSearchReplicas) == 0 {
return ss.GetReplica()
}
@@ -336,7 +343,7 @@ func (ss *SqlSupplier) GetSearchReplica() *gorp.DbMap {
}
func (ss *SqlSupplier) GetReplica() *gorp.DbMap {
if len(ss.settings.DataSourceReplicas) == 0 || ss.lockedToMaster {
if len(ss.settings.DataSourceReplicas) == 0 || ss.lockedToMaster || ss.license == nil {
return ss.GetMaster()
}
@@ -1178,6 +1185,12 @@ func (ss *SqlSupplier) CheckIntegrity() <-chan store.IntegrityCheckResult {
return results
}
func (ss *SqlSupplier) UpdateLicense(license *model.License) {
ss.licenseMutex.Lock()
defer ss.licenseMutex.Unlock()
ss.license = license
}
type mattermConverter struct{}
func (me mattermConverter) ToDb(val interface{}) (interface{}, error) {

56
store/sqlstore/supplier_test.go
Просмотреть файл

@@ -75,13 +75,14 @@ func TestGetReplica(t *testing.T) {
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.Description, func(t *testing.T) {
t.Run(testCase.Description+" with license", func(t *testing.T) {
t.Parallel()
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = testCase.DataSourceReplicas
settings.DataSourceSearchReplicas = testCase.DataSourceSearchReplicas
supplier := sqlstore.NewSqlSupplier(*settings, nil)
supplier.UpdateLicense(&model.License{})
replicas := make(map[*gorp.DbMap]bool)
for i := 0; i < 5; i++ {
@@ -130,6 +131,59 @@ func TestGetReplica(t *testing.T) {
}
}
})
t.Run(testCase.Description+" without license", func(t *testing.T) {
t.Parallel()
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = testCase.DataSourceReplicas
settings.DataSourceSearchReplicas = testCase.DataSourceSearchReplicas
supplier := sqlstore.NewSqlSupplier(*settings, nil)
replicas := make(map[*gorp.DbMap]bool)
for i := 0; i < 5; i++ {
replicas[supplier.GetReplica()] = true
}
searchReplicas := make(map[*gorp.DbMap]bool)
for i := 0; i < 5; i++ {
searchReplicas[supplier.GetSearchReplica()] = true
}
if len(testCase.DataSourceReplicas) > 0 {
// If replicas were defined, ensure none are the master.
assert.Len(t, replicas, 1)
for replica := range replicas {
assert.Same(t, supplier.GetMaster(), replica)
}
} else if assert.Len(t, replicas, 1) {
// Otherwise ensure the replicas contains only the master.
for replica := range replicas {
assert.Equal(t, supplier.GetMaster(), replica)
}
}
if len(testCase.DataSourceSearchReplicas) > 0 {
// If search replicas were defined, ensure none are the master nor the replicas.
assert.Len(t, searchReplicas, 1)
for searchReplica := range searchReplicas {
assert.Same(t, supplier.GetMaster(), searchReplica)
}
} else if len(testCase.DataSourceReplicas) > 0 {
// If no search replicas were defined, but replicas were, ensure they are equal.
assert.Equal(t, replicas, searchReplicas)
} else if assert.Len(t, searchReplicas, 1) {
// Otherwise ensure the search replicas contains the master.
for searchReplica := range searchReplicas {
assert.Equal(t, supplier.GetMaster(), searchReplica)
}
}
})
}
}