MM-28859 Add feature flag managment system using split.io and remove viper. (#15954)

* Add feature flag managment system using split.io and remove viper. * Fixing tests. * Attempt to fix postgres tests. * Fix watch filepath for advanced logging. * Review fixes. * Some error wrapping. * Remove unessisary store interface. * Desanitize SplitKey * Simplify. * Review feedback. * Rename split mlog adatper to split logger. * fsInner * Style. * Restore oldcfg test. * Downgrading non-actionable feature flag errors to warnings. Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2020-10-29 15:54:39 -07:00
--- a/app/feature_flags.go
+++ b/app/feature_flags.go
@@ -0,0 +1,102 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/config"
+	"github.com/mattermost/mattermost-server/v5/mlog"
+)
+
+// setupFeatureFlags called on startup and when the cluster leader changes.
+// Starts or stops the synchronization of feature flags from upstream management.
+func (s *Server) setupFeatureFlags() {
+	s.featureFlagSynchronizerMutex.Lock()
+	defer s.featureFlagSynchronizerMutex.Unlock()
+	license := s.License()
+	inCloud := license != nil && *license.Features.Cloud
+	splitKey := *s.Config().ServiceSettings.SplitKey
+	syncFeatureFlags := inCloud && splitKey != "" && s.IsLeader()
+
+	s.configStore.PersistFeatures(inCloud)
+
+	if syncFeatureFlags {
+		if err := s.startFeatureFlagUpdateJob(); err != nil {
+			s.Log.Warn("Unable to setup synchronization with feature flag management. Will fallback to cloud cache.", mlog.Err(err))
+		}
+	} else {
+		s.stopFeatureFlagUpdateJob()
+	}
+
+	if err := s.configStore.Load(); err != nil {
+		s.Log.Warn("Unable to load config store after feature flag setup.", mlog.Err(err))
+	}
+}
+
+func (s *Server) updateFeatureFlagValuesFromManagment() {
+	newCfg := s.configStore.GetNoEnv().Clone()
+	oldFlags := *newCfg.FeatureFlags
+	newFlags := s.featureFlagSynchronizer.UpdateFeatureFlagValues(oldFlags)
+	if oldFlags != newFlags {
+		*newCfg.FeatureFlags = newFlags
+		s.SaveConfig(newCfg, true)
+	}
+}
+
+func (s *Server) startFeatureFlagUpdateJob() error {
+	// Can be run multiple times
+	if s.featureFlagSynchronizer != nil {
+		return nil
+	}
+
+	var log *mlog.Logger
+	if *s.Config().ServiceSettings.DebugSplit {
+		log = s.Log
+	}
+
+	synchronizer, err := config.NewFeatureFlagSynchronizer(config.FeatureFlagSyncParams{
+		ServerID: s.TelemetryId(),
+		SplitKey: *s.Config().ServiceSettings.SplitKey,
+		Log:      log,
+	})
+	if err != nil {
+		return err
+	}
+
+	s.featureFlagStop = make(chan struct{})
+	s.featureFlagStopped = make(chan struct{})
+	s.featureFlagSynchronizer = synchronizer
+	syncInterval := *s.Config().ServiceSettings.FeatureFlagSyncIntervalSeconds
+
+	go func() {
+		ticker := time.NewTicker(time.Duration(syncInterval) * time.Second)
+		defer ticker.Stop()
+		defer close(s.featureFlagStopped)
+		if err := synchronizer.EnsureReady(); err != nil {
+			s.Log.Warn("Problem connecting to feature flag management. Will fallback to cloud cache.", mlog.Err(err))
+			return
+		}
+		s.updateFeatureFlagValuesFromManagment()
+		for {
+			select {
+			case <-s.featureFlagStop:
+				return
+			case <-ticker.C:
+				s.updateFeatureFlagValuesFromManagment()
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (s *Server) stopFeatureFlagUpdateJob() {
+	if s.featureFlagSynchronizer != nil {
+		close(s.featureFlagStop)
+		<-s.featureFlagStopped
+		s.featureFlagSynchronizer.Close()
+		s.featureFlagSynchronizer = nil
+	}
+}
--- a/app/helper_test.go
+++ b/app/helper_test.go
@@ -48,12 +48,9 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 		panic(err)
 	}

-	memoryStore, err := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{IgnoreEnvironmentOverrides: true})
-	if err != nil {
-		panic("failed to initialize memory store: " + err.Error())
-	}
+	configStore := config.NewTestMemoryStore()

-	config := memoryStore.Get()
+	config := configStore.Get()
 	if configSet != nil {
 		configSet(config)
 	}
@@ -62,12 +59,12 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 	*config.LogSettings.EnableSentry = false // disable error reporting during tests
 	*config.AnnouncementSettings.AdminNoticesEnabled = false
 	*config.AnnouncementSettings.UserNoticesEnabled = false
-	memoryStore.Set(config)
+	configStore.Set(config)

 	buffer := &bytes.Buffer{}

 	var options []Option
-	options = append(options, ConfigStore(memoryStore))
+	options = append(options, ConfigStore(configStore))
 	options = append(options, StoreOverride(dbStore))
 	options = append(options, SetLogger(mlog.NewTestingLogger(tb, buffer)))

--- a/app/notification_push_test.go
+++ b/app/notification_push_test.go
@@ -1365,12 +1365,7 @@ func TestAllPushNotifications(t *testing.T) {
 }

 func TestPushNotificationRace(t *testing.T) {
-	memoryStore, err := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{
-		IgnoreEnvironmentOverrides: true,
-	})
-	require.NoError(t, err, "failed to initialize memory store")
-	defer memoryStore.Close()
-
+	memoryStore := config.NewTestMemoryStore()
 	mockStore := testlib.GetMockStoreForSetupFunctions()
 	mockPreferenceStore := mocks.PreferenceStore{}
 	mockPreferenceStore.On("Get",
--- a/app/options.go
+++ b/app/options.go
@@ -52,7 +52,7 @@ func Config(dsn string, watch bool) Option {
 }

 // ConfigStore applies the given config store, typically to replace the traditional sources with a memory store for testing.
-func ConfigStore(configStore config.Store) Option {
+func ConfigStore(configStore *config.Store) Option {
 	return func(s *Server) error {
 		s.configStore = configStore

--- a/app/server.go
+++ b/app/server.go
@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -127,7 +128,7 @@ type Server struct {
 	searchConfigListenerId  string
 	searchLicenseListenerId string
 	loggerLicenseListenerId string
-	configStore             config.Store
+	configStore             *config.Store
 	asymmetricSigningKey    *ecdsa.PrivateKey
 	postActionCookieSecret  []byte

@@ -178,6 +179,11 @@ type Server struct {
 	// and data corruption.
 	uploadLockMapMut sync.Mutex
 	uploadLockMap    map[string]bool
+
+	featureFlagSynchronizer      *config.FeatureFlagSynchronizer
+	featureFlagStop              chan struct{}
+	featureFlagStopped           chan struct{}
+	featureFlagSynchronizerMutex sync.Mutex
 }

 func NewServer(options ...Option) (*Server, error) {
@@ -200,7 +206,11 @@ func NewServer(options ...Option) (*Server, error) {
 	}

 	if s.configStore == nil {
-		configStore, err := config.NewFileStore("config.json", true)
+		innerStore, err := config.NewFileStore("config.json", true)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load config")
+		}
+		configStore, err := config.NewStoreFromBacking(innerStore)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to load config")
 		}
@@ -292,27 +302,6 @@ func NewServer(options ...Option) (*Server, error) {
 		return nil, errors.Wrapf(err, "unable to load Mattermost translation files")
 	}

-	s.configListenerId = s.AddConfigListener(func(_, _ *model.Config) {
-		s.configOrLicenseListener()
-
-		message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_CONFIG_CHANGED, "", "", "", nil)
-
-		message.Add("config", s.ClientConfigWithComputed())
-		s.Go(func() {
-			s.Publish(message)
-		})
-	})
-	s.licenseListenerId = s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
-		s.configOrLicenseListener()
-
-		message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_LICENSE_CHANGED, "", "", "", nil)
-		message.Add("license", s.GetSanitizedClientLicense())
-		s.Go(func() {
-			s.Publish(message)
-		})
-
-	})
-
 	s.initEnterprise()

 	if s.newStore == nil {
@@ -353,6 +342,27 @@ func NewServer(options ...Option) (*Server, error) {

 	s.Store = s.newStore()

+	s.configListenerId = s.AddConfigListener(func(_, _ *model.Config) {
+		s.configOrLicenseListener()
+
+		message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_CONFIG_CHANGED, "", "", "", nil)
+
+		message.Add("config", s.ClientConfigWithComputed())
+		s.Go(func() {
+			s.Publish(message)
+		})
+	})
+	s.licenseListenerId = s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+		s.configOrLicenseListener()
+
+		message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_LICENSE_CHANGED, "", "", "", nil)
+		message.Add("license", s.GetSanitizedClientLicense())
+		s.Go(func() {
+			s.Publish(message)
+		})
+
+	})
+
 	s.telemetryService = telemetry.New(s, s.Store, s.SearchEngine, s.Log)

 	emailService, err := NewEmailService(s)
@@ -365,8 +375,18 @@ func NewServer(options ...Option) (*Server, error) {
 		s.LoadLicense()
 	}

+	s.setupFeatureFlags()
+
 	s.initJobs()

+	s.clusterLeaderListenerId = s.AddClusterLeaderChangedListener(func() {
+		mlog.Info("Cluster leader changed. Determining if job schedulers should be running:", mlog.Bool("isLeader", s.IsLeader()))
+		if s.Jobs != nil && s.Jobs.Schedulers != nil {
+			s.Jobs.Schedulers.HandleClusterLeaderChange(s.IsLeader())
+		}
+		s.setupFeatureFlags()
+	})
+
 	if s.joinCluster && s.Cluster != nil {
 		s.Cluster.StartInterNodeCommunication()
 	}
@@ -389,13 +409,6 @@ func NewServer(options ...Option) (*Server, error) {

 	s.regenerateClientConfig()

-	s.clusterLeaderListenerId = s.AddClusterLeaderChangedListener(func() {
-		mlog.Info("Cluster leader changed. Determining if job schedulers should be running:", mlog.Bool("isLeader", s.IsLeader()))
-		if s.Jobs != nil && s.Jobs.Schedulers != nil {
-			s.Jobs.Schedulers.HandleClusterLeaderChange(s.IsLeader())
-		}
-	})
-
 	subpath, err := utils.GetSubpathFromConfig(s.Config())
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse SiteURL subpath")
@@ -480,7 +493,6 @@ func NewServer(options ...Option) (*Server, error) {
 	s.checkPushNotificationServerUrl()

 	license := s.License()
-
 	if license == nil {
 		s.UpdateConfig(func(cfg *model.Config) {
 			cfg.TeamSettings.MaxNotificationsPerChannel = &MaxNotificationsPerChannelDefault
@@ -618,10 +630,9 @@ func (s *Server) initLogging() error {
 		isJson := config.IsJsonMap(dsn)

 		// If this is a file based config we need the full path so it can be watched.
-		if !isJson {
-			if fs, ok := s.configStore.(*config.FileStore); ok {
-				dsn = fs.GetFilePath(dsn)
-			}
+		if !isJson && strings.HasPrefix(s.configStore.String(), "file://") && !filepath.IsAbs(dsn) {
+			configPath := strings.TrimPrefix(s.configStore.String(), "file://")
+			dsn = filepath.Join(filepath.Dir(configPath), dsn)
 		}

 		cfg, err := config.NewLogConfigSrc(dsn, isJson, s.configStore)
@@ -752,6 +763,8 @@ func (s *Server) Shutdown() error {

 	s.Audit.Shutdown()

+	s.stopFeatureFlagUpdateJob()
+
 	s.configStore.Close()

 	if s.Cluster != nil {
--- a/app/server_test.go
+++ b/app/server_test.go
@@ -65,7 +65,8 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {

 	t.Run("Read Replicas with no License", func(t *testing.T) {
 		s, err := NewServer(func(server *Server) error {
-			configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
+			configStore := config.NewTestMemoryStore()
+			configStore.Set(&cfg)
 			server.configStore = configStore
 			return nil
 		})
@@ -77,8 +78,8 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {

 	t.Run("Read Replicas With License", func(t *testing.T) {
 		s, err := NewServer(func(server *Server) error {
-			configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
-			server.configStore = configStore
+			configStore := config.NewTestMemoryStore()
+			configStore.Set(&cfg)
 			server.licenseValue.Store(model.NewTestLicense())
 			return nil
 		})
@@ -90,7 +91,8 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {

 	t.Run("Search Replicas with no License", func(t *testing.T) {
 		s, err := NewServer(func(server *Server) error {
-			configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
+			configStore := config.NewTestMemoryStore()
+			configStore.Set(&cfg)
 			server.configStore = configStore
 			return nil
 		})
@@ -102,7 +104,8 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {

 	t.Run("Search Replicas With License", func(t *testing.T) {
 		s, err := NewServer(func(server *Server) error {
-			configStore, _ := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{InitialConfig: cfg.Clone()})
+			configStore := config.NewTestMemoryStore()
+			configStore.Set(&cfg)
 			server.configStore = configStore
 			server.licenseValue.Store(model.NewTestLicense())
 			return nil
@@ -114,27 +117,6 @@ func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {
 	})
 }

-func TestStartServerRateLimiterCriticalError(t *testing.T) {
-	// Attempt to use Rate Limiter with an invalid config
-	ms, err := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{
-		SkipValidation: true,
-	})
-	require.NoError(t, err)
-
-	config := ms.Get()
-	*config.RateLimitSettings.Enable = true
-	*config.RateLimitSettings.MaxBurst = -100
-	_, err = ms.Set(config)
-	require.NoError(t, err)
-
-	s, err := NewServer(ConfigStore(ms))
-	require.NoError(t, err)
-
-	serverErr := s.Start()
-	s.Shutdown()
-	require.Error(t, serverErr)
-}
-
 func TestStartServerPortUnavailable(t *testing.T) {
 	s, err := NewServer()
 	require.NoError(t, err)
@@ -398,7 +380,8 @@ func TestSentry(t *testing.T) {
 	testDir, _ := fileutils.FindDir("tests")
 	s, err := NewServer(func(server *Server) error {
 		configStore, _ := config.NewFileStore("config.json", true)
-		server.configStore = configStore
+		store, _ := config.NewStoreFromBacking(configStore)
+		server.configStore = store
 		server.UpdateConfig(func(cfg *model.Config) {
 			*cfg.ServiceSettings.ListenAddress = ":0"
 			*cfg.LogSettings.EnableSentry = false
@@ -441,7 +424,8 @@ func TestSentry(t *testing.T) {
 	SENTRY_DSN = fmt.Sprintf("http://test:test@localhost:%s/123", port)
 	s2, err := NewServer(func(server *Server) error {
 		configStore, _ := config.NewFileStore("config.json", true)
-		server.configStore = configStore
+		store, _ := config.NewStoreFromBacking(configStore)
+		server.configStore = store
 		server.UpdateConfig(func(cfg *model.Config) {
 			*cfg.ServiceSettings.ListenAddress = ":0"
 			*cfg.ServiceSettings.ConnectionSecurity = "TLS"
--- a/app/slashcommands/helper_test.go
+++ b/app/slashcommands/helper_test.go
@@ -45,10 +45,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 		panic(err)
 	}

-	memoryStore, err := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{IgnoreEnvironmentOverrides: true})
-	if err != nil {
-		panic("failed to initialize memory store: " + err.Error())
-	}
+	memoryStore := config.NewTestMemoryStore()

 	config := memoryStore.Get()
 	if configSet != nil {