[MM-32389] Fix FeatureFlags section erroneously getting written to config (#16836)

* Fix FeatureFlags section erroneously getting written to config

* Avoid invoking config listeners if config has not changed

* Avoid resetting feature flags on store creation

config/store.go

@@ -214,6 +214,8 @@ func (s *Store) loadLockedWithOld(oldCfg *model.Config, unlockOnce *sync.Once) e
 		}
 	}
 
+	loadedFeatureFlags := loadedConfig.FeatureFlags
+
 	// If we have custom defaults set, the initial config is merged on
 	// top of them and we delete them not to be used again in the
 	// configuration reloads
@@ -248,9 +250,28 @@ func (s *Store) loadLockedWithOld(oldCfg *model.Config, unlockOnce *sync.Once) e
 	if err != nil {
 		return errors.Wrap(err, "failed to marshal loaded config")
 	}
-	if !s.readOnly && (len(configBytes) == 0 || !bytes.Equal(oldCfgBytes, newCfgBytes)) {
-		if err := s.backingStore.Set(s.configNoEnv); err != nil {
-			if !errors.Is(err, ErrReadOnlyConfiguration) {
+
+	var shouldStore bool
+	hasChanged := len(configBytes) == 0 || !bytes.Equal(oldCfgBytes, newCfgBytes)
+	if hasChanged {
+		featureFlags := s.configNoEnv.FeatureFlags
+		// Don't persist feature flags unless we are on MM cloud
+		// MM cloud uses config in the DB as a cache of the feature flag
+		// settings in case the management system is down when a pod starts.
+		if !s.persistFeatureFlags {
+			s.configNoEnv.FeatureFlags = loadedFeatureFlags
+		}
+		toStoreBytes, err := json.Marshal(s.configNoEnv)
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal old config")
+		}
+		shouldStore = !bytes.Equal(toStoreBytes, configBytes)
+		// We write back to the backing store only if
+		// the config has changed and the store is not read-only.
+		if !s.readOnly && shouldStore {
+			err := s.backingStore.Set(s.configNoEnv)
+			s.configNoEnv.FeatureFlags = featureFlags
+			if err != nil && !errors.Is(err, ErrReadOnlyConfiguration) {
 				return errors.Wrap(err, "failed to persist")
 			}
 		}
@@ -260,7 +281,9 @@ func (s *Store) loadLockedWithOld(oldCfg *model.Config, unlockOnce *sync.Once) e
 
 	unlockOnce.Do(s.configLock.Unlock)
 
-	s.invokeConfigListeners(oldCfg, loadedConfig)
+	if hasChanged {
+		s.invokeConfigListeners(oldCfg, loadedConfig)
+	}
 
 	return nil
 }