[MM-28692] Include config diffs in audit record for config changing API calls (#17623)

* Replace config generator

* Cleanup

* Some renaming and docs additions to add clarity

* Cleanup logging related methods

* Cleanup emitter

* Fix TestDefaultsGenerator

* Move feature flags synchronization logic out of config package

* Remove unnecessary util functions

* Simplify load/set logic

* Refine semantics and add some test to cover them

* Remove unnecessary deep copies

* Improve logic further

* Fix license header

* Review file store tests

* Fix test

* Fix test

* Avoid additional write during initialization

* More consistent naming

* Update app/feature_flags.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

* Make ConfigStore.Set() return both old and new configs

* Implement config diff function

* Make app.SaveConfig return previous and current configs

* Add config diff to audit record

* Fix returned configs

* Include high level test

* Move FF synchronizer to its own package

* Remove unidiomatic use of sync.Once

* Add some comments

* Rename function

* More comment

* Save config diff in audit record for local endpoints

* Enable audit for config set/reset commands

* Improve tests output

Co-authored-by: Christopher Speller <crspeller@gmail.com>
Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

config/store.go

@@ -170,12 +170,13 @@ func (s *Store) SetReadOnlyFF(readOnly bool) {
 }
 
 // Set replaces the current configuration in its entirety and updates the backing store.
-func (s *Store) Set(newCfg *model.Config) (*model.Config, error) {
+// It returns both old and new versions of the config.
+func (s *Store) Set(newCfg *model.Config) (*model.Config, *model.Config, error) {
 	s.configLock.Lock()
 	defer s.configLock.Unlock()
 
 	if s.readOnly {
-		return nil, ErrReadOnlyStore
+		return nil, nil, ErrReadOnlyStore
 	}
 
 	newCfg = newCfg.Clone()
@@ -190,7 +191,7 @@ func (s *Store) Set(newCfg *model.Config) (*model.Config, error) {
 	desanitize(oldCfg, newCfg)
 
 	if err := newCfg.IsValid(); err != nil {
-		return nil, errors.Wrap(err, "new configuration is invalid")
+		return nil, nil, errors.Wrap(err, "new configuration is invalid")
 	}
 
 	// We attempt to remove any environment override that may be present in the input config.
@@ -211,7 +212,7 @@ func (s *Store) Set(newCfg *model.Config) (*model.Config, error) {
 	}
 
 	if err := s.backingStore.Set(newCfgNoEnv); err != nil {
-		return nil, errors.Wrap(err, "failed to persist")
+		return nil, nil, errors.Wrap(err, "failed to persist")
 	}
 
 	// We apply back environment overrides since the input config may or
@@ -219,12 +220,12 @@ func (s *Store) Set(newCfg *model.Config) (*model.Config, error) {
 	newCfg = applyEnvironmentMap(newCfgNoEnv, GetEnvironment())
 	fixConfig(newCfg)
 	if err := newCfg.IsValid(); err != nil {
-		return nil, errors.Wrap(err, "new configuration is invalid")
+		return nil, nil, errors.Wrap(err, "new configuration is invalid")
 	}
 
 	hasChanged, err := equal(oldCfg, newCfg)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to compare configs")
+		return nil, nil, errors.Wrap(err, "failed to compare configs")
 	}
 
 	// We restore the previously cleared feature flags sections back.
@@ -237,13 +238,15 @@ func (s *Store) Set(newCfg *model.Config) (*model.Config, error) {
 	s.configNoEnv = newCfgNoEnv
 	s.config = newCfg
 
+	newCfgCopy := newCfg.Clone()
+
 	if hasChanged {
 		s.configLock.Unlock()
-		s.invokeConfigListeners(oldCfg, newCfg.Clone())
+		s.invokeConfigListeners(oldCfg, newCfgCopy.Clone())
 		s.configLock.Lock()
 	}
 
-	return oldCfg, nil
+	return oldCfg, newCfgCopy, nil
 }
 
 // Load updates the current configuration from the backing store, possibly initializing.
@@ -337,9 +340,11 @@ func (s *Store) Load() error {
 	s.config = loadedCfg
 	s.configNoEnv = loadedCfgNoEnv
 
+	loadedCfgCopy := loadedCfg.Clone()
+
 	if hasChanged {
 		s.configLock.Unlock()
-		s.invokeConfigListeners(oldCfg, loadedCfg.Clone())
+		s.invokeConfigListeners(oldCfg, loadedCfgCopy)
 		s.configLock.Lock()
 	}