[MM-28692] Include config diffs in audit record for config changing API calls (#17623)

* Replace config generator

* Cleanup

* Some renaming and docs additions to add clarity

* Cleanup logging related methods

* Cleanup emitter

* Fix TestDefaultsGenerator

* Move feature flags synchronization logic out of config package

* Remove unnecessary util functions

* Simplify load/set logic

* Refine semantics and add some test to cover them

* Remove unnecessary deep copies

* Improve logic further

* Fix license header

* Review file store tests

* Fix test

* Fix test

* Avoid additional write during initialization

* More consistent naming

* Update app/feature_flags.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Christopher Speller <crspeller@gmail.com>

* Update config/store.go

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

* Make ConfigStore.Set() return both old and new configs

* Implement config diff function

* Make app.SaveConfig return previous and current configs

* Add config diff to audit record

* Fix returned configs

* Include high level test

* Move FF synchronizer to its own package

* Remove unidiomatic use of sync.Once

* Add some comments

* Rename function

* More comment

* Save config diff in audit record for local endpoints

* Enable audit for config set/reset commands

* Improve tests output

Co-authored-by: Christopher Speller <crspeller@gmail.com>
Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

api4/config_test.go

@@ -4,6 +4,8 @@
 package api4
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
@@ -12,6 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mattermost/mattermost-server/v5/app"
 	"github.com/mattermost/mattermost-server/v5/config"
 	"github.com/mattermost/mattermost-server/v5/model"
 )
@@ -436,6 +439,45 @@ func TestUpdateConfigRestrictSystemAdmin(t *testing.T) {
 	})
 }
 
+func TestUpdateConfigDiffInAuditRecord(t *testing.T) {
+	logFile, err := ioutil.TempFile("", "adv.log")
+	require.NoError(t, err)
+	defer os.Remove(logFile.Name())
+
+	os.Setenv("MM_EXPERIMENTALAUDITSETTINGS_FILEENABLED", "true")
+	os.Setenv("MM_EXPERIMENTALAUDITSETTINGS_FILENAME", logFile.Name())
+	defer os.Unsetenv("MM_EXPERIMENTALAUDITSETTINGS_FILEENABLED")
+	defer os.Unsetenv("MM_EXPERIMENTALAUDITSETTINGS_FILENAME")
+
+	options := []app.Option{
+		func(s *app.Server) error {
+			s.SetLicense(model.NewTestLicense("advanced_logging"))
+			return nil
+		},
+	}
+	th := SetupWithServerOptions(t, options)
+	defer th.TearDown()
+
+	cfg, resp := th.SystemAdminClient.GetConfig()
+	CheckNoError(t, resp)
+
+	timeoutVal := *cfg.ServiceSettings.ReadTimeout
+	cfg.ServiceSettings.ReadTimeout = model.NewInt(timeoutVal + 1)
+	cfg, resp = th.SystemAdminClient.UpdateConfig(cfg)
+	CheckNoError(t, resp)
+	defer th.App.UpdateConfig(func(cfg *model.Config) {
+		cfg.ServiceSettings.ReadTimeout = model.NewInt(timeoutVal)
+	})
+	require.Equal(t, timeoutVal+1, *cfg.ServiceSettings.ReadTimeout)
+
+	data, err := ioutil.ReadAll(logFile)
+	require.NoError(t, err)
+	require.NotEmpty(t, data)
+	require.Contains(t, string(data),
+		fmt.Sprintf(`"diff":"[{Path:ServiceSettings.ReadTimeout BaseVal:%d ActualVal:%d}]"`,
+			timeoutVal, timeoutVal+1))
+}
+
 func TestGetEnvironmentConfig(t *testing.T) {
 	os.Setenv("MM_SERVICESETTINGS_SITEURL", "http://example.mattermost.com")
 	os.Setenv("MM_SERVICESETTINGS_ENABLECUSTOMEMOJI", "true")