Allow inline JSON in config.json for advanced logging config (#20954)

* Allow embedded JSON in config.json for AdvancedLoggingConfig

* fix escaped JSON case

* Add unit test cases for escaped JSON

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/audit.go

@@ -109,14 +109,14 @@ func (s *Server) configureAudit(adt *audit.Audit, bAllowAdvancedLogging bool) er
 	adt.OnError = s.onAuditError
 
 	var logConfigSrc config.LogConfigSrc
-	dsn := *s.platform.Config().ExperimentalAuditSettings.AdvancedLoggingConfig
+	auditSettings := s.platform.Config().ExperimentalAuditSettings
-	if bAllowAdvancedLogging && dsn != "" {
+	if bAllowAdvancedLogging && !config.IsEmptyDSN(auditSettings.AdvancedLoggingConfig) {
 		var err error
-		logConfigSrc, err = config.NewLogConfigSrc(dsn, s.platform.GetConfigStore())
+		logConfigSrc, err = config.NewLogConfigSrc(auditSettings.AdvancedLoggingConfig, s.platform.GetConfigStore())
 		if err != nil {
 			return fmt.Errorf("invalid config source for audit, %w", err)
 		}
-		mlog.Debug("Loaded audit configuration", mlog.String("source", dsn))
+		mlog.Debug("Loaded audit configuration", mlog.String("source", string(auditSettings.AdvancedLoggingConfig)))
 	}
 
 	// ExperimentalAuditSettings provides basic file audit (E0, E10); logConfigSrc provides advanced config (E20).

app/platform/config.go

@@ -122,14 +122,14 @@ func (ps *PlatformService) ConfigureLogger(name string, logger *mlog.Logger, log
 	// file is loaded.  If no valid E20 license exists then advanced logging will be
 	// shutdown once license is loaded/checked.
 	var err error
-	dsn := *logSettings.AdvancedLoggingConfig
 	var logConfigSrc config.LogConfigSrc
-	if dsn != "" {
+
-		logConfigSrc, err = config.NewLogConfigSrc(dsn, ps.configStore)
+	if !config.IsEmptyDSN(logSettings.AdvancedLoggingConfig) {
+		logConfigSrc, err = config.NewLogConfigSrc(logSettings.AdvancedLoggingConfig, ps.configStore)
 		if err != nil {
 			return fmt.Errorf("invalid config source for %s, %w", name, err)
 		}
-		ps.logger.Info("Loaded configuration for "+name, mlog.String("source", dsn))
+		ps.logger.Info("Loaded configuration for "+name, mlog.String("source", string(logSettings.AdvancedLoggingConfig)))
 	}
 
 	cfg, err := config.MloggerConfigFromLoggerConfig(logSettings, logConfigSrc, getPath)

config/logconfigsrc.go

@@ -4,9 +4,11 @@
 package config
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -22,16 +24,23 @@ type LogConfigSrc interface {
 	Get() mlog.LoggerConfiguration
 
 	// Set updates the dsn specifying the source and reloads
-	Set(dsn string, configStore *Store) (err error)
+	Set(dsn []byte, configStore *Store) (err error)
 
 	// Close cleans up resources.
 	Close() error
 }
 
+func IsEmptyDSN(dsn json.RawMessage) bool {
+	if len(dsn) == 0 || bytes.Equal(dsn, []byte("{}")) || bytes.Equal(dsn, []byte("\"\"")) {
+		return true
+	}
+	return false
+}
+
 // NewLogConfigSrc creates an advanced logging configuration source, backed by a
-// file, JSON string, or database.
+// file, JSON, or database.
-func NewLogConfigSrc(dsn string, configStore *Store) (LogConfigSrc, error) {
+func NewLogConfigSrc(dsn json.RawMessage, configStore *Store) (LogConfigSrc, error) {
-	if dsn == "" {
+	if len(dsn) == 0 {
 		return nil, errors.New("dsn should not be empty")
 	}
 
@@ -39,17 +48,28 @@ func NewLogConfigSrc(dsn string, configStore *Store) (LogConfigSrc, error) {
 		return nil, errors.New("configStore should not be nil")
 	}
 
-	dsn = strings.TrimSpace(dsn)
+	// check if embedded JSON
-
 	if isJSONMap(dsn) {
 		return newJSONSrc(dsn)
 	}
 
-	path := dsn
+	// Now we're treating the DSN as a string which may contain escaped JSON or be a filespec.
+	str := strings.TrimSpace(string(dsn))
+	if s, err := strconv.Unquote(str); err == nil {
+		str = s
+	}
+
+	// check if escaped JSON
+	strBytes := []byte(str)
+	if isJSONMap(strBytes) {
+		return newJSONSrc(strBytes)
+	}
+
 	// If this is a file based config we need the full path so it can be watched.
-	if strings.HasPrefix(configStore.String(), "file://") && !filepath.IsAbs(dsn) {
+	path := str
+	if strings.HasPrefix(configStore.String(), "file://") && !filepath.IsAbs(path) {
 		configPath := strings.TrimPrefix(configStore.String(), "file://")
-		path = filepath.Join(filepath.Dir(configPath), dsn)
+		path = filepath.Join(filepath.Dir(configPath), path)
 	}
 
 	return newFileSrc(path, configStore)
@@ -63,7 +83,7 @@ type jsonSrc struct {
 	cfg   mlog.LoggerConfiguration
 }
 
-func newJSONSrc(data string) (*jsonSrc, error) {
+func newJSONSrc(data json.RawMessage) (*jsonSrc, error) {
 	src := &jsonSrc{}
 	return src, src.Set(data, nil)
 }
@@ -76,8 +96,8 @@ func (src *jsonSrc) Get() mlog.LoggerConfiguration {
 }
 
 // Set updates the JSON specifying the source and reloads
-func (src *jsonSrc) Set(data string, _ *Store) error {
+func (src *jsonSrc) Set(data []byte, _ *Store) error {
-	cfg, err := logTargetCfgFromJSON([]byte(data))
+	cfg, err := logTargetCfgFromJSON(data)
 	if err != nil {
 		return err
 	}
@@ -112,7 +132,7 @@ func newFileSrc(path string, configStore *Store) (*fileSrc, error) {
 	src := &fileSrc{
 		path: path,
 	}
-	if err := src.Set(path, configStore); err != nil {
+	if err := src.Set([]byte(path), configStore); err != nil {
 		return nil, err
 	}
 	return src, nil
@@ -128,8 +148,8 @@ func (src *fileSrc) Get() mlog.LoggerConfiguration {
 // Set updates the dsn specifying the file source and reloads.
 // The file will be watched for changes and reloaded as needed,
 // and all listeners notified.
-func (src *fileSrc) Set(path string, configStore *Store) error {
+func (src *fileSrc) Set(path []byte, configStore *Store) error {
-	data, err := configStore.GetFile(path)
+	data, err := configStore.GetFile(string(path))
 	if err != nil {
 		return err
 	}

config/logconfigsrc_test.go

@@ -4,36 +4,41 @@
 package config
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const (
+var (
-	validJSON = `{"file":{ "Type":"file"}}`
+	validJSON        = []byte(`{"file":{ "Type":"file"}}`)
-	badJSON   = `{"file":{ Type="file"}}`
+	badJSON          = []byte(`{"file":{ Type="file"}}`)
+	validEscapedJSON = []byte(`"{\"file\":{ \"Type\":\"file\"}}"`)
+	badEscapedJSON   = []byte(`"{\"file\":{ Type:\"file\"}}"`)
 )
 
 func TestNewLogConfigSrc(t *testing.T) {
 	store := NewTestMemoryStore()
 	require.NotNil(t, store)
-	err := store.SetFile("advancedlogging.conf", []byte(validJSON))
+	err := store.SetFile("advancedlogging.conf", validJSON)
 	require.NoError(t, err)
 
 	tests := []struct {
 		name        string
-		dsn         string
+		dsn         json.RawMessage
 		configStore *Store
 		wantErr     bool
 		wantType    LogConfigSrc
 	}{
-		{name: "empty dsn", dsn: "", configStore: store, wantErr: true, wantType: nil},
+		{name: "empty dsn", dsn: []byte(""), configStore: store, wantErr: true, wantType: nil},
-		{name: "garbage dsn", dsn: "!@wfejwcevioj", configStore: store, wantErr: true, wantType: nil},
+		{name: "garbage dsn", dsn: []byte("!@wfejwcevioj"), configStore: store, wantErr: true, wantType: nil},
 		{name: "valid json dsn", dsn: validJSON, configStore: store, wantErr: false, wantType: &jsonSrc{}},
 		{name: "invalid json dsn", dsn: badJSON, configStore: store, wantErr: true, wantType: nil},
-		{name: "valid filespec dsn", dsn: "advancedlogging.conf", configStore: store, wantErr: false, wantType: &fileSrc{}},
+		{name: "valid escaped json dsn", dsn: validEscapedJSON, configStore: store, wantErr: false, wantType: &jsonSrc{}},
-		{name: "invalid filespec dsn", dsn: "/nobody/here.conf", configStore: store, wantErr: true, wantType: nil},
+		{name: "invalid escaped json dsn", dsn: badEscapedJSON, configStore: store, wantErr: true, wantType: nil},
+		{name: "valid filespec dsn", dsn: []byte("advancedlogging.conf"), configStore: store, wantErr: false, wantType: &fileSrc{}},
+		{name: "invalid filespec dsn", dsn: []byte("/nobody/here.conf"), configStore: store, wantErr: true, wantType: nil},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

config/migrate.go

@@ -33,8 +33,8 @@ func Migrate(from, to string) error {
 	}
 
 	// Only migrate advanced logging config if it is not embedded JSON.
-	if !isJSONMap(*sourceConfig.LogSettings.AdvancedLoggingConfig) {
+	if !isJSONMap(sourceConfig.LogSettings.AdvancedLoggingConfig) {
-		files = append(files, *sourceConfig.LogSettings.AdvancedLoggingConfig)
+		files = append(files, string(sourceConfig.LogSettings.AdvancedLoggingConfig))
 	}
 
 	files = append(files, sourceConfig.PluginSettings.SignaturePublicKeyFiles...)

config/utils.go

@@ -200,9 +200,10 @@ func stripPassword(dsn, schema string) string {
 	return prefix + dsn[:i+1] + dsn[j:]
 }
 
-func isJSONMap(data string) bool {
+func isJSONMap(data []byte) bool {
 	var m map[string]any
-	return json.Unmarshal([]byte(data), &m) == nil
+	err := json.Unmarshal(data, &m)
+	return err == nil
 }
 
 func GetValueByPath(path []string, obj any) (any, bool) {

config/utils_test.go

@@ -276,7 +276,7 @@ func TestIsJSONMap(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := isJSONMap(tt.data); got != tt.want {
+			if got := isJSONMap([]byte(tt.data)); got != tt.want {
 				t.Errorf("isJSONMap() = %v, want %v", got, tt.want)
 			}
 		})

model/config.go

@@ -1221,7 +1221,7 @@ type LogSettings struct {
 	EnableWebhookDebugging *bool           `access:"environment_logging,write_restrictable,cloud_restrictable"`
 	EnableDiagnostics      *bool           `access:"environment_logging,write_restrictable,cloud_restrictable"` // telemetry: none
 	EnableSentry           *bool           `access:"environment_logging,write_restrictable,cloud_restrictable"` // telemetry: none
-	AdvancedLoggingConfig  *string `access:"environment_logging,write_restrictable,cloud_restrictable"`
+	AdvancedLoggingConfig  json.RawMessage `access:"environment_logging,write_restrictable,cloud_restrictable"`
 }
 
 func NewLogSettings() *LogSettings {
@@ -1276,7 +1276,7 @@ func (s *LogSettings) SetDefaults() {
 	}
 
 	if s.AdvancedLoggingConfig == nil {
-		s.AdvancedLoggingConfig = NewString("")
+		s.AdvancedLoggingConfig = []byte("{}")
 	}
 }
 
@@ -1288,7 +1288,7 @@ type ExperimentalAuditSettings struct {
 	FileMaxBackups        *int            `access:"experimental_features,write_restrictable,cloud_restrictable"`
 	FileCompress          *bool           `access:"experimental_features,write_restrictable,cloud_restrictable"`
 	FileMaxQueueSize      *int            `access:"experimental_features,write_restrictable,cloud_restrictable"`
-	AdvancedLoggingConfig *string `access:"experimental_features,write_restrictable,cloud_restrictable"`
+	AdvancedLoggingConfig json.RawMessage `access:"experimental_features,write_restrictable,cloud_restrictable"`
 }
 
 func (s *ExperimentalAuditSettings) SetDefaults() {
@@ -1321,7 +1321,7 @@ func (s *ExperimentalAuditSettings) SetDefaults() {
 	}
 
 	if s.AdvancedLoggingConfig == nil {
-		s.AdvancedLoggingConfig = NewString("")
+		s.AdvancedLoggingConfig = []byte("{}")
 	}
 }
 
@@ -1334,7 +1334,7 @@ type NotificationLogSettings struct {
 	FileLevel             *string         `access:"write_restrictable,cloud_restrictable"`
 	FileJson              *bool           `access:"write_restrictable,cloud_restrictable"`
 	FileLocation          *string         `access:"write_restrictable,cloud_restrictable"`
-	AdvancedLoggingConfig *string `access:"write_restrictable,cloud_restrictable"`
+	AdvancedLoggingConfig json.RawMessage `access:"write_restrictable,cloud_restrictable"`
 }
 
 func (s *NotificationLogSettings) SetDefaults() {
@@ -1371,7 +1371,7 @@ func (s *NotificationLogSettings) SetDefaults() {
 	}
 
 	if s.AdvancedLoggingConfig == nil {
-		s.AdvancedLoggingConfig = NewString("")
+		s.AdvancedLoggingConfig = []byte("{}")
 	}
 }
 

services/telemetry/telemetry.go

@@ -503,7 +503,7 @@ func (ts *TelemetryService) trackConfig() {
 		"file_json":                cfg.LogSettings.FileJson,
 		"enable_webhook_debugging": cfg.LogSettings.EnableWebhookDebugging,
 		"isdefault_file_location":  isDefault(cfg.LogSettings.FileLocation, ""),
-		"advanced_logging_config":  *cfg.LogSettings.AdvancedLoggingConfig != "",
+		"advanced_logging_config":  len(cfg.LogSettings.AdvancedLoggingConfig) != 0,
 	})
 
 	ts.SendTelemetry(TrackConfigAudit, map[string]any{
@@ -513,7 +513,7 @@ func (ts *TelemetryService) trackConfig() {
 		"file_max_backups":        *cfg.ExperimentalAuditSettings.FileMaxBackups,
 		"file_compress":           *cfg.ExperimentalAuditSettings.FileCompress,
 		"file_max_queue_size":     *cfg.ExperimentalAuditSettings.FileMaxQueueSize,
-		"advanced_logging_config": *cfg.ExperimentalAuditSettings.AdvancedLoggingConfig != "",
+		"advanced_logging_config": len(cfg.ExperimentalAuditSettings.AdvancedLoggingConfig) != 0,
 	})
 
 	ts.SendTelemetry(TrackConfigNotificationLog, map[string]any{
@@ -524,7 +524,7 @@ func (ts *TelemetryService) trackConfig() {
 		"file_level":              *cfg.NotificationLogSettings.FileLevel,
 		"file_json":               *cfg.NotificationLogSettings.FileJson,
 		"isdefault_file_location": isDefault(*cfg.NotificationLogSettings.FileLocation, ""),
-		"advanced_logging_config": *cfg.NotificationLogSettings.AdvancedLoggingConfig != "",
+		"advanced_logging_config": len(cfg.NotificationLogSettings.AdvancedLoggingConfig) != 0,
 	})
 
 	ts.SendTelemetry(TrackConfigPassword, map[string]any{