MM-36764 mlog refactor (#18118)

Refactor mlog
- simplify mlog by removing redundant code
- remove Zap dependency
- update unit test helpers
- update logging config
- update auditing

config/emitter.go

@@ -57,7 +57,7 @@ func (e *logSrcEmitter) RemoveListener(id string) {
 }
 
 // invokeConfigListeners synchronously notifies all listeners about the configuration change.
-func (e *logSrcEmitter) invokeConfigListeners(oldCfg, newCfg mlog.LogTargetCfg) {
+func (e *logSrcEmitter) invokeConfigListeners(oldCfg, newCfg mlog.LoggerConfiguration) {
 	e.listeners.Range(func(key, value interface{}) bool {
 		listener := value.(LogSrcListener)
 		listener(oldCfg, newCfg)

config/emitter_test.go

@@ -56,18 +56,18 @@ func TestEmitter(t *testing.T) {
 func TestLogSrcEmitter(t *testing.T) {
 	var e logSrcEmitter
 
-	expectedOldCfg := make(mlog.LogTargetCfg)
-	expectedNewCfg := make(mlog.LogTargetCfg)
+	expectedOldCfg := make(mlog.LoggerConfiguration)
+	expectedNewCfg := make(mlog.LoggerConfiguration)
 
 	listener1 := false
-	id1 := e.AddListener(func(oldCfg, newCfg mlog.LogTargetCfg) {
+	id1 := e.AddListener(func(oldCfg, newCfg mlog.LoggerConfiguration) {
 		assert.Equal(t, expectedOldCfg, oldCfg)
 		assert.Equal(t, expectedNewCfg, newCfg)
 		listener1 = true
 	})
 
 	listener2 := false
-	id2 := e.AddListener(func(oldCfg, newCfg mlog.LogTargetCfg) {
+	id2 := e.AddListener(func(oldCfg, newCfg mlog.LoggerConfiguration) {
 		assert.Equal(t, expectedOldCfg, oldCfg)
 		assert.Equal(t, expectedNewCfg, newCfg)
 		listener2 = true

config/logconfigsrc.go

@@ -6,7 +6,6 @@ package config
 import (
 	"encoding/json"
 	"errors"
-	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -14,23 +13,17 @@ import (
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 )
 
-type LogSrcListener func(old, new mlog.LogTargetCfg)
+type LogSrcListener func(old, new mlog.LoggerConfiguration)
 
 // LogConfigSrc abstracts the Advanced Logging configuration so that implementations can
 // fetch from file, database, etc.
 type LogConfigSrc interface {
 	// Get fetches the current, cached configuration.
-	Get() mlog.LogTargetCfg
+	Get() mlog.LoggerConfiguration
 
 	// Set updates the dsn specifying the source and reloads
 	Set(dsn string, configStore *Store) (err error)
 
-	// AddListener adds a callback function to invoke when the configuration is modified.
-	AddListener(listener LogSrcListener) string
-
-	// RemoveListener removes a callback function using an id returned from AddListener.
-	RemoveListener(id string)
-
 	// Close cleans up resources.
 	Close() error
 }
@@ -38,6 +31,10 @@ type LogConfigSrc interface {
 // NewLogConfigSrc creates an advanced logging configuration source, backed by a
 // file, JSON string, or database.
 func NewLogConfigSrc(dsn string, configStore *Store) (LogConfigSrc, error) {
+	if dsn == "" {
+		return nil, errors.New("dsn should not be empty")
+	}
+
 	if configStore == nil {
 		return nil, errors.New("configStore should not be nil")
 	}
@@ -63,7 +60,7 @@ func NewLogConfigSrc(dsn string, configStore *Store) (LogConfigSrc, error) {
 type jsonSrc struct {
 	logSrcEmitter
 	mutex sync.RWMutex
-	cfg   mlog.LogTargetCfg
+	cfg   mlog.LoggerConfiguration
 }
 
 func newJSONSrc(data string) (*jsonSrc, error) {
@@ -72,7 +69,7 @@ func newJSONSrc(data string) (*jsonSrc, error) {
 }
 
 // Get fetches the current, cached configuration
-func (src *jsonSrc) Get() mlog.LogTargetCfg {
+func (src *jsonSrc) Get() mlog.LoggerConfiguration {
 	src.mutex.RLock()
 	defer src.mutex.RUnlock()
 	return src.cfg
@@ -89,7 +86,7 @@ func (src *jsonSrc) Set(data string, _ *Store) error {
 	return nil
 }
 
-func (src *jsonSrc) set(cfg mlog.LogTargetCfg) {
+func (src *jsonSrc) set(cfg mlog.LoggerConfiguration) {
 	src.mutex.Lock()
 	defer src.mutex.Unlock()
 
@@ -106,11 +103,9 @@ func (src *jsonSrc) Close() error {
 // fileSrc
 
 type fileSrc struct {
-	logSrcEmitter
 	mutex sync.RWMutex
-	cfg   mlog.LogTargetCfg
-
-	path string
+	cfg   mlog.LoggerConfiguration
+	path  string
 }
 
 func newFileSrc(path string, configStore *Store) (*fileSrc, error) {
@@ -124,7 +119,7 @@ func newFileSrc(path string, configStore *Store) (*fileSrc, error) {
 }
 
 // Get fetches the current, cached configuration
-func (src *fileSrc) Get() mlog.LogTargetCfg {
+func (src *fileSrc) Get() mlog.LoggerConfiguration {
 	src.mutex.RLock()
 	defer src.mutex.RUnlock()
 	return src.cfg
@@ -145,26 +140,14 @@ func (src *fileSrc) Set(path string, configStore *Store) error {
 	}
 
 	src.set(cfg)
-
-	// If path is a real file and not just the name of a database resource then watch it for changes.
-	// Absolute paths are explicit and require no resolution.
-	if _, err = os.Stat(path); os.IsNotExist(err) {
-		return nil
-	}
-
-	src.mutex.Lock()
-	defer src.mutex.Unlock()
-
 	return nil
 }
 
-func (src *fileSrc) set(cfg mlog.LogTargetCfg) {
+func (src *fileSrc) set(cfg mlog.LoggerConfiguration) {
 	src.mutex.Lock()
 	defer src.mutex.Unlock()
 
-	old := src.cfg
 	src.cfg = cfg
-	src.invokeConfigListeners(old, cfg)
 }
 
 // Close cleans up resources.
@@ -172,8 +155,8 @@ func (src *fileSrc) Close() error {
 	return nil
 }
 
-func logTargetCfgFromJSON(data []byte) (mlog.LogTargetCfg, error) {
-	cfg := make(mlog.LogTargetCfg)
+func logTargetCfgFromJSON(data []byte) (mlog.LoggerConfiguration, error) {
+	cfg := make(mlog.LoggerConfiguration)
 	err := json.Unmarshal(data, &cfg)
 	if err != nil {
 		return nil, err

config/logger.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/utils/fileutils"
+)
+
+const (
+	LogRotateSize           = 10000
+	LogCompress             = true
+	LogRotateMaxAge         = 0
+	LogRotateMaxBackups     = 0
+	LogFilename             = "mattermost.log"
+	LogNotificationFilename = "notifications.log"
+	LogMinLevelLen          = 5
+	LogMinMsgLen            = 45
+	LogDelim                = " "
+	LogEnableCaller         = true
+)
+
+type fileLocationFunc func(string) string
+
+func MloggerConfigFromLoggerConfig(s *model.LogSettings, configSrc LogConfigSrc, getFileFunc fileLocationFunc) (mlog.LoggerConfiguration, error) {
+	cfg := make(mlog.LoggerConfiguration)
+
+	var targetCfg mlog.TargetCfg
+	var err error
+
+	// add the simple logging config
+	if *s.EnableConsole {
+		targetCfg, err = makeSimpleConsoleTarget(*s.ConsoleLevel, *s.ConsoleJson, *s.EnableColor)
+		if err != nil {
+			return cfg, err
+		}
+		cfg["_defConsole"] = targetCfg
+	}
+
+	if *s.EnableFile {
+		targetCfg, err = makeSimpleFileTarget(getFileFunc(*s.FileLocation), *s.FileLevel, *s.FileJson)
+		if err != nil {
+			return cfg, err
+		}
+		cfg["_defFile"] = targetCfg
+	}
+
+	if configSrc == nil {
+		return cfg, nil
+	}
+
+	// add advanced logging config
+	cfgAdv := configSrc.Get()
+	cfg.Append(cfgAdv)
+
+	return cfg, nil
+}
+
+func MloggerConfigFromAuditConfig(auditSettings model.ExperimentalAuditSettings, configSrc LogConfigSrc) (mlog.LoggerConfiguration, error) {
+	cfg := make(mlog.LoggerConfiguration)
+
+	var targetCfg mlog.TargetCfg
+	var err error
+
+	// add the simple audit config
+	if *auditSettings.FileEnabled {
+		targetCfg, err = makeSimpleFileTarget(*auditSettings.FileName, "error", true)
+		if err != nil {
+			return nil, err
+		}
+
+		// apply audit specific levels
+		targetCfg.Levels = []mlog.Level{mlog.LvlAuditAPI, mlog.LvlAuditContent, mlog.LvlAuditPerms, mlog.LvlAuditCLI}
+
+		// apply audit specific formatting
+		targetCfg.FormatOptions = json.RawMessage(`{"disable_timestamp": true, "disable_msg": true, "disable_stacktrace": true, "disable_level": true}`)
+
+		cfg["_defAudit"] = targetCfg
+	}
+
+	if configSrc == nil {
+		return cfg, nil
+	}
+
+	// add advanced audit config
+	cfgAdv := configSrc.Get()
+	cfg.Append(cfgAdv)
+
+	return cfg, nil
+}
+
+func GetLogFileLocation(fileLocation string) string {
+	if fileLocation == "" {
+		fileLocation, _ = fileutils.FindDir("logs")
+	}
+
+	return filepath.Join(fileLocation, LogFilename)
+}
+
+func GetNotificationsLogFileLocation(fileLocation string) string {
+	if fileLocation == "" {
+		fileLocation, _ = fileutils.FindDir("logs")
+	}
+
+	return filepath.Join(fileLocation, LogNotificationFilename)
+}
+
+func GetLogSettingsFromNotificationsLogSettings(notificationLogSettings *model.NotificationLogSettings) *model.LogSettings {
+	settings := &model.LogSettings{}
+	settings.SetDefaults()
+	settings.ConsoleJson = notificationLogSettings.ConsoleJson
+	settings.ConsoleLevel = notificationLogSettings.ConsoleLevel
+	settings.EnableConsole = notificationLogSettings.EnableConsole
+	settings.EnableFile = notificationLogSettings.EnableFile
+	settings.FileJson = notificationLogSettings.FileJson
+	settings.FileLevel = notificationLogSettings.FileLevel
+	settings.FileLocation = notificationLogSettings.FileLocation
+	settings.AdvancedLoggingConfig = notificationLogSettings.AdvancedLoggingConfig
+	settings.EnableColor = notificationLogSettings.EnableColor
+	return settings
+}
+
+func makeSimpleConsoleTarget(level string, outputJSON bool, color bool) (mlog.TargetCfg, error) {
+	levels, err := stdLevels(level)
+	if err != nil {
+		return mlog.TargetCfg{}, err
+	}
+
+	target := mlog.TargetCfg{
+		Type:         "console",
+		Levels:       levels,
+		Options:      json.RawMessage(`{"out": "stdout"}`),
+		MaxQueueSize: 1000,
+	}
+
+	if outputJSON {
+		target.Format = "json"
+		target.FormatOptions = makeJSONFormatOptions()
+	} else {
+		target.Format = "plain"
+		target.FormatOptions = makePlainFormatOptions(color)
+	}
+	return target, nil
+}
+
+func makeSimpleFileTarget(filename string, level string, json bool) (mlog.TargetCfg, error) {
+	levels, err := stdLevels(level)
+	if err != nil {
+		return mlog.TargetCfg{}, err
+	}
+
+	target := mlog.TargetCfg{
+		Type:         "file",
+		Levels:       levels,
+		Options:      makeFileOptions(filename),
+		MaxQueueSize: 1000,
+	}
+
+	if json {
+		target.Format = "json"
+		target.FormatOptions = makeJSONFormatOptions()
+	} else {
+		target.Format = "plain"
+		target.FormatOptions = makePlainFormatOptions(false)
+	}
+	return target, nil
+}
+
+func stdLevels(level string) ([]mlog.Level, error) {
+	stdLevel, err := stringToStdLevel(level)
+	if err != nil {
+		return nil, err
+	}
+
+	var levels []mlog.Level
+	for _, l := range mlog.StdAll {
+		if l.ID <= stdLevel.ID {
+			levels = append(levels, l)
+		}
+	}
+	return levels, nil
+}
+
+func stringToStdLevel(level string) (mlog.Level, error) {
+	level = strings.ToLower(level)
+	for _, l := range mlog.StdAll {
+		if l.Name == level {
+			return l, nil
+		}
+	}
+	return mlog.Level{}, fmt.Errorf("%s is not a standard level", level)
+}
+
+func makeJSONFormatOptions() json.RawMessage {
+	str := fmt.Sprintf(`{"enable_caller": %t}`, LogEnableCaller)
+	return json.RawMessage(str)
+}
+
+func makePlainFormatOptions(enableColor bool) json.RawMessage {
+	str := fmt.Sprintf(`{"delim": "%s", "min_level_len": %d, "min_msg_len": %d, "enable_color": %t, "enable_caller": %t}`,
+		LogDelim, LogMinLevelLen, LogMinMsgLen, enableColor, LogEnableCaller)
+	return json.RawMessage(str)
+}
+
+func makeFileOptions(filename string) json.RawMessage {
+	str := fmt.Sprintf(`{"filename": "%s", "max_size": %d, "max_age": %d, "max_backups": %d, "compress": %t}`,
+		filename, LogRotateSize, LogRotateMaxAge, LogRotateMaxBackups, LogCompress)
+	return json.RawMessage(str)
+}

config/main_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v6/model"
-	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 	"github.com/mattermost/mattermost-server/v6/testlib"
 )
 
@@ -23,8 +22,6 @@ func TestMain(m *testing.M) {
 		EnableStore: true,
 	}
 
-	mlog.DisableZap()
-
 	mainHelper = testlib.NewMainHelperWithOptions(&options)
 	defer mainHelper.Close()