MM-22784 Advanced logging config for audit (#15076)

Adds the advanced logging config for audit. Existing support for auditing to a single file remains for E0 and E10 licenses instances, and a new config item ExperimentalAuditSettings.AdvancedLoggingConfig is added that behaves like LogSettings.AdvancedLoggingConfig.

Supported destinations:

- file
- syslog (with out without TLS)
- raw TCP socket (with out without TLS)

ExperimentalAuditSettings.AdvancedLoggingConfig can contain a filespec to a config file, a database DSN, or JSON.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Claudio Costa <cstcld91@gmail.com>

app/audit.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"os/user"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/config"
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/store"
@@ -23,10 +25,10 @@ const (
 )
 
 var (
-	RestLevel        = audit.Level{ID: RestLevelID, Name: "audit-rest", Stacktrace: false}
-	RestContentLevel = audit.Level{ID: RestContentLevelID, Name: "audit-rest-content", Stacktrace: false}
-	RestPermsLevel   = audit.Level{ID: RestPermsLevelID, Name: "audit-rest-perms", Stacktrace: false}
-	CLILevel         = audit.Level{ID: CLILevelID, Name: "audit-cli", Stacktrace: false}
+	LevelAPI     = mlog.LvlAuditAPI
+	LevelContent = mlog.LvlAuditContent
+	LevelPerms   = mlog.LvlAuditPerms
+	LevelCLI     = mlog.LvlAuditCLI
 )
 
 func (a *App) GetAudits(userId string, limit int) (model.Audits, *model.AppError) {
@@ -57,13 +59,13 @@ func (a *App) GetAuditsPage(userId string, page int, perPage int) (model.Audits,
 	return audits, nil
 }
 
-// LogAuditRec logs an audit record using default CLILevel.
+// LogAuditRec logs an audit record using default LvlAuditCLI.
 func (a *App) LogAuditRec(rec *audit.Record, err error) {
-	a.LogAuditRecWithLevel(rec, CLILevel, err)
+	a.LogAuditRecWithLevel(rec, mlog.LvlAuditCLI, err)
 }
 
 // LogAuditRecWithLevel logs an audit record using specified Level.
-func (a *App) LogAuditRecWithLevel(rec *audit.Record, level audit.Level, err error) {
+func (a *App) LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error) {
 	if rec == nil {
 		return
 	}
@@ -102,46 +104,13 @@ func (a *App) MakeAuditRecord(event string, initialStatus string) *audit.Record
 	return rec
 }
 
-func (s *Server) configureAudit(adt *audit.Audit) {
+func (s *Server) configureAudit(adt *audit.Audit, bAllowAdvancedLogging bool) error {
+	var errs error
+
 	adt.OnQueueFull = s.onAuditTargetQueueFull
 	adt.OnError = s.onAuditError
 
-	// Configure target for SysLog via TLS.
-	// See https://www.rsyslog.com/doc/v8-stable/tutorials/tls_cert_summary.html
-	if *s.Config().ExperimentalAuditSettings.SysLogEnabled {
-		IP := *s.Config().ExperimentalAuditSettings.SysLogIP
-		if IP == "" {
-			IP = "localhost"
-		}
-		port := *s.Config().ExperimentalAuditSettings.SysLogPort
-		if port <= 0 {
-			port = 6514
-		}
-		maxQSize := *s.Config().ExperimentalAuditSettings.SysLogMaxQueueSize
-		if maxQSize <= 0 {
-			maxQSize = audit.DefMaxQueueSize
-		}
-
-		params := &mlog.SyslogParams{
-			IP:       IP,
-			Port:     port,
-			Cert:     *s.Config().ExperimentalAuditSettings.SysLogCert,
-			Tag:      *s.Config().ExperimentalAuditSettings.SysLogTag,
-			Insecure: *s.Config().ExperimentalAuditSettings.SysLogInsecure,
-		}
-
-		filter := adt.MakeFilter(RestLevel, RestContentLevel, RestPermsLevel, CLILevel)
-		formatter := adt.MakeJSONFormatter()
-		target, err := mlog.NewSyslogTarget(filter, formatter, params, maxQSize)
-		if err != nil {
-			mlog.Error("cannot configure SysLogTLS audit target", mlog.Err(err))
-		} else {
-			mlog.Debug("SysLogTLS audit target connected successfully", mlog.String("IP", IP), mlog.Int("Port", port))
-			adt.AddTarget(target)
-		}
-	}
-
-	// Configure target for rotating file output
+	// Configure target for rotating file output (E0, E10)
 	if *s.Config().ExperimentalAuditSettings.FileEnabled {
 		opts := audit.FileOptions{
 			Filename:   *s.Config().ExperimentalAuditSettings.FileName,
@@ -156,21 +125,50 @@ func (s *Server) configureAudit(adt *audit.Audit) {
 			maxQueueSize = audit.DefMaxQueueSize
 		}
 
-		filter := adt.MakeFilter(RestLevel, RestContentLevel, RestPermsLevel, CLILevel)
+		filter := adt.MakeFilter(LevelAPI, LevelContent, LevelPerms, LevelCLI)
 		formatter := adt.MakeJSONFormatter()
 		formatter.DisableTimestamp = false
 		target, err := audit.NewFileTarget(filter, formatter, opts, maxQueueSize)
 		if err != nil {
-			mlog.Error("cannot configure File audit target", mlog.Err(err))
+			errs = multierror.Append(err)
 		} else {
 			mlog.Debug("File audit target created successfully", mlog.String("filename", opts.Filename))
 			adt.AddTarget(target)
 		}
 	}
+
+	// Advanced logging for audit requires license.
+	dsn := *s.Config().ExperimentalAuditSettings.AdvancedLoggingConfig
+	if !bAllowAdvancedLogging || dsn == "" {
+		return errs
+	}
+	isJson := config.IsJsonMap(dsn)
+	cfg, err := config.NewLogConfigSrc(dsn, isJson, s.configStore)
+	if err != nil {
+		errs = multierror.Append(fmt.Errorf("invalid config for audit, %w", err))
+		return errs
+	}
+	if !isJson {
+		mlog.Debug("Loaded audit configuration", mlog.String("filename", dsn))
+	}
+
+	for name, t := range cfg.Get() {
+		if len(t.Levels) == 0 {
+			t.Levels = mlog.MLvlAuditAll
+		}
+		target, err := mlog.NewLogrTarget(name, t)
+		if err != nil {
+			errs = multierror.Append(err)
+			continue
+		}
+		adt.AddTarget(target)
+	}
+	return errs
 }
 
-func (s *Server) onAuditTargetQueueFull(qname string, maxQSize int) {
-	mlog.Warn("Audit Queue Full", mlog.String("qname", qname), mlog.Int("maxQSize", maxQSize))
+func (s *Server) onAuditTargetQueueFull(qname string, maxQSize int) bool {
+	mlog.Error("Audit queue full, dropping record.", mlog.String("qname", qname), mlog.Int("queueSize", maxQSize))
+	return true // drop it
 }
 
 func (s *Server) onAuditError(err error) {