MM-22784 Advanced logging config for audit (#15076)

Adds the advanced logging config for audit. Existing support for auditing to a single file remains for E0 and E10 licenses instances, and a new config item ExperimentalAuditSettings.AdvancedLoggingConfig is added that behaves like LogSettings.AdvancedLoggingConfig. Supported destinations: - file - syslog (with out without TLS) - raw TCP socket (with out without TLS) ExperimentalAuditSettings.AdvancedLoggingConfig can contain a filespec to a config file, a database DSN, or JSON. Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Claudio Costa <cstcld91@gmail.com>
2020-07-22 18:48:46 -04:00
--- a/app/app_iface.go
+++ b/app/app_iface.go
@@ -214,10 +214,10 @@ type AppIface interface {
 	IsUsernameTaken(name string) bool
 	// LimitedClientConfigWithComputed gets the configuration in a format suitable for sending to the client.
 	LimitedClientConfigWithComputed() map[string]string
-	// LogAuditRec logs an audit record using default CLILevel.
+	// LogAuditRec logs an audit record using default LvlAuditCLI.
 	LogAuditRec(rec *audit.Record, err error)
 	// LogAuditRecWithLevel logs an audit record using specified Level.
-	LogAuditRecWithLevel(rec *audit.Record, level audit.Level, err error)
+	LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error)
 	// MakeAuditRecord creates a audit record pre-populated with defaults.
 	MakeAuditRecord(event string, initialStatus string) *audit.Record
 	// MarkChanelAsUnreadFromPost will take a post and set the channel as unread from that one.
--- a/app/audit.go
+++ b/app/audit.go
@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"os/user"

+	"github.com/hashicorp/go-multierror"
 	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/config"
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/store"
@@ -23,10 +25,10 @@ const (
 )

 var (
-	RestLevel        = audit.Level{ID: RestLevelID, Name: "audit-rest", Stacktrace: false}
-	RestContentLevel = audit.Level{ID: RestContentLevelID, Name: "audit-rest-content", Stacktrace: false}
-	RestPermsLevel   = audit.Level{ID: RestPermsLevelID, Name: "audit-rest-perms", Stacktrace: false}
-	CLILevel         = audit.Level{ID: CLILevelID, Name: "audit-cli", Stacktrace: false}
+	LevelAPI     = mlog.LvlAuditAPI
+	LevelContent = mlog.LvlAuditContent
+	LevelPerms   = mlog.LvlAuditPerms
+	LevelCLI     = mlog.LvlAuditCLI
 )

 func (a *App) GetAudits(userId string, limit int) (model.Audits, *model.AppError) {
@@ -57,13 +59,13 @@ func (a *App) GetAuditsPage(userId string, page int, perPage int) (model.Audits,
 	return audits, nil
 }

-// LogAuditRec logs an audit record using default CLILevel.
+// LogAuditRec logs an audit record using default LvlAuditCLI.
 func (a *App) LogAuditRec(rec *audit.Record, err error) {
-	a.LogAuditRecWithLevel(rec, CLILevel, err)
+	a.LogAuditRecWithLevel(rec, mlog.LvlAuditCLI, err)
 }

 // LogAuditRecWithLevel logs an audit record using specified Level.
-func (a *App) LogAuditRecWithLevel(rec *audit.Record, level audit.Level, err error) {
+func (a *App) LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error) {
 	if rec == nil {
 		return
 	}
@@ -102,46 +104,13 @@ func (a *App) MakeAuditRecord(event string, initialStatus string) *audit.Record
 	return rec
 }

-func (s *Server) configureAudit(adt *audit.Audit) {
+func (s *Server) configureAudit(adt *audit.Audit, bAllowAdvancedLogging bool) error {
+	var errs error
+
 	adt.OnQueueFull = s.onAuditTargetQueueFull
 	adt.OnError = s.onAuditError

-	// Configure target for SysLog via TLS.
-	// See https://www.rsyslog.com/doc/v8-stable/tutorials/tls_cert_summary.html
-	if *s.Config().ExperimentalAuditSettings.SysLogEnabled {
-		IP := *s.Config().ExperimentalAuditSettings.SysLogIP
-		if IP == "" {
-			IP = "localhost"
-		}
-		port := *s.Config().ExperimentalAuditSettings.SysLogPort
-		if port <= 0 {
-			port = 6514
-		}
-		maxQSize := *s.Config().ExperimentalAuditSettings.SysLogMaxQueueSize
-		if maxQSize <= 0 {
-			maxQSize = audit.DefMaxQueueSize
-		}
-
-		params := &mlog.SyslogParams{
-			IP:       IP,
-			Port:     port,
-			Cert:     *s.Config().ExperimentalAuditSettings.SysLogCert,
-			Tag:      *s.Config().ExperimentalAuditSettings.SysLogTag,
-			Insecure: *s.Config().ExperimentalAuditSettings.SysLogInsecure,
-		}
-
-		filter := adt.MakeFilter(RestLevel, RestContentLevel, RestPermsLevel, CLILevel)
-		formatter := adt.MakeJSONFormatter()
-		target, err := mlog.NewSyslogTarget(filter, formatter, params, maxQSize)
-		if err != nil {
-			mlog.Error("cannot configure SysLogTLS audit target", mlog.Err(err))
-		} else {
-			mlog.Debug("SysLogTLS audit target connected successfully", mlog.String("IP", IP), mlog.Int("Port", port))
-			adt.AddTarget(target)
-		}
-	}
-
-	// Configure target for rotating file output
+	// Configure target for rotating file output (E0, E10)
 	if *s.Config().ExperimentalAuditSettings.FileEnabled {
 		opts := audit.FileOptions{
 			Filename:   *s.Config().ExperimentalAuditSettings.FileName,
@@ -156,21 +125,50 @@ func (s *Server) configureAudit(adt *audit.Audit) {
 			maxQueueSize = audit.DefMaxQueueSize
 		}

-		filter := adt.MakeFilter(RestLevel, RestContentLevel, RestPermsLevel, CLILevel)
+		filter := adt.MakeFilter(LevelAPI, LevelContent, LevelPerms, LevelCLI)
 		formatter := adt.MakeJSONFormatter()
 		formatter.DisableTimestamp = false
 		target, err := audit.NewFileTarget(filter, formatter, opts, maxQueueSize)
 		if err != nil {
-			mlog.Error("cannot configure File audit target", mlog.Err(err))
+			errs = multierror.Append(err)
 		} else {
 			mlog.Debug("File audit target created successfully", mlog.String("filename", opts.Filename))
 			adt.AddTarget(target)
 		}
 	}
+
+	// Advanced logging for audit requires license.
+	dsn := *s.Config().ExperimentalAuditSettings.AdvancedLoggingConfig
+	if !bAllowAdvancedLogging || dsn == "" {
+		return errs
+	}
+	isJson := config.IsJsonMap(dsn)
+	cfg, err := config.NewLogConfigSrc(dsn, isJson, s.configStore)
+	if err != nil {
+		errs = multierror.Append(fmt.Errorf("invalid config for audit, %w", err))
+		return errs
+	}
+	if !isJson {
+		mlog.Debug("Loaded audit configuration", mlog.String("filename", dsn))
+	}
+
+	for name, t := range cfg.Get() {
+		if len(t.Levels) == 0 {
+			t.Levels = mlog.MLvlAuditAll
+		}
+		target, err := mlog.NewLogrTarget(name, t)
+		if err != nil {
+			errs = multierror.Append(err)
+			continue
+		}
+		adt.AddTarget(target)
+	}
+	return errs
 }

-func (s *Server) onAuditTargetQueueFull(qname string, maxQSize int) {
-	mlog.Warn("Audit Queue Full", mlog.String("qname", qname), mlog.Int("maxQSize", maxQSize))
+func (s *Server) onAuditTargetQueueFull(qname string, maxQSize int) bool {
+	mlog.Error("Audit queue full, dropping record.", mlog.String("qname", qname), mlog.Int("queueSize", maxQSize))
+	return true // drop it
 }

 func (s *Server) onAuditError(err error) {
--- a/app/diagnostics.go
+++ b/app/diagnostics.go
@@ -405,15 +405,13 @@ func (s *Server) trackConfig() {
 	})

 	s.SendDiagnostic(TRACK_CONFIG_AUDIT, map[string]interface{}{
-		"syslog_enabled":        *cfg.ExperimentalAuditSettings.SysLogEnabled,
-		"syslog_insecure":       *cfg.ExperimentalAuditSettings.SysLogInsecure,
-		"syslog_max_queue_size": *cfg.ExperimentalAuditSettings.SysLogMaxQueueSize,
-		"file_enabled":          *cfg.ExperimentalAuditSettings.FileEnabled,
-		"file_max_size_mb":      *cfg.ExperimentalAuditSettings.FileMaxSizeMB,
-		"file_max_age_days":     *cfg.ExperimentalAuditSettings.FileMaxAgeDays,
-		"file_max_backups":      *cfg.ExperimentalAuditSettings.FileMaxBackups,
-		"file_compress":         *cfg.ExperimentalAuditSettings.FileCompress,
-		"file_max_queue_size":   *cfg.ExperimentalAuditSettings.FileMaxQueueSize,
+		"file_enabled":            *cfg.ExperimentalAuditSettings.FileEnabled,
+		"file_max_size_mb":        *cfg.ExperimentalAuditSettings.FileMaxSizeMB,
+		"file_max_age_days":       *cfg.ExperimentalAuditSettings.FileMaxAgeDays,
+		"file_max_backups":        *cfg.ExperimentalAuditSettings.FileMaxBackups,
+		"file_compress":           *cfg.ExperimentalAuditSettings.FileCompress,
+		"file_max_queue_size":     *cfg.ExperimentalAuditSettings.FileMaxQueueSize,
+		"advanced_logging_config": *cfg.ExperimentalAuditSettings.AdvancedLoggingConfig != "",
 	})

 	s.SendDiagnostic(TRACK_CONFIG_NOTIFICATION_LOG, map[string]interface{}{
@@ -424,6 +422,7 @@ func (s *Server) trackConfig() {
 		"file_level":              *cfg.NotificationLogSettings.FileLevel,
 		"file_json":               *cfg.NotificationLogSettings.FileJson,
 		"isdefault_file_location": isDefault(*cfg.NotificationLogSettings.FileLocation, ""),
+		"advanced_logging_config": *cfg.NotificationLogSettings.AdvancedLoggingConfig != "",
 	})

 	s.SendDiagnostic(TRACK_CONFIG_PASSWORD, map[string]interface{}{
--- a/app/opentracing/opentracing_layer.go
+++ b/app/opentracing/opentracing_layer.go
@@ -9945,7 +9945,7 @@ func (a *OpenTracingAppLayer) LogAuditRec(rec *audit.Record, err error) {
 	a.app.LogAuditRec(rec, err)
 }

-func (a *OpenTracingAppLayer) LogAuditRecWithLevel(rec *audit.Record, level audit.Level, err error) {
+func (a *OpenTracingAppLayer) LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.LogAuditRecWithLevel")

--- a/app/server.go
+++ b/app/server.go
@@ -459,13 +459,17 @@ func NewServer(options ...Option) (*Server, error) {

 	s.ReloadConfig()

+	allowAdvancedLogging := license != nil && *license.Features.AdvancedLogging
+
 	if s.Audit == nil {
 		s.Audit = &audit.Audit{}
 		s.Audit.Init(audit.DefMaxQueueSize)
-		s.configureAudit(s.Audit)
+		if err := s.configureAudit(s.Audit, allowAdvancedLogging); err != nil {
+			mlog.Error("Error configuring audit", mlog.Err(err))
+		}
 	}

-	if license == nil || !*license.Features.AdvancedLogging {
+	if !allowAdvancedLogging {
 		timeoutCtx, cancelCtx := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancelCtx()
 		mlog.Info("Shutting down advanced logging")