MM-22273 New auditing system (phase 1) (#13967)

* New auditing API outputting to syslog via TLS

* New config section for specifying remote syslog server IP, port, and cert.

* Legacy audit API retained for access history feature

audit/audit.go

@@ -0,0 +1,119 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+import (
+	"fmt"
+
+	"github.com/wiggin77/logr"
+	"github.com/wiggin77/logr/format"
+)
+
+type Level logr.Level
+
+type Audit struct {
+	lgr    *logr.Logr
+	logger logr.Logger
+
+	// OnQueueFull is called on an attempt to add an audit record to a full queue.
+	// On return the calling goroutine will block until the audit record can be added.
+	OnQueueFull func(qname string, maxQueueSize int)
+
+	// OnError is called when an error occurs while writing an audit record.
+	OnError func(err error)
+}
+
+func (a *Audit) Init(maxQueueSize int) {
+	a.lgr = &logr.Logr{MaxQueueSize: maxQueueSize}
+	a.logger = a.lgr.NewLogger()
+
+	a.lgr.OnQueueFull = a.onQueueFull
+	a.lgr.OnTargetQueueFull = a.onTargetQueueFull
+	a.lgr.OnLoggerError = a.onLoggerError
+}
+
+// MakeFilter creates a filter which only allows the specified audit levels to be output.
+func (a *Audit) MakeFilter(level ...Level) *logr.CustomFilter {
+	filter := &logr.CustomFilter{}
+	for _, l := range level {
+		filter.Add(logr.Level(l))
+	}
+	return filter
+}
+
+// MakeJSONFormatter creates a formatter that outputs JSON suitable for audit records.
+func (a *Audit) MakeJSONFormatter() *format.JSON {
+	f := &format.JSON{
+		DisableTimestamp:  true,
+		DisableStacktrace: true,
+		DisableLevel:      true,
+	}
+	return f
+}
+
+// LogRecord emits an audit record with complete info.
+func (a *Audit) LogRecord(level Level, rec Record) {
+	flds := logr.Fields{}
+	flds[KeyAPIPath] = rec.APIPath
+	flds[KeyEvent] = rec.Event
+	flds[KeyStatus] = rec.Status
+	flds[KeyUserID] = rec.UserID
+	flds[KeySessionID] = rec.SessionID
+	flds[KeyClient] = rec.Client
+	flds[KeyIPAddress] = rec.IPAddress
+
+	for k, v := range rec.Meta {
+		flds[k] = v
+	}
+
+	l := a.logger.WithFields(flds)
+	l.Log(logr.Level(level))
+}
+
+// Log emits an audit record based on minimum required info.
+func (a *Audit) Log(level Level, path string, evt string, status string, userID string, sessionID string, meta Meta) {
+	a.LogRecord(level, Record{
+		APIPath:   path,
+		Event:     evt,
+		Status:    status,
+		UserID:    userID,
+		SessionID: sessionID,
+		Meta:      meta,
+	})
+}
+
+// AddTarget adds a Logr target to the list of targets each audit record will be output to.
+func (a *Audit) AddTarget(target logr.Target) {
+	a.lgr.AddTarget(target)
+}
+
+// Shutdown cleanly stops the audit engine after making best efforts to flush all targets.
+func (a *Audit) Shutdown() {
+	err := a.lgr.Shutdown()
+	if err != nil {
+		a.onLoggerError(err)
+	}
+}
+
+func (a *Audit) onQueueFull(rec *logr.LogRec, maxQueueSize int) bool {
+	if a.OnQueueFull != nil {
+		a.OnQueueFull("main", maxQueueSize)
+	}
+	// block until record can be added.
+	return false
+}
+
+func (a *Audit) onTargetQueueFull(target logr.Target, rec *logr.LogRec, maxQueueSize int) bool {
+	if a.OnQueueFull != nil {
+		a.OnQueueFull(fmt.Sprintf("%v", target), maxQueueSize)
+	}
+	// block until record can be added.
+	return false
+}
+
+func (a *Audit) onLoggerError(err error) {
+	if a.OnError != nil {
+		a.OnError(err)
+	}
+}

audit/const.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+const (
+	DefMaxQueueSize = 1000
+
+	KeyAPIPath   = "api_path"
+	KeyEvent     = "event"
+	KeyStatus    = "status"
+	KeyUserID    = "user_id"
+	KeySessionID = "session_id"
+	KeyClient    = "client"
+	KeyIPAddress = "ip_address"
+
+	Success = "success"
+	Attempt = "attempt"
+	Fail    = "fail"
+)

audit/record.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+// Meta represents metadata that can be added to a audit record as name/value pairs.
+type Meta map[string]interface{}
+
+// Record provides a consistent set of fields used for all audit logging.
+type Record struct {
+	APIPath   string
+	Event     string
+	Status    string
+	UserID    string
+	SessionID string
+	Client    string
+	IPAddress string
+	Meta      Meta
+}
+
+// Success marks the audit record status as successful.
+func (rec *Record) Success() {
+	rec.Status = Success
+}
+
+// Success marks the audit record status as failed.
+func (rec *Record) Fail() {
+	rec.Status = Fail
+}
+
+// AddMeta adds a single name/value pair to this audit record's metadata.
+func (rec *Record) AddMeta(name string, val interface{}) {
+	if rec.Meta == nil {
+		rec.Meta = Meta{}
+	}
+	rec.Meta[name] = val
+}

audit/syslogtls.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+
+	syslog "github.com/RackSec/srslog"
+	"github.com/wiggin77/logr"
+	"github.com/wiggin77/merror"
+)
+
+// Syslog outputs log records to local or remote syslog.
+type SyslogTLS struct {
+	logr.Basic
+	w *syslog.Writer
+}
+
+// SyslogParams provides parameters for dialing a syslogTLS daemon.
+type SyslogParams struct {
+	Raddr    string
+	Cert     string
+	Tag      string
+	Insecure bool
+}
+
+// NewSyslogTLSTarget creates a target capable of outputting log records to remote or local syslog via TLS.
+func NewSyslogTLSTarget(filter logr.Filter, formatter logr.Formatter, params *SyslogParams, maxQueue int) (*SyslogTLS, error) {
+	config := tls.Config{InsecureSkipVerify: params.Insecure}
+
+	if params.Cert != "" {
+		serverCert, err := ioutil.ReadFile(params.Cert)
+		if err != nil {
+			return nil, err
+		}
+		pool := x509.NewCertPool()
+		pool.AppendCertsFromPEM(serverCert)
+		config.RootCAs = pool
+	}
+
+	writer, err := syslog.DialWithTLSConfig("tcp+tls", params.Raddr, syslog.LOG_INFO, params.Tag, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &SyslogTLS{w: writer}
+	s.Basic.Start(s, s, filter, formatter, maxQueue)
+
+	return s, nil
+}
+
+// Shutdown stops processing log records after making best
+// effort to flush queue.
+func (s *SyslogTLS) Shutdown(ctx context.Context) error {
+	errs := merror.New()
+
+	err := s.Basic.Shutdown(ctx)
+	errs.Append(err)
+
+	err = s.w.Close()
+	errs.Append(err)
+
+	return errs.ErrorOrNil()
+}
+
+// Write converts the log record to bytes, via the Formatter,
+// and outputs to syslog via TLS.
+func (s *SyslogTLS) Write(rec *logr.LogRec) error {
+	_, stacktrace := s.IsLevelEnabled(rec.Level())
+
+	buf := rec.Logger().Logr().BorrowBuffer()
+	defer rec.Logger().Logr().ReleaseBuffer(buf)
+
+	buf, err := s.Formatter().Format(rec, stacktrace, buf)
+	if err != nil {
+		return err
+	}
+	txt := buf.String()
+
+	switch rec.Level() {
+	case logr.Panic, logr.Fatal:
+		err = s.w.Crit(txt)
+	case logr.Error:
+		err = s.w.Err(txt)
+	case logr.Warn:
+		err = s.w.Warning(txt)
+	case logr.Debug, logr.Trace:
+		err = s.w.Debug(txt)
+	default:
+		// logr.Info plus all custom levels.
+		err = s.w.Info(txt)
+	}
+
+	if err != nil {
+		reporter := rec.Logger().Logr().ReportError
+		reporter(fmt.Errorf("syslog write fail: %w", err))
+		// syslogTLS writer will try to reconnect.
+	}
+	return err
+}
+
+// String returns a string representation of this target.
+func (s *SyslogTLS) String() string {
+	return "SyslogTLSTarget"
+}