MM-23222 mirror audit logs to file (#14062)

* MM-23222 add file target (with rotation) to audit

* MM-23222 mirror syslog audits to local filesystem

    * provides config options for file name, max size, max age

    * rotates files based on max size and max age; delete as needed based on max backups

* include cluster id in log records

* sort meta data fields

audit/audit.go

@@ -5,6 +5,7 @@ package audit
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/wiggin77/logr"
 	"github.com/wiggin77/logr/format"
@@ -46,8 +47,10 @@ func (a *Audit) MakeFilter(level ...Level) *logr.CustomFilter {
 func (a *Audit) MakeJSONFormatter() *format.JSON {
 	f := &format.JSON{
 		DisableTimestamp:  true,
+		DisableMsg:        true,
 		DisableStacktrace: true,
 		DisableLevel:      true,
+		ContextSorter:     sortAuditFields,
 	}
 	return f
 }
@@ -117,3 +120,56 @@ func (a *Audit) onLoggerError(err error) {
 		a.OnError(err)
 	}
 }
+
+// sortAuditFields sorts the context fields of an audit record such that some fields
+// are prepended in order, some are appended in order, and the rest are sorted alphabetically.
+// This is done to make reading the records easier since common fields will appear in the same order.
+func sortAuditFields(fields logr.Fields) []format.ContextField {
+	prependKeys := []string{KeyEvent, KeyStatus, KeyUserID, KeySessionID, KeyIPAddress}
+	appendKeys := []string{KeyClusterID, KeyClient}
+
+	// sort alphabetically any fields not in the prepend/append lists.
+	keys := make([]string, 0, len(fields))
+	for k := range fields {
+		if !findIn(k, prependKeys, appendKeys) {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+
+	allKeys := make([]string, 0, len(fields))
+
+	// add any prepends that exist in fields
+	for _, k := range prependKeys {
+		if _, ok := fields[k]; ok {
+			allKeys = append(allKeys, k)
+		}
+	}
+
+	// sorted
+	allKeys = append(allKeys, keys...)
+
+	// add any appends that exist in fields
+	for _, k := range appendKeys {
+		if _, ok := fields[k]; ok {
+			allKeys = append(allKeys, k)
+		}
+	}
+
+	cfs := make([]format.ContextField, 0, len(allKeys))
+	for _, k := range allKeys {
+		cfs = append(cfs, format.ContextField{Key: k, Val: fields[k]})
+	}
+	return cfs
+}
+
+func findIn(s string, arrs ...[]string) bool {
+	for _, list := range arrs {
+		for _, key := range list {
+			if s == key {
+				return true
+			}
+		}
+	}
+	return false
+}

audit/audit_test.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/logr"
+	"github.com/wiggin77/logr/format"
+)
+
+func Test_sortAuditFields(t *testing.T) {
+	type args struct {
+		fields logr.Fields
+	}
+	tests := []struct {
+		name string
+		args args
+		want []format.ContextField
+	}{
+		{name: "empty list",
+			args: args{fields: logr.Fields{}},
+			want: []format.ContextField{},
+		},
+		{name: "partial list",
+			args: args{fields: logr.Fields{"zProp": "x", "xProp": "x", "yProp": "x", KeyClusterID: "x", KeyEvent: "x"}},
+			want: []format.ContextField{
+				{Key: KeyEvent, Val: "x"},
+				{Key: "xProp", Val: "x"},
+				{Key: "yProp", Val: "x"},
+				{Key: "zProp", Val: "x"},
+				{Key: KeyClusterID, Val: "x"},
+			},
+		},
+		{name: "append/prepend only list",
+			args: args{fields: logr.Fields{KeyClusterID: "x", KeyEvent: "x", KeySessionID: "x", KeyIPAddress: "x", KeyClient: "x",
+				KeyUserID: "x", KeyStatus: "x"}},
+			want: []format.ContextField{
+				// prepend: KeyEvent, KeyStatus, KeyUserID, KeySessionID, KeyIPAddress
+				// append: KeyClusterID, KeyClient
+				{Key: KeyEvent, Val: "x"},
+				{Key: KeyStatus, Val: "x"},
+				{Key: KeyUserID, Val: "x"},
+				{Key: KeySessionID, Val: "x"},
+				{Key: KeyIPAddress, Val: "x"},
+				{Key: KeyClusterID, Val: "x"},
+				{Key: KeyClient, Val: "x"},
+			},
+		},
+		{name: "sortables only list",
+			args: args{fields: logr.Fields{"zProp": "x", "xProp": "x", "yProp": "x"}},
+			want: []format.ContextField{
+				{Key: "xProp", Val: "x"},
+				{Key: "yProp", Val: "x"},
+				{Key: "zProp", Val: "x"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := sortAuditFields(tt.args.fields)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

audit/const.go

@@ -13,6 +13,7 @@ const (
 	KeySessionID = "session_id"
 	KeyClient    = "client"
 	KeyIPAddress = "ip_address"
+	KeyClusterID = "cluster_id"
 
 	Success = "success"
 	Attempt = "attempt"

audit/file.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package audit
+
+import (
+	"os"
+
+	"github.com/wiggin77/logr"
+	"github.com/wiggin77/logr/target"
+)
+
+type FileOptions target.FileOptions
+
+// NewFileTarget creates a target capable of outputting log records to a rotated file.
+func NewFileTarget(filter logr.Filter, formatter logr.Formatter, opts FileOptions, maxQSize int) (*target.File, error) {
+	fopts := target.FileOptions(opts)
+	err := checkFileWritable(fopts.Filename)
+	if err != nil {
+		return nil, err
+	}
+	target := target.NewFileTarget(filter, formatter, fopts, maxQSize)
+	return target, nil
+}
+
+func checkFileWritable(filename string) error {
+	// try opening/creating the file for writing
+	file, err := os.OpenFile(filename, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	file.Close()
+	return nil
+}