MM-25731 Advanced Logging (#14888)

Adds Advanced Logging to server. Advanced Logging is an optional logging capability that allows customers to send log records to any number of destinations.

Supported destinations:
- file
- syslog (with out without TLS)
- raw TCP socket (with out without TLS)

Allows developers to specify discrete log levels as well as the standard trace, debug, info, ... panic. Existing code and logging API usage is unchanged.

Log records are emitted asynchronously to reduce latency to the caller. Supports hot-reloading of logger config, including adding removing targets.

Advanced Logging is configured within config.json via "LogSettings.AdvancedLoggingConfig" which can contain a filespec to another config file, a database DSN, or JSON.

vendor/github.com/mattermost/logr/target/file.go

@@ -0,0 +1,92 @@
+package target
+
+import (
+	"context"
+	"io"
+
+	"github.com/mattermost/logr"
+	"github.com/wiggin77/merror"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+type FileOptions struct {
+	// Filename is the file to write logs to.  Backup log files will be retained
+	// in the same directory.  It uses <processname>-lumberjack.log in
+	// os.TempDir() if empty.
+	Filename string
+
+	// MaxSize is the maximum size in megabytes of the log file before it gets
+	// rotated. It defaults to 100 megabytes.
+	MaxSize int
+
+	// MaxAge is the maximum number of days to retain old log files based on the
+	// timestamp encoded in their filename.  Note that a day is defined as 24
+	// hours and may not exactly correspond to calendar days due to daylight
+	// savings, leap seconds, etc. The default is not to remove old log files
+	// based on age.
+	MaxAge int
+
+	// MaxBackups is the maximum number of old log files to retain.  The default
+	// is to retain all old log files (though MaxAge may still cause them to get
+	// deleted.)
+	MaxBackups int
+
+	// Compress determines if the rotated log files should be compressed
+	// using gzip. The default is not to perform compression.
+	Compress bool
+}
+
+// File outputs log records to a file which can be log rotated based on size or age.
+// Uses `https://github.com/natefinch/lumberjack` for rotation.
+type File struct {
+	logr.Basic
+	out io.WriteCloser
+}
+
+// NewFileTarget creates a target capable of outputting log records to a rotated file.
+func NewFileTarget(filter logr.Filter, formatter logr.Formatter, opts FileOptions, maxQueue int) *File {
+	lumber := &lumberjack.Logger{
+		Filename:   opts.Filename,
+		MaxSize:    opts.MaxSize,
+		MaxBackups: opts.MaxBackups,
+		MaxAge:     opts.MaxAge,
+		Compress:   opts.Compress,
+	}
+	f := &File{out: lumber}
+	f.Basic.Start(f, f, filter, formatter, maxQueue)
+	return f
+}
+
+// Write converts the log record to bytes, via the Formatter,
+// and outputs to a file.
+func (f *File) Write(rec *logr.LogRec) error {
+	_, stacktrace := f.IsLevelEnabled(rec.Level())
+
+	buf := rec.Logger().Logr().BorrowBuffer()
+	defer rec.Logger().Logr().ReleaseBuffer(buf)
+
+	buf, err := f.Formatter().Format(rec, stacktrace, buf)
+	if err != nil {
+		return err
+	}
+	_, err = f.out.Write(buf.Bytes())
+	return err
+}
+
+// Shutdown flushes any remaining log records and closes the file.
+func (f *File) Shutdown(ctx context.Context) error {
+	errs := merror.New()
+
+	err := f.Basic.Shutdown(ctx)
+	errs.Append(err)
+
+	err = f.out.Close()
+	errs.Append(err)
+
+	return errs.ErrorOrNil()
+}
+
+// String returns a string representation of this target.
+func (f *File) String() string {
+	return "FileTarget"
+}

vendor/github.com/mattermost/logr/target/syslog.go

@@ -0,0 +1,94 @@
+// +build !windows,!nacl,!plan9
+
+package target
+
+import (
+	"context"
+	"fmt"
+	"log/syslog"
+
+	"github.com/mattermost/logr"
+	"github.com/wiggin77/merror"
+)
+
+// Syslog outputs log records to local or remote syslog.
+type Syslog struct {
+	logr.Basic
+	w *syslog.Writer
+}
+
+// SyslogParams provides parameters for dialing a syslog daemon.
+type SyslogParams struct {
+	Network  string
+	Raddr    string
+	Priority syslog.Priority
+	Tag      string
+}
+
+// NewSyslogTarget creates a target capable of outputting log records to remote or local syslog.
+func NewSyslogTarget(filter logr.Filter, formatter logr.Formatter, params *SyslogParams, maxQueue int) (*Syslog, error) {
+	writer, err := syslog.Dial(params.Network, params.Raddr, params.Priority, params.Tag)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Syslog{w: writer}
+	s.Basic.Start(s, s, filter, formatter, maxQueue)
+
+	return s, nil
+}
+
+// Shutdown stops processing log records after making best
+// effort to flush queue.
+func (s *Syslog) Shutdown(ctx context.Context) error {
+	errs := merror.New()
+
+	err := s.Basic.Shutdown(ctx)
+	errs.Append(err)
+
+	err = s.w.Close()
+	errs.Append(err)
+
+	return errs.ErrorOrNil()
+}
+
+// Write converts the log record to bytes, via the Formatter,
+// and outputs to syslog.
+func (s *Syslog) Write(rec *logr.LogRec) error {
+	_, stacktrace := s.IsLevelEnabled(rec.Level())
+
+	buf := rec.Logger().Logr().BorrowBuffer()
+	defer rec.Logger().Logr().ReleaseBuffer(buf)
+
+	buf, err := s.Formatter().Format(rec, stacktrace, buf)
+	if err != nil {
+		return err
+	}
+	txt := buf.String()
+
+	switch rec.Level() {
+	case logr.Panic, logr.Fatal:
+		err = s.w.Crit(txt)
+	case logr.Error:
+		err = s.w.Err(txt)
+	case logr.Warn:
+		err = s.w.Warning(txt)
+	case logr.Debug, logr.Trace:
+		err = s.w.Debug(txt)
+	default:
+		// logr.Info plus all custom levels.
+		err = s.w.Info(txt)
+	}
+
+	if err != nil {
+		reporter := rec.Logger().Logr().ReportError
+		reporter(fmt.Errorf("syslog write fail: %w", err))
+		// syslog writer will try to reconnect.
+	}
+	return err
+}
+
+// String returns a string representation of this target.
+func (s *Syslog) String() string {
+	return "SyslogTarget"
+}

vendor/github.com/mattermost/logr/target/writer.go

@@ -0,0 +1,45 @@
+package target
+
+import (
+	"io"
+	"io/ioutil"
+
+	"github.com/mattermost/logr"
+)
+
+// Writer outputs log records to any `io.Writer`.
+type Writer struct {
+	logr.Basic
+	out io.Writer
+}
+
+// NewWriterTarget creates a target capable of outputting log records to an io.Writer.
+func NewWriterTarget(filter logr.Filter, formatter logr.Formatter, out io.Writer, maxQueue int) *Writer {
+	if out == nil {
+		out = ioutil.Discard
+	}
+	w := &Writer{out: out}
+	w.Basic.Start(w, w, filter, formatter, maxQueue)
+	return w
+}
+
+// Write converts the log record to bytes, via the Formatter,
+// and outputs to the io.Writer.
+func (w *Writer) Write(rec *logr.LogRec) error {
+	_, stacktrace := w.IsLevelEnabled(rec.Level())
+
+	buf := rec.Logger().Logr().BorrowBuffer()
+	defer rec.Logger().Logr().ReleaseBuffer(buf)
+
+	buf, err := w.Formatter().Format(rec, stacktrace, buf)
+	if err != nil {
+		return err
+	}
+	_, err = w.out.Write(buf.Bytes())
+	return err
+}
+
+// String returns a string representation of this target.
+func (w *Writer) String() string {
+	return "WriterTarget"
+}