MM-22786 enterprise metrics for logging (#15313)

Adds metrics for logging engine which are collected by Prometheus.
- current queue level(s)
- rate of logging records emitted
- rate of logging errors

mlog/default.go

@@ -81,3 +81,9 @@ func defaultAddTarget(target logr.Target) error {
 	// logger is replaced with mlog.Logger instance.
 	return errors.New("cannot AddTarget on default logger")
 }
+
+func defaultEnableMetrics(collector logr.MetricsCollector) error {
+	// mlog.EnableMetrics should not be called until default
+	// logger is replaced with mlog.Logger instance.
+	return errors.New("cannot EnableMetrics on default logger")
+}

mlog/global.go

@@ -32,6 +32,7 @@ func InitGlobalLogger(logger *Logger) {
 	ConfigAdvancedLogging = globalLogger.ConfigAdvancedLogging
 	ShutdownAdvancedLogging = globalLogger.ShutdownAdvancedLogging
 	AddTarget = globalLogger.AddTarget
+	EnableMetrics = globalLogger.EnableMetrics
 }
 
 func RedirectStdLog(logger *Logger) {
@@ -45,6 +46,7 @@ type FlushFunc func(context.Context) error
 type ConfigFunc func(cfg LogTargetCfg) error
 type ShutdownFunc func(context.Context) error
 type AddTargetFunc func(logr.Target) error
+type EnableMetricsFunc func(logr.MetricsCollector) error
 
 // DON'T USE THIS Modify the level on the app logger
 func GloballyDisableDebugLogForTest() {
@@ -68,3 +70,4 @@ var Flush FlushFunc = defaultFlush
 var ConfigAdvancedLogging ConfigFunc = defaultAdvancedConfig
 var ShutdownAdvancedLogging ShutdownFunc = defaultAdvancedShutdown
 var AddTarget AddTargetFunc = defaultAddTarget
+var EnableMetrics EnableMetricsFunc = defaultEnableMetrics

mlog/log.go

@@ -87,6 +87,7 @@ func NewLogger(config *LoggerConfiguration) *Logger {
 	logger := &Logger{
 		consoleLevel: zap.NewAtomicLevelAt(getZapLevel(config.ConsoleLevel)),
 		fileLevel:    zap.NewAtomicLevelAt(getZapLevel(config.FileLevel)),
+		logrLogger:   newLogr(),
 	}
 
 	if config.EnableConsole {
@@ -101,6 +102,7 @@ func NewLogger(config *LoggerConfiguration) *Logger {
 			MaxSize:  100,
 			Compress: true,
 		})
+
 		core := zapcore.NewCore(makeEncoder(config.FileJson), writer, logger.fileLevel)
 		cores = append(cores, core)
 	}
@@ -167,77 +169,67 @@ func (l *Logger) Sugar() *SugarLogger {
 
 func (l *Logger) Debug(message string, fields ...Field) {
 	l.zap.Debug(message, fields...)
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Debug) {
+	if isLevelEnabled(l.logrLogger, logr.Debug) {
 		l.logrLogger.WithFields(zapToLogr(fields)).Debug(message)
 	}
 }
 
 func (l *Logger) Info(message string, fields ...Field) {
 	l.zap.Info(message, fields...)
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Info) {
+	if isLevelEnabled(l.logrLogger, logr.Info) {
 		l.logrLogger.WithFields(zapToLogr(fields)).Info(message)
 	}
 }
 
 func (l *Logger) Warn(message string, fields ...Field) {
 	l.zap.Warn(message, fields...)
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Warn) {
+	if isLevelEnabled(l.logrLogger, logr.Warn) {
 		l.logrLogger.WithFields(zapToLogr(fields)).Warn(message)
 	}
 }
 
 func (l *Logger) Error(message string, fields ...Field) {
 	l.zap.Error(message, fields...)
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Error) {
+	if isLevelEnabled(l.logrLogger, logr.Error) {
 		l.logrLogger.WithFields(zapToLogr(fields)).Error(message)
 	}
 }
 
 func (l *Logger) Critical(message string, fields ...Field) {
 	l.zap.Error(message, fields...)
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Error) {
+	if isLevelEnabled(l.logrLogger, logr.Error) {
 		l.logrLogger.WithFields(zapToLogr(fields)).Error(message)
 	}
 }
 
 func (l *Logger) Log(level LogLevel, message string, fields ...Field) {
-	if l.logrLogger != nil && isLevelEnabled(l.logrLogger, logr.Level(level)) {
-		l.logrLogger.WithFields(zapToLogr(fields)).Log(logr.Level(level), message)
-	}
+	l.logrLogger.WithFields(zapToLogr(fields)).Log(logr.Level(level), message)
 }
 
 func (l *Logger) LogM(levels []LogLevel, message string, fields ...Field) {
-	if l.logrLogger != nil {
-		var logger *logr.Logger
-		for _, lvl := range levels {
-			if isLevelEnabled(l.logrLogger, logr.Level(lvl)) {
-				// don't create logger with fields unless at least one level is active.
-				if logger == nil {
-					l := l.logrLogger.WithFields(zapToLogr(fields))
-					logger = &l
-				}
-				logger.Log(logr.Level(lvl), message)
+	var logger *logr.Logger
+	for _, lvl := range levels {
+		if isLevelEnabled(l.logrLogger, logr.Level(lvl)) {
+			// don't create logger with fields unless at least one level is active.
+			if logger == nil {
+				l := l.logrLogger.WithFields(zapToLogr(fields))
+				logger = &l
 			}
+			logger.Log(logr.Level(lvl), message)
 		}
 	}
 }
 
 func (l *Logger) Flush(cxt context.Context) error {
-	if l.logrLogger != nil {
-		return l.logrLogger.Logr().Flush() // TODO: use context when Logr lib supports it.
-	}
-	return nil
+	return l.logrLogger.Logr().Flush() // TODO: use context when Logr lib supports it.
 }
 
 // ShutdownAdvancedLogging stops the logger from accepting new log records and tries to
 // flush queues within the context timeout. Once complete all targets are shutdown
 // and any resources released.
 func (l *Logger) ShutdownAdvancedLogging(cxt context.Context) error {
-	var err error
-	if l.logrLogger != nil {
-		err = l.logrLogger.Logr().Shutdown() // TODO: use context when Logr lib supports it.
-		l.logrLogger = nil
-	}
+	err := l.logrLogger.Logr().Shutdown() // TODO: use context when Logr lib supports it.
+	l.logrLogger = newLogr()
 	return err
 }
 
@@ -245,14 +237,11 @@ func (l *Logger) ShutdownAdvancedLogging(cxt context.Context) error {
 // specified log targets. This is the easiest way to get the advanced logger
 // configured via a config source such as file.
 func (l *Logger) ConfigAdvancedLogging(targets LogTargetCfg) error {
-	if l.logrLogger != nil {
-		if err := l.ShutdownAdvancedLogging(context.Background()); err != nil {
-			Error("error shutting down previous logger", Err(err))
-		}
+	if err := l.ShutdownAdvancedLogging(context.Background()); err != nil {
+		Error("error shutting down previous logger", Err(err))
 	}
 
-	logr, err := newLogr(targets)
-	l.logrLogger = logr
+	err := logrAddTargets(l.logrLogger, targets)
 	return err
 }
 
@@ -262,3 +251,9 @@ func (l *Logger) ConfigAdvancedLogging(targets LogTargetCfg) error {
 func (l *Logger) AddTarget(target logr.Target) error {
 	return l.logrLogger.Logr().AddTarget(target)
 }
+
+// EnableMetrics enables metrics collection by supplying a MetricsCollector.
+// The MetricsCollector provides counters and gauges that are updated by log targets.
+func (l *Logger) EnableMetrics(collector logr.MetricsCollector) error {
+	return l.logrLogger.Logr().SetMetricsCollector(collector)
+}

mlog/logr.go

@@ -28,7 +28,7 @@ type LogLevel struct {
 }
 
 type LogTarget struct {
-	Type         string // one of "console", "file", "tcp", "syslog".
+	Type         string // one of "console", "file", "tcp", "syslog", "none".
 	Format       string // one of "json", "plain"
 	Levels       []LogLevel
 	Options      json.RawMessage
@@ -38,7 +38,7 @@ type LogTarget struct {
 type LogTargetCfg map[string]*LogTarget
 type LogrCleanup func() error
 
-func newLogr(targets LogTargetCfg) (*logr.Logger, error) {
+func newLogr() *logr.Logger {
 	lgr := &logr.Logr{}
 	lgr.OnExit = func(int) {}
 	lgr.OnPanic = func(interface{}) {}
@@ -46,12 +46,12 @@ func newLogr(targets LogTargetCfg) (*logr.Logger, error) {
 	lgr.OnQueueFull = onQueueFull
 	lgr.OnTargetQueueFull = onTargetQueueFull
 
-	err := logrAddTargets(lgr, targets)
 	logger := lgr.NewLogger()
-	return &logger, err
+	return &logger
 }
 
-func logrAddTargets(lgr *logr.Logr, targets LogTargetCfg) error {
+func logrAddTargets(logger *logr.Logger, targets LogTargetCfg) error {
+	lgr := logger.Logr()
 	var errs error
 	for name, t := range targets {
 		target, err := NewLogrTarget(name, t)
@@ -59,7 +59,9 @@ func logrAddTargets(lgr *logr.Logr, targets LogTargetCfg) error {
 			errs = multierror.Append(err)
 			continue
 		}
-		lgr.AddTarget(target)
+		if target != nil {
+			lgr.AddTarget(target)
+		}
 	}
 	return errs
 }
@@ -90,6 +92,8 @@ func NewLogrTarget(name string, t *LogTarget) (logr.Target, error) {
 		return newSyslogTarget(name, t, filter, formatter)
 	case "tcp":
 		return newTCPTarget(name, t, filter, formatter)
+	case "none":
+		return nil, nil
 	}
 	return nil, fmt.Errorf("invalid type '%s' for target %s", t.Type, name)
 }
@@ -201,6 +205,10 @@ func checkFileWritable(filename string) error {
 }
 
 func isLevelEnabled(logger *logr.Logger, level logr.Level) bool {
+	if logger == nil || logger.Logr() == nil {
+		return false
+	}
+
 	status := logger.Logr().IsLevelEnabled(level)
 	return status.Enabled
 }

mlog/tcp_test.go

@@ -38,15 +38,16 @@ func TestNewTcpTarget(t *testing.T) {
 
 		data := []string{"I drink your milkshake!", "We don't need no badges!", "You can't fight in here! This is the war room!"}
 
-		logr, err := newLogr(targets)
+		logger := newLogr()
+		err = logrAddTargets(logger, targets)
 		require.NoError(t, err)
 
 		for _, s := range data {
-			logr.Info(s)
+			logger.Info(s)
 		}
-		err = logr.Logr().Flush()
+		err = logger.Logr().Flush()
 		require.NoError(t, err)
-		err = logr.Logr().Shutdown()
+		err = logger.Logr().Shutdown()
 		require.NoError(t, err)
 
 		err = server.waitForAnyConnection()

mlog/testing.go

@@ -32,6 +32,7 @@ func NewTestingLogger(tb testing.TB, writer io.Writer) *Logger {
 	testingLogger := &Logger{
 		consoleLevel: zap.NewAtomicLevelAt(getZapLevel("debug")),
 		fileLevel:    zap.NewAtomicLevelAt(getZapLevel("info")),
+		logrLogger:   newLogr(),
 	}
 
 	logWriterCore := zapcore.NewCore(makeEncoder(true), logWriterSync, testingLogger.consoleLevel)