MM-27744 disable Zap for unit tests. (#15398)

MM-27744 disable Zap for unit tests.

Zap has no concept of shutdown or close. Zap is only shutdown when the app exits. Not a problem for console logging, but when creating a new Zap logger that outputs to files on every unit test, that leaves no easy way to clean up until process exit. Depending on what else is running this can exhaust all file handles and cause unit tests to fail.

Zap is now disabled unit tests and uses Logr instead, regardless of config settings. `make test-server` peak file handle usage dropped from ~5K to less than 100.

mlog/log.go

@@ -5,9 +5,12 @@ package mlog
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"log"
 	"os"
+	"sync/atomic"
+	"time"
 
 	"github.com/mattermost/logr"
 	"go.uber.org/zap"
@@ -24,6 +27,19 @@ const (
 	LevelWarn = "warn"
 	// Errors are messages about things we know are problems
 	LevelError = "error"
+
+	// DefaultFlushTimeout is the default amount of time mlog.Flush will wait
+	// before timing out.
+	DefaultFlushTimeout = time.Second * 5
+)
+
+var (
+	// disableZap is set when Zap should be disabled and Logr used instead.
+	// This is needed for unit testing as Zap has no shutdown capabilities
+	// and holds file handles until process exit. Currently unit test create
+	// many server instances, and thus many Zap log files.
+	// This flag will be removed when Zap is permanently replaced.
+	disableZap int32
 )
 
 // Type and function aliases from zap to limit the libraries scope into MM code
@@ -40,6 +56,8 @@ var NamedErr = zap.NamedError
 var Bool = zap.Bool
 var Duration = zap.Duration
 
+type TargetInfo logr.TargetInfo
+
 type LoggerConfiguration struct {
 	EnableConsole bool
 	ConsoleJson   bool
@@ -97,14 +115,33 @@ func NewLogger(config *LoggerConfiguration) *Logger {
 	}
 
 	if config.EnableFile {
-		writer := zapcore.AddSync(&lumberjack.Logger{
-			Filename: config.FileLocation,
-			MaxSize:  100,
-			Compress: true,
-		})
+		if atomic.LoadInt32(&disableZap) != 0 {
+			t := &LogTarget{
+				Type:         "file",
+				Format:       "json",
+				Levels:       mlogLevelToLogrLevels(config.FileLevel),
+				MaxQueueSize: DefaultMaxTargetQueue,
+				Options: []byte(fmt.Sprintf(`{"Filename":"%s", "MaxSizeMB":%d, "Compress":%t}`,
+					config.FileLocation, 100, true)),
+			}
+			if !config.FileJson {
+				t.Format = "plain"
+			}
+			if tgt, err := NewLogrTarget("mlogFile", t); err == nil {
+				logger.logrLogger.Logr().AddTarget(tgt)
+			} else {
+				Error("error creating mlogFile", Err(err))
+			}
+		} else {
+			writer := zapcore.AddSync(&lumberjack.Logger{
+				Filename: config.FileLocation,
+				MaxSize:  100,
+				Compress: true,
+			})
 
-		core := zapcore.NewCore(makeEncoder(config.FileJson), writer, logger.fileLevel)
-		cores = append(cores, core)
+			core := zapcore.NewCore(makeEncoder(config.FileJson), writer, logger.fileLevel)
+			cores = append(cores, core)
+		}
 	}
 
 	combinedCore := zapcore.NewTee(cores...)
@@ -221,14 +258,14 @@ func (l *Logger) LogM(levels []LogLevel, message string, fields ...Field) {
 }
 
 func (l *Logger) Flush(cxt context.Context) error {
-	return l.logrLogger.Logr().Flush() // TODO: use context when Logr lib supports it.
+	return l.logrLogger.Logr().FlushWithTimeout(cxt)
 }
 
 // ShutdownAdvancedLogging stops the logger from accepting new log records and tries to
 // flush queues within the context timeout. Once complete all targets are shutdown
 // and any resources released.
 func (l *Logger) ShutdownAdvancedLogging(cxt context.Context) error {
-	err := l.logrLogger.Logr().Shutdown() // TODO: use context when Logr lib supports it.
+	err := l.logrLogger.Logr().ShutdownWithTimeout(cxt)
 	l.logrLogger = newLogr()
 	return err
 }
@@ -245,11 +282,22 @@ func (l *Logger) ConfigAdvancedLogging(targets LogTargetCfg) error {
 	return err
 }
 
-// AddTarget adds a logr.Target to the advanced logger. This is the preferred method
+// AddTarget adds one or more logr.Target to the advanced logger. This is the preferred method
 // to add custom targets or provide configuration that cannot be expressed via a
-//config source.
-func (l *Logger) AddTarget(target logr.Target) error {
-	return l.logrLogger.Logr().AddTarget(target)
+// config source.
+func (l *Logger) AddTarget(targets ...logr.Target) error {
+	return l.logrLogger.Logr().AddTarget(targets...)
+}
+
+// RemoveTargets selectively removes targets that were previously added to this logger instance
+// using the passed in filter function. The filter function should return true to remove the target
+// and false to keep it.
+func (l *Logger) RemoveTargets(ctx context.Context, f func(ti TargetInfo) bool) error {
+	// Use locally defined TargetInfo type so we don't spread Logr dependencies.
+	fc := func(tic logr.TargetInfo) bool {
+		return f(TargetInfo(tic))
+	}
+	return l.logrLogger.Logr().RemoveTargets(ctx, fc)
 }
 
 // EnableMetrics enables metrics collection by supplying a MetricsCollector.
@@ -257,3 +305,21 @@ func (l *Logger) AddTarget(target logr.Target) error {
 func (l *Logger) EnableMetrics(collector logr.MetricsCollector) error {
 	return l.logrLogger.Logr().SetMetricsCollector(collector)
 }
+
+// DisableZap is called to disable Zap, and Logr will be used instead. Any Logger
+// instances created after this call will only use Logr.
+//
+// This is needed for unit testing as Zap has no shutdown capabilities
+// and holds file handles until process exit. Currently unit tests create
+// many server instances, and thus many Zap log file handles.
+//
+// This method will be removed when Zap is permanently replaced.
+func DisableZap() {
+	atomic.StoreInt32(&disableZap, 1)
+}
+
+// EnableZap re-enables Zap such that any Logger instances created after this
+// call will allow Zap targets.
+func EnableZap() {
+	atomic.StoreInt32(&disableZap, 0)
+}