mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-36764 mlog refactor (#18118)

Просмотр исходного кода 
Refactor mlog
- simplify mlog by removing redundant code
- remove Zap dependency
- update unit test helpers
- update logging config
- update auditing
Этот коммит содержится в:
Doug Lauder
2021-08-17 16:08:04 -04:00
коммит произвёл GitHub
родитель 04b27ce93c
Коммит a4507327a7
216 изменённых файлов: 4940 добавлений и 14674 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
app/admin.go
Просмотреть файл

@@ -4,7 +4,6 @@
package app
import (
"context"
"fmt"
"io"
"io/ioutil"
@@ -13,11 +12,11 @@ import (
"runtime/debug"
"time"
"github.com/mattermost/mattermost-server/v6/config"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/i18n"
"github.com/mattermost/mattermost-server/v6/shared/mail"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/utils"
)
func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) {
@@ -63,11 +62,8 @@ func (s *Server) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError)
var lines []string
if *s.Config().LogSettings.EnableFile {
timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), mlog.DefaultFlushTimeout)
defer timeoutCancel()
mlog.Flush(timeoutCtx)
logFile := utils.GetLogFileLocation(*s.Config().LogSettings.FileLocation)
s.Log.Flush()
logFile := config.GetLogFileLocation(*s.Config().LogSettings.FileLocation)
file, err := os.Open(logFile)
if err != nil {
return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, err.Error(), http.StatusInternalServerError)

2
app/app_iface.go
Просмотреть файл

@@ -232,7 +232,7 @@ type AppIface interface {
// LogAuditRec logs an audit record using default LvlAuditCLI.
LogAuditRec(rec *audit.Record, err error)
// LogAuditRecWithLevel logs an audit record using specified Level.
LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error)
LogAuditRecWithLevel(rec *audit.Record, level mlog.Level, err error)
// MakeAuditRecord creates a audit record pre-populated with defaults.
MakeAuditRecord(event string, initialStatus string) *audit.Record
// MarkChanelAsUnreadFromPost will take a post and set the channel as unread from that one.

69
app/audit.go
Просмотреть файл

@@ -9,8 +9,6 @@ import (
"net/http"
"os/user"
"github.com/hashicorp/go-multierror"
"github.com/mattermost/mattermost-server/v6/audit"
"github.com/mattermost/mattermost-server/v6/config"
"github.com/mattermost/mattermost-server/v6/model"
@@ -66,7 +64,7 @@ func (a *App) LogAuditRec(rec *audit.Record, err error) {
}
// LogAuditRecWithLevel logs an audit record using specified Level.
func (a *App) LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error) {
func (a *App) LogAuditRecWithLevel(rec *audit.Record, level mlog.Level, err error) {
if rec == nil {
return
}
@@ -106,64 +104,27 @@ func (a *App) MakeAuditRecord(event string, initialStatus string) *audit.Record
}
func (s *Server) configureAudit(adt *audit.Audit, bAllowAdvancedLogging bool) error {
var errs error
adt.OnQueueFull = s.onAuditTargetQueueFull
adt.OnError = s.onAuditError
// Configure target for rotating file output (E0, E10)
if *s.Config().ExperimentalAuditSettings.FileEnabled {
opts := audit.FileOptions{
Filename: *s.Config().ExperimentalAuditSettings.FileName,
MaxSize: *s.Config().ExperimentalAuditSettings.FileMaxSizeMB,
MaxAge: *s.Config().ExperimentalAuditSettings.FileMaxAgeDays,
MaxBackups: *s.Config().ExperimentalAuditSettings.FileMaxBackups,
Compress: *s.Config().ExperimentalAuditSettings.FileCompress,
}
maxQueueSize := *s.Config().ExperimentalAuditSettings.FileMaxQueueSize
if maxQueueSize <= 0 {
maxQueueSize = audit.DefMaxQueueSize
}
filter := adt.MakeFilter(LevelAPI, LevelContent, LevelPerms, LevelCLI)
formatter := adt.MakeJSONFormatter()
formatter.DisableTimestamp = false
target, err := audit.NewFileTarget(filter, formatter, opts, maxQueueSize)
if err != nil {
errs = multierror.Append(err)
} else {
mlog.Debug("File audit target created successfully", mlog.String("filename", opts.Filename))
adt.AddTarget(target)
}
}
// Advanced logging for audit requires license.
var logConfigSrc config.LogConfigSrc
dsn := *s.Config().ExperimentalAuditSettings.AdvancedLoggingConfig
if !bAllowAdvancedLogging || dsn == "" {
return errs
}
cfg, err := config.NewLogConfigSrc(dsn, s.configStore)
if err != nil {
errs = multierror.Append(fmt.Errorf("invalid config for audit, %w", err))
return errs
}
mlog.Debug("Loaded audit configuration", mlog.String("source", dsn))
for name, t := range cfg.Get() {
if len(t.Levels) == 0 {
t.Levels = mlog.MLvlAuditAll
}
target, err := mlog.NewLogrTarget(name, t)
if bAllowAdvancedLogging && dsn != "" {
var err error
logConfigSrc, err = config.NewLogConfigSrc(dsn, s.configStore)
if err != nil {
errs = multierror.Append(err)
continue
}
if target != nil {
adt.AddTarget(target)
return fmt.Errorf("invalid config source for audit, %w", err)
}
mlog.Debug("Loaded audit configuration", mlog.String("source", dsn))
}
return errs
// ExperimentalAuditSettings provides basic file audit (E0, E10); logConfigSrc provides advanced config (E20).
cfg, err := config.MloggerConfigFromAuditConfig(s.Config().ExperimentalAuditSettings, logConfigSrc)
if err != nil {
return fmt.Errorf("invalid config for audit, %w", err)
}
return adt.Configure(cfg)
}
func (s *Server) onAuditTargetQueueFull(qname string, maxQSize int) bool {

5
app/email/helper_test.go
Просмотреть файл

@@ -19,7 +19,6 @@ import (
"github.com/mattermost/mattermost-server/v6/store"
"github.com/mattermost/mattermost-server/v6/store/storetest/mocks"
"github.com/mattermost/mattermost-server/v6/testlib"
"github.com/mattermost/mattermost-server/v6/utils"
)
type TestHelper struct {
@@ -172,12 +171,10 @@ func (th *TestHelper) CreateTeam() *model.Team {
Type: model.TeamOpen,
}
utils.DisableDebugLogForTest()
var err error
if team, err = th.store.Team().Save(team); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return team
}
@@ -192,13 +189,11 @@ func (th *TestHelper) createChannel(team *model.Team, channelType string) *model
CreatorId: th.BasicUser.Id,
}
utils.DisableDebugLogForTest()
var err error
if channel, err = th.store.Channel().Save(channel, *th.configStore.Get().TeamSettings.MaxChannelsPerTeam); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}

3
app/email/main_test.go
Просмотреть файл

@@ -7,7 +7,6 @@ import (
"flag"
"testing"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/testlib"
)
@@ -26,8 +25,6 @@ func TestMain(m *testing.M) {
WithReadReplica: replicaFlag,
}
mlog.DisableZap()
mainHelper = testlib.NewMainHelperWithOptions(&options)
defer mainHelper.Close()

4
app/file_bench_test.go
Просмотреть файл

@@ -16,7 +16,6 @@ import (
"time"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
var randomJPEG []byte
@@ -59,8 +58,7 @@ func BenchmarkUploadFile(b *testing.B) {
prepareTestImages(b)
th := Setup(b).InitBasic()
defer th.TearDown()
// disable logging in the benchmark, as best we can
th.App.Log().SetConsoleLevel(mlog.LevelError)
teamID := model.NewId()
channelID := model.NewId()
userID := model.NewId()

95
app/helper_test.go
Просмотреть файл

@@ -4,7 +4,6 @@
package app
import (
"bytes"
"context"
"io/ioutil"
"os"
@@ -27,7 +26,6 @@ import (
"github.com/mattermost/mattermost-server/v6/store/sqlstore"
"github.com/mattermost/mattermost-server/v6/store/storetest/mocks"
"github.com/mattermost/mattermost-server/v6/testlib"
"github.com/mattermost/mattermost-server/v6/utils"
)
type TestHelper struct {
@@ -41,7 +39,8 @@ type TestHelper struct {
BasicPost *model.Post
SystemAdminUser *model.User
LogBuffer *bytes.Buffer
LogBuffer *mlog.Buffer
TestLogger *mlog.Logger
IncludeCacheLayer bool
tempWorkspace string
@@ -55,16 +54,16 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
configStore := config.NewTestMemoryStore()
config := configStore.Get()
*config.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
*config.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
*config.PluginSettings.AutomaticPrepackagedPlugins = false
*config.LogSettings.EnableSentry = false // disable error reporting during tests
*config.AnnouncementSettings.AdminNoticesEnabled = false
*config.AnnouncementSettings.UserNoticesEnabled = false
configStore.Set(config)
memoryConfig := configStore.Get()
*memoryConfig.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
*memoryConfig.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
*memoryConfig.PluginSettings.AutomaticPrepackagedPlugins = false
*memoryConfig.LogSettings.EnableSentry = false // disable error reporting during tests
*memoryConfig.AnnouncementSettings.AdminNoticesEnabled = false
*memoryConfig.AnnouncementSettings.UserNoticesEnabled = false
configStore.Set(memoryConfig)
buffer := &bytes.Buffer{}
buffer := &mlog.Buffer{}
var options []Option
options = append(options, ConfigStore(configStore))
@@ -80,7 +79,18 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
} else {
options = append(options, StoreOverride(dbStore))
}
options = append(options, SetLogger(mlog.NewTestingLogger(tb, buffer)))
testLogger, _ := mlog.NewLogger()
logCfg, _ := config.MloggerConfigFromLoggerConfig(&memoryConfig.LogSettings, nil, config.GetLogFileLocation)
if errCfg := testLogger.ConfigureTargets(logCfg); errCfg != nil {
panic("failed to configure test logger: " + errCfg.Error())
}
if errW := mlog.AddWriterTarget(testLogger, buffer, true, mlog.StdAll...); errW != nil {
panic("failed to add writer target to test logger: " + errW.Error())
}
// lock logger config so server init cannot override it during testing.
testLogger.LockConfiguration()
options = append(options, SetLogger(testLogger))
s, err := NewServer(options...)
if err != nil {
@@ -92,6 +102,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
Context: &request.Context{},
Server: s,
LogBuffer: buffer,
TestLogger: testLogger,
IncludeCacheLayer: includeCacheLayer,
}
@@ -240,12 +251,10 @@ func (th *TestHelper) CreateTeam() *model.Team {
Type: model.TeamOpen,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if team, err = th.App.CreateTeam(th.Context, team); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return team
}
@@ -268,7 +277,6 @@ func (th *TestHelper) CreateUserOrGuest(guest bool) *model.User {
EmailVerified: true,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if guest {
if user, err = th.App.CreateGuest(th.Context, user); err != nil {
@@ -279,7 +287,6 @@ func (th *TestHelper) CreateUserOrGuest(guest bool) *model.User {
panic(err)
}
}
utils.EnableDebugLogForTest()
return user
}
@@ -331,7 +338,6 @@ func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelT
option(channel)
}
utils.DisableDebugLogForTest()
var appErr *model.AppError
if channel, appErr = th.App.CreateChannel(th.Context, channel, true); appErr != nil {
panic(appErr)
@@ -353,29 +359,24 @@ func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelT
panic(err)
}
}
utils.EnableDebugLogForTest()
return channel
}
func (th *TestHelper) CreateDmChannel(user *model.User) *model.Channel {
utils.DisableDebugLogForTest()
var err *model.AppError
var channel *model.Channel
if channel, err = th.App.GetOrCreateDirectChannel(th.Context, th.BasicUser.Id, user.Id); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
func (th *TestHelper) CreateGroupChannel(user1 *model.User, user2 *model.User) *model.Channel {
utils.DisableDebugLogForTest()
var err *model.AppError
var channel *model.Channel
if channel, err = th.App.CreateGroupChannel([]string{th.BasicUser.Id, user1.Id, user2.Id}, th.BasicUser.Id); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
@@ -389,12 +390,10 @@ func (th *TestHelper) CreatePost(channel *model.Channel) *model.Post {
CreateAt: model.GetMillis() - 10000,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if post, err = th.App.CreatePost(th.Context, post, channel, false, true); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return post
}
@@ -406,47 +405,32 @@ func (th *TestHelper) CreateMessagePost(channel *model.Channel, message string)
CreateAt: model.GetMillis() - 10000,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if post, err = th.App.CreatePost(th.Context, post, channel, false, true); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return post
}
func (th *TestHelper) LinkUserToTeam(user *model.User, team *model.Team) {
utils.DisableDebugLogForTest()
_, err := th.App.JoinUserToTeam(th.Context, team, user, "")
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
}
func (th *TestHelper) RemoveUserFromTeam(user *model.User, team *model.Team) {
utils.DisableDebugLogForTest()
err := th.App.RemoveUserFromTeam(th.Context, team.Id, user.Id, "")
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
}
func (th *TestHelper) AddUserToChannel(user *model.User, channel *model.Channel) *model.ChannelMember {
utils.DisableDebugLogForTest()
member, err := th.App.AddUserToChannel(user, channel, false)
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return member
}
@@ -456,8 +440,6 @@ func (th *TestHelper) CreateRole(roleName string) *model.Role {
}
func (th *TestHelper) CreateScheme() (*model.Scheme, []*model.Role) {
utils.DisableDebugLogForTest()
scheme, err := th.App.CreateScheme(&model.Scheme{
DisplayName: "Test Scheme Display Name",
Name: model.NewId(),
@@ -485,9 +467,6 @@ func (th *TestHelper) CreateScheme() (*model.Scheme, []*model.Role) {
}
roles = append(roles, role)
}
utils.EnableDebugLogForTest()
return scheme, roles
}
@@ -501,18 +480,14 @@ func (th *TestHelper) CreateGroup() *model.Group {
RemoteId: model.NewId(),
}
utils.DisableDebugLogForTest()
var err *model.AppError
if group, err = th.App.CreateGroup(group); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return group
}
func (th *TestHelper) CreateEmoji() *model.Emoji {
utils.DisableDebugLogForTest()
emoji, err := th.App.Srv().Store.Emoji().Save(&model.Emoji{
CreatorId: th.BasicUser.Id,
Name: model.NewRandomString(10),
@@ -520,15 +495,10 @@ func (th *TestHelper) CreateEmoji() *model.Emoji {
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return emoji
}
func (th *TestHelper) AddReactionToPost(post *model.Post, user *model.User, emojiName string) *model.Reaction {
utils.DisableDebugLogForTest()
reaction, err := th.App.SaveReactionForPost(th.Context, &model.Reaction{
UserId: user.Id,
PostId: post.Id,
@@ -537,9 +507,6 @@ func (th *TestHelper) AddReactionToPost(post *model.Post, user *model.User, emoj
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return reaction
}
@@ -668,11 +635,8 @@ func (th *TestHelper) SetupPluginAPI() *PluginAPI {
}
func (th *TestHelper) RemovePermissionFromRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := th.App.GetRoleByName(context.Background(), roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
@@ -684,7 +648,6 @@ func (th *TestHelper) RemovePermissionFromRole(permission string, roleName strin
}
if strings.Join(role.Permissions, " ") == strings.Join(newPermissions, " ") {
utils.EnableDebugLogForTest()
return
}
@@ -692,25 +655,18 @@ func (th *TestHelper) RemovePermissionFromRole(permission string, roleName strin
_, err2 := th.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}
func (th *TestHelper) AddPermissionToRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := th.App.GetRoleByName(context.Background(), roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
for _, existingPermission := range role.Permissions {
if existingPermission == permission {
utils.EnableDebugLogForTest()
return
}
}
@@ -719,11 +675,8 @@ func (th *TestHelper) AddPermissionToRole(permission string, roleName string) {
_, err2 := th.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}
// This function is copy of storetest/NewTestId

3
app/main_test.go
Просмотреть файл

@@ -7,7 +7,6 @@ import (
"flag"
"testing"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/testlib"
)
@@ -26,8 +25,6 @@ func TestMain(m *testing.M) {
WithReadReplica: replicaFlag,
}
mlog.DisableZap()
mainHelper = testlib.NewMainHelperWithOptions(&options)
defer mainHelper.Close()

2
app/opentracing/opentracing_layer.go
Просмотреть файл

@@ -11251,7 +11251,7 @@ func (a *OpenTracingAppLayer) LogAuditRec(rec *audit.Record, err error) {
a.app.LogAuditRec(rec, err)
}
func (a *OpenTracingAppLayer) LogAuditRecWithLevel(rec *audit.Record, level mlog.LogLevel, err error) {
func (a *OpenTracingAppLayer) LogAuditRecWithLevel(rec *audit.Record, level mlog.Level, err error) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.LogAuditRecWithLevel")

4
app/plugin_api.go
Просмотреть файл

@@ -25,7 +25,7 @@ type PluginAPI struct {
id string
app *App
ctx *request.Context
logger *mlog.SugarLogger
logger mlog.Sugar
manifest *model.Manifest
}
@@ -35,7 +35,7 @@ func NewPluginAPI(a *App, c *request.Context, manifest *model.Manifest) *PluginA
manifest: manifest,
ctx: c,
app: a,
logger: a.Log().With(mlog.String("plugin_id", manifest.Id)).Sugar(),
logger: a.Log().Sugar(mlog.String("plugin_id", manifest.Id)),
}
}

5
app/plugin_test.go
Просмотреть файл

@@ -724,11 +724,14 @@ func TestPluginPanicLogs(t *testing.T) {
}
_, err := th.App.CreatePost(th.Context, post, th.BasicChannel, false, true)
assert.Nil(t, err)
th.TestLogger.Flush()
// We shutdown plugins first so that the read on the log buffer is race-free.
th.App.Srv().ShutDownPlugins()
tearDown()
testlib.AssertLog(t, th.LogBuffer, mlog.LevelDebug, "panic: some text from panic")
testlib.AssertLog(t, th.LogBuffer, mlog.LvlDebug.Name, "panic: some text from panic")
})
}

4
app/post_test.go
Просмотреть файл

@@ -970,7 +970,7 @@ func TestCreatePostAsUser(t *testing.T) {
_, appErr := th.App.CreatePostAsUser(th.Context, post, "", true)
require.Nil(t, appErr)
testlib.AssertLog(t, th.LogBuffer, mlog.LevelWarn, "Failed to get membership")
testlib.AssertLog(t, th.LogBuffer, mlog.LvlWarn.Name, "Failed to get membership")
})
t.Run("does not log warning for bot user not in channel", func(t *testing.T) {
@@ -993,7 +993,7 @@ func TestCreatePostAsUser(t *testing.T) {
_, appErr = th.App.CreatePostAsUser(th.Context, post, "", true)
require.Nil(t, appErr)
testlib.AssertNoLog(t, th.LogBuffer, mlog.LevelWarn, "Failed to get membership")
testlib.AssertNoLog(t, th.LogBuffer, mlog.LvlWarn.Name, "Failed to get membership")
})
t.Run("marks channel as viewed for reply post when CRT is off", func(t *testing.T) {

176
app/server.go
Просмотреть файл

@@ -138,7 +138,6 @@ type Server struct {
statusCache cache.Cache
configListenerId string
licenseListenerId string
logListenerId string
clusterLeaderListenerId string
searchConfigListenerId string
searchLicenseListenerId string
@@ -146,8 +145,6 @@ type Server struct {
configStore *config.Store
postActionCookieSecret []byte
advancedLogListenerCleanup func()
pluginCommands []*PluginCommand
pluginCommandsLock sync.RWMutex
@@ -787,74 +784,79 @@ func (s *Server) DatabaseTypeAndMattermostVersion() (string, string) {
return *s.Config().SqlSettings.DriverName, mattermostVersion.Value
}
// initLogging initializes and configures the logger. This may be called more than once.
// initLogging initializes and configures the logger(s). This may be called more than once.
func (s *Server) initLogging() error {
var err error
// create the app logger if needed
if s.Log == nil {
s.Log = mlog.NewLogger(utils.MloggerConfigFromLoggerConfig(&s.Config().LogSettings, utils.GetLogFileLocation))
}
// Use this app logger as the global logger (eventually remove all instances of global logging).
// This is deferred because a copy is made of the logger and it must be fully configured before
// the copy is made.
defer mlog.InitGlobalLogger(s.Log)
// Redirect default Go logger to this logger.
defer mlog.RedirectStdLog(s.Log)
if s.NotificationsLog == nil {
notificationLogSettings := utils.GetLogSettingsFromNotificationsLogSettings(&s.Config().NotificationLogSettings)
s.NotificationsLog = mlog.NewLogger(utils.MloggerConfigFromLoggerConfig(notificationLogSettings, utils.GetNotificationsLogFileLocation)).
WithCallerSkip(1).With(mlog.String("logSource", "notifications"))
}
if s.logListenerId != "" {
s.RemoveConfigListener(s.logListenerId)
}
s.logListenerId = s.AddConfigListener(func(_, after *model.Config) {
s.Log.ChangeLevels(utils.MloggerConfigFromLoggerConfig(&after.LogSettings, utils.GetLogFileLocation))
notificationLogSettings := utils.GetLogSettingsFromNotificationsLogSettings(&after.NotificationLogSettings)
s.NotificationsLog.ChangeLevels(utils.MloggerConfigFromLoggerConfig(notificationLogSettings, utils.GetNotificationsLogFileLocation))
})
// Configure advanced logging.
// Advanced logging is E20 only, however logging must be initialized before the license
// file is loaded. If no valid E20 license exists then advanced logging will be
// shutdown once license is loaded/checked.
if *s.Config().LogSettings.AdvancedLoggingConfig != "" {
dsn := *s.Config().LogSettings.AdvancedLoggingConfig
cfg, err := config.NewLogConfigSrc(dsn, s.configStore)
s.Log, err = mlog.NewLogger()
if err != nil {
return fmt.Errorf("invalid advanced logging config, %w", err)
return err
}
}
if err := s.Log.ConfigAdvancedLogging(cfg.Get()); err != nil {
return fmt.Errorf("error configuring advanced logging, %w", err)
// create notification logger if needed
if s.NotificationsLog == nil {
l, err := mlog.NewLogger()
if err != nil {
return err
}
s.NotificationsLog = l.With(mlog.String("logSource", "notifications"))
}
mlog.Info("Loaded advanced logging config", mlog.String("source", dsn))
listenerId := cfg.AddListener(func(_, newCfg mlog.LogTargetCfg) {
if err := s.Log.ConfigAdvancedLogging(newCfg); err != nil {
mlog.Error("Error re-configuring advanced logging", mlog.Err(err))
} else {
mlog.Info("Re-configured advanced logging")
}
})
// In case initLogging is called more than once.
if s.advancedLogListenerCleanup != nil {
s.advancedLogListenerCleanup()
if err := s.configureLogger("logging", s.Log, &s.Config().LogSettings, s.configStore, config.GetLogFileLocation); err != nil {
// if the config is locked then a unit test has already configured and locked the logger; not an error.
if !errors.Is(err, mlog.ErrConfigurationLock) {
// revert to default logger if the config is invalid
mlog.InitGlobalLogger(nil)
return err
}
}
s.advancedLogListenerCleanup = func() {
cfg.RemoveListener(listenerId)
// Redirect default Go logger to app logger.
s.Log.RedirectStdLog(mlog.LvlStdLog)
// Use the app logger as the global logger (eventually remove all instances of global logging).
mlog.InitGlobalLogger(s.Log)
notificationLogSettings := config.GetLogSettingsFromNotificationsLogSettings(&s.Config().NotificationLogSettings)
if err := s.configureLogger("notification logging", s.NotificationsLog, notificationLogSettings, s.configStore, config.GetNotificationsLogFileLocation); err != nil {
if !errors.Is(err, mlog.ErrConfigurationLock) {
mlog.Error("Error configuring notification logger", mlog.Err(err))
return err
}
}
return nil
}
// configureLogger applies the specified configuration to a logger.
func (s *Server) configureLogger(name string, logger *mlog.Logger, logSettings *model.LogSettings, configStore *config.Store, getPath func(string) string) error {
// Advanced logging is E20 only, however logging must be initialized before the license
// file is loaded. If no valid E20 license exists then advanced logging will be
// shutdown once license is loaded/checked.
var err error
dsn := *logSettings.AdvancedLoggingConfig
var logConfigSrc config.LogConfigSrc
if dsn != "" {
logConfigSrc, err = config.NewLogConfigSrc(dsn, configStore)
if err != nil {
return fmt.Errorf("invalid config source for %s, %w", name, err)
}
mlog.Info("Loaded configuration for "+name, mlog.String("source", dsn))
}
cfg, err := config.MloggerConfigFromLoggerConfig(logSettings, logConfigSrc, getPath)
if err != nil {
return fmt.Errorf("invalid config source for %s, %w", name, err)
}
if err := logger.ConfigureTargets(cfg); err != nil {
return fmt.Errorf("invalid config for %s, %w", name, err)
}
return nil
}
// removeUnlicensedLogTargets removes any unlicensed log target types.
func (s *Server) removeUnlicensedLogTargets(license *model.License) {
if license != nil && *license.Features.AdvancedLogging {
// advanced logging enabled via license; no need to remove any targets
@@ -864,8 +866,12 @@ func (s *Server) removeUnlicensedLogTargets(license *model.License) {
timeoutCtx, cancelCtx := context.WithTimeout(context.Background(), time.Second*10)
defer cancelCtx()
mlog.RemoveTargets(timeoutCtx, func(ti mlog.TargetInfo) bool {
return ti.Type != "*target.Writer" && ti.Type != "*target.File"
s.Log.RemoveTargets(timeoutCtx, func(ti mlog.TargetInfo) bool {
return ti.Type != "*targets.Writer" && ti.Type != "*targets.File"
})
s.NotificationsLog.RemoveTargets(timeoutCtx, func(ti mlog.TargetInfo) bool {
return ti.Type != "*targets.Writer" && ti.Type != "*targets.File"
})
}
@@ -939,11 +945,15 @@ func (s *Server) enableLoggingMetrics() {
return
}
if err := mlog.EnableMetrics(s.Metrics.GetLoggerMetricsCollector()); err != nil {
mlog.Error("Failed to enable advanced logging metrics", mlog.Err(err))
} else {
mlog.Debug("Advanced logging metrics enabled")
s.Log.SetMetricsCollector(s.Metrics.GetLoggerMetricsCollector(), mlog.DefaultMetricsUpdateFreqMillis)
// logging config needs to be reloaded when metrics collector is added or changed.
if err := s.initLogging(); err != nil {
mlog.Error("Error re-configuring logging for metrics")
return
}
mlog.Debug("Logging metrics enabled")
}
const TimeToWaitForConnectionsToCloseOnServerShutdown = time.Second
@@ -1014,13 +1024,7 @@ func (s *Server) Shutdown() {
s.WaitForGoroutines()
if s.advancedLogListenerCleanup != nil {
s.advancedLogListenerCleanup()
s.advancedLogListenerCleanup = nil
}
s.RemoveConfigListener(s.configListenerId)
s.RemoveConfigListener(s.logListenerId)
s.stopSearchEngine()
s.Audit.Shutdown()
@@ -1058,12 +1062,6 @@ func (s *Server) Shutdown() {
}
}
timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), time.Second*15)
defer timeoutCancel()
if err := mlog.Flush(timeoutCtx); err != nil {
mlog.Warn("Error flushing logs", mlog.Err(err))
}
s.dndTaskMut.Lock()
if s.dndTask != nil {
s.dndTask.Cancel()
@@ -1072,10 +1070,15 @@ func (s *Server) Shutdown() {
mlog.Info("Server stopped")
// this should just write the "server stopped" record, the rest are already flushed.
timeoutCtx2, timeoutCancel2 := context.WithTimeout(context.Background(), time.Second*5)
defer timeoutCancel2()
_ = mlog.ShutdownAdvancedLogging(timeoutCtx2)
// shutdown main and notification loggers which will flush any remaining log records.
timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), time.Second*15)
defer timeoutCancel()
if err = s.NotificationsLog.ShutdownWithTimeout(timeoutCtx); err != nil {
fmt.Fprintf(os.Stderr, "Error shutting down notification logger: %v", err)
}
if err = s.Log.ShutdownWithTimeout(timeoutCtx); err != nil {
fmt.Fprintf(os.Stderr, "Error shutting down main logger: %v", err)
}
}
func (s *Server) Restart() error {
@@ -1202,7 +1205,7 @@ func (s *Server) Start() error {
// If we have debugging of CORS turned on then forward messages to logs
if debug {
corsWrapper.Log = s.Log.StdLog(mlog.String("source", "cors"))
corsWrapper.Log = s.Log.With(mlog.String("source", "cors")).StdLogger(mlog.LvlDebug)
}
handler = corsWrapper.Handler(handler)
@@ -1222,10 +1225,7 @@ func (s *Server) Start() error {
s.Busy = NewBusy(s.Cluster)
// Creating a logger for logging errors from http.Server at error level
errStdLog, err := s.Log.StdLogAt(mlog.LevelError, mlog.String("source", "httpserver"))
if err != nil {
return err
}
errStdLog := s.Log.With(mlog.String("source", "httpserver")).StdLogger(mlog.LvlError)
s.Server = &http.Server{
Handler: handler,
@@ -1270,7 +1270,7 @@ func (s *Server) Start() error {
server := &http.Server{
Addr: httpListenAddress,
Handler: m.HTTPHandler(nil),
ErrorLog: s.Log.StdLog(mlog.String("source", "le_forwarder_server")),
ErrorLog: s.Log.With(mlog.String("source", "le_forwarder_server")).StdLogger(mlog.LvlError),
}
go server.ListenAndServe()
} else {
@@ -1284,7 +1284,7 @@ func (s *Server) Start() error {
server := &http.Server{
Handler: http.HandlerFunc(handleHTTPRedirect),
ErrorLog: s.Log.StdLog(mlog.String("source", "forwarder_server")),
ErrorLog: s.Log.With(mlog.String("source", "forwarder_server")).StdLogger(mlog.LvlError),
}
server.Serve(redirectListener)
}()
@@ -2026,7 +2026,7 @@ func (a *App) getNotificationsLog() (*model.FileData, string) {
// Getting notifications.log
if *a.Srv().Config().NotificationLogSettings.EnableFile {
// notifications.log
notificationsLog := utils.GetNotificationsLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
notificationsLog := config.GetNotificationsLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
notificationsLogFileData, notificationsLogFileDataErr := ioutil.ReadFile(notificationsLog)
@@ -2053,7 +2053,7 @@ func (a *App) getMattermostLog() (*model.FileData, string) {
// Getting mattermost.log
if *a.Srv().Config().LogSettings.EnableFile {
// mattermost.log
mattermostLog := utils.GetLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
mattermostLog := config.GetLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
mattermostLogFileData, mattermostLogFileDataErr := ioutil.ReadFile(mattermostLog)

48
app/server_test.go
Просмотреть файл

@@ -516,29 +516,29 @@ func checkEndpoint(t *testing.T, client *http.Client, url string) error {
}
func TestPanicLog(t *testing.T) {
// Creating a temp file to collect logs
tmpfile, err := ioutil.TempFile("", "mlog")
if err != nil {
require.NoError(t, err)
}
// Creating a temp dir for log
tmpDir, err := os.MkdirTemp("", "mlog-test")
require.NoError(t, err, "cannot create tmp dir for log file")
defer func() {
require.NoError(t, tmpfile.Close())
require.NoError(t, os.Remove(tmpfile.Name()))
err2 := os.RemoveAll(tmpDir)
assert.NoError(t, err2)
}()
// This test requires Zap file target for now.
mlog.EnableZap()
defer mlog.DisableZap()
// Creating logger to log to console and temp file
logger := mlog.NewLogger(&mlog.LoggerConfiguration{
EnableConsole: true,
ConsoleJson: true,
EnableFile: true,
FileLocation: tmpfile.Name(),
FileLevel: mlog.LevelInfo,
})
logger, _ := mlog.NewLogger()
logSettings := model.NewLogSettings()
logSettings.EnableConsole = model.NewBool(true)
logSettings.ConsoleJson = model.NewBool(true)
logSettings.EnableFile = model.NewBool(true)
logSettings.FileLocation = &tmpDir
logSettings.FileLevel = &mlog.LvlInfo.Name
cfg, err := config.MloggerConfigFromLoggerConfig(logSettings, nil, config.GetLogFileLocation)
require.NoError(t, err)
err = logger.ConfigureTargets(cfg)
require.NoError(t, err)
logger.LockConfiguration()
// Creating a server with logger
s, err := NewServer(SetLogger(logger))
@@ -567,16 +567,22 @@ func TestPanicLog(t *testing.T) {
client := &http.Client{Transport: tr}
client.Get("https://localhost:" + strconv.Itoa(s.ListenAddr.Port) + "/panic")
err = logger.Flush()
assert.NoError(t, err, "flush should succeed")
s.Shutdown()
// Checking whether panic was logged
var panicLogged = false
var infoLogged = false
_, err = tmpfile.Seek(0, 0)
logFile, err := os.Open(config.GetLogFileLocation(tmpDir))
require.NoError(t, err, "cannot open log file")
_, err = logFile.Seek(0, 0)
require.NoError(t, err)
scanner := bufio.NewScanner(tmpfile)
scanner := bufio.NewScanner(logFile)
for scanner.Scan() {
if !infoLogged && strings.Contains(scanner.Text(), "inside panic handler") {
infoLogged = true

65
app/slashcommands/helper_test.go
Просмотреть файл

@@ -21,7 +21,6 @@ import (
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
"github.com/mattermost/mattermost-server/v6/utils"
)
type TestHelper struct {
@@ -36,6 +35,7 @@ type TestHelper struct {
SystemAdminUser *model.User
LogBuffer *bytes.Buffer
TestLogger *mlog.Logger
IncludeCacheLayer bool
tempWorkspace string
@@ -49,15 +49,15 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
memoryStore := config.NewTestMemoryStore()
config := memoryStore.Get()
memoryConfig := memoryStore.Get()
if configSet != nil {
configSet(config)
configSet(memoryConfig)
}
*config.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
*config.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
*config.PluginSettings.AutomaticPrepackagedPlugins = false
*config.LogSettings.EnableSentry = false // disable error reporting during tests
memoryStore.Set(config)
*memoryConfig.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
*memoryConfig.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
*memoryConfig.PluginSettings.AutomaticPrepackagedPlugins = false
*memoryConfig.LogSettings.EnableSentry = false // disable error reporting during tests
memoryStore.Set(memoryConfig)
buffer := &bytes.Buffer{}
@@ -74,7 +74,15 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
} else {
options = append(options, app.StoreOverride(dbStore))
}
options = append(options, app.SetLogger(mlog.NewTestingLogger(tb, buffer)))
testLogger, _ := mlog.NewLogger()
logCfg, _ := config.MloggerConfigFromLoggerConfig(&memoryConfig.LogSettings, nil, config.GetLogFileLocation)
if errCfg := testLogger.ConfigureTargets(logCfg); errCfg != nil {
panic("failed to configure test logger: " + errCfg.Error())
}
// lock logger config so server init cannot override it during testing.
testLogger.LockConfiguration()
options = append(options, app.SetLogger(testLogger))
s, err := app.NewServer(options...)
if err != nil {
@@ -86,6 +94,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
Context: &request.Context{},
Server: s,
LogBuffer: buffer,
TestLogger: testLogger,
IncludeCacheLayer: includeCacheLayer,
}
@@ -188,12 +197,11 @@ func (th *TestHelper) createTeam() *model.Team {
Type: model.TeamOpen,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if team, err = th.App.CreateTeam(th.Context, team); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return team
}
@@ -216,7 +224,6 @@ func (th *TestHelper) createUserOrGuest(guest bool) *model.User {
EmailVerified: true,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if guest {
if user, err = th.App.CreateGuest(th.Context, user); err != nil {
@@ -227,7 +234,6 @@ func (th *TestHelper) createUserOrGuest(guest bool) *model.User {
panic(err)
}
}
utils.EnableDebugLogForTest()
return user
}
@@ -262,7 +268,6 @@ func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelT
option(channel)
}
utils.DisableDebugLogForTest()
var err *model.AppError
if channel, err = th.App.CreateChannel(th.Context, channel, true); err != nil {
panic(err)
@@ -284,7 +289,6 @@ func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelT
panic(err)
}
}
utils.EnableDebugLogForTest()
return channel
}
@@ -299,34 +303,28 @@ func (th *TestHelper) createChannelWithAnotherUser(team *model.Team, channelType
CreatorId: userID,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if channel, err = th.App.CreateChannel(th.Context, channel, true); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
func (th *TestHelper) createDmChannel(user *model.User) *model.Channel {
utils.DisableDebugLogForTest()
var err *model.AppError
var channel *model.Channel
if channel, err = th.App.GetOrCreateDirectChannel(th.Context, th.BasicUser.Id, user.Id); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
func (th *TestHelper) createGroupChannel(user1 *model.User, user2 *model.User) *model.Channel {
utils.DisableDebugLogForTest()
var err *model.AppError
var channel *model.Channel
if channel, err = th.App.CreateGroupChannel([]string{th.BasicUser.Id, user1.Id, user2.Id}, th.BasicUser.Id); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
@@ -340,36 +338,25 @@ func (th *TestHelper) createPost(channel *model.Channel) *model.Post {
CreateAt: model.GetMillis() - 10000,
}
utils.DisableDebugLogForTest()
var err *model.AppError
if post, err = th.App.CreatePost(th.Context, post, channel, false, true); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return post
}
func (th *TestHelper) linkUserToTeam(user *model.User, team *model.Team) {
utils.DisableDebugLogForTest()
_, err := th.App.JoinUserToTeam(th.Context, team, user, "")
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
}
func (th *TestHelper) addUserToChannel(user *model.User, channel *model.Channel) *model.ChannelMember {
utils.DisableDebugLogForTest()
member, err := th.App.AddUserToChannel(user, channel, false)
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return member
}
@@ -401,11 +388,8 @@ func (th *TestHelper) tearDown() {
}
func (th *TestHelper) removePermissionFromRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := th.App.GetRoleByName(context.Background(), roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
@@ -417,7 +401,6 @@ func (th *TestHelper) removePermissionFromRole(permission string, roleName strin
}
if strings.Join(role.Permissions, " ") == strings.Join(newPermissions, " ") {
utils.EnableDebugLogForTest()
return
}
@@ -425,25 +408,18 @@ func (th *TestHelper) removePermissionFromRole(permission string, roleName strin
_, err2 := th.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}
func (th *TestHelper) addPermissionToRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := th.App.GetRoleByName(context.Background(), roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
for _, existingPermission := range role.Permissions {
if existingPermission == permission {
utils.EnableDebugLogForTest()
return
}
}
@@ -452,9 +428,6 @@ func (th *TestHelper) addPermissionToRole(permission string, roleName string) {
_, err2 := th.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}

3
app/slashcommands/main_test.go
Просмотреть файл

@@ -6,7 +6,6 @@ package slashcommands
import (
"testing"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/testlib"
)
@@ -18,8 +17,6 @@ func TestMain(m *testing.M) {
EnableResources: true,
}
mlog.DisableZap()
mainHelper = testlib.NewMainHelperWithOptions(&options)
defer mainHelper.Close()