Admin log table (#21153)

* Reformat Log Output for SysAdmin UI * Restore old code * Update * Backend first pass * PR feedback * Set helpers to private * Lint fix * Fix mocks * Fix another mock * Fix lint again * Fix i18n * Update app/admin.go Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * PR feedback * FIx lint issue --------- Co-authored-by: Daniel Schalla <daniel@schalla.me> Co-authored-by: Nevyana Angelova <nevyangelova@Nevyanas-MacBook-Pro.local> Co-authored-by: Devin Binnie <devin.binnie@mattermost.com> Co-authored-by: Devin Binnie <52460000+devinbinnie@users.noreply.github.com> Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Mattermost Build <build@mattermost.com>
2023-02-14 12:39:10 +02:00
--- a/app/admin.go
+++ b/app/admin.go
@@ -37,7 +37,7 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) {
 		}
 	}

-	melines, err := s.GetLogsSkipSend(page, perPage)
+	melines, err := s.GetLogsSkipSend(page, perPage, &model.LogFilter{})
 	if err != nil {
 		return nil, err
 	}
@@ -56,16 +56,75 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) {
 	return lines, nil
 }

+func (s *Server) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	logData := make(map[string][]string)
+
+	serverName := "default"
+
+	license := s.License()
+	if license != nil && *license.Features.Cluster && s.platform.Cluster() != nil && *s.platform.Config().ClusterSettings.Enable {
+		if info := s.platform.Cluster().GetMyClusterInfo(); info != nil {
+			serverName = info.Hostname
+		} else {
+			mlog.Error("Could not get cluster info")
+		}
+	}
+
+	serverNames := logFilter.ServerNames
+	if len(serverNames) > 0 {
+		for _, nodeName := range serverNames {
+			if nodeName == "default" {
+				AddLocalLogs(logData, s, page, perPage, nodeName, logFilter)
+			}
+		}
+	} else {
+		AddLocalLogs(logData, s, page, perPage, serverName, logFilter)
+	}
+
+	if s.platform.Cluster() != nil && *s.Config().ClusterSettings.Enable {
+		clusterLogs, err := s.platform.Cluster().QueryLogs(page, perPage)
+		if err != nil {
+			return nil, err
+		}
+
+		if clusterLogs != nil && len(serverNames) > 0 {
+			for _, filteredNodeName := range serverNames {
+				logData[filteredNodeName] = clusterLogs[filteredNodeName]
+			}
+		} else {
+			for nodeName, logs := range clusterLogs {
+				logData[nodeName] = logs
+			}
+		}
+	}
+
+	return logData, nil
+}
+
+func AddLocalLogs(logData map[string][]string, s *Server, page, perPage int, serverName string, logFilter *model.LogFilter) *model.AppError {
+	currentServerLogs, err := s.GetLogsSkipSend(page, perPage, logFilter)
+	if err != nil {
+		return err
+	}
+
+	logData[serverName] = currentServerLogs
+	return nil
+}
+
+func (a *App) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	return a.Srv().QueryLogs(page, perPage, logFilter)
+}
+
 func (a *App) GetLogs(page, perPage int) ([]string, *model.AppError) {
 	return a.Srv().GetLogs(page, perPage)
 }

-func (s *Server) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
-	return s.platform.GetLogsSkipSend(page, perPage)
+func (s *Server) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
+	return s.platform.GetLogsSkipSend(page, perPage, logFilter)
 }

-func (a *App) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
-	return a.Srv().GetLogsSkipSend(page, perPage)
+func (a *App) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
+	return a.Srv().GetLogsSkipSend(page, perPage, logFilter)
 }

 func (a *App) GetClusterStatus() []*model.ClusterInfo {
--- a/app/app_iface.go
+++ b/app/app_iface.go
@@ -682,7 +682,7 @@ type AppIface interface {
 	GetLatestTermsOfService() (*model.TermsOfService, *model.AppError)
 	GetLatestVersion(latestVersionUrl string) (*model.GithubReleaseInfo, *model.AppError)
 	GetLogs(page, perPage int) ([]string, *model.AppError)
-	GetLogsSkipSend(page, perPage int) ([]string, *model.AppError)
+	GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError)
 	GetMemberCountsByGroup(ctx context.Context, channelID string, includeTimezones bool) ([]*model.ChannelMemberCountByGroup, *model.AppError)
 	GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string
 	GetMultipleEmojiByName(c request.CTX, names []string) ([]*model.Emoji, *model.AppError)
@@ -961,6 +961,7 @@ type AppIface interface {
 	PublishUserTyping(userID, channelID, parentId string) *model.AppError
 	PurgeBleveIndexes() *model.AppError
 	PurgeElasticsearchIndexes() *model.AppError
+	QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError)
 	ReadFile(path string) ([]byte, *model.AppError)
 	RecycleDatabaseConnection()
 	RegenCommandToken(cmd *model.Command) (*model.Command, *model.AppError)
--- a/app/busy_test.go
+++ b/app/busy_test.go
@@ -134,13 +134,18 @@ func (c *ClusterMock) StartInterNodeCommunication() {}
 func (c *ClusterMock) StopInterNodeCommunication()  {}
 func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
 }
-func (c *ClusterMock) GetClusterId() string                                       { return "cluster_mock" }
-func (c *ClusterMock) IsLeader() bool                                             { return false }
-func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                       { return nil }
-func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                      { return nil }
-func (c *ClusterMock) NotifyMsg(buf []byte)                                       {}
-func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError)  { return nil, nil }
-func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)      { return nil, nil }
+func (c *ClusterMock) GetClusterId() string                                      { return "cluster_mock" }
+func (c *ClusterMock) IsLeader() bool                                            { return false }
+func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                      { return nil }
+func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                     { return nil }
+func (c *ClusterMock) NotifyMsg(buf []byte)                                      {}
+func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil }
+func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) {
+	return nil, nil
+}
+func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) {
+	return nil, nil
+}
 func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil }
 func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
 	return nil
--- a/app/opentracing/opentracing_layer.go
+++ b/app/opentracing/opentracing_layer.go
@@ -7138,7 +7138,7 @@ func (a *OpenTracingAppLayer) GetLogs(page int, perPage int) ([]string, *model.A
 	return resultVar0, resultVar1
 }

-func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string, *model.AppError) {
+func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLogsSkipSend")

@@ -7150,7 +7150,7 @@ func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string,
 	}()

 	defer span.Finish()
-	resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage)
+	resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage, logFilter)

 	if resultVar1 != nil {
 		span.LogFields(spanlog.Error(resultVar1))
@@ -13646,6 +13646,28 @@ func (a *OpenTracingAppLayer) PurgeElasticsearchIndexes() *model.AppError {
 	return resultVar0
 }

+func (a *OpenTracingAppLayer) QueryLogs(page int, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.QueryLogs")
+
+	a.ctx = newCtx
+	a.app.Srv().Store().SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store().SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.QueryLogs(page, perPage, logFilter)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) ReadFile(path string) ([]byte, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ReadFile")
--- a/app/platform/busy_test.go
+++ b/app/platform/busy_test.go
@@ -134,13 +134,16 @@ func (c *ClusterMock) StartInterNodeCommunication() {}
 func (c *ClusterMock) StopInterNodeCommunication()  {}
 func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
 }
-func (c *ClusterMock) GetClusterId() string                                       { return "cluster_mock" }
-func (c *ClusterMock) IsLeader() bool                                             { return false }
-func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                       { return nil }
-func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                      { return nil }
-func (c *ClusterMock) NotifyMsg(buf []byte)                                       {}
-func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError)  { return nil, nil }
-func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)      { return nil, nil }
+func (c *ClusterMock) GetClusterId() string                                      { return "cluster_mock" }
+func (c *ClusterMock) IsLeader() bool                                            { return false }
+func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                      { return nil }
+func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                     { return nil }
+func (c *ClusterMock) NotifyMsg(buf []byte)                                      {}
+func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil }
+func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)     { return nil, nil }
+func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) {
+	return nil, nil
+}
 func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil }
 func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
 	return nil
--- a/app/platform/log.go
+++ b/app/platform/log.go
@@ -5,6 +5,7 @@ package platform

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -123,7 +124,7 @@ func (ps *PlatformService) RemoveUnlicensedLogTargets(license *model.License) {
 	})
 }

-func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
+func (ps *PlatformService) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
 	var lines []string

 	if *ps.Config().LogSettings.EnableFile {
@@ -172,7 +173,22 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.
 					if err != nil {
 						return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
 					}
-					lines = append(lines, string(line))
+
+					filtered := false
+					var entry *model.LogEntry
+					err = json.Unmarshal(line, &entry)
+					if err != nil {
+						mlog.Debug("Failed to parse line, skipping")
+					} else {
+						filtered = isLogFilteredByLevel(logFilter, entry) || filtered
+						filtered = isLogFilteredByDate(logFilter, entry) || filtered
+					}
+
+					if filtered {
+						lineCount--
+					} else {
+						lines = append(lines, string(line))
+					}
 				}
 				if pos == 0 {
 					break
@@ -194,3 +210,48 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.

 	return lines, nil
 }
+
+func isLogFilteredByLevel(logFilter *model.LogFilter, entry *model.LogEntry) bool {
+	logLevels := logFilter.LogLevels
+	if len(logLevels) == 0 {
+		return false
+	}
+
+	for _, level := range logLevels {
+		if entry.Level == level {
+			return false
+		}
+	}
+
+	return true
+}
+
+func isLogFilteredByDate(logFilter *model.LogFilter, entry *model.LogEntry) bool {
+	if logFilter.DateFrom == "" && logFilter.DateTo == "" {
+		return false
+	}
+
+	dateFrom, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateFrom)
+	if err != nil {
+		dateFrom = time.Time{}
+	}
+	dateTo, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateTo)
+	if err != nil {
+		dateTo = time.Now()
+	}
+
+	timestamp, err := time.Parse("2006-01-02 15:04:05.999 -07:00", entry.Timestamp)
+	if err != nil {
+		mlog.Debug("Cannot parse timestamp, skipping")
+		return false
+	}
+
+	if timestamp.Equal(dateFrom) || timestamp.Equal(dateTo) {
+		return false
+	}
+	if timestamp.After(dateFrom) && timestamp.Before(dateTo) {
+		return false
+	}
+
+	return true
+}