diff --git a/api4/system.go b/api4/system.go index fbba869222..5220e66e60 100644 --- a/api4/system.go +++ b/api4/system.go @@ -48,6 +48,7 @@ func (api *API) InitSystem() { api.BaseRoutes.APIRoot.Handle("/caches/invalidate", api.APISessionRequired(invalidateCaches)).Methods("POST") api.BaseRoutes.APIRoot.Handle("/logs", api.APISessionRequired(getLogs)).Methods("GET") + api.BaseRoutes.APIRoot.Handle("/logs/query", api.APISessionRequired(queryLogs)).Methods("POST") api.BaseRoutes.APIRoot.Handle("/logs", api.APIHandler(postLog)).Methods("POST") api.BaseRoutes.APIRoot.Handle("/analytics/old", api.APISessionRequired(getAnalytics)).Methods("GET") @@ -326,6 +327,52 @@ func invalidateCaches(c *Context, w http.ResponseWriter, r *http.Request) { ReturnStatusOK(w) } +func queryLogs(c *Context, w http.ResponseWriter, r *http.Request) { + auditRec := c.MakeAuditRecord("queryLogs", audit.Fail) + defer c.LogAuditRec(auditRec) + + if *c.App.Config().ExperimentalSettings.RestrictSystemAdmin { + c.Err = model.NewAppError("queryLogs", "api.restricted_system_admin", nil, "", http.StatusForbidden) + return + } + + if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionGetLogs) { + c.SetPermissionError(model.PermissionGetLogs) + return + } + + var logFilter *model.LogFilter + err := json.NewDecoder(r.Body).Decode(&logFilter) + if err != nil { + c.Err = model.NewAppError("queryLogs", "api.system.logs.invalidFilter", nil, "", http.StatusInternalServerError) + return + } + + logs, logerr := c.App.QueryLogs(c.Params.Page, c.Params.LogsPerPage, logFilter) + if logerr != nil { + c.Err = logerr + return + } + + logsJSON := make(map[string][]interface{}) + var result interface{} + for node, logLines := range logs { + for _, log := range logLines { + err2 := json.Unmarshal([]byte(log), &result) + if err2 == nil { + logsJSON[node] = append(logsJSON[node], result) + } else { + mlog.Warn("Error parsing log line in Server Logs") + } + } + } + + auditRec.AddMeta("page", c.Params.Page) + auditRec.AddMeta("logs_per_page", c.Params.LogsPerPage) + + w.Write(model.ToJSON(logsJSON)) +} + func getLogs(c *Context, w http.ResponseWriter, r *http.Request) { auditRec := c.MakeAuditRecord("getLogs", audit.Fail) defer c.LogAuditRec(auditRec) diff --git a/app/admin.go b/app/admin.go index 84be5e71d9..5d7556d65b 100644 --- a/app/admin.go +++ b/app/admin.go @@ -37,7 +37,7 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) { } } - melines, err := s.GetLogsSkipSend(page, perPage) + melines, err := s.GetLogsSkipSend(page, perPage, &model.LogFilter{}) if err != nil { return nil, err } @@ -56,16 +56,75 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) { return lines, nil } +func (s *Server) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) { + logData := make(map[string][]string) + + serverName := "default" + + license := s.License() + if license != nil && *license.Features.Cluster && s.platform.Cluster() != nil && *s.platform.Config().ClusterSettings.Enable { + if info := s.platform.Cluster().GetMyClusterInfo(); info != nil { + serverName = info.Hostname + } else { + mlog.Error("Could not get cluster info") + } + } + + serverNames := logFilter.ServerNames + if len(serverNames) > 0 { + for _, nodeName := range serverNames { + if nodeName == "default" { + AddLocalLogs(logData, s, page, perPage, nodeName, logFilter) + } + } + } else { + AddLocalLogs(logData, s, page, perPage, serverName, logFilter) + } + + if s.platform.Cluster() != nil && *s.Config().ClusterSettings.Enable { + clusterLogs, err := s.platform.Cluster().QueryLogs(page, perPage) + if err != nil { + return nil, err + } + + if clusterLogs != nil && len(serverNames) > 0 { + for _, filteredNodeName := range serverNames { + logData[filteredNodeName] = clusterLogs[filteredNodeName] + } + } else { + for nodeName, logs := range clusterLogs { + logData[nodeName] = logs + } + } + } + + return logData, nil +} + +func AddLocalLogs(logData map[string][]string, s *Server, page, perPage int, serverName string, logFilter *model.LogFilter) *model.AppError { + currentServerLogs, err := s.GetLogsSkipSend(page, perPage, logFilter) + if err != nil { + return err + } + + logData[serverName] = currentServerLogs + return nil +} + +func (a *App) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) { + return a.Srv().QueryLogs(page, perPage, logFilter) +} + func (a *App) GetLogs(page, perPage int) ([]string, *model.AppError) { return a.Srv().GetLogs(page, perPage) } -func (s *Server) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) { - return s.platform.GetLogsSkipSend(page, perPage) +func (s *Server) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) { + return s.platform.GetLogsSkipSend(page, perPage, logFilter) } -func (a *App) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) { - return a.Srv().GetLogsSkipSend(page, perPage) +func (a *App) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) { + return a.Srv().GetLogsSkipSend(page, perPage, logFilter) } func (a *App) GetClusterStatus() []*model.ClusterInfo { diff --git a/app/app_iface.go b/app/app_iface.go index aa0fea139d..c7029bcb56 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -682,7 +682,7 @@ type AppIface interface { GetLatestTermsOfService() (*model.TermsOfService, *model.AppError) GetLatestVersion(latestVersionUrl string) (*model.GithubReleaseInfo, *model.AppError) GetLogs(page, perPage int) ([]string, *model.AppError) - GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) + GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) GetMemberCountsByGroup(ctx context.Context, channelID string, includeTimezones bool) ([]*model.ChannelMemberCountByGroup, *model.AppError) GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string GetMultipleEmojiByName(c request.CTX, names []string) ([]*model.Emoji, *model.AppError) @@ -961,6 +961,7 @@ type AppIface interface { PublishUserTyping(userID, channelID, parentId string) *model.AppError PurgeBleveIndexes() *model.AppError PurgeElasticsearchIndexes() *model.AppError + QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) ReadFile(path string) ([]byte, *model.AppError) RecycleDatabaseConnection() RegenCommandToken(cmd *model.Command) (*model.Command, *model.AppError) diff --git a/app/busy_test.go b/app/busy_test.go index 4b076fee8d..358a144ccf 100644 --- a/app/busy_test.go +++ b/app/busy_test.go @@ -134,13 +134,18 @@ func (c *ClusterMock) StartInterNodeCommunication() {} func (c *ClusterMock) StopInterNodeCommunication() {} func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) { } -func (c *ClusterMock) GetClusterId() string { return "cluster_mock" } -func (c *ClusterMock) IsLeader() bool { return false } -func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo { return nil } -func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo { return nil } -func (c *ClusterMock) NotifyMsg(buf []byte) {} -func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil } -func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) { return nil, nil } +func (c *ClusterMock) GetClusterId() string { return "cluster_mock" } +func (c *ClusterMock) IsLeader() bool { return false } +func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo { return nil } +func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo { return nil } +func (c *ClusterMock) NotifyMsg(buf []byte) {} +func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil } +func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) { + return nil, nil +} +func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) { + return nil, nil +} func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil } func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError { return nil diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index b93056bc89..d9a1e3f723 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -7138,7 +7138,7 @@ func (a *OpenTracingAppLayer) GetLogs(page int, perPage int) ([]string, *model.A return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string, *model.AppError) { +func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLogsSkipSend") @@ -7150,7 +7150,7 @@ func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string, }() defer span.Finish() - resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage) + resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage, logFilter) if resultVar1 != nil { span.LogFields(spanlog.Error(resultVar1)) @@ -13646,6 +13646,28 @@ func (a *OpenTracingAppLayer) PurgeElasticsearchIndexes() *model.AppError { return resultVar0 } +func (a *OpenTracingAppLayer) QueryLogs(page int, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.QueryLogs") + + a.ctx = newCtx + a.app.Srv().Store().SetContext(newCtx) + defer func() { + a.app.Srv().Store().SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.QueryLogs(page, perPage, logFilter) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) ReadFile(path string) ([]byte, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ReadFile") diff --git a/app/platform/busy_test.go b/app/platform/busy_test.go index 69695f56d4..88cb195595 100644 --- a/app/platform/busy_test.go +++ b/app/platform/busy_test.go @@ -134,13 +134,16 @@ func (c *ClusterMock) StartInterNodeCommunication() {} func (c *ClusterMock) StopInterNodeCommunication() {} func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) { } -func (c *ClusterMock) GetClusterId() string { return "cluster_mock" } -func (c *ClusterMock) IsLeader() bool { return false } -func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo { return nil } -func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo { return nil } -func (c *ClusterMock) NotifyMsg(buf []byte) {} -func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil } -func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) { return nil, nil } +func (c *ClusterMock) GetClusterId() string { return "cluster_mock" } +func (c *ClusterMock) IsLeader() bool { return false } +func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo { return nil } +func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo { return nil } +func (c *ClusterMock) NotifyMsg(buf []byte) {} +func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil } +func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) { return nil, nil } +func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) { + return nil, nil +} func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil } func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError { return nil diff --git a/app/platform/log.go b/app/platform/log.go index 42ad85b54e..5988d5ee36 100644 --- a/app/platform/log.go +++ b/app/platform/log.go @@ -5,6 +5,7 @@ package platform import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -123,7 +124,7 @@ func (ps *PlatformService) RemoveUnlicensedLogTargets(license *model.License) { }) } -func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) { +func (ps *PlatformService) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) { var lines []string if *ps.Config().LogSettings.EnableFile { @@ -172,7 +173,22 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model. if err != nil { return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err) } - lines = append(lines, string(line)) + + filtered := false + var entry *model.LogEntry + err = json.Unmarshal(line, &entry) + if err != nil { + mlog.Debug("Failed to parse line, skipping") + } else { + filtered = isLogFilteredByLevel(logFilter, entry) || filtered + filtered = isLogFilteredByDate(logFilter, entry) || filtered + } + + if filtered { + lineCount-- + } else { + lines = append(lines, string(line)) + } } if pos == 0 { break @@ -194,3 +210,48 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model. return lines, nil } + +func isLogFilteredByLevel(logFilter *model.LogFilter, entry *model.LogEntry) bool { + logLevels := logFilter.LogLevels + if len(logLevels) == 0 { + return false + } + + for _, level := range logLevels { + if entry.Level == level { + return false + } + } + + return true +} + +func isLogFilteredByDate(logFilter *model.LogFilter, entry *model.LogEntry) bool { + if logFilter.DateFrom == "" && logFilter.DateTo == "" { + return false + } + + dateFrom, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateFrom) + if err != nil { + dateFrom = time.Time{} + } + dateTo, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateTo) + if err != nil { + dateTo = time.Now() + } + + timestamp, err := time.Parse("2006-01-02 15:04:05.999 -07:00", entry.Timestamp) + if err != nil { + mlog.Debug("Cannot parse timestamp, skipping") + return false + } + + if timestamp.Equal(dateFrom) || timestamp.Equal(dateTo) { + return false + } + if timestamp.After(dateFrom) && timestamp.Before(dateTo) { + return false + } + + return true +} diff --git a/einterfaces/cluster.go b/einterfaces/cluster.go index 6d52f35b5a..6ba89cd299 100644 --- a/einterfaces/cluster.go +++ b/einterfaces/cluster.go @@ -26,6 +26,7 @@ type ClusterInterface interface { NotifyMsg(buf []byte) GetClusterStats() ([]*model.ClusterStats, *model.AppError) GetLogs(page, perPage int) ([]string, *model.AppError) + QueryLogs(page, perPage int) (map[string][]string, *model.AppError) GetPluginStatuses() (model.PluginStatuses, *model.AppError) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError } diff --git a/einterfaces/mocks/ClusterInterface.go b/einterfaces/mocks/ClusterInterface.go index 5621a03e0a..0708057c3f 100644 --- a/einterfaces/mocks/ClusterInterface.go +++ b/einterfaces/mocks/ClusterInterface.go @@ -186,6 +186,31 @@ func (_m *ClusterInterface) NotifyMsg(buf []byte) { _m.Called(buf) } +// QueryLogs provides a mock function with given fields: page, perPage +func (_m *ClusterInterface) QueryLogs(page int, perPage int) (map[string][]string, *model.AppError) { + ret := _m.Called(page, perPage) + + var r0 map[string][]string + if rf, ok := ret.Get(0).(func(int, int) map[string][]string); ok { + r0 = rf(page, perPage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string][]string) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(int, int) *model.AppError); ok { + r1 = rf(page, perPage) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + // RegisterClusterMessageHandler provides a mock function with given fields: event, crm func (_m *ClusterInterface) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) { _m.Called(event, crm) diff --git a/i18n/en.json b/i18n/en.json index dd65c2cbfd..caf02fdd55 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -2947,6 +2947,10 @@ "id": "api.system.id_loaded.not_available.app_error", "translation": "ID Loaded Push Notifications are not configured or supported on this server." }, + { + "id": "api.system.logs.invalidFilter", + "translation": "Invalid log filter" + }, { "id": "api.system.update_notices.clear_failed", "translation": "Clearing old product notices failed" diff --git a/model/system.go b/model/system.go index f978473fd5..7a5d2ee696 100644 --- a/model/system.go +++ b/model/system.go @@ -181,3 +181,15 @@ type AppliedMigration struct { Version int `json:"version"` Name string `json:"name"` } + +type LogFilter struct { + ServerNames []string `json:"server_names"` + LogLevels []string `json:"log_levels"` + DateFrom string `json:"date_from"` + DateTo string `json:"date_to"` +} + +type LogEntry struct { + Timestamp string + Level string +} diff --git a/testlib/cluster.go b/testlib/cluster.go index fd1ce73697..037bf69076 100644 --- a/testlib/cluster.go +++ b/testlib/cluster.go @@ -59,6 +59,10 @@ func (c *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppE return []string{}, nil } +func (c *FakeClusterInterface) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) { + return make(map[string][]string), nil +} + func (c *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError { return nil }