Adding the debug bar logic in the server (#22410)

* Adding debugbar layer

* Adding sql debugbar info

* Make duration consistent across the debugbar lines

* Adding the debugbar/systeminfo endpoint

* Adding logs to the debugbar

* Improve the debugbar logger fields info

* Improving the debug bar architecture

* Allow to enable/disable debugbar in the backend

* Exposing the Debug Bar enable in the client config

* Adding more system information to the debugbar

* Adding params info to the store layer

* Organizing a bit the debugbar code in the server and adding some extra data to the system info api

* Adding debugbar email traces

* Changing the socket event name to 'debugbar'

* Adding explain support for the debugbar

* Adding missed file

* Omitting data related to the debugbar itself

* Removing unneeded functions

* Avoid arbitrary execution in explain api

* Moving debugbar inside the platform directory

* Replacing debugbar logger with a new logger Target

* Removed uneeded changes

* Fixing some linter errors

* Adding a debugbar log level to use it later for log events strictly related to the debug bar

* Fixing linter errors

* Fixing tests

* Adding i18n strings

app/platform/config.go

@@ -286,6 +286,7 @@ func (ps *PlatformService) LimitedClientConfigWithComputed() map[string]string {
 	// These properties are not configurable, but nevertheless represent configuration expected
 	// by the client.
 	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+	respCfg["DebugBar"] = strconv.FormatBool(ps.DebugBar.IsEnabled())
 
 	return respCfg
 }
@@ -300,6 +301,7 @@ func (ps *PlatformService) ClientConfigWithComputed() map[string]string {
 	// These properties are not configurable, but nevertheless represent configuration expected
 	// by the client.
 	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+	respCfg["DebugBar"] = strconv.FormatBool(ps.DebugBar.IsEnabled())
 	respCfg["MaxPostSize"] = strconv.Itoa(ps.MaxPostSize())
 	respCfg["UpgradedFromTE"] = strconv.FormatBool(ps.isUpgradedFromTE())
 	respCfg["InstallationDate"] = ""

app/platform/debugbar/debugbar.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package debugbar
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mail"
+)
+
+const (
+	socketEventName = "debugbar"
+)
+
+type DebugBar struct {
+	publish func(*model.WebSocketEvent)
+	enabled bool
+}
+
+func New(publish func(*model.WebSocketEvent)) *DebugBar {
+	return &DebugBar{
+		publish: publish,
+		enabled: os.Getenv("MM_ENABLE_DEBUG_BAR") == "true",
+	}
+}
+
+func (db *DebugBar) IsEnabled() bool {
+	return db.enabled
+}
+
+func (db *DebugBar) SendLogEvent(logLevel string, logMessage string, fields map[string]string) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "log-line")
+	event.Add("level", logLevel)
+	event.Add("message", logMessage)
+	event.Add("fields", fields)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendApiCall(endpoint, method, statusCode string, elapsed float64) {
+	if endpoint == "getSystemInfo" || endpoint == "getQueryExplain" {
+		return
+	}
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "api-call")
+	event.Add("endpoint", endpoint)
+	event.Add("method", method)
+	event.Add("statusCode", statusCode)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendStoreCall(method string, success bool, elapsed float64, params map[string]any) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "store-call")
+	event.Add("method", method)
+	event.Add("params", params)
+	event.Add("success", success)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendSqlQuery(query string, elapsed float64, args ...any) {
+	if strings.HasPrefix(query, "EXPLAIN ") {
+		return
+	}
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "sql-query")
+	event.Add("query", query)
+	event.Add("args", args)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendEmailSent(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *mail.SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string, category string, err error) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "email-sent")
+	event.Add("to", to)
+	event.Add("subject", subject)
+	event.Add("htmlBody", htmlBody)
+	event.Add("embeddedFiles", embeddedFiles)
+	event.Add("SMTPConfig", config)
+	event.Add("enableComplianceFeatures", enableComplianceFeatures)
+	event.Add("messageID", messageID)
+	event.Add("inReplyTo", inReplyTo)
+	event.Add("references", references)
+	event.Add("cc", ccMail)
+	event.Add("category", category)
+	event.Add("err", err)
+	db.publish(event)
+}

app/platform/debugbar/logger.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package debugbar
+
+import (
+	"bytes"
+
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+type DebugBarLogTarget struct {
+	debugBar *DebugBar
+}
+
+type DebugBarLogFilter struct{}
+
+func (_ *DebugBarLogFilter) GetEnabledLevel(level logr.Level) (logr.Level, bool) {
+	return level, true
+}
+
+type DebugBarLogFormatter struct{}
+
+func (_ *DebugBarLogFormatter) IsStacktraceNeeded() bool {
+	return false
+}
+
+func (_ *DebugBarLogFormatter) Format(rec *logr.LogRec, level logr.Level, buf *bytes.Buffer) (*bytes.Buffer, error) {
+	return bytes.NewBuffer([]byte{}), nil
+}
+
+func NewDebugBarLogTarget(debugBar *DebugBar) *DebugBarLogTarget {
+	return &DebugBarLogTarget{debugBar: debugBar}
+}
+
+func (dblt *DebugBarLogTarget) Init() error {
+	return nil
+}
+
+func (dblt *DebugBarLogTarget) Shutdown() error {
+	return nil
+}
+
+func (dblt *DebugBarLogTarget) Write(p []byte, rec *logr.LogRec) (int, error) {
+	dblt.debugBar.SendLogEvent(rec.Level().Name, rec.Msg(), dblt.fieldsToStringsMap(rec.Fields()...))
+	return len(p), nil
+}
+
+func (dblt *DebugBarLogTarget) fieldsToStringsMap(fields ...mlog.Field) map[string]string {
+	result := map[string]string{}
+	for _, field := range fields {
+		value := &bytes.Buffer{}
+		field.ValueString(value, nil)
+		result[field.Key] = value.String()
+	}
+	return result
+}

app/platform/log.go

@@ -13,6 +13,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
@@ -78,6 +80,13 @@ func (ps *PlatformService) initLogging() error {
 		}
 	}
 
+	if ps.DebugBar.IsEnabled() {
+		err := ps.logger.AddTarget(debugbar.NewDebugBarLogTarget(ps.DebugBar), "debugbar", &debugbar.DebugBarLogFilter{}, &debugbar.DebugBarLogFormatter{}, logr.DefaultMaxQueueSize)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

app/platform/service.go

@@ -12,6 +12,7 @@ import (
 	"sync/atomic"
 
 	"github.com/mattermost/mattermost-server/v6/app/featureflag"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/einterfaces"
 	"github.com/mattermost/mattermost-server/v6/jobs"
@@ -23,6 +24,7 @@ import (
 	"github.com/mattermost/mattermost-server/v6/shared/filestore"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 	"github.com/mattermost/mattermost-server/v6/store"
+	"github.com/mattermost/mattermost-server/v6/store/debugbarlayer"
 	"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
 	"github.com/mattermost/mattermost-server/v6/store/retrylayer"
 	"github.com/mattermost/mattermost-server/v6/store/searchlayer"
@@ -34,9 +36,11 @@ import (
 // responsible for non-entity related functionalities that are required
 // by a product such as database access, configuration access, licensing etc.
 type PlatformService struct {
-	sqlStore *sqlstore.SqlStore
-	Store    store.Store
-	newStore func() (store.Store, error)
+	sqlStore   *sqlstore.SqlStore
+	DebugBar   *debugbar.DebugBar
+	Store      store.Store
+	newStore   func() (store.Store, error)
+	LastUserID string
 
 	WebSocketRouter *WebSocketRouter
 
@@ -125,6 +129,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 		licenseListeners:          map[string]func(*model.License, *model.License){},
 		additionalClusterHandlers: map[model.ClusterEvent]einterfaces.ClusterMessageHandler{},
 	}
+	ps.DebugBar = debugbar.New(ps.Publish)
 
 	// Step 1: Cache provider.
 	// At the moment we only have this implementation
@@ -186,7 +191,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 	// Depends on Step 0 (config), 1 (cacheProvider), 3 (search engine), 5 (metrics) and cluster.
 	if ps.newStore == nil {
 		ps.newStore = func() (store.Store, error) {
-			ps.sqlStore = sqlstore.New(ps.Config().SqlSettings, ps.metricsIFace)
+			ps.sqlStore = sqlstore.New(ps.Config().SqlSettings, ps.metricsIFace, ps.DebugBar.SendSqlQuery)
 
 			lcl, err2 := localcachelayer.NewLocalCacheLayer(
 				retrylayer.New(ps.sqlStore),
@@ -214,10 +219,15 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 				ps.sqlStore.UpdateLicense(newLicense)
 			})
 
-			return timerlayer.New(
+			timerStore := timerlayer.New(
 				searchStore,
 				ps.metricsIFace,
-			), nil
+			)
+
+			if ps.DebugBar.IsEnabled() {
+				return debugbarlayer.New(timerStore, ps.DebugBar), nil
+			}
+			return timerStore, nil
 		}
 	}
 

app/platform/service_test.go

@@ -142,6 +142,8 @@ func TestMetrics(t *testing.T) {
 		mockMetricsImpl.On("Register").Return()
 		mockMetricsImpl.On("ObserveStoreMethodDuration", mock.Anything, mock.Anything, mock.Anything).Return()
 		mockMetricsImpl.On("RegisterDBCollector", mock.AnythingOfType("*sql.DB"), "master")
+		mockMetricsImpl.On("IncrementWebsocketEvent", "debugbar")
+		mockMetricsImpl.On("IncrementWebSocketBroadcastBufferSize", mock.AnythingOfType("string"), float64(1))
 
 		th := Setup(t, StartMetrics(), func(ps *PlatformService) error {
 			ps.metricsIFace = mockMetricsImpl