Adding the debug bar logic in the server (#22410)

* Adding debugbar layer

* Adding sql debugbar info

* Make duration consistent across the debugbar lines

* Adding the debugbar/systeminfo endpoint

* Adding logs to the debugbar

* Improve the debugbar logger fields info

* Improving the debug bar architecture

* Allow to enable/disable debugbar in the backend

* Exposing the Debug Bar enable in the client config

* Adding more system information to the debugbar

* Adding params info to the store layer

* Organizing a bit the debugbar code in the server and adding some extra data to the system info api

* Adding debugbar email traces

* Changing the socket event name to 'debugbar'

* Adding explain support for the debugbar

* Adding missed file

* Omitting data related to the debugbar itself

* Removing unneeded functions

* Avoid arbitrary execution in explain api

* Moving debugbar inside the platform directory

* Replacing debugbar logger with a new logger Target

* Removed uneeded changes

* Fixing some linter errors

* Adding a debugbar log level to use it later for log events strictly related to the debug bar

* Fixing linter errors

* Fixing tests

* Adding i18n strings

api4/api.go

@@ -103,6 +103,8 @@ type Routes struct {
 
 	Jobs *mux.Router // 'api/v4/jobs'
 
+	DebugBar *mux.Router // 'api/v4/debugbar'
+
 	Preferences *mux.Router // 'api/v4/users/{user_id:[A-Za-z0-9]+}/preferences'
 
 	License *mux.Router // 'api/v4/license'
@@ -235,6 +237,7 @@ func Init(srv *app.Server) (*API, error) {
 	api.BaseRoutes.Public = api.BaseRoutes.APIRoot.PathPrefix("/public").Subrouter()
 	api.BaseRoutes.Reactions = api.BaseRoutes.APIRoot.PathPrefix("/reactions").Subrouter()
 	api.BaseRoutes.Jobs = api.BaseRoutes.APIRoot.PathPrefix("/jobs").Subrouter()
+	api.BaseRoutes.DebugBar = api.BaseRoutes.APIRoot.PathPrefix("/debugbar").Subrouter()
 	api.BaseRoutes.Elasticsearch = api.BaseRoutes.APIRoot.PathPrefix("/elasticsearch").Subrouter()
 	api.BaseRoutes.Bleve = api.BaseRoutes.APIRoot.PathPrefix("/bleve").Subrouter()
 	api.BaseRoutes.DataRetention = api.BaseRoutes.APIRoot.PathPrefix("/data_retention").Subrouter()
@@ -298,6 +301,7 @@ func Init(srv *app.Server) (*API, error) {
 	api.InitDataRetention()
 	api.InitBrand()
 	api.InitJob()
+	api.InitDebugBar()
 	api.InitCommand()
 	api.InitStatus()
 	api.InitWebSocket()

api4/debugbar.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+func (api *API) InitDebugBar() {
+	api.BaseRoutes.DebugBar.Handle("/systeminfo", api.APISessionRequired(getSystemInfo)).Methods("GET")
+	api.BaseRoutes.DebugBar.Handle("/queryexplain", api.APISessionRequired(getQueryExplain)).Methods("POST")
+}
+
+func getSystemInfo(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.Srv().DebugBar().IsEnabled() {
+		c.Err = model.NewAppError("Api4.GetSystemInfo", "api.debugbar.getSystemInfo.disabled_debugbar.error", nil, "", http.StatusNotImplemented)
+		return
+	}
+
+	info, err := c.App.GetDebugBarInfo()
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(info); err != nil {
+		c.Logger.Warn("Error while writing response", mlog.Err(err))
+	}
+}
+
+func getQueryExplain(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.Srv().DebugBar().IsEnabled() {
+		c.Err = model.NewAppError("Api4.GetSystemInfo", "api.debugbar.getSystemInfo.disabled_debugbar.error", nil, "", http.StatusNotImplemented)
+		return
+	}
+
+	var requestBody struct {
+		Query string
+		Args  []any
+	}
+	if jsonErr := json.NewDecoder(r.Body).Decode(&requestBody); jsonErr != nil {
+		c.SetInvalidParamWithErr("explain_request", jsonErr)
+		return
+	}
+
+	explain, err := c.App.GetQueryExplain(requestBody.Query, requestBody.Args)
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(map[string]string{"explain": explain}); err != nil {
+		c.Logger.Warn("Error while writing response", mlog.Err(err))
+	}
+}

app/app_iface.go

@@ -633,6 +633,7 @@ type AppIface interface {
 	GetComplianceReports(page, perPage int) (model.Compliances, *model.AppError)
 	GetCookieDomain() string
 	GetCustomStatus(userID string) (*model.CustomStatus, *model.AppError)
+	GetDebugBarInfo() (*model.DebugBarInfo, *model.AppError)
 	GetDefaultProfileImage(user *model.User) ([]byte, *model.AppError)
 	GetDeletedChannels(c request.CTX, teamID string, offset int, limit int, userID string) (model.ChannelList, *model.AppError)
 	GetDraft(userID, channelID, rootID string) (*model.Draft, *model.AppError)
@@ -740,6 +741,7 @@ type AppIface interface {
 	GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
 	GetPublicChannelsByIdsForTeam(c request.CTX, teamID string, channelIDs []string) (model.ChannelList, *model.AppError)
 	GetPublicChannelsForTeam(c request.CTX, teamID string, offset int, limit int) (model.ChannelList, *model.AppError)
+	GetQueryExplain(query string, args []interface{}) (string, *model.AppError)
 	GetReactionsForPost(postID string) ([]*model.Reaction, *model.AppError)
 	GetRecentSearchesForUser(userID string) ([]*model.SearchParams, *model.AppError)
 	GetRecentlyActiveUsersForTeam(teamID string) (map[string]*model.User, *model.AppError)

app/debugbar.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"net/http"
+	"runtime"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func (a *App) GetDebugBarInfo() (*model.DebugBarInfo, *model.AppError) {
+	sessionsCount, err := a.Srv().Store().Session().AnalyticsSessionCount()
+	if err != nil {
+		return nil, model.NewAppError("GetDebugBarInfo", "debugbar.info.session-count.error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	// Here we are getting information regarding Elastic Search
+	var elasticServerVersion string
+	var elasticServerPlugins []string
+	if a.Srv().Platform().SearchEngine.ElasticsearchEngine != nil {
+		elasticServerVersion = a.Srv().Platform().SearchEngine.ElasticsearchEngine.GetFullVersion()
+		elasticServerPlugins = a.Srv().Platform().SearchEngine.ElasticsearchEngine.GetPlugins()
+	}
+
+	// Here we are getting information regarding LDAP
+	ldapInterface := a.Channels().Ldap
+	var vendorName, vendorVersion string
+	if ldapInterface != nil {
+		vendorName, vendorVersion = ldapInterface.GetVendorNameAndVendorVersion()
+	}
+
+	// Here we are getting information regarding the database (mysql/postgres + current schema version)
+	databaseType, databaseVersion := a.Srv().DatabaseTypeAndSchemaVersion()
+
+	info := model.DebugBarInfo{
+		SessionsCount:        sessionsCount,
+		GoVersion:            runtime.Version(),
+		Goroutines:           runtime.NumGoroutine(),
+		Cpus:                 runtime.NumCPU(),
+		CgoCalls:             runtime.NumCgoCall(),
+		ServerOS:             runtime.GOOS,
+		ServerArchitecture:   runtime.GOARCH,
+		ServerVersion:        model.CurrentVersion,
+		BuildHash:            model.BuildHash,
+		DatabaseType:         databaseType,
+		DatabaseVersion:      databaseVersion,
+		LdapVendorName:       vendorName,
+		LdapVendorVersion:    vendorVersion,
+		ElasticServerVersion: elasticServerVersion,
+		ElasticServerPlugins: elasticServerPlugins,
+	}
+
+	runtime.ReadMemStats(&info.GoMemStats)
+
+	totalSockets := a.TotalWebsocketConnections()
+	totalMasterDb := a.Srv().Store().TotalMasterDbConnections()
+	totalReadDb := a.Srv().Store().TotalReadDbConnections()
+
+	// If in HA mode then aggregate all the stats
+	if a.Cluster() != nil && *a.Config().ClusterSettings.Enable {
+		stats, appErr := a.Cluster().GetClusterStats()
+		if appErr != nil {
+			return nil, appErr
+		}
+
+		for _, stat := range stats {
+			totalSockets = totalSockets + stat.TotalWebsocketConnections
+			totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections
+			totalReadDb = totalReadDb + stat.TotalReadDbConnections
+		}
+	}
+
+	info.WebSocketConnections = totalSockets
+	info.MasterDBConnections = totalMasterDb
+	info.ReadDBConnections = totalReadDb
+
+	return &info, nil
+}
+
+func (a *App) GetQueryExplain(query string, args []interface{}) (string, *model.AppError) {
+	explain, err := a.Srv().Store().Explain(query, args)
+	if err != nil {
+		return "", model.NewAppError("GetQueryExplain", "debugbar.query_explain.error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	return explain, nil
+}

app/email/email.go

@@ -861,7 +861,11 @@ func (es *Service) sendEmailWithCustomReplyTo(to, subject, htmlBody, replyToAddr
 
 	category = getSendGridCategory(category, license.IsCloud())
 
-	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category)
+	err := mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category)
+	if es.debugBar != nil && es.debugBar() != nil && es.debugBar().IsEnabled() {
+		es.debugBar().SendEmailSent(to, subject, htmlBody, nil, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category, err)
+	}
+	return err
 }
 
 func (es *Service) sendMailWithCC(to, subject, htmlBody, ccMail, category string) error {
@@ -870,7 +874,11 @@ func (es *Service) sendMailWithCC(to, subject, htmlBody, ccMail, category string
 
 	category = getSendGridCategory(category, license.IsCloud())
 
-	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", ccMail, category)
+	err := mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", ccMail, category)
+	if es.debugBar != nil && es.debugBar() != nil && es.debugBar().IsEnabled() {
+		es.debugBar().SendEmailSent(to, subject, htmlBody, nil, mailConfig, license != nil && *license.Features.Compliance, "", "", "", ccMail, category, err)
+	}
+	return err
 }
 
 func (es *Service) SendMailWithEmbeddedFilesAndCustomReplyTo(to, subject, htmlBody, replyToAddress string, embeddedFiles map[string]io.Reader, category string) error {
@@ -879,7 +887,11 @@ func (es *Service) SendMailWithEmbeddedFilesAndCustomReplyTo(to, subject, htmlBo
 
 	category = getSendGridCategory(category, license.IsCloud())
 
-	return mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category)
+	err := mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category)
+	if es.debugBar != nil && es.debugBar() != nil && es.debugBar().IsEnabled() {
+		es.debugBar().SendEmailSent(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "", category, err)
+	}
+	return err
 }
 
 func (es *Service) SendMailWithEmbeddedFiles(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, messageID string, inReplyTo string, references string, category string) error {
@@ -888,7 +900,11 @@ func (es *Service) SendMailWithEmbeddedFiles(to, subject, htmlBody string, embed
 
 	category = getSendGridCategory(category, license.IsCloud())
 
-	return mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, messageID, inReplyTo, references, "", category)
+	err := mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, messageID, inReplyTo, references, "", category)
+	if es.debugBar != nil && es.debugBar() != nil && es.debugBar().IsEnabled() {
+		es.debugBar().SendEmailSent(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, messageID, inReplyTo, references, "", category, err)
+	}
+	return err
 }
 
 func (es *Service) InvalidateVerifyEmailTokensForUser(userID string) *model.AppError {

app/email/service.go

@@ -12,6 +12,7 @@ import (
 	"github.com/throttled/throttled"
 	"github.com/throttled/throttled/store/memstore"
 
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/app/users"
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/shared/i18n"
@@ -42,8 +43,9 @@ func condenseSiteURL(siteURL string) string {
 }
 
 type Service struct {
-	config  func() *model.Config
+	config   func() *model.Config
-	license func() *model.License
+	license  func() *model.License
+	debugBar func() *debugbar.DebugBar
 
 	userService *users.UserService
 	store       store.Store
@@ -57,6 +59,7 @@ type Service struct {
 type ServiceConfig struct {
 	ConfigFn  func() *model.Config
 	LicenseFn func() *model.License
+	DebugBar  func() *debugbar.DebugBar
 
 	TemplatesContainer *templates.Container
 	UserService        *users.UserService
@@ -71,6 +74,7 @@ func NewService(config ServiceConfig) (*Service, error) {
 		config:             config.ConfigFn,
 		templatesContainer: config.TemplatesContainer,
 		license:            config.LicenseFn,
+		debugBar:           config.DebugBar,
 		store:              config.Store,
 		userService:        config.UserService,
 	}

app/opentracing/opentracing_layer.go

@@ -5872,6 +5872,28 @@ func (a *OpenTracingAppLayer) GetCustomStatus(userID string) (*model.CustomStatu
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) GetDebugBarInfo() (*model.DebugBarInfo, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetDebugBarInfo")
+
+	a.ctx = newCtx
+	a.app.Srv().Store().SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store().SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.GetDebugBarInfo()
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) GetDefaultProfileImage(user *model.User) ([]byte, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetDefaultProfileImage")
@@ -8560,6 +8582,28 @@ func (a *OpenTracingAppLayer) GetPublicKey(name string) ([]byte, *model.AppError
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) GetQueryExplain(query string, args []interface{}) (string, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetQueryExplain")
+
+	a.ctx = newCtx
+	a.app.Srv().Store().SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store().SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.GetQueryExplain(query, args)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) GetReactionsForPost(postID string) ([]*model.Reaction, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetReactionsForPost")

app/platform/config.go

@@ -286,6 +286,7 @@ func (ps *PlatformService) LimitedClientConfigWithComputed() map[string]string {
 	// These properties are not configurable, but nevertheless represent configuration expected
 	// by the client.
 	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+	respCfg["DebugBar"] = strconv.FormatBool(ps.DebugBar.IsEnabled())
 
 	return respCfg
 }
@@ -300,6 +301,7 @@ func (ps *PlatformService) ClientConfigWithComputed() map[string]string {
 	// These properties are not configurable, but nevertheless represent configuration expected
 	// by the client.
 	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+	respCfg["DebugBar"] = strconv.FormatBool(ps.DebugBar.IsEnabled())
 	respCfg["MaxPostSize"] = strconv.Itoa(ps.MaxPostSize())
 	respCfg["UpgradedFromTE"] = strconv.FormatBool(ps.isUpgradedFromTE())
 	respCfg["InstallationDate"] = ""

app/platform/debugbar/debugbar.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package debugbar
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mail"
+)
+
+const (
+	socketEventName = "debugbar"
+)
+
+type DebugBar struct {
+	publish func(*model.WebSocketEvent)
+	enabled bool
+}
+
+func New(publish func(*model.WebSocketEvent)) *DebugBar {
+	return &DebugBar{
+		publish: publish,
+		enabled: os.Getenv("MM_ENABLE_DEBUG_BAR") == "true",
+	}
+}
+
+func (db *DebugBar) IsEnabled() bool {
+	return db.enabled
+}
+
+func (db *DebugBar) SendLogEvent(logLevel string, logMessage string, fields map[string]string) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "log-line")
+	event.Add("level", logLevel)
+	event.Add("message", logMessage)
+	event.Add("fields", fields)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendApiCall(endpoint, method, statusCode string, elapsed float64) {
+	if endpoint == "getSystemInfo" || endpoint == "getQueryExplain" {
+		return
+	}
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "api-call")
+	event.Add("endpoint", endpoint)
+	event.Add("method", method)
+	event.Add("statusCode", statusCode)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendStoreCall(method string, success bool, elapsed float64, params map[string]any) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "store-call")
+	event.Add("method", method)
+	event.Add("params", params)
+	event.Add("success", success)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendSqlQuery(query string, elapsed float64, args ...any) {
+	if strings.HasPrefix(query, "EXPLAIN ") {
+		return
+	}
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "sql-query")
+	event.Add("query", query)
+	event.Add("args", args)
+	event.Add("duration", elapsed)
+	db.publish(event)
+}
+
+func (db *DebugBar) SendEmailSent(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *mail.SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string, category string, err error) {
+	event := model.NewWebSocketEvent(socketEventName, "", "", "", nil, "")
+	event.Add("time", model.GetMillis())
+	event.Add("type", "email-sent")
+	event.Add("to", to)
+	event.Add("subject", subject)
+	event.Add("htmlBody", htmlBody)
+	event.Add("embeddedFiles", embeddedFiles)
+	event.Add("SMTPConfig", config)
+	event.Add("enableComplianceFeatures", enableComplianceFeatures)
+	event.Add("messageID", messageID)
+	event.Add("inReplyTo", inReplyTo)
+	event.Add("references", references)
+	event.Add("cc", ccMail)
+	event.Add("category", category)
+	event.Add("err", err)
+	db.publish(event)
+}

app/platform/debugbar/logger.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package debugbar
+
+import (
+	"bytes"
+
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+type DebugBarLogTarget struct {
+	debugBar *DebugBar
+}
+
+type DebugBarLogFilter struct{}
+
+func (_ *DebugBarLogFilter) GetEnabledLevel(level logr.Level) (logr.Level, bool) {
+	return level, true
+}
+
+type DebugBarLogFormatter struct{}
+
+func (_ *DebugBarLogFormatter) IsStacktraceNeeded() bool {
+	return false
+}
+
+func (_ *DebugBarLogFormatter) Format(rec *logr.LogRec, level logr.Level, buf *bytes.Buffer) (*bytes.Buffer, error) {
+	return bytes.NewBuffer([]byte{}), nil
+}
+
+func NewDebugBarLogTarget(debugBar *DebugBar) *DebugBarLogTarget {
+	return &DebugBarLogTarget{debugBar: debugBar}
+}
+
+func (dblt *DebugBarLogTarget) Init() error {
+	return nil
+}
+
+func (dblt *DebugBarLogTarget) Shutdown() error {
+	return nil
+}
+
+func (dblt *DebugBarLogTarget) Write(p []byte, rec *logr.LogRec) (int, error) {
+	dblt.debugBar.SendLogEvent(rec.Level().Name, rec.Msg(), dblt.fieldsToStringsMap(rec.Fields()...))
+	return len(p), nil
+}
+
+func (dblt *DebugBarLogTarget) fieldsToStringsMap(fields ...mlog.Field) map[string]string {
+	result := map[string]string{}
+	for _, field := range fields {
+		value := &bytes.Buffer{}
+		field.ValueString(value, nil)
+		result[field.Key] = value.String()
+	}
+	return result
+}

app/platform/log.go

@@ -13,6 +13,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
@@ -78,6 +80,13 @@ func (ps *PlatformService) initLogging() error {
 		}
 	}
 
+	if ps.DebugBar.IsEnabled() {
+		err := ps.logger.AddTarget(debugbar.NewDebugBarLogTarget(ps.DebugBar), "debugbar", &debugbar.DebugBarLogFilter{}, &debugbar.DebugBarLogFormatter{}, logr.DefaultMaxQueueSize)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

app/platform/service.go

@@ -12,6 +12,7 @@ import (
 	"sync/atomic"
 
 	"github.com/mattermost/mattermost-server/v6/app/featureflag"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/einterfaces"
 	"github.com/mattermost/mattermost-server/v6/jobs"
@@ -23,6 +24,7 @@ import (
 	"github.com/mattermost/mattermost-server/v6/shared/filestore"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 	"github.com/mattermost/mattermost-server/v6/store"
+	"github.com/mattermost/mattermost-server/v6/store/debugbarlayer"
 	"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
 	"github.com/mattermost/mattermost-server/v6/store/retrylayer"
 	"github.com/mattermost/mattermost-server/v6/store/searchlayer"
@@ -34,9 +36,11 @@ import (
 // responsible for non-entity related functionalities that are required
 // by a product such as database access, configuration access, licensing etc.
 type PlatformService struct {
-	sqlStore *sqlstore.SqlStore
+	sqlStore   *sqlstore.SqlStore
-	Store    store.Store
+	DebugBar   *debugbar.DebugBar
-	newStore func() (store.Store, error)
+	Store      store.Store
+	newStore   func() (store.Store, error)
+	LastUserID string
 
 	WebSocketRouter *WebSocketRouter
 
@@ -125,6 +129,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 		licenseListeners:          map[string]func(*model.License, *model.License){},
 		additionalClusterHandlers: map[model.ClusterEvent]einterfaces.ClusterMessageHandler{},
 	}
+	ps.DebugBar = debugbar.New(ps.Publish)
 
 	// Step 1: Cache provider.
 	// At the moment we only have this implementation
@@ -186,7 +191,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 	// Depends on Step 0 (config), 1 (cacheProvider), 3 (search engine), 5 (metrics) and cluster.
 	if ps.newStore == nil {
 		ps.newStore = func() (store.Store, error) {
-			ps.sqlStore = sqlstore.New(ps.Config().SqlSettings, ps.metricsIFace)
+			ps.sqlStore = sqlstore.New(ps.Config().SqlSettings, ps.metricsIFace, ps.DebugBar.SendSqlQuery)
 
 			lcl, err2 := localcachelayer.NewLocalCacheLayer(
 				retrylayer.New(ps.sqlStore),
@@ -214,10 +219,15 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 				ps.sqlStore.UpdateLicense(newLicense)
 			})
 
-			return timerlayer.New(
+			timerStore := timerlayer.New(
 				searchStore,
 				ps.metricsIFace,
-			), nil
+			)
+
+			if ps.DebugBar.IsEnabled() {
+				return debugbarlayer.New(timerStore, ps.DebugBar), nil
+			}
+			return timerStore, nil
 		}
 	}
 

app/platform/service_test.go

@@ -142,6 +142,8 @@ func TestMetrics(t *testing.T) {
 		mockMetricsImpl.On("Register").Return()
 		mockMetricsImpl.On("ObserveStoreMethodDuration", mock.Anything, mock.Anything, mock.Anything).Return()
 		mockMetricsImpl.On("RegisterDBCollector", mock.AnythingOfType("*sql.DB"), "master")
+		mockMetricsImpl.On("IncrementWebsocketEvent", "debugbar")
+		mockMetricsImpl.On("IncrementWebSocketBroadcastBufferSize", mock.AnythingOfType("string"), float64(1))
 
 		th := Setup(t, StartMetrics(), func(ps *PlatformService) error {
 			ps.metricsIFace = mockMetricsImpl

app/plugin_api_tests/test_db_driver/main.go

@@ -31,7 +31,7 @@ func (p *MyPlugin) OnConfigurationChange() error {
 func (p *MyPlugin) MessageWillBePosted(_ *plugin.Context, _ *model.Post) (*model.Post, string) {
 	settings := p.API.GetUnsanitizedConfig().SqlSettings
 	settings.Trace = model.NewBool(false)
-	store := sqlstore.New(settings, nil)
+	store := sqlstore.New(settings, nil, nil)
 	store.GetMasterX().Close()
 
 	for _, isMaster := range []bool{true, false} {

app/server.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/mattermost/mattermost-server/v6/app/email"
 	"github.com/mattermost/mattermost-server/v6/app/platform"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
 	"github.com/mattermost/mattermost-server/v6/app/request"
 	"github.com/mattermost/mattermost-server/v6/app/teams"
 	"github.com/mattermost/mattermost-server/v6/app/users"
@@ -143,6 +144,14 @@ type Server struct {
 	hooksManager *product.HooksManager
 }
 
+func (s *Server) DebugBar() *debugbar.DebugBar {
+	if s.platform != nil {
+		return s.platform.DebugBar
+	}
+
+	return nil
+}
+
 func (s *Server) Store() store.Store {
 	if s.platform != nil {
 		return s.platform.Store
@@ -366,6 +375,7 @@ func NewServer(options ...Option) (*Server, error) {
 	emailService, err := email.NewService(email.ServiceConfig{
 		ConfigFn:           s.platform.Config,
 		LicenseFn:          s.License,
+		DebugBar:           s.DebugBar,
 		TemplatesContainer: s.TemplatesContainer(),
 		UserService:        s.userService,
 		Store:              s.GetStore(),

cmd/mattermost/commands/db.go

@@ -92,7 +92,7 @@ func initDbCmdF(command *cobra.Command, _ []string) error {
 	}
 	defer configStore.Close()
 
-	sqlStore := sqlstore.New(configStore.Get().SqlSettings, nil)
+	sqlStore := sqlstore.New(configStore.Get().SqlSettings, nil, nil)
 	defer sqlStore.Close()
 
 	fmt.Println("Database store correctly initialised")
@@ -140,7 +140,7 @@ func migrateCmdF(command *cobra.Command, args []string) error {
 	}
 	config := cfgStore.Get()
 
-	store := sqlstore.New(config.SqlSettings, nil)
+	store := sqlstore.New(config.SqlSettings, nil, nil)
 	defer store.Close()
 
 	CommandPrettyPrintln("Database successfully migrated")
@@ -156,7 +156,7 @@ func dbVersionCmdF(command *cobra.Command, args []string) error {
 	}
 	config := cfgStore.Get()
 
-	store := sqlstore.New(config.SqlSettings, nil)
+	store := sqlstore.New(config.SqlSettings, nil, nil)
 	defer store.Close()
 
 	allFlag, _ := command.Flags().GetBool("all")

i18n/en.json

@@ -1658,6 +1658,10 @@
     "id": "api.custom_status.set_custom_statuses.update.app_error",
     "translation": "Failed to update the custom status. Please add either emoji or custom text status or both."
   },
+  {
+    "id": "api.debugbar.getSystemInfo.disabled_debugbar.error",
+    "translation": "DebugBar feature is disabled."
+  },
   {
     "id": "api.draft.create_draft.can_not_draft_to_deleted.error",
     "translation": "Can not save draft to deleted channel"
@@ -7347,6 +7351,14 @@
     "id": "common.parse_error_int64",
     "translation": "Failed to parse the value:{{.Value}} to int64"
   },
+  {
+    "id": "debugbar.info.session-count.error",
+    "translation": "Unable to count sessions."
+  },
+  {
+    "id": "debugbar.query_explain.error",
+    "translation": "Unable to execute the query explain."
+  },
   {
     "id": "ent.account_migration.get_all_failed",
     "translation": "Unable to get users."

model/debugbar.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package model
+
+import "runtime"
+
+type DebugBarInfo struct {
+	ServerOS             string
+	ServerArchitecture   string
+	ServerVersion        string
+	BuildHash            string
+	DatabaseType         string
+	DatabaseVersion      string
+	LdapVendorName       string
+	LdapVendorVersion    string
+	ElasticServerVersion string
+	ElasticServerPlugins []string
+	WebSocketConnections int
+	MasterDBConnections  int
+	ReadDBConnections    int
+	SessionsCount        int64
+	Goroutines           int
+	Cpus                 int
+	CgoCalls             int64
+	GoVersion            string
+	GoMemStats           runtime.MemStats
+}

services/searchengine/bleveengine/bleve_test.go

@@ -49,7 +49,7 @@ func (s *BleveEngineTestSuite) setupStore() {
 		driverName = model.DatabaseDriverPostgres
 	}
 	s.SQLSettings = storetest.MakeSqlSettings(driverName, false)
-	s.SQLStore = sqlstore.New(*s.SQLSettings, nil)
+	s.SQLStore = sqlstore.New(*s.SQLSettings, nil, nil)
 
 	cfg := &model.Config{}
 	cfg.SetDefaults()

shared/mlog/levels.go

@@ -47,6 +47,9 @@ var (
 	LvlSharedChannelServiceMessagesInbound  = Level{ID: 203, Name: "SharedChannelServiceMsgInbound"}
 	LvlSharedChannelServiceMessagesOutbound = Level{ID: 204, Name: "SharedChannelServiceMsgOutbound"}
 
+	// DebugBar
+	LvlDebugBar = Level{ID: 300, Name: "DebugBar"}
+
 	// Focalboard
 	LvlFBTelemetry = Level{ID: 9000, Name: "telemetry"}
 	LvlFBMetrics   = Level{ID: 9001, Name: "metrics"}

shared/mlog/mlog.go

@@ -48,6 +48,8 @@ type Field = logr.Field
 type Level = logr.Level
 type Option = logr.Option
 type Target = logr.Target
+type Filter = logr.Filter
+type Formatter = logr.Formatter
 type TargetInfo = logr.TargetInfo
 type LogRec = logr.LogRec
 type LogCloner = logr.LogCloner
@@ -178,6 +180,11 @@ func NewLogger(options ...Option) (*Logger, error) {
 	}, nil
 }
 
+// AddTarget adds a new logr.Target to the Logger object.
+func (l *Logger) AddTarget(target Target, name string, filter Filter, formatter Formatter, maxQueueSize int) error {
+	return l.log.Logr().AddTarget(target, name, filter, formatter, maxQueueSize)
+}
+
 // Configure provides a new configuration for this logger.
 // Zero or more sources of config can be provided:
 //

store/layer_generators/debugbar_layer.go.tmpl

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "make store-layers"
+// DO NOT EDIT
+
+package debugbarlayer
+
+import (
+	"context"
+    "time"
+    //"fmt"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/app/platform/debugbar"
+	"github.com/mattermost/mattermost-server/v6/store"
+)
+
+type {{.Name}} struct {
+	store.Store
+    debugBar *debugbar.DebugBar
+    eventPublish func(event *model.WebSocketEvent)
+{{range $index, $element := .SubStores}}	{{$index}}Store store.{{$index}}Store
+{{end}}
+}
+
+{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
+	return s.{{$index}}Store
+}
+
+{{end}}
+
+{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
+	store.{{$index}}Store
+	Root *{{$.Name}}
+}
+
+{{end}}
+
+{{range $substoreName, $substore := .SubStores}}
+{{range $index, $element := $substore.Methods}}
+func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	start := time.Now()
+	{{if $element.Results | len | eq 0}}
+	s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}
+	{{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+    success := false
+    if {{$element.Results | errorToBoolean}} {
+        success = true
+    }
+
+    debugBarLayerParams := map[string]any{}
+    {{range $paramIndex, $param := $element.Params}}
+    debugBarLayerParams["{{$param.Name}}"] = {{$param.Name}}
+    {{end}}
+    s.Root.debugBar.SendStoreCall("{{$substoreName}}Store.{{$index}}", success, elapsed, debugBarLayerParams)
+
+	{{ with (genResultsVars $element.Results false ) -}}
+	return {{ . }}
+	{{- end }}
+}
+{{end}}
+{{end}}
+
+{{range $index, $element := .Methods}}
+func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}}
+{{end}}
+
+func New(childStore store.Store, debugBar *debugbar.DebugBar) *{{.Name}} {
+	newStore := {{.Name}}{
+		Store: childStore,
+        debugBar: debugBar,
+	}
+	{{range $substoreName, $substore := .SubStores}}
+	newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
+	return &newStore
+}

store/layer_generators/main.go

@@ -28,6 +28,9 @@ func isError(typeName string) bool {
 }
 
 func main() {
+	if err := buildDebugBarLayer(); err != nil {
+		log.Fatal(err)
+	}
 	if err := buildTimerLayer(); err != nil {
 		log.Fatal(err)
 	}
@@ -65,6 +68,19 @@ func buildTimerLayer() error {
 	return os.WriteFile(path.Join("timerlayer", "timerlayer.go"), formatedCode, 0644)
 }
 
+func buildDebugBarLayer() error {
+	code, err := generateLayer("DebugBarLayer", "debugbar_layer.go.tmpl")
+	if err != nil {
+		return err
+	}
+	formatedCode, err := format.Source(code)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(path.Join("debugbarlayer", "debugbarlayer.go"), formatedCode, 0644)
+}
+
 func buildOpenTracingLayer() error {
 	code, err := generateLayer("OpenTracingLayer", "opentracing_layer.go.tmpl")
 	if err != nil {

store/localcachelayer/layer_test.go

@@ -98,7 +98,7 @@ func initStores() {
 		go func() {
 			var err error
 			defer wg.Done()
-			st.SqlStore = sqlstore.New(*st.SqlSettings, nil)
+			st.SqlStore = sqlstore.New(*st.SqlSettings, nil, nil)
 			st.Store, err = NewLocalCacheLayer(st.SqlStore, nil, nil, getMockCacheProvider())
 			if err != nil {
 				panic(err)

store/searchlayer/layer_test.go

@@ -24,7 +24,7 @@ func TestUpdateConfigRace(t *testing.T) {
 		driverName = model.DatabaseDriverPostgres
 	}
 	settings := storetest.MakeSqlSettings(driverName, false)
-	store := sqlstore.New(*settings, nil)
+	store := sqlstore.New(*settings, nil, nil)
 
 	cfg := &model.Config{}
 	cfg.SetDefaults()

store/sqlstore/sqlx_wrapper.go

@@ -64,15 +64,17 @@ var namedParamRegex = regexp.MustCompile(`:\w+`)
 
 type sqlxDBWrapper struct {
 	*sqlx.DB
-	queryTimeout time.Duration
+	queryTimeout    time.Duration
-	trace        bool
+	trace           bool
+	debugbarPublish func(string, float64, ...any)
 }
 
-func newSqlxDBWrapper(db *sqlx.DB, timeout time.Duration, trace bool) *sqlxDBWrapper {
+func newSqlxDBWrapper(db *sqlx.DB, timeout time.Duration, trace bool, debugbarPublish func(string, float64, ...any)) *sqlxDBWrapper {
 	return &sqlxDBWrapper{
-		DB:           db,
+		DB:              db,
-		queryTimeout: timeout,
+		queryTimeout:    timeout,
-		trace:        trace,
+		trace:           trace,
+		debugbarPublish: debugbarPublish,
 	}
 }
 
@@ -86,7 +88,7 @@ func (w *sqlxDBWrapper) Beginx() (*sqlxTxWrapper, error) {
 		return nil, err
 	}
 
-	return newSqlxTxWrapper(tx, w.queryTimeout, w.trace), nil
+	return newSqlxTxWrapper(tx, w.queryTimeout, w.trace, w.debugbarPublish), nil
 }
 
 func (w *sqlxDBWrapper) BeginXWithIsolation(opts *sql.TxOptions) (*sqlxTxWrapper, error) {
@@ -95,7 +97,7 @@ func (w *sqlxDBWrapper) BeginXWithIsolation(opts *sql.TxOptions) (*sqlxTxWrapper
 		return nil, err
 	}
 
-	return newSqlxTxWrapper(tx, w.queryTimeout, w.trace), nil
+	return newSqlxTxWrapper(tx, w.queryTimeout, w.trace, w.debugbarPublish), nil
 }
 
 func (w *sqlxDBWrapper) Get(dest any, query string, args ...any) error {
@@ -108,6 +110,11 @@ func (w *sqlxDBWrapper) Get(dest any, query string, args ...any) error {
 			printArgs(query, time.Since(then), args)
 		}(time.Now())
 	}
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
 
 	return w.DB.GetContext(ctx, dest, query, args...)
 }
@@ -134,6 +141,12 @@ func (w *sqlxDBWrapper) NamedExec(query string, arg any) (sql.Result, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), arg)
+		}(time.Now())
+	}
+
 	return w.DB.NamedExecContext(ctx, query, arg)
 }
 
@@ -161,6 +174,12 @@ func (w *sqlxDBWrapper) ExecNoTimeout(query string, args ...any) (sql.Result, er
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.DB.ExecContext(context.Background(), query, args...)
 }
 
@@ -176,6 +195,12 @@ func (w *sqlxDBWrapper) ExecRaw(query string, args ...any) (sql.Result, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.DB.ExecContext(ctx, query, args...)
 }
 
@@ -192,6 +217,12 @@ func (w *sqlxDBWrapper) NamedQuery(query string, arg any) (*sqlx.Rows, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), arg)
+		}(time.Now())
+	}
+
 	return w.DB.NamedQueryContext(ctx, query, arg)
 }
 
@@ -206,6 +237,12 @@ func (w *sqlxDBWrapper) QueryRowX(query string, args ...any) *sqlx.Row {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.DB.QueryRowxContext(ctx, query, args...)
 }
 
@@ -220,6 +257,12 @@ func (w *sqlxDBWrapper) QueryX(query string, args ...any) (*sqlx.Rows, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.DB.QueryxContext(ctx, query, args)
 }
 
@@ -238,6 +281,12 @@ func (w *sqlxDBWrapper) SelectCtx(ctx context.Context, dest any, query string, a
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.DB.SelectContext(ctx, dest, query, args...)
 }
 
@@ -252,15 +301,17 @@ func (w *sqlxDBWrapper) SelectBuilder(dest any, builder Builder) error {
 
 type sqlxTxWrapper struct {
 	*sqlx.Tx
-	queryTimeout time.Duration
+	queryTimeout    time.Duration
-	trace        bool
+	trace           bool
+	debugbarPublish func(string, float64, ...any)
 }
 
-func newSqlxTxWrapper(tx *sqlx.Tx, timeout time.Duration, trace bool) *sqlxTxWrapper {
+func newSqlxTxWrapper(tx *sqlx.Tx, timeout time.Duration, trace bool, debugbarPublish func(string, float64, ...any)) *sqlxTxWrapper {
 	return &sqlxTxWrapper{
-		Tx:           tx,
+		Tx:              tx,
-		queryTimeout: timeout,
+		queryTimeout:    timeout,
-		trace:        trace,
+		trace:           trace,
+		debugbarPublish: debugbarPublish,
 	}
 }
 
@@ -275,6 +326,12 @@ func (w *sqlxTxWrapper) Get(dest any, query string, args ...any) error {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.GetContext(ctx, dest, query, args...)
 }
 
@@ -302,6 +359,12 @@ func (w *sqlxTxWrapper) ExecNoTimeout(query string, args ...any) (sql.Result, er
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.ExecContext(context.Background(), query, args...)
 }
 
@@ -326,6 +389,12 @@ func (w *sqlxTxWrapper) ExecRaw(query string, args ...any) (sql.Result, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.ExecContext(ctx, query, args...)
 }
 
@@ -342,6 +411,12 @@ func (w *sqlxTxWrapper) NamedExec(query string, arg any) (sql.Result, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), arg)
+		}(time.Now())
+	}
+
 	return w.Tx.NamedExecContext(ctx, query, arg)
 }
 
@@ -358,6 +433,12 @@ func (w *sqlxTxWrapper) NamedQuery(query string, arg any) (*sqlx.Rows, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), arg)
+		}(time.Now())
+	}
+
 	// There is no tx.NamedQueryContext support in the sqlx API. (https://github.com/jmoiron/sqlx/issues/447)
 	// So we need to implement this ourselves.
 	type result struct {
@@ -400,6 +481,12 @@ func (w *sqlxTxWrapper) QueryRowX(query string, args ...any) *sqlx.Row {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.QueryRowxContext(ctx, query, args...)
 }
 
@@ -414,6 +501,12 @@ func (w *sqlxTxWrapper) QueryX(query string, args ...any) (*sqlx.Rows, error) {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.QueryxContext(ctx, query, args)
 }
 
@@ -428,6 +521,12 @@ func (w *sqlxTxWrapper) Select(dest any, query string, args ...any) error {
 		}(time.Now())
 	}
 
+	if w.debugbarPublish != nil {
+		defer func(then time.Time) {
+			w.debugbarPublish(query, float64(time.Since(then))/float64(time.Second), args...)
+		}(time.Now())
+	}
+
 	return w.Tx.SelectContext(ctx, dest, query, args...)
 }
 

store/sqlstore/store.go

@@ -130,6 +130,7 @@ type SqlStore struct {
 	replicaLagHandles []*dbsql.DB
 	stores            SqlStoreStores
 	settings          *model.SqlSettings
+	debugbarPublish   func(string, float64, ...any)
 	lockedToMaster    bool
 	context           context.Context
 	license           *model.License
@@ -140,12 +141,13 @@ type SqlStore struct {
 	pgDefaultTextSearchConfig string
 }
 
-func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlStore {
+func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface, debugbarPublish func(string, float64, ...any)) *SqlStore {
 	store := &SqlStore{
-		rrCounter: 0,
+		rrCounter:       0,
-		srCounter: 0,
+		srCounter:       0,
-		settings:  &settings,
+		settings:        &settings,
-		metrics:   metrics,
+		metrics:         metrics,
+		debugbarPublish: debugbarPublish,
 	}
 
 	store.initConnection()
@@ -299,7 +301,9 @@ func (ss *SqlStore) initConnection() {
 	handle := SetupConnection("master", dataSource, ss.settings)
 	ss.masterX = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
 		time.Duration(*ss.settings.QueryTimeout)*time.Second,
-		*ss.settings.Trace)
+		*ss.settings.Trace,
+		ss.debugbarPublish,
+	)
 	if ss.DriverName() == model.DatabaseDriverMysql {
 		ss.masterX.MapperFunc(noOpMapper)
 	}
@@ -313,7 +317,9 @@ func (ss *SqlStore) initConnection() {
 			handle := SetupConnection(fmt.Sprintf("replica-%v", i), replica, ss.settings)
 			ss.ReplicaXs[i] = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
 				time.Duration(*ss.settings.QueryTimeout)*time.Second,
-				*ss.settings.Trace)
+				*ss.settings.Trace,
+				ss.debugbarPublish,
+			)
 			if ss.DriverName() == model.DatabaseDriverMysql {
 				ss.ReplicaXs[i].MapperFunc(noOpMapper)
 			}
@@ -329,7 +335,9 @@ func (ss *SqlStore) initConnection() {
 			handle := SetupConnection(fmt.Sprintf("search-replica-%v", i), replica, ss.settings)
 			ss.searchReplicaXs[i] = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
 				time.Duration(*ss.settings.QueryTimeout)*time.Second,
-				*ss.settings.Trace)
+				*ss.settings.Trace,
+				ss.debugbarPublish,
+			)
 			if ss.DriverName() == model.DatabaseDriverMysql {
 				ss.searchReplicaXs[i].MapperFunc(noOpMapper)
 			}
@@ -434,7 +442,9 @@ func (ss *SqlStore) GetMasterX() *sqlxDBWrapper {
 func (ss *SqlStore) SetMasterX(db *sql.DB) {
 	ss.masterX = newSqlxDBWrapper(sqlx.NewDb(db, ss.DriverName()),
 		time.Duration(*ss.settings.QueryTimeout)*time.Second,
-		*ss.settings.Trace)
+		*ss.settings.Trace,
+		ss.debugbarPublish,
+	)
 	if ss.DriverName() == model.DatabaseDriverMysql {
 		ss.masterX.MapperFunc(noOpMapper)
 	}
@@ -1284,3 +1294,17 @@ func (ss *SqlStore) GetAppliedMigrations() ([]model.AppliedMigration, error) {
 
 	return migrations, nil
 }
+
+func (ss *SqlStore) Explain(query string, args []any) (string, error) {
+	var explain []string
+
+	if strings.HasPrefix(query, "ANALYZE") {
+		return "", errors.New("not allowed to explain queries with analyze at the beginning")
+	}
+
+	if err := ss.GetMasterX().Select(&explain, "EXPLAIN "+query, args...); err != nil {
+		return "", errors.Wrap(err, "unable to run the explain query")
+	}
+
+	return strings.Join(explain, "\n"), nil
+}

store/sqlstore/store_test.go

@@ -132,7 +132,7 @@ func initStores() {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			st.SqlStore = New(*st.SqlSettings, nil)
+			st.SqlStore = New(*st.SqlSettings, nil, nil)
 			st.Store = st.SqlStore
 			st.Store.DropAllTables()
 			st.Store.MarkSystemRanUnitTests()
@@ -171,7 +171,7 @@ func tearDownStores() {
 // Keeping it here to help avoiding future regressions.
 func TestStoreLicenseRace(t *testing.T) {
 	settings := makeSqlSettings(model.DatabaseDriverPostgres)
-	store := New(*settings, nil)
+	store := New(*settings, nil, nil)
 	defer func() {
 		store.Close()
 		storetest.CleanupSqlSettings(settings)
@@ -268,7 +268,7 @@ func TestGetReplica(t *testing.T) {
 
 			settings.DataSourceReplicas = dataSourceReplicas
 			settings.DataSourceSearchReplicas = dataSourceSearchReplicas
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 			defer func() {
 				store.Close()
 				storetest.CleanupSqlSettings(settings)
@@ -338,7 +338,7 @@ func TestGetReplica(t *testing.T) {
 
 			settings.DataSourceReplicas = dataSourceReplicas
 			settings.DataSourceSearchReplicas = dataSourceSearchReplicas
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 			defer func() {
 				store.Close()
 				storetest.CleanupSqlSettings(settings)
@@ -402,7 +402,7 @@ func TestGetDbVersion(t *testing.T) {
 		t.Run("Should return db version for "+driver, func(t *testing.T) {
 			t.Parallel()
 			settings := makeSqlSettings(driver)
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 
 			version, err := store.GetDbVersion(false)
 			require.NoError(t, err)
@@ -546,7 +546,7 @@ func TestUpAndDownMigrations(t *testing.T) {
 	for _, driver := range testDrivers {
 		t.Run("Should be reversible for "+driver, func(t *testing.T) {
 			settings := makeSqlSettings(driver)
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 			defer store.Close()
 
 			err := store.migrate(migrationsDirectionDown)
@@ -635,7 +635,7 @@ func TestGetAllConns(t *testing.T) {
 
 			settings.DataSourceReplicas = dataSourceReplicas
 			settings.DataSourceSearchReplicas = dataSourceSearchReplicas
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 			defer func() {
 				store.Close()
 				storetest.CleanupSqlSettings(settings)
@@ -819,7 +819,7 @@ func TestGetDBSchemaVersion(t *testing.T) {
 		t.Run("Should return latest version number of applied migrations for "+driver, func(t *testing.T) {
 			t.Parallel()
 			settings := makeSqlSettings(driver)
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 
 			assetsList, err := assets.ReadDir(filepath.Join("migrations", driver))
 			require.NoError(t, err)
@@ -853,7 +853,7 @@ func TestGetAppliedMigrations(t *testing.T) {
 		t.Run("Should return db applied migrations for "+driver, func(t *testing.T) {
 			t.Parallel()
 			settings := makeSqlSettings(driver)
-			store := New(*settings, nil)
+			store := New(*settings, nil, nil)
 
 			assetsList, err := assets.ReadDir(filepath.Join("migrations", driver))
 			require.NoError(t, err)

store/store.go

@@ -88,6 +88,7 @@ type Store interface {
 	PostPriority() PostPriorityStore
 	PostAcknowledgement() PostAcknowledgementStore
 	TrueUpReview() TrueUpReviewStore
+	Explain(query string, args []interface{}) (string, error)
 }
 
 type RetentionPolicyStore interface {

store/storetest/mocks/Store.go

@@ -224,6 +224,27 @@ func (_m *Store) Emoji() store.EmojiStore {
 	return r0
 }
 
+// Explain provides a mock function with given fields: query, args
+func (_m *Store) Explain(query string, args []interface{}) (string, error) {
+	ret := _m.Called(query, args)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, []interface{}) string); ok {
+		r0 = rf(query, args)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, []interface{}) error); ok {
+		r1 = rf(query, args)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // FileInfo provides a mock function with given fields:
 func (_m *Store) FileInfo() store.FileInfoStore {
 	ret := _m.Called()

store/storetest/store.go

@@ -133,6 +133,8 @@ func (s *Store) CheckIntegrity() <-chan model.IntegrityCheckResult {
 func (s *Store) ReplicaLagAbs() error  { return nil }
 func (s *Store) ReplicaLagTime() error { return nil }
 
+func (s *Store) Explain(query string, args []any) (string, error) { return "", nil }
+
 func (s *Store) AssertExpectations(t mock.TestingT) bool {
 	return mock.AssertExpectationsForObjects(t,
 		&s.TeamStore,

testlib/helper.go

@@ -103,7 +103,7 @@ func (h *MainHelper) setupStore(withReadReplica bool) {
 
 	h.SearchEngine = searchengine.NewBroker(config)
 	h.ClusterInterface = &FakeClusterInterface{}
-	h.SQLStore = sqlstore.New(*h.Settings, nil)
+	h.SQLStore = sqlstore.New(*h.Settings, nil, nil)
 	h.Store = searchlayer.NewSearchLayer(&TestStore{
 		h.SQLStore,
 	}, h.SearchEngine, config)
@@ -115,7 +115,7 @@ func (h *MainHelper) ToggleReplicasOff() {
 	}
 	h.Settings.DataSourceReplicas = []string{}
 	lic := h.SQLStore.GetLicense()
-	h.SQLStore = sqlstore.New(*h.Settings, nil)
+	h.SQLStore = sqlstore.New(*h.Settings, nil, nil)
 	h.SQLStore.UpdateLicense(lic)
 }
 
@@ -125,7 +125,7 @@ func (h *MainHelper) ToggleReplicasOn() {
 	}
 	h.Settings.DataSourceReplicas = h.replicas
 	lic := h.SQLStore.GetLicense()
-	h.SQLStore = sqlstore.New(*h.Settings, nil)
+	h.SQLStore = sqlstore.New(*h.Settings, nil, nil)
 	h.SQLStore.UpdateLicense(lic)
 }
 

web/handlers.go

@@ -401,6 +401,20 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	statusCode = strconv.Itoa(w.(*responseWriterWrapper).StatusCode())
+
+	if c.App.Srv().DebugBar().IsEnabled() {
+		elapsed := float64(time.Since(now)) / float64(time.Second)
+		var endpoint string
+		if strings.HasPrefix(r.URL.Path, model.APIURLSuffixV5) {
+			// It's a graphQL query, so use the operation name.
+			endpoint = c.GraphQLOperationName
+		} else {
+			endpoint = h.HandlerName
+		}
+
+		c.App.Srv().DebugBar().SendApiCall(endpoint, r.Method, statusCode, elapsed)
+	}
+
 	if c.App.Metrics() != nil {
 		c.App.Metrics().IncrementHTTPRequest()