MM-61887: Log the userID if a metric exceeds the last histogram bucket (#29448)

We create a custom histogram metric that logs the userID
when the observed value is greater or equal to the last bucket value.

This allows us to start tracking the slowest users of a system
while at the same time not polluting the Prometheus metrics
by storing a userID for every observation.

https://mattermost.atlassian.net/browse/MM-61887

```release-note
NONE
```

server/channels/app/metrics.go

@@ -14,6 +14,7 @@ func (a *App) RegisterPerformanceReport(rctx request.CTX, report *model.Performa
 	}
 
 	commonLabels := report.ProcessLabels()
+	userID := rctx.Session().UserId
 
 	for _, c := range report.Counters {
 		switch c.Metric {
@@ -27,60 +28,78 @@ func (a *App) RegisterPerformanceReport(rctx request.CTX, report *model.Performa
 	for _, h := range report.Histograms {
 		switch h.Metric {
 		case model.ClientTimeToFirstByte:
-			a.Metrics().ObserveClientTimeToFirstByte(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientTimeToFirstByte(
+				commonLabels["platform"],
+				commonLabels["agent"],
+				userID, h.Value/1000)
 		case model.ClientTimeToLastByte:
-			a.Metrics().ObserveClientTimeToLastByte(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientTimeToLastByte(
+				commonLabels["platform"],
+				commonLabels["agent"],
+				userID, h.Value/1000)
 		case model.ClientTimeToDOMInteractive:
-			a.Metrics().ObserveClientTimeToDomInteractive(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientTimeToDomInteractive(
+				commonLabels["platform"],
+				commonLabels["agent"],
+				userID, h.Value/1000)
 		case model.ClientSplashScreenEnd:
 			a.Metrics().ObserveClientSplashScreenEnd(commonLabels["platform"],
 				commonLabels["agent"],
 				h.GetLabelValue("page_type", model.AcceptedSplashScreenOrigins, "team_controller"),
-				h.Value/1000)
+				userID, h.Value/1000)
 		case model.ClientFirstContentfulPaint:
-			a.Metrics().ObserveClientFirstContentfulPaint(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientFirstContentfulPaint(commonLabels["platform"],
+				commonLabels["agent"],
+				h.Value/1000)
 		case model.ClientLargestContentfulPaint:
 			a.Metrics().ObserveClientLargestContentfulPaint(
 				commonLabels["platform"],
 				commonLabels["agent"],
 				h.GetLabelValue("region", model.AcceptedLCPRegions, "other"),
-				h.Value/1000,
-			)
+				h.Value/1000)
 		case model.ClientInteractionToNextPaint:
 			a.Metrics().ObserveClientInteractionToNextPaint(
 				commonLabels["platform"],
 				commonLabels["agent"],
 				h.GetLabelValue("interaction", model.AcceptedInteractions, "other"),
-				h.Value/1000,
-			)
+				h.Value/1000)
 		case model.ClientCumulativeLayoutShift:
-			a.Metrics().ObserveClientCumulativeLayoutShift(commonLabels["platform"], commonLabels["agent"], h.Value)
+			a.Metrics().ObserveClientCumulativeLayoutShift(commonLabels["platform"],
+				commonLabels["agent"],
+				h.Value)
 		case model.ClientPageLoadDuration:
-			a.Metrics().ObserveClientPageLoadDuration(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientPageLoadDuration(commonLabels["platform"],
+				commonLabels["agent"],
+				userID, h.Value/1000)
 		case model.ClientChannelSwitchDuration:
 			a.Metrics().ObserveClientChannelSwitchDuration(
 				commonLabels["platform"],
 				commonLabels["agent"],
 				h.GetLabelValue("fresh", model.AcceptedTrueFalseLabels, ""),
-				h.Value/1000,
-			)
+				h.Value/1000)
 		case model.ClientTeamSwitchDuration:
 			a.Metrics().ObserveClientTeamSwitchDuration(
 				commonLabels["platform"],
 				commonLabels["agent"],
 				h.GetLabelValue("fresh", model.AcceptedTrueFalseLabels, ""),
-				h.Value/1000,
-			)
+				h.Value/1000)
 		case model.ClientRHSLoadDuration:
-			a.Metrics().ObserveClientRHSLoadDuration(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveClientRHSLoadDuration(commonLabels["platform"],
+				commonLabels["agent"],
+				h.Value/1000)
 		case model.ClientGlobalThreadsLoadDuration:
-			a.Metrics().ObserveGlobalThreadsLoadDuration(commonLabels["platform"], commonLabels["agent"], h.Value/1000)
+			a.Metrics().ObserveGlobalThreadsLoadDuration(commonLabels["platform"],
+				commonLabels["agent"],
+				h.Value/1000)
 		case model.MobileClientLoadDuration:
-			a.Metrics().ObserveMobileClientLoadDuration(commonLabels["platform"], h.Value/1000)
+			a.Metrics().ObserveMobileClientLoadDuration(commonLabels["platform"],
+				h.Value/1000)
 		case model.MobileClientChannelSwitchDuration:
-			a.Metrics().ObserveMobileClientChannelSwitchDuration(commonLabels["platform"], h.Value/1000)
+			a.Metrics().ObserveMobileClientChannelSwitchDuration(commonLabels["platform"],
+				h.Value/1000)
 		case model.MobileClientTeamSwitchDuration:
-			a.Metrics().ObserveMobileClientTeamSwitchDuration(commonLabels["platform"], h.Value/1000)
+			a.Metrics().ObserveMobileClientTeamSwitchDuration(commonLabels["platform"],
+				h.Value/1000)
 		case model.DesktopClientCPUUsage:
 			a.Metrics().ObserveDesktopCpuUsage(commonLabels["platform"], commonLabels["desktop_app_version"], h.Labels["process"], h.Value)
 		case model.DesktopClientMemoryUsage:

server/channels/testlib/assertions.go

@@ -15,7 +15,7 @@ import (
 func AssertLog(t *testing.T, logs io.Reader, level, message string) {
 	t.Helper()
 	if !hasMsg(t, logs, level, message) {
-		assert.Failf(t, "failed to find %s log message: %s", level, message)
+		assert.Failf(t, "failed to find", "Expected log_level: %s, log_message: %s", level, message)
 	}
 }
 
@@ -23,7 +23,7 @@ func AssertLog(t *testing.T, logs io.Reader, level, message string) {
 func AssertNoLog(t *testing.T, logs io.Reader, level, message string) {
 	t.Helper()
 	if hasMsg(t, logs, level, message) {
-		assert.Failf(t, "found %s log message: %s", level, message)
+		assert.Failf(t, "found", "Not expected log_level: %s log_message: %s", level, message)
 	}
 }