mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-61887: Log the userID if a metric exceeds the last histogram bucket (#29448)

Просмотр исходного кода 
We create a custom histogram metric that logs the userID
when the observed value is greater or equal to the last bucket value.

This allows us to start tracking the slowest users of a system
while at the same time not polluting the Prometheus metrics
by storing a userID for every observation.

https://mattermost.atlassian.net/browse/MM-61887

```release-note
NONE
```
Этот коммит содержится в:
Agniva De Sarker
2024-12-05 09:12:54 +05:30
коммит произвёл GitHub
родитель 421001c981
Коммит a6d37fa14c
7 изменённых файлов: 203 добавлений и 63 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

76
server/enterprise/metrics/histogram.go Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.enterprise for license information.
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// HistogramVec is a wrapper of prometheus.HistogramVec that stores the buckets
// which are later passed down to WrappedObserver.
type HistogramVec struct {
*prometheus.HistogramVec
buckets []float64
fqName string
logger mlog.LoggerIFace
}
// WrappedObserver is a wrapper of prometheus.Observer which addtionally
// logs userIDs when the value exceeds or equals to the one in the highest bucket.
type WrappedObserver struct {
prometheus.Observer
labels prometheus.Labels
buckets []float64
logger mlog.LoggerIFace
metricName string
userID string
}
// NewHistogramVec returns a custom-purpose HistogramVec.
func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string, logger mlog.LoggerIFace) *HistogramVec {
if len(opts.Buckets) == 0 {
opts.Buckets = prometheus.DefBuckets
}
return &HistogramVec{
HistogramVec: prometheus.NewHistogramVec(opts, labelNames),
buckets: opts.Buckets,
fqName: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
logger: logger,
}
}
func (v *HistogramVec) With(labels prometheus.Labels, userID string) prometheus.Observer {
h, err := v.GetMetricWith(labels)
if err != nil {
panic(err)
}
return &WrappedObserver{
Observer: h,
labels: labels,
buckets: v.buckets,
logger: v.logger,
metricName: v.fqName,
userID: userID,
}
}
func (o *WrappedObserver) Observe(v float64) {
if v >= o.buckets[len(o.buckets)-1] {
fields := []mlog.Field{
mlog.String("metric_name", o.metricName),
mlog.String("user_id", o.userID),
mlog.Float("observed_value", v),
mlog.Float("highest_bucket_value", o.buckets[len(o.buckets)-1]),
}
for k, v := range o.labels {
fields = append(fields,
mlog.String("label_name", k),
mlog.String("label_value", v))
}
o.logger.Warn("Metric observation exceeded.", fields...)
}
o.Observer.Observe(v)
}

35
server/enterprise/metrics/histogram_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,35 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.enterprise for license information.
package metrics
import (
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/api4"
"github.com/mattermost/mattermost/server/v8/channels/testlib"
)
func TestWrappedObserver(t *testing.T) {
th := api4.Setup(t)
defer th.TearDown()
h := NewHistogramVec(prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
Name: "test",
Buckets: []float64{0, 5, 10},
}, []string{"l1"}, th.TestLogger)
h.With(prometheus.Labels{"l1": "hello"}, th.BasicUser.Id).Observe(6)
require.NoError(t, th.TestLogger.Flush())
testlib.AssertNoLog(t, th.LogBuffer, mlog.LvlWarn.Name, "Metric observation exceeded.")
h.With(prometheus.Labels{"l1": "hello"}, th.BasicUser.Id).Observe(10)
require.NoError(t, th.TestLogger.Flush())
testlib.AssertLog(t, th.LogBuffer, mlog.LvlWarn.Name, "Metric observation exceeded.")
}

52
server/enterprise/metrics/metrics.go
Просмотреть файл

@@ -201,16 +201,16 @@ type MetricsInterfaceImpl struct {
NotificationNotSentCounters *prometheus.CounterVec
NotificationUnsupportedCounters *prometheus.CounterVec
ClientTimeToFirstByte *prometheus.HistogramVec
ClientTimeToLastByte *prometheus.HistogramVec
ClientTimeToDOMInteractive *prometheus.HistogramVec
ClientSplashScreenEnd *prometheus.HistogramVec
ClientTimeToFirstByte *HistogramVec
ClientTimeToLastByte *HistogramVec
ClientTimeToDOMInteractive *HistogramVec
ClientSplashScreenEnd *HistogramVec
ClientFirstContentfulPaint *prometheus.HistogramVec
ClientLargestContentfulPaint *prometheus.HistogramVec
ClientInteractionToNextPaint *prometheus.HistogramVec
ClientCumulativeLayoutShift *prometheus.HistogramVec
ClientLongTasks *prometheus.CounterVec
ClientPageLoadDuration *prometheus.HistogramVec
ClientPageLoadDuration *HistogramVec
ClientChannelSwitchDuration *prometheus.HistogramVec
ClientTeamSwitchDuration *prometheus.HistogramVec
ClientRHSLoadDuration *prometheus.HistogramVec
@@ -1179,7 +1179,7 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
)
m.Registry.MustRegister(m.NotificationUnsupportedCounters)
m.ClientTimeToFirstByte = prometheus.NewHistogramVec(
m.ClientTimeToFirstByte = NewHistogramVec(
prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
@@ -1187,10 +1187,11 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
Help: "Duration from when a browser starts to request a page from a server until when it starts to receive data in response (seconds)",
},
[]string{"platform", "agent"},
m.Platform.Log(),
)
m.Registry.MustRegister(m.ClientTimeToFirstByte)
m.ClientTimeToLastByte = prometheus.NewHistogramVec(
m.ClientTimeToLastByte = NewHistogramVec(
prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
@@ -1198,28 +1199,33 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
Help: "Duration from when a browser starts to request a page from a server until when it receives the last byte of the resource or immediately before the transport connection is closed, whichever comes first. (seconds)",
},
[]string{"platform", "agent"},
m.Platform.Log(),
)
m.Registry.MustRegister(m.ClientTimeToLastByte)
m.ClientTimeToDOMInteractive = prometheus.NewHistogramVec(
m.ClientTimeToDOMInteractive = NewHistogramVec(
prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
Name: "dom_interactive",
Help: "Duration from when a browser starts to request a page from a server until when it sets the document's readyState to interactive. (seconds)",
Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 7.5, 10, 12.5, 15},
},
[]string{"platform", "agent"},
m.Platform.Log(),
)
m.Registry.MustRegister(m.ClientTimeToDOMInteractive)
m.ClientSplashScreenEnd = prometheus.NewHistogramVec(
m.ClientSplashScreenEnd = NewHistogramVec(
prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
Name: "splash_screen",
Help: "Duration from when a browser starts to request a page from a server until when the splash screen ends. (seconds)",
Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 7.5, 10, 12.5, 15},
},
[]string{"platform", "agent", "page_type"},
m.Platform.Log(),
)
m.Registry.MustRegister(m.ClientSplashScreenEnd)
@@ -1284,7 +1290,7 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
)
m.Registry.MustRegister(m.ClientLongTasks)
m.ClientPageLoadDuration = prometheus.NewHistogramVec(
m.ClientPageLoadDuration = NewHistogramVec(
prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Subsystem: MetricsSubsystemClientsWeb,
@@ -1293,6 +1299,7 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 20, 40},
},
[]string{"platform", "agent"},
m.Platform.Log(),
)
m.Registry.MustRegister(m.ClientPageLoadDuration)
@@ -1870,20 +1877,20 @@ func (mi *MetricsInterfaceImpl) DecrementHTTPWebSockets(originClient string) {
mi.HTTPWebsocketsGauge.With(prometheus.Labels{"origin_client": originClient}).Dec()
}
func (mi *MetricsInterfaceImpl) ObserveClientTimeToFirstByte(platform, agent string, elapsed float64) {
mi.ClientTimeToFirstByte.With(prometheus.Labels{"platform": platform, "agent": agent}).Observe(elapsed)
func (mi *MetricsInterfaceImpl) ObserveClientTimeToFirstByte(platform, agent, userID string, elapsed float64) {
mi.ClientTimeToFirstByte.With(prometheus.Labels{"platform": platform, "agent": agent}, userID).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveClientTimeToLastByte(platform, agent string, elapsed float64) {
mi.ClientTimeToLastByte.With(prometheus.Labels{"platform": platform, "agent": agent}).Observe(elapsed)
func (mi *MetricsInterfaceImpl) ObserveClientTimeToLastByte(platform, agent, userID string, elapsed float64) {
mi.ClientTimeToLastByte.With(prometheus.Labels{"platform": platform, "agent": agent}, userID).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveClientTimeToDomInteractive(platform, agent string, elapsed float64) {
mi.ClientTimeToDOMInteractive.With(prometheus.Labels{"platform": platform, "agent": agent}).Observe(elapsed)
func (mi *MetricsInterfaceImpl) ObserveClientTimeToDomInteractive(platform, agent, userID string, elapsed float64) {
mi.ClientTimeToDOMInteractive.With(prometheus.Labels{"platform": platform, "agent": agent}, userID).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveClientSplashScreenEnd(platform, agent, pageType string, elapsed float64) {
mi.ClientSplashScreenEnd.With(prometheus.Labels{"platform": platform, "agent": agent, "page_type": pageType}).Observe(elapsed)
func (mi *MetricsInterfaceImpl) ObserveClientSplashScreenEnd(platform, agent, pageType, userID string, elapsed float64) {
mi.ClientSplashScreenEnd.With(prometheus.Labels{"platform": platform, "agent": agent, "page_type": pageType}, userID).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveClientFirstContentfulPaint(platform, agent string, elapsed float64) {
@@ -1906,8 +1913,11 @@ func (mi *MetricsInterfaceImpl) IncrementClientLongTasks(platform, agent string,
mi.ClientLongTasks.With(prometheus.Labels{"platform": platform, "agent": agent}).Add(inc)
}
func (mi *MetricsInterfaceImpl) ObserveClientPageLoadDuration(platform, agent string, elapsed float64) {
mi.ClientPageLoadDuration.With(prometheus.Labels{"platform": platform, "agent": agent}).Observe(elapsed)
func (mi *MetricsInterfaceImpl) ObserveClientPageLoadDuration(platform, agent, userID string, elapsed float64) {
mi.ClientPageLoadDuration.With(prometheus.Labels{
"platform": platform,
"agent": agent,
}, userID).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveClientChannelSwitchDuration(platform, agent, fresh string, elapsed float64) {
@@ -1946,7 +1956,7 @@ func (mi *MetricsInterfaceImpl) ObserveMobileClientTeamSwitchDuration(platform s
mi.MobileClientTeamSwitchDuration.With(prometheus.Labels{"platform": platform}).Observe(elapsed)
}
func (mi *MetricsInterfaceImpl) ObserveMobileClientSessionMetadata(version string, platform string, value float64, notificationDisabled string) {
func (mi *MetricsInterfaceImpl) ObserveMobileClientSessionMetadata(version, platform string, value float64, notificationDisabled string) {
mi.MobileClientSessionMetadataGauge.With(prometheus.Labels{"version": version, "platform": platform, "notifications_disabled": notificationDisabled}).Set(value)
}