* MM-33818: Add replica lag metric We add two new metrics for monitoring replica lag: - Monitor absolute lag based on binlog distance/transaction queue length. - Monitor time taken for the replica to catch up. To achieve this, we add a config setting to run a user defined SQL query on the database. We need to specify a separate datasource field as part of the config because in some databases, querying the replica lag value requires elevated credentials which are not needed for usual running of the application, and can even be a security risk. Arguably, a peculiar part of the design is the requirement of the query output to be in a (node, value) format. But since from the application, the SQL query is a black box and the user can set any query they want, we cannot, in any way templatize this. And as an extra note, freno also does it in a similar way. The last bit is because we need to have a separate datasources, now we consume one extra connection rather than sharing it with the pool. This is an unfortunate result of the design, and while one extra connection doesn't make much of a difference in a single-tenant scenario. It does make so, in a multi-tenant scenario. But in a multi-tenant scenario, the expectation would already be to use a connection pool. So this is not a big concern. https://mattermost.atlassian.net/browse/MM-33818 ```release-note Two new gauge metrics were added: mattermost_db_replica_lag_abs and mattermost_db_replica_lag_time, both containing a label of "node", signifying which db host is the metric from. These metrics signify the replica lag in absolute terms and in the time dimension capturing the whole picture of replica lag. To use these metrics, a separate config section ReplicaLagSettings was added under SqlSettings. This is an array of maps which contain three keys: DataSource, QueryAbsoluteLag, and QueryTimeLag. Each map entry is for a single replica instance. DataSource contains the DB credentials to connect to the replica instance. QueryAbsoluteLag is a plain SQL query that must return a single row of which the first column must be the node value of the Prometheus metric, and the second column must be the value of the lag. QueryTimeLag is the same as above, but used to measure the time lag. As an example, for AWS Aurora instances, the QueryAbsoluteLag can be: select server_id, highest_lsn_rcvd-durable_lsn as bindiff from aurora_global_db_instance_status() where server_id=<> and QueryTimeLag can be: select server_id, visibility_lag_in_msec from aurora_global_db_instance_status() where server_id=<> For MySQL Group Replication, the absolute lag can be measured from the number of pending transactions in the applier queue: select member_id, count_transaction_remote_in_applier_queue FROM performance_schema.replication_group_member_stats where member_id=<> Overall, what query to choose is left to the administrator, and depending on the database and need, an appropriate query can be chosen. ``` * Trigger CI * Fix tests * address review comments * Remove t.Parallel It was spawning too many connections, and overloading the docker container. Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
277 строки
9.2 KiB
Go
277 строки
9.2 KiB
Go
// Code generated by mockery v1.0.0. DO NOT EDIT.
|
|
|
|
// Regenerate this file using `make einterfaces-mocks`.
|
|
|
|
package mocks
|
|
|
|
import (
|
|
logr "github.com/mattermost/logr"
|
|
mock "github.com/stretchr/testify/mock"
|
|
)
|
|
|
|
// MetricsInterface is an autogenerated mock type for the MetricsInterface type
|
|
type MetricsInterface struct {
|
|
mock.Mock
|
|
}
|
|
|
|
// AddMemCacheHitCounter provides a mock function with given fields: cacheName, amount
|
|
func (_m *MetricsInterface) AddMemCacheHitCounter(cacheName string, amount float64) {
|
|
_m.Called(cacheName, amount)
|
|
}
|
|
|
|
// AddMemCacheMissCounter provides a mock function with given fields: cacheName, amount
|
|
func (_m *MetricsInterface) AddMemCacheMissCounter(cacheName string, amount float64) {
|
|
_m.Called(cacheName, amount)
|
|
}
|
|
|
|
// DecrementJobActive provides a mock function with given fields: jobType
|
|
func (_m *MetricsInterface) DecrementJobActive(jobType string) {
|
|
_m.Called(jobType)
|
|
}
|
|
|
|
// DecrementWebSocketBroadcastBufferSize provides a mock function with given fields: hub, amount
|
|
func (_m *MetricsInterface) DecrementWebSocketBroadcastBufferSize(hub string, amount float64) {
|
|
_m.Called(hub, amount)
|
|
}
|
|
|
|
// DecrementWebSocketBroadcastUsersRegistered provides a mock function with given fields: hub, amount
|
|
func (_m *MetricsInterface) DecrementWebSocketBroadcastUsersRegistered(hub string, amount float64) {
|
|
_m.Called(hub, amount)
|
|
}
|
|
|
|
// GetLoggerMetricsCollector provides a mock function with given fields:
|
|
func (_m *MetricsInterface) GetLoggerMetricsCollector() logr.MetricsCollector {
|
|
ret := _m.Called()
|
|
|
|
var r0 logr.MetricsCollector
|
|
if rf, ok := ret.Get(0).(func() logr.MetricsCollector); ok {
|
|
r0 = rf()
|
|
} else {
|
|
if ret.Get(0) != nil {
|
|
r0 = ret.Get(0).(logr.MetricsCollector)
|
|
}
|
|
}
|
|
|
|
return r0
|
|
}
|
|
|
|
// IncrementChannelIndexCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementChannelIndexCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementClusterEventType provides a mock function with given fields: eventType
|
|
func (_m *MetricsInterface) IncrementClusterEventType(eventType string) {
|
|
_m.Called(eventType)
|
|
}
|
|
|
|
// IncrementClusterRequest provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementClusterRequest() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementEtagHitCounter provides a mock function with given fields: route
|
|
func (_m *MetricsInterface) IncrementEtagHitCounter(route string) {
|
|
_m.Called(route)
|
|
}
|
|
|
|
// IncrementEtagMissCounter provides a mock function with given fields: route
|
|
func (_m *MetricsInterface) IncrementEtagMissCounter(route string) {
|
|
_m.Called(route)
|
|
}
|
|
|
|
// IncrementFileIndexCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementFileIndexCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementFilesSearchCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementFilesSearchCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementHttpError provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementHttpError() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementHttpRequest provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementHttpRequest() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementJobActive provides a mock function with given fields: jobType
|
|
func (_m *MetricsInterface) IncrementJobActive(jobType string) {
|
|
_m.Called(jobType)
|
|
}
|
|
|
|
// IncrementLogin provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementLogin() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementLoginFail provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementLoginFail() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementMemCacheHitCounter provides a mock function with given fields: cacheName
|
|
func (_m *MetricsInterface) IncrementMemCacheHitCounter(cacheName string) {
|
|
_m.Called(cacheName)
|
|
}
|
|
|
|
// IncrementMemCacheHitCounterSession provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementMemCacheHitCounterSession() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementMemCacheInvalidationCounter provides a mock function with given fields: cacheName
|
|
func (_m *MetricsInterface) IncrementMemCacheInvalidationCounter(cacheName string) {
|
|
_m.Called(cacheName)
|
|
}
|
|
|
|
// IncrementMemCacheInvalidationCounterSession provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementMemCacheInvalidationCounterSession() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementMemCacheMissCounter provides a mock function with given fields: cacheName
|
|
func (_m *MetricsInterface) IncrementMemCacheMissCounter(cacheName string) {
|
|
_m.Called(cacheName)
|
|
}
|
|
|
|
// IncrementMemCacheMissCounterSession provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementMemCacheMissCounterSession() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostBroadcast provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostBroadcast() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostCreate provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostCreate() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostFileAttachment provides a mock function with given fields: count
|
|
func (_m *MetricsInterface) IncrementPostFileAttachment(count int) {
|
|
_m.Called(count)
|
|
}
|
|
|
|
// IncrementPostIndexCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostIndexCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostSentEmail provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostSentEmail() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostSentPush provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostSentPush() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementPostsSearchCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementPostsSearchCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementUserIndexCounter provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementUserIndexCounter() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementWebSocketBroadcast provides a mock function with given fields: eventType
|
|
func (_m *MetricsInterface) IncrementWebSocketBroadcast(eventType string) {
|
|
_m.Called(eventType)
|
|
}
|
|
|
|
// IncrementWebSocketBroadcastBufferSize provides a mock function with given fields: hub, amount
|
|
func (_m *MetricsInterface) IncrementWebSocketBroadcastBufferSize(hub string, amount float64) {
|
|
_m.Called(hub, amount)
|
|
}
|
|
|
|
// IncrementWebSocketBroadcastUsersRegistered provides a mock function with given fields: hub, amount
|
|
func (_m *MetricsInterface) IncrementWebSocketBroadcastUsersRegistered(hub string, amount float64) {
|
|
_m.Called(hub, amount)
|
|
}
|
|
|
|
// IncrementWebhookPost provides a mock function with given fields:
|
|
func (_m *MetricsInterface) IncrementWebhookPost() {
|
|
_m.Called()
|
|
}
|
|
|
|
// IncrementWebsocketEvent provides a mock function with given fields: eventType
|
|
func (_m *MetricsInterface) IncrementWebsocketEvent(eventType string) {
|
|
_m.Called(eventType)
|
|
}
|
|
|
|
// ObserveApiEndpointDuration provides a mock function with given fields: endpoint, method, statusCode, elapsed
|
|
func (_m *MetricsInterface) ObserveApiEndpointDuration(endpoint string, method string, statusCode string, elapsed float64) {
|
|
_m.Called(endpoint, method, statusCode, elapsed)
|
|
}
|
|
|
|
// ObserveClusterRequestDuration provides a mock function with given fields: elapsed
|
|
func (_m *MetricsInterface) ObserveClusterRequestDuration(elapsed float64) {
|
|
_m.Called(elapsed)
|
|
}
|
|
|
|
// ObserveEnabledUsers provides a mock function with given fields: users
|
|
func (_m *MetricsInterface) ObserveEnabledUsers(users int64) {
|
|
_m.Called(users)
|
|
}
|
|
|
|
// ObserveFilesSearchDuration provides a mock function with given fields: elapsed
|
|
func (_m *MetricsInterface) ObserveFilesSearchDuration(elapsed float64) {
|
|
_m.Called(elapsed)
|
|
}
|
|
|
|
// ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed
|
|
func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) {
|
|
_m.Called(pluginID, apiName, success, elapsed)
|
|
}
|
|
|
|
// ObservePluginHookDuration provides a mock function with given fields: pluginID, hookName, success, elapsed
|
|
func (_m *MetricsInterface) ObservePluginHookDuration(pluginID string, hookName string, success bool, elapsed float64) {
|
|
_m.Called(pluginID, hookName, success, elapsed)
|
|
}
|
|
|
|
// ObservePluginMultiHookDuration provides a mock function with given fields: elapsed
|
|
func (_m *MetricsInterface) ObservePluginMultiHookDuration(elapsed float64) {
|
|
_m.Called(elapsed)
|
|
}
|
|
|
|
// ObservePluginMultiHookIterationDuration provides a mock function with given fields: pluginID, elapsed
|
|
func (_m *MetricsInterface) ObservePluginMultiHookIterationDuration(pluginID string, elapsed float64) {
|
|
_m.Called(pluginID, elapsed)
|
|
}
|
|
|
|
// ObservePostsSearchDuration provides a mock function with given fields: elapsed
|
|
func (_m *MetricsInterface) ObservePostsSearchDuration(elapsed float64) {
|
|
_m.Called(elapsed)
|
|
}
|
|
|
|
// ObserveStoreMethodDuration provides a mock function with given fields: method, success, elapsed
|
|
func (_m *MetricsInterface) ObserveStoreMethodDuration(method string, success string, elapsed float64) {
|
|
_m.Called(method, success, elapsed)
|
|
}
|
|
|
|
// Register provides a mock function with given fields:
|
|
func (_m *MetricsInterface) Register() {
|
|
_m.Called()
|
|
}
|
|
|
|
// SetReplicaLagAbsolute provides a mock function with given fields: node, value
|
|
func (_m *MetricsInterface) SetReplicaLagAbsolute(node string, value float64) {
|
|
_m.Called(node, value)
|
|
}
|
|
|
|
// SetReplicaLagTime provides a mock function with given fields: node, value
|
|
func (_m *MetricsInterface) SetReplicaLagTime(node string, value float64) {
|
|
_m.Called(node, value)
|
|
}
|