[MM-57066][MM-57329] Added metrics for all notification stopping points, consolidated categories between metrics and logging (#26799)

* [MM-57066] Add metric counters for notification events

* Some small changes

* Account for Metrics() sometimes being nil

* Fix test (again)

* Fix more tests

* A few changes from testing - added success counter

* Missed a mock

* Lint

* Add feature flag for notification monitoring

server/einterfaces/metrics.go

@@ -95,4 +95,10 @@ type MetricsInterface interface {
 
 	SetReplicaLagAbsolute(node string, value float64)
 	SetReplicaLagTime(node string, value float64)
+
+	IncrementNotificationCounter(notificationType model.NotificationType)
+	IncrementNotificationAckCounter(notificationType model.NotificationType)
+	IncrementNotificationSuccessCounter(notificationType model.NotificationType)
+	IncrementNotificationErrorCounter(notificationType model.NotificationType, errorReason model.NotificationReason)
+	IncrementNotificationNotSentCounter(notificationType model.NotificationType, notSentReason model.NotificationReason)
 }

server/einterfaces/mocks/MetricsInterface.go

@@ -163,6 +163,31 @@ func (_m *MetricsInterface) IncrementMemCacheMissCounterSession() {
 	_m.Called()
 }
 
+// IncrementNotificationAckCounter provides a mock function with given fields: notificationType
+func (_m *MetricsInterface) IncrementNotificationAckCounter(notificationType model.NotificationType) {
+	_m.Called(notificationType)
+}
+
+// IncrementNotificationCounter provides a mock function with given fields: notificationType
+func (_m *MetricsInterface) IncrementNotificationCounter(notificationType model.NotificationType) {
+	_m.Called(notificationType)
+}
+
+// IncrementNotificationErrorCounter provides a mock function with given fields: notificationType, errorReason
+func (_m *MetricsInterface) IncrementNotificationErrorCounter(notificationType model.NotificationType, errorReason model.NotificationReason) {
+	_m.Called(notificationType, errorReason)
+}
+
+// IncrementNotificationNotSentCounter provides a mock function with given fields: notificationType, notSentReason
+func (_m *MetricsInterface) IncrementNotificationNotSentCounter(notificationType model.NotificationType, notSentReason model.NotificationReason) {
+	_m.Called(notificationType, notSentReason)
+}
+
+// IncrementNotificationSuccessCounter provides a mock function with given fields: notificationType
+func (_m *MetricsInterface) IncrementNotificationSuccessCounter(notificationType model.NotificationType) {
+	_m.Called(notificationType)
+}
+
 // IncrementPostBroadcast provides a mock function with given fields:
 func (_m *MetricsInterface) IncrementPostBroadcast() {
 	_m.Called()