[MM-57066][MM-57329] Added metrics for all notification stopping points, consolidated categories between metrics and logging (#26799)

* [MM-57066] Add metric counters for notification events

* Some small changes

* Account for Metrics() sometimes being nil

* Fix test (again)

* Fix more tests

* A few changes from testing - added success counter

* Missed a mock

* Lint

* Add feature flag for notification monitoring

server/enterprise/metrics/metrics.go

@@ -41,6 +41,7 @@ const (
 	MetricsSubsystemSharedChannels     = "shared_channels"
 	MetricsSubsystemSystem             = "system"
 	MetricsSubsystemJobs               = "jobs"
+	MetricsSubsystemNotifications      = "notifications"
 	MetricsCloudInstallationLabel      = "installationId"
 	MetricsCloudDatabaseClusterLabel   = "databaseClusterName"
 	MetricsCloudInstallationGroupLabel = "installationGroupId"
@@ -201,6 +202,12 @@ type MetricsInterfaceImpl struct {
 	ServerStartTime prometheus.Gauge
 
 	JobsActive *prometheus.GaugeVec
+
+	NotificationTotalCounters   *prometheus.CounterVec
+	NotificationAckCounters     *prometheus.CounterVec
+	NotificationSuccessCounters *prometheus.CounterVec
+	NotificationErrorCounters   *prometheus.CounterVec
+	NotificationNotSentCounters *prometheus.CounterVec
 }
 
 func init() {
@@ -1041,6 +1048,67 @@ func New(ps *platform.PlatformService, driver, dataSource string) *MetricsInterf
 		[]string{"type"},
 	)
 	m.Registry.MustRegister(m.JobsActive)
+
+	m.NotificationTotalCounters = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemNotifications,
+			Name:        "total",
+			Help:        "Total number of notification events",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"type"},
+	)
+	m.Registry.MustRegister(m.NotificationTotalCounters)
+
+	m.NotificationAckCounters = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemNotifications,
+			Name:        "total_ack",
+			Help:        "Total number of notification events acknowledged",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"type"},
+	)
+	m.Registry.MustRegister(m.NotificationAckCounters)
+
+	m.NotificationSuccessCounters = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemNotifications,
+			Name:        "success",
+			Help:        "Total number of successfully sent notifications",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"type"},
+	)
+	m.Registry.MustRegister(m.NotificationSuccessCounters)
+
+	m.NotificationErrorCounters = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemNotifications,
+			Name:        "error",
+			Help:        "Total number of errors that stop the notification flow",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"type", "reason"},
+	)
+	m.Registry.MustRegister(m.NotificationErrorCounters)
+
+	m.NotificationNotSentCounters = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   MetricsNamespace,
+			Subsystem:   MetricsSubsystemNotifications,
+			Name:        "not_sent",
+			Help:        "Total number of notifications the system deliberately did not send",
+			ConstLabels: additionalLabels,
+		},
+		[]string{"type", "reason"},
+	)
+	m.Registry.MustRegister(m.NotificationNotSentCounters)
+
 	return m
 }
 
@@ -1467,6 +1535,26 @@ func (mi *MetricsInterfaceImpl) SetReplicaLagTime(node string, value float64) {
 	mi.DbReplicaLagGaugeTime.With(prometheus.Labels{"node": node}).Set(value)
 }
 
+func (mi *MetricsInterfaceImpl) IncrementNotificationCounter(notificationType model.NotificationType) {
+	mi.NotificationTotalCounters.With(prometheus.Labels{"type": string(notificationType)}).Inc()
+}
+
+func (mi *MetricsInterfaceImpl) IncrementNotificationAckCounter(notificationType model.NotificationType) {
+	mi.NotificationAckCounters.With(prometheus.Labels{"type": string(notificationType)}).Inc()
+}
+
+func (mi *MetricsInterfaceImpl) IncrementNotificationSuccessCounter(notificationType model.NotificationType) {
+	mi.NotificationSuccessCounters.With(prometheus.Labels{"type": string(notificationType)}).Inc()
+}
+
+func (mi *MetricsInterfaceImpl) IncrementNotificationErrorCounter(notificationType model.NotificationType, errorReason model.NotificationReason) {
+	mi.NotificationErrorCounters.With(prometheus.Labels{"type": string(notificationType), "reason": string(errorReason)}).Inc()
+}
+
+func (mi *MetricsInterfaceImpl) IncrementNotificationNotSentCounter(notificationType model.NotificationType, notSentReason model.NotificationReason) {
+	mi.NotificationNotSentCounters.With(prometheus.Labels{"type": string(notificationType), "reason": string(notSentReason)}).Inc()
+}
+
 func (mi *MetricsInterfaceImpl) IncrementHTTPWebSockets(originClient string) {
 	mi.HTTPWebsocketsGauge.With(prometheus.Labels{"origin_client": originClient}).Inc()
 }