prometheus: add service_id to created alerts and sent notification co…

…unters (#3603)

* breakdown created alerts by service id

* msg sent by service id

* track status messages by service id

alert/metrics.go

@@ -6,10 +6,10 @@ import (
 )
 
 var (
-	metricCreatedTotal = promauto.NewCounter(prometheus.CounterOpts{
+	metricCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "goalert",
 		Subsystem: "alert",
 		Name:      "created_total",
 		Help:      "The total number of created alerts.",
-	})
+	}, []string{"service_id"})
 )

alert/store.go

@@ -514,7 +514,7 @@ func (s *Store) Create(ctx context.Context, a *Alert) (*Alert, error) {
 
 	ctx = log.WithFields(ctx, log.Fields{"AlertID": n.ID, "ServiceID": n.ServiceID})
 	log.Logf(ctx, "Alert created.")
-	metricCreatedTotal.Inc()
+	metricCreatedTotal.WithLabelValues(n.ServiceID).Inc()
 
 	return n, nil
 }
@@ -652,7 +652,7 @@ func (s *Store) CreateOrUpdate(ctx context.Context, a *Alert) (*Alert, bool, err
 	if isNew {
 		ctx = log.WithFields(ctx, log.Fields{"AlertID": n.ID, "ServiceID": n.ServiceID})
 		log.Logf(ctx, "Alert created.")
-		metricCreatedTotal.Inc()
+		metricCreatedTotal.WithLabelValues(n.ServiceID).Inc()
 	}
 
 	return n, isNew, nil

engine/sendmessage.go

@@ -113,6 +113,7 @@ func (p *Engine) sendMessage(ctx context.Context, msg *message.Message) (*notifi
 		notifMsg = notification.AlertStatus{
 			Dest:           msg.Dest,
 			AlertID:        e.AlertID(),
+			ServiceID:      a.ServiceID,
 			CallbackID:     msg.ID,
 			LogEntry:       e.String(ctx),
 			Summary:        a.Summary,

notification/alertstatus.go

@@ -16,6 +16,7 @@ type AlertStatus struct {
 	CallbackID string
 	AlertID    int
 	LogEntry   string
+	ServiceID  string
 
 	// Summary of the alert that this status is in regards to.
 	Summary string

notification/manager.go

@@ -150,7 +150,7 @@ func (mgr *Manager) SendMessage(ctx context.Context, msg Message) (*SendResult,
 		}
 		log.Logf(sendCtx, "notification sent")
 		metricSentTotal.
-			WithLabelValues(msg.Destination().Type.String(), msg.Type().String()).
+			WithLabelValues(msg.Destination().Type.String(), msg.Type().String(), msgSvcID(msg)).
 			Inc()
 		// status already wrapped via namedSender
 		return res, nil
@@ -161,3 +161,16 @@ func (mgr *Manager) SendMessage(ctx context.Context, msg Message) (*SendResult,
 
 	return nil, errors.New("all notification senders failed")
 }
+
+func msgSvcID(msg Message) string {
+	switch msg := msg.(type) {
+	case Alert:
+		return msg.ServiceID
+	case AlertBundle:
+		return msg.ServiceID
+	case AlertStatus:
+		return msg.ServiceID
+	}
+
+	return ""
+}

notification/metrics.go

@@ -11,7 +11,7 @@ var (
 		Subsystem: "notification",
 		Name:      "sent_total",
 		Help:      "Total number of sent notifications.",
-	}, []string{"dest_type", "message_type"})
+	}, []string{"dest_type", "message_type", "service_id"})
 	metricRecvTotal = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "goalert",
 		Subsystem: "notification",