notification: slack status updates

engine/message/ratelimit.go

@@ -15,8 +15,10 @@ var PerCMThrottle ThrottleConfig
 func init() {
 	var perCM ThrottleConfigBuilder
 
-	// all message types
-	perCM.AddRules([]ThrottleRule{{Count: 1, Per: time.Minute}})
+	// Rate limit sms, voice and email types
+	perCM.
+		WithDestTypes(notification.DestTypeVoice, notification.DestTypeSMS, notification.DestTypeUserEmail).
+		AddRules([]ThrottleRule{{Count: 1, Per: time.Minute}})
 
 	// On-Call Status Notifications
 	perCM.
@@ -29,6 +31,7 @@ func init() {
 	// status notifications
 	perCM.
 		WithMsgTypes(notification.MessageTypeAlertStatus).
+		WithDestTypes(notification.DestTypeVoice, notification.DestTypeSMS, notification.DestTypeUserEmail).
 		AddRules([]ThrottleRule{
 			{Count: 1, Per: 3 * time.Minute},
 			{Count: 3, Per: 20 * time.Minute},

engine/sendmessage.go

@@ -91,11 +91,25 @@ func (p *Engine) sendMessage(ctx context.Context, msg *message.Message) (*notifi
 		if err != nil {
 			return nil, errors.Wrap(err, "lookup alert log entry")
 		}
+		a, err := p.cfg.AlertStore.FindOne(ctx, msg.AlertID)
+		if err != nil {
+			return nil, fmt.Errorf("lookup original alert: %w", err)
+		}
+		stat, err := p.cfg.NotificationStore.OriginalMessageStatus(ctx, msg.AlertID, msg.Dest)
+		if err != nil {
+			return nil, fmt.Errorf("lookup original message: %w", err)
+		}
+		if stat == nil {
+			return nil, fmt.Errorf("could not find original notification for alert %d to %s", msg.AlertID, msg.Dest.String())
+		}
 		notifMsg = notification.AlertStatus{
-			Dest:       msg.Dest,
-			AlertID:    e.AlertID(),
-			CallbackID: msg.ID,
-			LogEntry:   e.String(),
+			Dest:           msg.Dest,
+			AlertID:        e.AlertID(),
+			CallbackID:     msg.ID,
+			LogEntry:       e.String(),
+			Summary:        a.Summary,
+			Details:        a.Details,
+			OriginalStatus: *stat,
 		}
 	case notification.MessageTypeTest:
 		notifMsg = notification.Test{

notification/alertstatus.go

@@ -5,6 +5,14 @@ type AlertStatus struct {
 	CallbackID string
 	AlertID    int
 	LogEntry   string
+
+	// Summary of the alert that this status is in regards to.
+	Summary string
+	// Details of the alert that this status is in regards to.
+	Details string
+
+	// OriginalStatus is the status of the first Alert notification to this Dest for this AlertID.
+	OriginalStatus SendResult
 }
 
 var _ Message = &AlertStatus{}

notification/slack/channel.go

@@ -321,12 +321,16 @@ func (s *ChannelSender) Send(ctx context.Context, msg notification.Message) (str
 		if t.OriginalStatus != nil {
 			// Reply in thread if we already sent a message for this alert.
 			vals.Set("thread_ts", t.OriginalStatus.ProviderMessageID.ExternalID)
-			vals.Set("text", "Escalated.")
+			// only if escalation results in second notification to channel
+			vals.Set("text", "Notified.")
 			vals.Set("reply_broadcast", "true")
 			break
 		}
 
 		vals.Set("text", fmt.Sprintf("Alert: %s\n\n<%s>", t.Summary, cfg.CallbackURL("/alerts/"+strconv.Itoa(t.AlertID))))
+	case notification.AlertStatus:
+		vals.Set("thread_ts", t.OriginalStatus.ProviderMessageID.ExternalID)
+		vals.Set("text", t.LogEntry)
 	case notification.AlertBundle:
 		vals.Set("text", fmt.Sprintf("Service '%s' has %d unacknowledged alerts.\n\n<%s>", t.ServiceName, t.Count, cfg.CallbackURL("/services/"+t.ServiceID+"/alerts")))
 	case notification.ScheduleOnCallUsers:

smoketest/statusinprogress_test.go

@@ -47,6 +47,10 @@ func TestStatusInProgress(t *testing.T) {
 		insert into user_last_alert_log (user_id, alert_id, log_id, next_log_id)
 		values
 			({{uuid "u1"}}, 1, 102, 103);
+
+		insert into outgoing_messages(message_type, user_id, contact_method_id, alert_id, service_id, escalation_policy_id, last_status, sent_at)
+		values
+			('alert_notification', {{uuid "u1"}}, {{uuid "c1"}}, 1, {{uuid "sid"}}, {{uuid "eid"}}, 'sent', now());
 	`
 
 	h := harness.NewHarness(t, sql, "sched-module-v3")