Files
worker/app/models/status_page_delivery.go
Gleb Tv 2c7a0236da feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
2026-07-13 17:55:14 +03:00

409 строки
18 KiB
Go

package models
import (
"context"
"encoding/json"
"errors"
"fmt"
"html"
"log"
"strings"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
const (
statusPageDigestHour = 9
statusPageOutboxPending = "pending"
statusPageOutboxDispatched = "dispatched"
statusPageOutboxCanceled = "canceled"
)
// StatusPageDelivery is an outbox row. It does not retain email addresses;
// confirmation links are the only transient body data and are redacted on
// cancellation. The subscriber/contact data is read while dispatching.
type StatusPageDelivery struct {
ID int64 `gorm:"primarykey"`
StatusPageID int64 `gorm:"not null;index"`
SubscriberID int64 `gorm:"not null;index"`
IncidentID *int64 `gorm:"index"`
Kind string `gorm:"size:32;not null"`
Version string `gorm:"size:64;not null"`
LocalDate string `gorm:"size:10"`
State string `gorm:"size:16;not null;index"`
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
MessageID *int64 `gorm:"index"`
TaskID *int64 `gorm:"index"`
LastError string `gorm:"type:text"`
Subject string `gorm:"type:text"`
BodyText string `gorm:"type:text"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
type StatusPageDigestSchedule struct {
ID int64 `gorm:"primarykey"`
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
DueAt time.Time `gorm:"not null;index"`
CreatedAt time.Time
}
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
if subscriber.ContactID != nil {
return nil
}
accountID := page.AccountID
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
if err := tx.Create(contact).Error; err != nil {
return err
}
subscriber.ContactID = &contact.ID
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
}
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
}
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
}
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
}
func valueOrZero(v *int64) int64 {
if v == nil {
return 0
}
return *v
}
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
// transaction, so an incident can never become visible without its delivery
// intent being recoverable by the dispatcher.
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
return err
}
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
for _, subscriber := range subscribers {
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
return err
}
}
return nil
}
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
var notification Notification
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(&notification).Error
if err == nil {
return &notification, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
return &notification, tx.Create(&notification).Error
}
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
// Message and Task rows in the same transaction, and preserves a pending row on
// transient worker/capability failure for the next tick.
func DispatchStatusPageDeliveries(now time.Time) error {
var rows []StatusPageDelivery
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
return err
}
var errs []error
for _, row := range rows {
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func dispatchStatusPageDelivery(id int64, now time.Time) error {
err := DB().Transaction(func(tx *gorm.DB) error {
var outbox StatusPageDelivery
if err := tx.First(&outbox, id).Error; err != nil {
return err
}
var subscriber StatusPageSubscriber
var page StatusPage
// Subscriber then outbox is the global lock order shared with
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
}
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
return err
}
if outbox.State != statusPageOutboxPending {
return nil
}
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
}
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
return err
}
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
return err
}
notification, err := statusPageNotificationTx(tx, page.AccountID)
if err != nil {
return err
}
subject, text := outbox.Subject, outbox.BodyText
if subject == "" {
subject, text = "["+page.Name+"] status update", "Status update"
}
if outbox.IncidentID != nil {
var incident StatusPageIncident
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
return err
}
subject = "[" + page.Name + "] " + incident.Title
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
} else if outbox.BodyText == "" {
text = "Open incidents daily digest"
}
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
if err := tx.Create(message).Error; err != nil {
return err
}
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
if err != nil {
return err
}
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
if err != nil {
return err
}
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
})
if err != nil {
// This update intentionally runs after rollback: diagnostic state must not
// disappear with the failed message/task transaction.
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
}
return err
}
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
return err
}
if outbox.TaskID != nil {
// Redact every task, including terminal audit rows. State is preserved for
// terminal rows, while the payload can no longer disclose the address.
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
return err
}
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
}
if outbox.MessageID != nil {
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
}
return nil
}
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
var subscriber StatusPageSubscriber
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
var rows []StatusPageDelivery
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
return err
}
}
return nil
}
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
// retained status-page lineage without touching unrelated account messages.
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
return err
}
for i := range rows {
row := &rows[i]
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if row.TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if row.MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if rows[i].TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if rows[i].MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
// endpoint inert and irreversibly removing subscriber PII.
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
return err
}
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
return err
}
now := time.Now()
for i := range subscribers {
s := &subscribers[i]
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
return err
}
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
return err
}
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
return err
}
if s.ContactID != nil {
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
return err
}
}
}
return tx.Delete(page).Error
}
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
func EnqueueStatusPageDailyDigests(now time.Time) error {
var subscribers []StatusPageSubscriber
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
return err
}
var errs []error
for i := range subscribers {
s := &subscribers[i]
if s.StatusPage == nil || s.StatusPage.Account == nil {
continue
}
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
if err != nil {
loc = time.UTC
}
localNow := now.In(loc)
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
// Only persist today's actual due date. Older pending outbox rows are
// retried by DispatchStatusPageDeliveries; never invent history from a
// current incident snapshot after downtime.
if now.Before(due) || s.ConfirmedAt.After(due) {
continue
}
var incidents []StatusPageIncident
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
errs = append(errs, err)
continue
}
if len(incidents) == 0 {
continue
} // No digest is better than a misleading outage summary.
lines := make([]string, 0, len(incidents))
for _, incident := range incidents {
lines = append(lines, "- "+incident.Title)
}
err = DB().Transaction(func(tx *gorm.DB) error {
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
return err
}
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
})
if err != nil {
errs = append(errs, err)
}
}
if err := DispatchStatusPageDeliveries(now); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func StartStatusPageDigestScheduler(ctx context.Context) {
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
log.Printf("status-page delivery dispatch: %v", err)
}
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
log.Printf("status-page digest: %v", err)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
}