Некоторые проверки не удались
CI / test (push) Successful in 2m5s
Docker / Build and publish worker image (push) Failing after 31s
409 строки
18 KiB
Go
409 строки
18 KiB
Go
package models
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
|
|
"rocketgit.ru/rsmon/worker/internal/wire"
|
|
)
|
|
|
|
const (
|
|
statusPageDigestHour = 9
|
|
statusPageOutboxPending = "pending"
|
|
statusPageOutboxDispatched = "dispatched"
|
|
statusPageOutboxCanceled = "canceled"
|
|
)
|
|
|
|
// StatusPageDelivery is an outbox row. It does not retain email addresses;
|
|
// confirmation links are the only transient body data and are redacted on
|
|
// cancellation. The subscriber/contact data is read while dispatching.
|
|
type StatusPageDelivery struct {
|
|
ID int64 `gorm:"primarykey"`
|
|
StatusPageID int64 `gorm:"not null;index"`
|
|
SubscriberID int64 `gorm:"not null;index"`
|
|
IncidentID *int64 `gorm:"index"`
|
|
Kind string `gorm:"size:32;not null"`
|
|
Version string `gorm:"size:64;not null"`
|
|
LocalDate string `gorm:"size:10"`
|
|
State string `gorm:"size:16;not null;index"`
|
|
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
|
|
MessageID *int64 `gorm:"index"`
|
|
TaskID *int64 `gorm:"index"`
|
|
LastError string `gorm:"type:text"`
|
|
Subject string `gorm:"type:text"`
|
|
BodyText string `gorm:"type:text"`
|
|
CreatedAt time.Time
|
|
UpdatedAt time.Time
|
|
}
|
|
|
|
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
|
|
|
|
type StatusPageDigestSchedule struct {
|
|
ID int64 `gorm:"primarykey"`
|
|
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
|
|
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
|
|
DueAt time.Time `gorm:"not null;index"`
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
|
|
|
|
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
|
|
if subscriber.ContactID != nil {
|
|
return nil
|
|
}
|
|
accountID := page.AccountID
|
|
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
|
|
if err := tx.Create(contact).Error; err != nil {
|
|
return err
|
|
}
|
|
subscriber.ContactID = &contact.ID
|
|
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
|
|
}
|
|
|
|
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
|
|
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
|
|
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
|
|
}
|
|
|
|
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
|
|
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
|
|
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
|
|
}
|
|
|
|
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
|
|
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
|
|
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
|
|
}
|
|
|
|
func valueOrZero(v *int64) int64 {
|
|
if v == nil {
|
|
return 0
|
|
}
|
|
return *v
|
|
}
|
|
|
|
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
|
|
// transaction, so an incident can never become visible without its delivery
|
|
// intent being recoverable by the dispatcher.
|
|
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
|
|
var subscribers []StatusPageSubscriber
|
|
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
|
|
return err
|
|
}
|
|
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
|
for _, subscriber := range subscribers {
|
|
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
|
|
var notification Notification
|
|
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(¬ification).Error
|
|
if err == nil {
|
|
return ¬ification, nil
|
|
}
|
|
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil, err
|
|
}
|
|
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
|
|
return ¬ification, tx.Create(¬ification).Error
|
|
}
|
|
|
|
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
|
|
// Message and Task rows in the same transaction, and preserves a pending row on
|
|
// transient worker/capability failure for the next tick.
|
|
func DispatchStatusPageDeliveries(now time.Time) error {
|
|
var rows []StatusPageDelivery
|
|
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
|
|
return err
|
|
}
|
|
var errs []error
|
|
for _, row := range rows {
|
|
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func dispatchStatusPageDelivery(id int64, now time.Time) error {
|
|
err := DB().Transaction(func(tx *gorm.DB) error {
|
|
var outbox StatusPageDelivery
|
|
if err := tx.First(&outbox, id).Error; err != nil {
|
|
return err
|
|
}
|
|
var subscriber StatusPageSubscriber
|
|
var page StatusPage
|
|
// Subscriber then outbox is the global lock order shared with
|
|
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
|
|
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
|
|
}
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
|
|
return err
|
|
}
|
|
if outbox.State != statusPageOutboxPending {
|
|
return nil
|
|
}
|
|
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
|
|
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
|
|
}
|
|
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
|
|
return err
|
|
}
|
|
notification, err := statusPageNotificationTx(tx, page.AccountID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
subject, text := outbox.Subject, outbox.BodyText
|
|
if subject == "" {
|
|
subject, text = "["+page.Name+"] status update", "Status update"
|
|
}
|
|
if outbox.IncidentID != nil {
|
|
var incident StatusPageIncident
|
|
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
|
|
return err
|
|
}
|
|
subject = "[" + page.Name + "] " + incident.Title
|
|
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
|
|
} else if outbox.BodyText == "" {
|
|
text = "Open incidents daily digest"
|
|
}
|
|
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
|
|
if err := tx.Create(message).Error; err != nil {
|
|
return err
|
|
}
|
|
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
|
|
})
|
|
if err != nil {
|
|
// This update intentionally runs after rollback: diagnostic state must not
|
|
// disappear with the failed message/task transaction.
|
|
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
|
|
}
|
|
return err
|
|
}
|
|
|
|
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
|
|
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
|
|
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
|
|
return err
|
|
}
|
|
if outbox.TaskID != nil {
|
|
// Redact every task, including terminal audit rows. State is preserved for
|
|
// terminal rows, while the payload can no longer disclose the address.
|
|
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if outbox.MessageID != nil {
|
|
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
|
|
var subscriber StatusPageSubscriber
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return err
|
|
}
|
|
var rows []StatusPageDelivery
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
|
|
return err
|
|
}
|
|
for i := range rows {
|
|
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
|
|
// retained status-page lineage without touching unrelated account messages.
|
|
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
|
|
var rows []StatusPageDelivery
|
|
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
|
|
return err
|
|
}
|
|
for i := range rows {
|
|
row := &rows[i]
|
|
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
if row.TaskID != nil {
|
|
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if row.MessageID != nil {
|
|
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
|
|
var rows []StatusPageDelivery
|
|
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
|
|
return err
|
|
}
|
|
for i := range rows {
|
|
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
if rows[i].TaskID != nil {
|
|
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if rows[i].MessageID != nil {
|
|
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
|
|
// endpoint inert and irreversibly removing subscriber PII.
|
|
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
|
|
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
|
|
return err
|
|
}
|
|
var subscribers []StatusPageSubscriber
|
|
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
|
|
return err
|
|
}
|
|
now := time.Now()
|
|
for i := range subscribers {
|
|
s := &subscribers[i]
|
|
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
|
|
return err
|
|
}
|
|
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
|
|
return err
|
|
}
|
|
if s.ContactID != nil {
|
|
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return tx.Delete(page).Error
|
|
}
|
|
|
|
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
|
|
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
|
|
func EnqueueStatusPageDailyDigests(now time.Time) error {
|
|
var subscribers []StatusPageSubscriber
|
|
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
|
|
return err
|
|
}
|
|
var errs []error
|
|
for i := range subscribers {
|
|
s := &subscribers[i]
|
|
if s.StatusPage == nil || s.StatusPage.Account == nil {
|
|
continue
|
|
}
|
|
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
|
|
if err != nil {
|
|
loc = time.UTC
|
|
}
|
|
localNow := now.In(loc)
|
|
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
|
|
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
|
|
// Only persist today's actual due date. Older pending outbox rows are
|
|
// retried by DispatchStatusPageDeliveries; never invent history from a
|
|
// current incident snapshot after downtime.
|
|
if now.Before(due) || s.ConfirmedAt.After(due) {
|
|
continue
|
|
}
|
|
var incidents []StatusPageIncident
|
|
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
if len(incidents) == 0 {
|
|
continue
|
|
} // No digest is better than a misleading outage summary.
|
|
lines := make([]string, 0, len(incidents))
|
|
for _, incident := range incidents {
|
|
lines = append(lines, "- "+incident.Title)
|
|
}
|
|
err = DB().Transaction(func(tx *gorm.DB) error {
|
|
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
|
|
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
|
|
return err
|
|
}
|
|
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
|
|
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
if err := DispatchStatusPageDeliveries(now); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func StartStatusPageDigestScheduler(ctx context.Context) {
|
|
go func() {
|
|
ticker := time.NewTicker(time.Minute)
|
|
defer ticker.Stop()
|
|
for {
|
|
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
|
|
log.Printf("status-page delivery dispatch: %v", err)
|
|
}
|
|
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
|
|
log.Printf("status-page digest: %v", err)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}()
|
|
}
|