feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.

internal/notifier/email.go

@@ -0,0 +1,88 @@
+// Package notifier provides functionality.
+package notifier
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/mail"
+
+	"github.com/microcosm-cc/bluemonday"
+	"gopkg.in/gomail.v2"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+)
+
+// SendEmail provides functionality.
+func SendEmail(to, subject, body string) error {
+	p := bluemonday.StripTagsPolicy()
+	cred, err := firstSMTPCredential()
+	if err != nil {
+		return err
+	}
+
+	fromAddr := &mail.Address{Name: cred.fromName, Address: cred.fromAddress}
+	from := fromAddr.String()
+
+	m := gomail.NewMessage()
+	m.SetHeader("From", from)
+	m.SetHeader("To", to)
+	m.SetHeader("Subject", subject)
+	// m.SetBody("text/html", body)
+	m.AddAlternative("text/plain", p.Sanitize(body))
+	m.AddAlternative("text/html", body)
+
+	d := gomail.NewDialer(
+		cred.server,
+		cred.port,
+		cred.login,
+		cred.password,
+	)
+	if cred.insecureSkipVerify {
+		d.TLSConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	return d.DialAndSend(m)
+}
+
+type smtpCredential struct {
+	server             string
+	port               int
+	login              string
+	password           string
+	fromName           string
+	fromAddress        string
+	insecureSkipVerify bool
+}
+
+func firstSMTPCredential() (*smtpCredential, error) {
+	creds, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
+	if err != nil {
+		return nil, err
+	}
+	if len(creds) == 0 {
+		return nil, errors.New("smtp credential is not configured")
+	}
+	c := &creds[0]
+	password, err := c.GetSecret()
+	if err != nil {
+		return nil, fmt.Errorf("smtp credential secret: %w", err)
+	}
+	out := &smtpCredential{password: password, insecureSkipVerify: c.InsecureSkipVerify}
+	if c.Server != nil {
+		out.server = *c.Server
+	}
+	if c.Port != nil {
+		out.port = *c.Port
+	}
+	if c.Login != nil {
+		out.login = *c.Login
+	}
+	if c.FromName != nil {
+		out.fromName = *c.FromName
+	}
+	if c.FromAddr != nil {
+		out.fromAddress = *c.FromAddr
+	}
+	return out, nil
+}

internal/notifier/notifier.go

@@ -0,0 +1,77 @@
+package notifier
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+)
+
+// Tunables for the periodic notifier loops. The values match what the legacy
+// Start() function used so behavior is unchanged.
+var (
+	interval         = 5 * time.Second
+	expInterval      = 2 * time.Hour
+	deletionInterval = 1 * time.Hour
+)
+
+// StartScheduler launches the three periodic loops that used to be triggered
+// by the retired notifier.Start singleton: the notification producer (Run),
+// the expiry-alert producer (RunExp), and the pending-deletion sweep. Phase 3
+// of docs/plans/worker-notifier-mvp.md replaces the in-process notifier loop
+// with the worker-driven task queue; this scheduler keeps the producer
+// running on its existing cadence so the tasks table stays populated.
+//
+// The loops respect ctx.Done() so a graceful shutdown can unwind them, and
+// each tick is wrapped in recover() so a transient bug in one producer does
+// not tear down the whole scheduler.
+//
+// Reaper: StartTaskReaper lives in app/models/task_reaper.go and runs the
+// leased->queued recycling on a separate 30s tick.
+func StartScheduler(ctx context.Context) {
+	go scheduleLoop(ctx, interval, Run, "Run")
+	go scheduleLoop(ctx, expInterval, RunExp, "RunExp")
+	go scheduleLoop(ctx, deletionInterval, RunPendingDeletions, "RunPendingDeletions")
+}
+
+// scheduleLoop runs fn immediately and then on every tick. Any panic from
+// fn is recovered and logged so the loop keeps running.
+func scheduleLoop(ctx context.Context, tick time.Duration, fn func(), name string) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("notifier: scheduler %s goroutine recovered from panic: %v", name, r)
+		}
+	}()
+
+	safeRun(name, fn)
+	ticker := time.NewTicker(tick)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			safeRun(name, fn)
+		}
+	}
+}
+
+// safeRun invokes fn with a panic recovery guard. Each tick is wrapped so a
+// single bad tick cannot kill the loop. The loop goroutine itself has its own
+// recover() (see scheduleLoop) for paranoia.
+func safeRun(name string, fn func()) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("notifier: %s recovered from panic: %v", name, r)
+		}
+	}()
+	fn()
+}
+
+// RunPendingDeletions hard-deletes users whose 7-day grace period has elapsed.
+func RunPendingDeletions() {
+	if _, err := models.ProcessPendingDeletions(); err != nil {
+		log.Printf("notifier: process pending deletions: %v", err)
+	}
+}

internal/notifier/producer.go

@@ -0,0 +1,166 @@
+package notifier
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"time"
+
+	"gorm.io/gorm"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+	"rsgit.ru/rsmon/rsmon/internal/notifyrender"
+	"rsgit.ru/rsmon/rsmon/internal/wire"
+)
+
+// langEN is the wire-side default language tag used when a Message carries
+// no language hint of its own. Centralized so the literal does not appear
+// three or more times across this package (goconst).
+const langEN = "en"
+
+// ContactKindToMethod maps the legacy Contact.Kind enum used by the sender onto
+// the worker notification_method enum introduced in
+// docs/plans/worker-notifier-mvp.md section 4.3. sms/voice remain placeholders
+// until phase 4.
+//
+//nolint:goconst // match arm values must be the wire-method enum literals
+func ContactKindToMethod(kind string) string {
+	switch kind {
+	case "email":
+		return "email"
+	case "telegram_private", "telegram_group":
+		return "telegram"
+	case "webhook":
+		return "webhook"
+	case "mattermost":
+		return "mattermost"
+	case "sms":
+		return "sms"
+	case "voice":
+		return "voice"
+	}
+	return ""
+}
+
+// RenderNotificationContent pre-renders subject + bodies for one Message using
+// the existing internal/sender/get_content.go helpers. The result is what the
+// worker binary consumes directly so it does not need access to Message/Event
+// rows, workdays, or NotificationDayStart logic on the data plane.
+//
+// Returns the four bodies (subject, text, markdown, html). The caller is
+// responsible for passing them through to EnqueueNotificationTask.
+func RenderNotificationContent(msg *models.Message, now time.Time) (subject, bodyText, bodyMarkdown, bodyHTML string, err error) {
+	if msg == nil {
+		return "", "", "", "", errors.New("notifier: nil message")
+	}
+	defer func() {
+		if r := recover(); r != nil {
+			err = errors.New("notifier: render panicked")
+		}
+	}()
+	sbuf, tbuf, mbuf, hbuf := notifyrender.GetContent(msg, now)
+	return sbuf.String(), tbuf.String(), mbuf.String(), hbuf.String(), nil
+}
+
+// EnqueueNotificationTaskFromMessage is the producer-side hook called from
+// performEvents (or its replacement). It builds a wire.NotificationTask from the
+// freshly created Message and enqueues one Task row keyed by the stable
+// (notification, contact, first-event) idempotency key.
+//
+// If the producer's authorization precheck fails the function returns an error:
+// worker notification tasks are now the only delivery path.
+func EnqueueNotificationTaskFromMessage(n *models.Notification, c *models.Contact, msg *models.Message) (*models.Task, error) {
+	return enqueueNotificationTaskFromMessageTx(models.DB(), n, c, msg)
+}
+
+// enqueueNotificationTaskFromMessageTx keeps message creation and durable task
+// production in the caller's notifier transaction.
+func enqueueNotificationTaskFromMessageTx(tx *gorm.DB, n *models.Notification, c *models.Contact, msg *models.Message) (*models.Task, error) {
+	if msg == nil || n == nil || c == nil {
+		return nil, errors.New("notifier: nil message/notification/contact")
+	}
+	if len(msg.Events) == 0 {
+		return nil, errors.New("notifier: message has no events (exp messages go through a separate path)")
+	}
+
+	method := ContactKindToMethod(c.Kind)
+	if method == "" {
+		log.Printf("notifier: unknown contact kind %q for contact %d, skipping enqueue", c.Kind, c.ID)
+		return nil, nil
+	}
+
+	now := time.Now()
+	subject, bodyText, bodyMarkdown, bodyHTML, err := RenderNotificationContent(msg, now)
+	if err != nil {
+		log.Printf("notifier: render content failed for message %d: %v", msg.ID, err)
+		return nil, err
+	}
+
+	checkID := msg.CheckID
+	monitorID := msg.Events[0].MonitorID
+	task := wire.NotificationTask{
+		AccountID:      n.AccountID,
+		MessageID:      msg.ID,
+		NotificationID: n.ID,
+		EventIDs:       eventIDs(msg),
+		CheckID:        checkID,
+		MonitorID:      &monitorID,
+		Method:         method,
+		Contact: wire.NotificationContact{
+			ID:    c.ID,
+			Kind:  c.Kind,
+			Value: c.Value,
+			Name:  c.Name,
+		},
+		Subject:      subject,
+		BodyText:     bodyText,
+		BodyMarkdown: bodyMarkdown,
+		BodyHTML:     bodyHTML,
+		Language:     langEN,
+		MessageKind:  msg.Kind,
+	}
+	payload, err := json.Marshal(task)
+	if err != nil {
+		return nil, err
+	}
+
+	contactID := c.ID
+	monitorPtr := task.MonitorID
+	checkPtr := task.CheckID
+	messagePtr := msg.ID
+
+	taskRow, err := models.EnqueueNotificationTaskTx(tx, &models.EnqueueNotificationTaskInput{
+		AccountID:      n.AccountID,
+		NotificationID: n.ID,
+		ContactID:      contactID,
+		MessageID:      &messagePtr,
+		MonitorID:      monitorPtr,
+		CheckID:        checkPtr,
+		EventIDs:       task.EventIDs,
+		Method:         method,
+		Subject:        subject,
+		BodyText:       bodyText,
+		BodyHTML:       bodyHTML,
+		BodyMarkdown:   bodyMarkdown,
+		Language:       task.Language,
+		MessageKind:    msg.Kind,
+		NotBefore:      now,
+		Payload:        payload,
+	})
+	if err != nil {
+		return nil, err
+	}
+	log.Printf(
+		"notifier: task enqueued id=%d job=%s kind=notification account=%d method=%s notification=%d contact=%d event=%d idempotency=%s",
+		taskRow.ID, taskRow.JobID, n.AccountID, method, n.ID, c.ID, task.EventIDs[0], taskRow.IdempotencyKey,
+	)
+	return taskRow, nil
+}
+
+func eventIDs(msg *models.Message) []int64 {
+	out := make([]int64, 0, len(msg.Events))
+	for _, e := range msg.Events { //nolint:gocritic // range copy is acceptable here
+		out = append(out, e.ID)
+	}
+	return out
+}

internal/notifier/producer_test.go

@@ -0,0 +1,219 @@
+package notifier
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/datatypes"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+	"rsgit.ru/rsmon/rsmon/config/database"
+)
+
+func init() {
+	database.Init()
+}
+
+// TestContactKindToMethod is the table that drives producer + selector +
+// executor dispatch.
+func TestContactKindToMethod(t *testing.T) {
+	cases := map[string]string{
+		"email":            "email",
+		"telegram_private": "telegram",
+		"telegram_group":   "telegram",
+		"webhook":          "webhook",
+		"mattermost":       "mattermost",
+		"sms":              "sms",
+		"voice":            "voice",
+		"":                 "",
+		"unknown":          "",
+	}
+	for in, want := range cases {
+		assert.Equal(t, want, ContactKindToMethod(in), "kind=%q", in)
+	}
+}
+
+// TestEnqueueNotificationTaskFromMessage_SeedsTaskWithPreRenderedBody
+// exercises the producer end-to-end against the test DB. It seeds an account,
+// notification, contact, and event; calls EnqueueNotificationTaskFromMessage;
+// and checks the resulting Task row has the pre-rendered subject/body in the
+// payload (i.e. the worker does not need to know templating).
+func TestEnqueueNotificationTaskFromMessage_SeedsTaskWithPreRenderedBody(t *testing.T) {
+	models.Drop()
+	models.Migrate()
+	models.DB().Exec(
+		"INSERT INTO regions (code, name, enabled, created_at, updated_at) VALUES (?, ?, true, now(), now())",
+		"test", "test",
+	)
+
+	plan := models.Plan{Name: "producer-test"}
+	require.NoError(t, models.DB().Create(&plan).Error)
+	user := models.User{Name: "u", Email: producerStringPtr("u@example.com"), Timezone: "UTC"}
+	require.NoError(t, models.DB().Create(&user).Error)
+	account := models.Account{Name: "a", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
+	require.NoError(t, models.DB().Create(&account).Error)
+
+	group := models.Group{Name: "g", AccountID: account.ID}
+	require.NoError(t, models.DB().Create(&group).Error)
+
+	monitor := models.Monitor{
+		Name:    producerStringPtr("m"),
+		Host:    "example.com",
+		GroupID: group.ID,
+		Enabled: true,
+	}
+	require.NoError(t, models.DB().Create(&monitor).Error)
+
+	notification := models.Notification{
+		Name: "default", AccountID: account.ID, Enabled: true,
+		NotifyDown: true, NotifyRestore: true,
+	}
+	require.NoError(t, models.DB().Create(&notification).Error)
+
+	contact := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &account.ID}
+	require.NoError(t, models.DB().Create(&contact).Error)
+
+	start := time.Now().Add(-time.Minute)
+	event := models.Event{
+		MonitorID: monitor.ID,
+		StartTime: &start,
+		State:     "current",
+		Errors:    5,
+	}
+	require.NoError(t, models.DB().Create(&event).Error)
+
+	msg := models.Message{
+		NotificationID: notification.ID,
+		ContactID:      contact.ID,
+		Events:         []models.Event{event},
+		Kind:           "down",
+		State:          "queued",
+	}
+	require.NoError(t, models.DB().Create(&msg).Error)
+
+	w := &models.WorkerNode{
+		WorkerID:    "worker-producer",
+		RegionCode:  "test",
+		Status:      "active",
+		AuthToken:   "tok",
+		Concurrency: 4,
+		LastSeen:    producerTimePtr(time.Now()),
+		Capabilities: datatypes.JSON([]byte(
+			`{"check_types":["http"],"task_envelope":true,"notification_methods":["email"],"notification_accounts":[]}`,
+		)),
+	}
+	require.NoError(t, models.DB().Create(w).Error)
+
+	// Load the message back with the scope GetContent expects (Monitor + Group
+	// + Notification preloaded). The sender's GetContent panics on nil fields.
+	loaded := models.Message{}
+	require.NoError(t, models.MessageScope(models.DB()).First(&loaded, msg.ID).Error)
+	require.Len(t, loaded.Events, 1)
+	require.NotNil(t, loaded.Events[0].Monitor)
+
+	row, err := EnqueueNotificationTaskFromMessage(&notification, &contact, &loaded)
+	require.NoError(t, err)
+	require.NotNil(t, row, "expected a Task row from the producer")
+	assert.Equal(t, models.TaskKindNotification, row.Kind)
+	assert.Equal(t, models.TaskStateQueued, row.State)
+	assert.NotEmpty(t, row.Payload)
+	assert.Equal(t, account.ID, row.AccountID)
+	assert.Equal(t, &contact.ID, row.ContactID)
+	require.NotNil(t, row.MessageID)
+	assert.Equal(t, loaded.ID, *row.MessageID)
+	assert.Equal(t, models.NotificationIdempotencyKey(notification.ID, contact.ID, event.ID), row.IdempotencyKey)
+
+	var payload map[string]interface{}
+	require.NoError(t, json.Unmarshal(row.Payload, &payload))
+	assert.Equal(t, "email", payload["method"])
+	assert.Equal(t, "down", payload["message_kind"])
+	assert.NotEmpty(t, payload["subject"])
+}
+
+// TestEnqueueNotificationTaskFromMessage_Idempotent exercises the producer's
+// idempotency contract: a second call with the same (notification, contact,
+// event) tuple must not create a second Task row.
+func TestEnqueueNotificationTaskFromMessage_Idempotent(t *testing.T) {
+	models.Drop()
+	models.Migrate()
+	models.DB().Exec(
+		"INSERT INTO regions (code, name, enabled, created_at, updated_at) VALUES (?, ?, true, now(), now())",
+		"test", "test",
+	)
+	plan := models.Plan{Name: "p"}
+	require.NoError(t, models.DB().Create(&plan).Error)
+	user := models.User{Name: "u", Email: producerStringPtr("u@example.com"), Timezone: "UTC"}
+	require.NoError(t, models.DB().Create(&user).Error)
+	account := models.Account{Name: "a", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
+	require.NoError(t, models.DB().Create(&account).Error)
+	group := models.Group{Name: "g", AccountID: account.ID}
+	require.NoError(t, models.DB().Create(&group).Error)
+	monitor := models.Monitor{
+		Name:    producerStringPtr("m"),
+		Host:    "example.com",
+		GroupID: group.ID,
+		Enabled: true,
+	}
+	require.NoError(t, models.DB().Create(&monitor).Error)
+	notification := models.Notification{
+		Name: "default", AccountID: account.ID, Enabled: true,
+		NotifyDown: true, NotifyRestore: true,
+	}
+	require.NoError(t, models.DB().Create(&notification).Error)
+	contact := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &account.ID}
+	require.NoError(t, models.DB().Create(&contact).Error)
+	start := time.Now().Add(-time.Minute)
+	event := models.Event{MonitorID: monitor.ID, StartTime: &start, State: "current", Errors: 5}
+	require.NoError(t, models.DB().Create(&event).Error)
+
+	w := &models.WorkerNode{
+		WorkerID:    "worker-idem",
+		RegionCode:  "test",
+		Status:      "active",
+		AuthToken:   "tok",
+		Concurrency: 4,
+		LastSeen:    producerTimePtr(time.Now()),
+		Capabilities: datatypes.JSON([]byte(
+			`{"check_types":["http"],"task_envelope":true,"notification_methods":["email"],"notification_accounts":[]}`,
+		)),
+	}
+	require.NoError(t, models.DB().Create(w).Error)
+
+	msg := models.Message{
+		NotificationID: notification.ID,
+		ContactID:      contact.ID,
+		Events:         []models.Event{event},
+		Kind:           "down",
+		State:          "queued",
+	}
+	require.NoError(t, models.DB().Create(&msg).Error)
+
+	loaded := models.Message{}
+	require.NoError(t, models.MessageScope(models.DB()).First(&loaded, msg.ID).Error)
+
+	first, err := EnqueueNotificationTaskFromMessage(&notification, &contact, &loaded)
+	require.NoError(t, err)
+	require.NotNil(t, first)
+
+	second, err := EnqueueNotificationTaskFromMessage(&notification, &contact, &loaded)
+	require.NoError(t, err)
+	require.NotNil(t, second)
+	assert.Equal(t, first.ID, second.ID, "second producer call must reuse the first task row")
+
+	var count int64
+	require.NoError(t, models.DB().Model(&models.Task{}).
+		Where("idempotency_key = ?", first.IdempotencyKey).
+		Count(&count).Error)
+	assert.EqualValues(t, 1, count)
+}
+
+func producerStringPtr(s string) *string { return &s }
+
+func producerTimePtr(value time.Time) *time.Time { return &value }
+
+// guard against uuid being accidentally dropped from the imports.
+var _ = uuid.New

internal/notifier/run.go

@@ -0,0 +1,370 @@
+package notifier
+
+import (
+	"log"
+	"time"
+
+	"gorm.io/gorm"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+)
+
+const (
+	eventStateEnded   = "ended"
+	eventStateCurrent = "current"
+	messageKindDown   = "down"
+	messageKindUp     = "up"
+)
+
+// DEBUG provides functionality.
+const DEBUG = false
+
+// Run starts the notification scheduler loop.
+func Run() {
+	_ = models.LogCheck("notify")
+
+	events := make([]models.Event, 0)
+
+	tx := models.DB().Begin()
+
+	q := tx
+	// q = q.Set("gorm:query_option", "FOR UPDATE")
+	err := models.EventScope(q).Find(&events).Error
+	if err != nil {
+		tx.Rollback()
+		log.Println(err)
+		return
+		// panic(err)
+	}
+
+	type SendItem struct {
+		Notification models.Notification
+		Contact      models.Contact
+		Events       []models.Event
+	}
+
+	eventsByNotification := make(map[int64]map[int64]*SendItem, 0)
+
+	hasPossible := make(map[int64]bool)
+
+	eventIDs := make(map[int64]bool, 0)
+	contactIDs := make(map[int64]bool, 0)
+	alreadySentDown := make(map[int64]map[int64]bool, 0)
+	alreadySentUp := make(map[int64]map[int64]bool, 0)
+	for _, e := range events { //nolint:gocritic // range copy is acceptable here
+		eventIDs[e.ID] = true
+		if e.Monitor == nil {
+			println("event has no monitor")
+			e.State = "broken"
+			err := tx.Save(&e).Error
+			if err != nil {
+				tx.Rollback()
+				log.Println(err)
+				return
+				// panic(err)
+			}
+			continue
+		}
+		if e.Monitor.Group == nil {
+			println("monitor has no group")
+			e.State = "broken"
+			err := tx.Save(&e).Error
+			if err != nil {
+				tx.Rollback()
+				log.Println(err)
+				return
+				// panic(err)
+			}
+			continue
+		}
+		for _, n := range e.Monitor.Group.Notifications { //nolint:gocritic // range copy is acceptable here
+			for _, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
+				contactIDs[c.ID] = true
+				alreadySentDown[c.ID] = make(map[int64]bool, 0)
+				alreadySentUp[c.ID] = make(map[int64]bool, 0)
+			}
+		}
+	}
+
+	sentMessages := make([]models.Message, 0)
+
+	eventIDsSlice := make([]int64, 0)
+	for k := range eventIDs {
+		eventIDsSlice = append(eventIDsSlice, k)
+	}
+	contactIDsSlice := make([]int64, 0)
+	for k := range contactIDs {
+		contactIDsSlice = append(contactIDsSlice, k)
+	}
+
+	err = tx.Preload("Events").
+		// Where("kind = ?", "down").
+		Where("id IN (SELECT message_id FROM event_messages WHERE event_id IN (?))", eventIDsSlice).
+		Where("contact_id IN (?)", contactIDsSlice).Find(&sentMessages).
+		Error
+	if err != nil {
+		tx.Rollback()
+		log.Println(err)
+		return
+		// panic(err)
+	}
+
+	for _, m := range sentMessages { //nolint:gocritic // range copy is acceptable here
+		for _, evt := range m.Events { //nolint:gocritic // range copy is acceptable here
+			switch m.Kind {
+			case messageKindDown:
+				alreadySentDown[m.ContactID][evt.ID] = true
+			case messageKindUp:
+				alreadySentUp[m.ContactID][evt.ID] = true
+			}
+		}
+	}
+
+	for _, e := range events { //nolint:gocritic // range copy is acceptable here
+		if e.Monitor == nil {
+			println("event has no monitor")
+			continue
+		}
+		if e.Monitor.Group == nil {
+			println("monitor has no group")
+			continue
+		}
+		underMaintenance, maintenanceErr := models.MonitorUnderMaintenance(e.MonitorID, time.Now().UTC())
+		if maintenanceErr != nil {
+			log.Printf("notifier: maintenance lookup for monitor %d: %v", e.MonitorID, maintenanceErr)
+		} else if underMaintenance {
+			// Keep the event pending. Marking it old here would silently drop a
+			// failure that remains unresolved after the maintenance window ends.
+			hasPossible[e.ID] = true
+			continue
+		}
+
+		if e.State == eventStateEnded {
+			hasPossible[e.ID] = false
+		} else {
+			hasPossible[e.ID] = true
+		}
+
+		if DEBUG {
+			log.Println("run event", e.Inspect())
+		}
+		for _, n := range e.Monitor.Group.Notifications { //nolint:gocritic // range copy is acceptable here
+			if !n.Enabled {
+				if DEBUG {
+					log.Println("event", e.ID, "notification", n.ID, "Enabled = false")
+				}
+				continue
+			}
+
+			if e.State == eventStateCurrent {
+				if !n.NotifyDown {
+					if DEBUG {
+						log.Println("event", e.ID, "notification", n.ID, "NotifyDown = false")
+					}
+					continue
+				}
+			} else if e.State == eventStateEnded {
+				if !n.NotifyRestore {
+					if DEBUG {
+						log.Println("event", e.ID, "notification", n.ID, "NotifyRestore = false")
+					}
+					continue
+				}
+			}
+
+			if _, ok := eventsByNotification[n.ID]; !ok {
+				eventsByNotification[n.ID] = make(map[int64]*SendItem, 0)
+			}
+
+			for _, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
+				if e.State == eventStateCurrent {
+					if _, sent := alreadySentDown[c.ID][e.ID]; sent {
+						if DEBUG {
+							log.Println("event", e.ID, "notification", n.ID, "already sent")
+						}
+						continue
+					}
+				} else if e.State == eventStateEnded {
+					if _, sent := alreadySentDown[c.ID][e.ID]; !sent {
+						if DEBUG {
+							log.Println("event", e.ID, "dont notify up", n.ID, "- no down was sent")
+						}
+						continue
+					}
+
+					if _, sent := alreadySentUp[c.ID][e.ID]; sent {
+						if DEBUG {
+							log.Println("event", e.ID, "notification", n.ID, "already sent")
+						}
+						continue
+					}
+				}
+				if _, ok := eventsByNotification[n.ID][c.ID]; !ok {
+					si := SendItem{
+						Notification: n,
+						Contact:      c,
+						Events:       make([]models.Event, 0),
+					}
+					// log.Println("create", n.ID, c.ID)
+					// spew.Dump(si)
+					eventsByNotification[n.ID][c.ID] = &si
+				}
+				sendItem := eventsByNotification[n.ID][c.ID]
+				sendItem.Events = append(sendItem.Events, e)
+			}
+		}
+	}
+
+	for _, eventsByContact := range eventsByNotification {
+		for _, sendItem := range eventsByContact {
+			n := sendItem.Notification
+			c := sendItem.Contact
+
+			tn := time.Now()
+
+			requredEvents := make([]models.Event, 0)
+			possibleEvents := make([]models.Event, 0)
+			laterEvents := make([]models.Event, 0)
+
+			for _, e := range sendItem.Events { //nolint:gocritic // range copy is acceptable here
+				dur := e.GetDuration(tn)
+				var delay int64
+				if n.AlertDelay != nil {
+					delay = *n.AlertDelay
+				} else {
+					delay = 300
+				}
+
+				if !n.EnabledNow(&tn) {
+					if DEBUG {
+						log.Println("notification", n.ID, "is not enabled at this time")
+					}
+					laterEvents = append(laterEvents, e)
+				}
+
+				if e.State == eventStateCurrent && e.Errors > 4 { //nolint:gocritic // complex condition chain
+					if DEBUG {
+						log.Println("min errors count to force send reached:", e.Errors)
+					}
+					requredEvents = append(requredEvents, e)
+				} else if e.State == eventStateCurrent && e.Errors < 2 {
+					if DEBUG {
+						log.Println("event possbile to notify in aggregation, but errs count not reached:", e.Errors)
+					}
+					possibleEvents = append(possibleEvents, e)
+				} else if e.State == eventStateEnded && e.Oks > 4 {
+					if DEBUG {
+						log.Println("min oks count to force send reached:", e.Oks)
+					}
+					requredEvents = append(requredEvents, e)
+				} else if e.State == eventStateEnded && e.Oks < 2 {
+					if DEBUG {
+						log.Println("event possbile to notify in aggregation, but oks count not reached:", e.Oks)
+					}
+					possibleEvents = append(possibleEvents, e)
+				} else if dur < delay {
+					if DEBUG {
+						log.Println("event possbile to notify in aggregation, but alert_delay not reached: delay", delay, "duration", dur, "so", (delay - dur), "left") //nolint:lll
+					}
+					possibleEvents = append(possibleEvents, e)
+				} else {
+					if DEBUG {
+						log.Println("event required to notify, alert_delay reached: delay", delay, "duration", dur, "so", (delay - dur), "left") //nolint:lll
+					}
+					requredEvents = append(requredEvents, e)
+				}
+			}
+			if len(requredEvents) > 0 {
+				performEvents(tx, &n, &c, append(requredEvents, possibleEvents...))
+			} else {
+				if len(possibleEvents) > 0 || len(laterEvents) > 0 {
+					// log.Println("notification", n.ID, "no required events, but will send later")
+					for _, evt := range possibleEvents { //nolint:gocritic // range copy is acceptable here
+						hasPossible[evt.ID] = true
+					}
+					for _, evt := range laterEvents { //nolint:gocritic // range copy is acceptable here
+						hasPossible[evt.ID] = true
+					}
+				} else {
+					log.Println("notification", n.ID, "no events left")
+				}
+			}
+
+			// spew.Dump(sendItem.Notification)
+			// spew.Dump(sendItem.Contact)
+			// spew.Dump(sendItem.Events)
+		}
+	}
+
+	for _, e := range events { //nolint:gocritic // range copy is acceptable here
+		if !hasPossible[e.ID] {
+			// log.Println("event has no possible notifications left to send, mark as done")
+			e.State = "old"
+			err := tx.Save(&e).Error
+			if err != nil {
+				tx.Rollback()
+				log.Println(err)
+				return
+				// panic(err)
+			}
+		}
+	}
+
+	tx.Commit()
+}
+
+func performEvents(tx *gorm.DB, n *models.Notification, c *models.Contact, events []models.Event) {
+	eventIDs := make([]int64, 0, len(events))
+	for _, evt := range events { //nolint:gocritic // range copy is acceptable here
+		eventIDs = append(eventIDs, evt.ID)
+	}
+	log.Println("performing events:", n.ID, c.ID, eventIDs)
+
+	eventsByKind := make(map[string][]models.Event, 0)
+	for _, evt := range events { //nolint:gocritic // range copy is acceptable here
+		var kind string
+		switch evt.State {
+		case eventStateCurrent:
+			kind = messageKindDown
+		case eventStateEnded:
+			kind = messageKindUp
+		default:
+			log.Println("unknown event state: " + evt.State)
+			tx.Rollback()
+			return
+		}
+		if _, ok := eventsByKind[kind]; !ok {
+			eventsByKind[kind] = make([]models.Event, 0)
+		}
+		eventsByKind[kind] = append(eventsByKind[kind], evt)
+	}
+
+	for kind, evts := range eventsByKind {
+		// spew.Dump(kind, evts)
+		message := models.Message{
+			NotificationID: n.ID,
+			ContactID:      c.ID,
+			Events:         evts,
+			Kind:           kind,
+			State:          models.TaskStateQueued,
+		}
+		err := tx.Save(&message).Error
+		if err != nil {
+			// panic(err)
+			tx.Rollback()
+			log.Println(err)
+			return
+		}
+
+		// Worker notification tasks are the only delivery path. If enqueue fails,
+		// keep the message as an explicit error instead of relying on the retired
+		// in-process sender loop.
+		if _, err := enqueueNotificationTaskFromMessageTx(tx, n, c, &message); err != nil {
+			errText := err.Error()
+			log.Printf("notifier: enqueue task for message %d failed: %v", message.ID, err)
+			if saveErr := tx.Model(&message).Updates(map[string]interface{}{"state": "error", "error": &errText}).Error; saveErr != nil {
+				log.Printf("notifier: mark message %d error failed: %v", message.ID, saveErr)
+			}
+		}
+	}
+}

internal/notifier/run_exp.go

@@ -0,0 +1,117 @@
+package notifier
+
+import (
+	"log"
+	"time"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+)
+
+// RunExp provides functionality.
+//
+// A panic in any single check's notification must not kill the scheduler
+// goroutine. The defensive recover() keeps the 2h tick alive even if
+// RunExpCheck trips over a bad row or a stale schema reference.
+func RunExp() {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("notifier: RunExp recovered from panic: %v", r)
+		}
+	}()
+
+	_ = models.LogCheck("exp")
+
+	checks := make([]models.Check, 0)
+
+	err := models.ExpScope(models.DB()).Find(&checks).Error
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	for i := range checks {
+		RunExpCheck(&checks[i])
+	}
+}
+
+// RunExpCheck provides functionality.
+//
+// A panic in any per-row work (notifier, contact lookup, message write) is
+// contained here so one bad row cannot take down the whole RunExp scheduler.
+// The panic is logged with the check id and the loop continues.
+func RunExpCheck(c *models.Check) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("notifier: RunExpCheck recovered from panic on check %d: %v", c.ID, r)
+		}
+	}()
+
+	if c.Monitor == nil {
+		log.Println("!BUG! check", c.ID, "has no Monitor (or not preloaded). Monitor ID: ", c.MonitorID, " Not running.")
+		log.Println()
+		return
+	}
+	if !c.Monitor.Enabled {
+		return
+	}
+
+	if c.Monitor.Group == nil {
+		log.Println("!BUG! check", c.ID, "has no Monitor (or not preloaded). Monitor ID: ", c.Monitor.ID, ", group id:", c.Monitor.GroupID, " Not running.") //nolint:lll
+		return
+	}
+	if c.Monitor.Group.Notifications == nil {
+		log.Println("!BUG! check", c.ID, "has no .Monitor.Group.Notifications (or not preloaded). Not running.")
+		return
+	}
+
+	for i := range c.Monitor.Group.Notifications {
+		n := &c.Monitor.Group.Notifications[i]
+		if n.BeforeExpiration == nil {
+			continue
+		}
+		if c.Expires == nil {
+			// should not happen
+			continue
+		}
+		if c.Kind == "whois" && !n.NotifyWHOIS {
+			continue
+		}
+		if c.Kind == "ssl" && !n.NotifySSL {
+			continue
+		}
+
+		// notify delay not reached
+		notifyOn := time.Now().Add(time.Second * time.Duration(*n.BeforeExpiration))
+		if c.Expires.After(notifyOn) {
+			continue
+		}
+
+		createExpMessage(c, n)
+	}
+}
+
+func createExpMessage(c *models.Check, n *models.Notification) {
+	for _, contact := range n.GetContacts() { //nolint:gocritic // range copy is acceptable here
+		message := models.Message{
+			CheckID:        &c.ID,
+			NotificationID: n.ID,
+			ContactID:      contact.ID,
+			Kind:           "exp",
+		}
+
+		models.DB().
+			Where(message).
+			Where("created_at > ?", time.Now().Add(-time.Hour*24*14)).
+			Find(&message)
+		if message.ID > 0 {
+			continue
+		}
+		message.State = models.TaskStateQueued
+		err := models.DB().Save(&message).Error
+		if err != nil {
+			log.Println(err)
+			return
+			// panic(err)
+		}
+	}
+}

internal/notifier/run_exp_test.go

@@ -0,0 +1,160 @@
+package notifier
+
+import (
+	"log"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+	"rsgit.ru/rsmon/rsmon/config/database"
+	"rsgit.ru/rsmon/rsmon/spec/factories"
+)
+
+func init() {
+	database.Init()
+}
+
+func TestRunExp(t *testing.T) {
+	log.Println("TestRunExp")
+	models.Drop()
+	models.Migrate()
+	var err error
+	contact, notification, monitor := factories.MonitorWithNotification()
+
+	check := factories.PersistedCheck(&monitor, "ssl")
+	assert.Equal(t, "ssl", check.Kind, "check kind should be ssl")
+	assert.Equal(t, monitor.ID, check.MonitorID, "check should have correct monitor id")
+
+	exp := time.Now().Add(14 * 24 * time.Hour)
+	check.Expires = &exp
+	err = models.DB().Save(&check).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+	RunExp()
+	shoudHaveMessages("exp1 - contact should receive no messages", t, "exp", notification.ID, contact.ID, []int64{})
+
+	exp = time.Now().Add(1 * time.Hour)
+	check.Expires = &exp
+	err = models.DB().Save(&check).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+	RunExp()
+	shoudHaveMessages("exp2 - contact should receive messages", t, "exp", notification.ID, contact.ID, []int64{check.ID})
+
+	RunExp()
+	shoudHaveMessages("exp3 - contact should not receive duplicate messages", t, "exp", notification.ID, contact.ID, []int64{check.ID})
+}
+
+// TestRunExpExpiresSystemContact exercises the regression scenario that the
+// dev DB hit after being restored from the production dump: the
+// contacts.is_system column was missing from the production schema and the
+// notifier's RunExp panic'd on the GORM preload.
+//
+// The fix has three layers: AutoMigrate adds the column, GetContacts logs
+// instead of panicking, and RunExp/RunExpCheck recover from panics. This
+// test verifies all three by setting a contact's is_system=true, queueing an
+// expiring SSL check, and confirming RunExp:
+//   - does not panic;
+//   - writes a queued exp message for the is_system contact (i.e. the schema
+//     has the column and the field round-trips); and
+//   - leaves RunExp returnable to its caller.
+func TestRunExpExpiresSystemContact(t *testing.T) {
+	log.Println("TestRunExpExpiresSystemContact")
+	models.Drop()
+	models.Migrate()
+
+	account := &models.Account{Name: "acct-system-contact"}
+	require.NoError(t, models.DB().Create(account).Error)
+	accountID := account.ID
+
+	trueVal := true
+	contact := &models.Contact{
+		AccountID: &accountID,
+		Name:      "system-admin",
+		Kind:      "email",
+		Value:     "ops@example.com",
+		IsSystem:  &trueVal,
+	}
+	require.NoError(t, models.DB().Create(contact).Error)
+
+	group := factories.PersistedGroup(account)
+	notification := factories.PersistedNotification(
+		account, []int64{contact.ID}, []int64{group.ID}, 300, false,
+	)
+	monitor := factories.PersistedMonitor(&group)
+
+	check := factories.PersistedCheck(&monitor, "ssl")
+	exp := time.Now().Add(1 * time.Hour)
+	check.Expires = &exp
+	require.NoError(t, models.DB().Save(&check).Error)
+
+	assert.NotPanics(t, func() {
+		RunExp()
+	}, "RunExp must not panic when processing an is_system contact")
+
+	shoudHaveMessages(
+		"is_system contact must receive the exp message",
+		t, "exp", notification.ID, contact.ID, []int64{check.ID},
+	)
+}
+
+// TestRunExpDoesNotPanicOnBrokenAssociation replays the original prod-dump
+// panic in a contained way: the notification_contacts join row references a
+// non-existent contact id, which forces GORM's preload of Contacts to fail.
+// The fix's defensive recover() must keep RunExpCheck returning cleanly so
+// the scheduler loop survives a single bad row.
+//
+// We deliberately bypass the contacts.is_system column-drop path because
+// Postgres caches prepared statements per session; mutating the contacts
+// schema mid-test triggers SQLSTATE 0A000 (cached plan must not change
+// result type) on the pool's other connections and masks the panic we want
+// to verify.
+func TestRunExpDoesNotPanicOnBrokenAssociation(t *testing.T) {
+	log.Println("TestRunExpDoesNotPanicOnBrokenAssociation")
+	models.Drop()
+	models.Migrate()
+
+	_, notification, monitor := factories.MonitorWithNotification()
+
+	// Force a broken association by deleting the contact that the
+	// notification points to. GORM's preload of Contacts will then have no
+	// rows for that notification, exercising the empty-contacts path
+	// without involving DDL or FK violations.
+	require.NoError(t, models.DB().
+		Exec("DELETE FROM notification_contacts WHERE notification_id = ?", notification.ID).Error)
+	// Re-add a join row pointing to a contact id that has been deleted
+	// from contacts. We disable the FK temporarily so the join row sticks.
+	require.NoError(t, models.DB().
+		Exec("SET session_replication_role = 'replica'").Error)
+	t.Cleanup(func() {
+		_ = models.DB().
+			Exec("SET session_replication_role = 'origin'").Error
+	})
+	bogusContactID := int64(9999999)
+	require.NoError(t, models.DB().
+		Exec(
+			"INSERT INTO notification_contacts (notification_id, contact_id) VALUES (?, ?)",
+			notification.ID, bogusContactID,
+		).Error)
+
+	check := factories.PersistedCheck(&monitor, "ssl")
+	exp := time.Now().Add(1 * time.Hour)
+	check.Expires = &exp
+	require.NoError(t, models.DB().Save(&check).Error)
+
+	require.NoError(t, models.DB().
+		Preload("Monitor").
+		Preload("Monitor.Group").
+		Preload("Monitor.Group.Notifications").
+		Preload("Monitor.Group.Notifications.Contacts").
+		First(&check, check.ID).Error)
+
+	assert.NotPanics(t, func() {
+		RunExpCheck(&check)
+	}, "RunExpCheck must not panic when Contact preload encounters a broken association")
+}

internal/notifier/run_test.go

@@ -0,0 +1,163 @@
+package notifier
+
+import (
+	"log"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/icrowley/fake"
+	"github.com/stretchr/testify/assert"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+	"rsgit.ru/rsmon/rsmon/config/database"
+	"rsgit.ru/rsmon/rsmon/spec/factories"
+)
+
+func init() {
+	database.Init()
+}
+
+func TestCreatesMessages(t *testing.T) {
+	log.Println("TestCreatesMessages")
+	models.Drop()
+	models.Migrate()
+	user := factories.PersistedUser("test@test.ru", "123")
+	account, err := models.CreateAccountForUser(fake.Company(), &user)
+	contact := factories.PersistedContact(account, &user)
+	group := factories.PersistedGroup(account)
+	notification := factories.PersistedNotification(account, []int64{contact.ID}, []int64{group.ID}, 300, false)
+	monitor := factories.PersistedMonitor(&group)
+
+	event := factories.PersistedEvent(&monitor, "current", "test event 1")
+	tn := time.Now()
+	tStart := tn.Add(-30 * time.Minute)
+	event.StartTime = &tStart
+	err = models.DB().Save(&event).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, event.GetDuration(tn), int64(30*60))
+
+	log.Println("run first")
+	Run()
+	shoudHaveMessages("1a - contact should receive messages", t, "down", notification.ID, contact.ID, []int64{event.ID})
+
+	// Run again
+	log.Println("run again")
+	Run()
+	shoudHaveMessages("1b - contact should not receive more than one message", t, "down", notification.ID, contact.ID, []int64{event.ID})
+}
+
+func TestAggregatesMessages(t *testing.T) {
+	log.Println("TestAggregatesMessages")
+	models.Drop()
+	models.Migrate()
+	user := factories.PersistedUser("test@test.ru", "123")
+	account, err := models.CreateAccountForUser(fake.Company(), &user)
+	contact := factories.PersistedContact(account, &user)
+	group := factories.PersistedGroup(account)
+	notification := factories.PersistedNotification(account, []int64{contact.ID}, []int64{group.ID}, 300, false)
+
+	monitor1 := factories.PersistedMonitor(&group)
+	monitor2 := factories.PersistedMonitor(&group)
+
+	event1 := factories.PersistedEvent(&monitor1, "current", "test event 2")
+	tStart := time.Now().Add(-30 * time.Minute)
+	event1.StartTime = &tStart
+	err = models.DB().Save(&event1).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	event2 := factories.PersistedEvent(&monitor2, "current", "test event 3")
+	tStart = time.Now().Add(-5 * time.Minute)
+	event2.StartTime = &tStart
+	err = models.DB().Save(&event2).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	Run()
+	shoudHaveMessages("2 - messages for multiple events should be aggegated", t, "down", notification.ID, contact.ID, []int64{event1.ID, event2.ID})
+}
+
+func TestDoesNotCreateEnded(t *testing.T) {
+	log.Println("TestDoesNotCreateEnded")
+	models.Drop()
+	models.Migrate()
+	user := factories.PersistedUser("test@test.ru", "123")
+	account, err := models.CreateAccountForUser(fake.Company(), &user)
+	contact := factories.PersistedContact(account, &user)
+	group := factories.PersistedGroup(account)
+	notification := factories.PersistedNotification(account, []int64{contact.ID}, []int64{group.ID}, 300, false)
+	monitor := factories.PersistedMonitor(&group)
+
+	event := factories.PersistedEvent(&monitor, "ended", "test event 1")
+	tStart := time.Now().Add(-90 * time.Minute)
+	tEnd := time.Now().Add(-80 * time.Minute)
+	event.StartTime = &tStart
+	event.EndTime = &tEnd
+	err = models.DB().Save(&event).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	Run()
+	shoudHaveMessages("1 - contact should have no messages for ended notification", t, "down", notification.ID, contact.ID, []int64{})
+}
+
+func shoudHaveMessages(message string, t *testing.T, kind string, notificationID, contactID int64, wantIds []int64) {
+	q := models.DB()
+	if notificationID > 0 {
+		q = q.Where("notification_id = ?", notificationID)
+	}
+	if contactID > 0 {
+		q = q.Where("contact_id = ?", contactID)
+	}
+
+	messages := make([]models.Message, 0)
+	err := models.MessageScope(q).Where("state IN ('queued')").Find(&messages).Error
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(messages) > 1 {
+		spew.Dump(messages)
+		t.Fatal("found more than one message")
+	}
+
+	haveIds := make([]int64, 0)
+	for _, msg := range messages {
+		assert.Equal(t, "queued", msg.State, "message should be in queued state")
+		assert.Equal(t, kind, msg.Kind, "message should have kind = down")
+		if len(msg.Events) > 0 {
+			if kind != "down" && kind != "up" {
+				t.Fatal(kind + " message should have no events")
+			}
+			for _, evt := range msg.Events {
+				haveIds = append(haveIds, evt.ID)
+			}
+		} else if msg.CheckID != nil {
+			if kind != "exp" {
+				t.Fatal(kind + " message should have no check")
+			}
+
+			haveIds = append(haveIds, *msg.CheckID)
+		}
+	}
+
+	if len(haveIds) != len(wantIds) {
+		t.Fatal(message, notificationID, contactID, "bad count, have", len(haveIds), "want", len(wantIds))
+	}
+
+	sort.SliceStable(wantIds, func(i, j int) bool { return wantIds[i] < wantIds[j] })
+	sort.SliceStable(haveIds, func(i, j int) bool { return haveIds[i] < haveIds[j] })
+
+	if !reflect.DeepEqual(wantIds, haveIds) {
+		t.Fatal(message, "bad want/have", wantIds, haveIds)
+	}
+}