2c884c5612
Некоторые проверки не удались
CI / test (push) Successful in 2m5s
Docker / Build and publish worker image (push) Failing after 31s
682 строки
27 KiB
Go
682 строки
27 KiB
Go
package models_test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"gorm.io/datatypes"
|
|
"gorm.io/gorm"
|
|
|
|
"rocketgit.ru/rsmon/worker/app/models"
|
|
"rocketgit.ru/rsmon/worker/config/database"
|
|
)
|
|
|
|
func init() {
|
|
database.Init()
|
|
}
|
|
|
|
// seedRegion creates a Region row before a worker fixture inserts, so the FK
|
|
// from worker_nodes -> regions holds. Idempotent: Drop() cleans up.
|
|
func seedRegion(t *testing.T, code string) models.Region {
|
|
t.Helper()
|
|
r := models.Region{}
|
|
err := models.DB().Where("code = ?", code).First(&r).Error
|
|
if err == nil {
|
|
return r
|
|
}
|
|
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
t.Fatalf("seed region lookup: %v", err)
|
|
}
|
|
r = models.Region{Code: code, Name: code, Enabled: true}
|
|
require.NoError(t, models.DB().Create(&r).Error)
|
|
return r
|
|
}
|
|
|
|
// seedAccountUserPlan returns an account with a default plan and the first user
|
|
// for the FK chain that contacts/notifications walk. Cleanup is the caller's
|
|
// responsibility (Drop() at end of test).
|
|
func seedAccountUserPlan(t *testing.T) (models.Account, models.User) {
|
|
t.Helper()
|
|
plan := models.Plan{Name: "test-plan", Default: false}
|
|
if err := models.DB().Create(&plan).Error; err != nil {
|
|
t.Fatalf("seed plan: %v", err)
|
|
}
|
|
|
|
user := models.User{Name: "test-user", Email: taskStringPtr("test-" + uuid.NewString() + "@example.com"), Timezone: "UTC"}
|
|
if err := models.DB().Create(&user).Error; err != nil {
|
|
t.Fatalf("seed user: %v", err)
|
|
}
|
|
|
|
account := models.Account{Name: "test-account", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
|
if err := models.DB().Create(&account).Error; err != nil {
|
|
t.Fatalf("seed account: %v", err)
|
|
}
|
|
return account, user
|
|
}
|
|
|
|
func seedNotification(t *testing.T, accountID int64) models.Notification {
|
|
t.Helper()
|
|
n := models.Notification{Name: "default", AccountID: accountID, Enabled: true, NotifyDown: true, NotifyRestore: true}
|
|
require.NoError(t, models.DB().Create(&n).Error)
|
|
return n
|
|
}
|
|
|
|
func seedEmailContact(t *testing.T, accountID int64) models.Contact {
|
|
t.Helper()
|
|
c := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &accountID}
|
|
require.NoError(t, models.DB().Create(&c).Error)
|
|
return c
|
|
}
|
|
|
|
func taskStringPtr(s string) *string { return &s }
|
|
|
|
func TestTaskSchemaMigration(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
|
|
require.True(t, models.DB().Migrator().HasTable(&models.Task{}), "tasks table must exist after Migrate()")
|
|
require.True(t, models.DB().Migrator().HasTable(&models.NotificationDelivery{}), "notification_deliveries table must exist after Migrate()")
|
|
}
|
|
|
|
func TestEnqueueNotificationTask_Idempotency(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
seedRegion(t, "test")
|
|
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
|
|
// Producer precheck requires at least one worker authorized for the
|
|
// (method, account) pair. Add an operated-style email worker.
|
|
now := time.Now()
|
|
w := &models.WorkerNode{
|
|
WorkerID: "worker-idempotency-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
LastSeen: &now,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"task_envelope": true,
|
|
"notification_methods": []string{"email"},
|
|
"notification_accounts": []int64{},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w).Error)
|
|
|
|
payload := []byte(`{"method":"email","subject":"[rsmon] x is down","body_text":"down","body_html":"<p>down</p>","body_markdown":"**down**","language":"en","message_kind":"down"}`)
|
|
|
|
input := models.EnqueueNotificationTaskInput{
|
|
AccountID: account.ID,
|
|
NotificationID: notification.ID,
|
|
ContactID: contact.ID,
|
|
Method: "email",
|
|
Subject: "[rsmon] x is down",
|
|
BodyText: "down",
|
|
BodyHTML: "<p>down</p>",
|
|
Language: "en",
|
|
MessageKind: "down",
|
|
EventIDs: []int64{42},
|
|
Payload: payload,
|
|
}
|
|
|
|
first, err := models.EnqueueNotificationTask(&input)
|
|
require.NoError(t, err)
|
|
require.NotZero(t, first.ID)
|
|
|
|
// Second call with the same logical event must not create a duplicate row.
|
|
second, err := models.EnqueueNotificationTask(&input)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, first.ID, second.ID, "idempotency: second enqueue should return the same row")
|
|
|
|
var count int64
|
|
require.NoError(t, models.DB().Model(&models.Task{}).Where("idempotency_key = ?", first.IdempotencyKey).Count(&count).Error)
|
|
assert.EqualValues(t, 1, count, "exactly one Task row per idempotency key")
|
|
}
|
|
|
|
// TestEnqueueNotificationTask_AuthorizationSkip makes sure the producer can
|
|
// observe ErrNotificationMethodNotAuthorized when no worker is eligible for
|
|
// the (method, account) pair. The producer uses this to avoid enqueueing work
|
|
// no operated worker could ever pick up.
|
|
func TestEnqueueNotificationTask_AuthorizationSkip(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
|
|
// Register a worker that only knows telegram. An email task must fail the
|
|
// precheck.
|
|
w := &models.WorkerNode{
|
|
WorkerID: "worker-tg-only-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"notification_methods": []string{"telegram"},
|
|
"notification_accounts": []int64{},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w).Error)
|
|
|
|
_, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
|
AccountID: account.ID,
|
|
NotificationID: notification.ID,
|
|
ContactID: contact.ID,
|
|
Method: "email",
|
|
EventIDs: []int64{1},
|
|
Payload: []byte(`{"method":"email"}`),
|
|
})
|
|
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
|
}
|
|
|
|
func TestEnqueueNotificationTask_RequiresTaskEnvelopeWorker(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
now := time.Now()
|
|
worker := &models.WorkerNode{WorkerID: "legacy-notify-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"}, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
|
}))}
|
|
require.NoError(t, models.DB().Create(worker).Error)
|
|
require.NoError(t, models.DB().Model(&models.WorkerNode{}).Where("id <> ?", worker.ID).Update("status", "dead").Error)
|
|
input := &models.EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notification.ID, ContactID: contact.ID, Method: "email", EventIDs: []int64{77}, Payload: []byte(`{"method":"email"}`)}
|
|
_, err := models.EnqueueNotificationTask(input)
|
|
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
|
var count int64
|
|
require.NoError(t, models.DB().Model(&models.Task{}).Count(&count).Error)
|
|
assert.Zero(t, count)
|
|
worker.Capabilities = datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"}, "task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
|
}))
|
|
require.NoError(t, models.DB().Save(worker).Error)
|
|
task, err := models.EnqueueNotificationTask(input)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, models.TaskStateQueued, task.State)
|
|
}
|
|
|
|
func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
|
|
now := time.Now()
|
|
w := &models.WorkerNode{
|
|
WorkerID: "worker-email-only-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
LastSeen: &now,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"task_envelope": true,
|
|
"notification_methods": []string{"email"},
|
|
"notification_accounts": []int64{},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w).Error)
|
|
|
|
// Enqueue 3 tasks of different methods; only the email ones should be
|
|
// picked up by the worker. We use the raw helper because the producer's
|
|
// precheck would refuse the telegram row when no worker handles telegram —
|
|
// the selector test must exercise the SELECT-side filter, not the
|
|
// producer-side authorization.
|
|
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{1})
|
|
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "telegram", []int64{2})
|
|
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{3})
|
|
|
|
picked, err := models.TasksForWorker(w, 10)
|
|
require.NoError(t, err)
|
|
require.Len(t, picked, 2, "only email tasks should be leased")
|
|
|
|
for _, p := range picked {
|
|
assert.Equal(t, models.TaskStateLeased, p.State)
|
|
assert.Equal(t, w.WorkerID, p.LeaseOwner)
|
|
assert.NotNil(t, p.LeaseExpiresAt)
|
|
assert.Equal(t, 1, p.Attempts)
|
|
}
|
|
|
|
// A second call must not return the same rows.
|
|
picked2, err := models.TasksForWorker(w, 10)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, picked2, "second selector poll should see an empty queue while leased")
|
|
}
|
|
|
|
func TestEnqueueDueCheckTasks_UsesGenericEnvelope(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
seedRegion(t, "test")
|
|
account, _ := seedAccountUserPlan(t)
|
|
group := models.Group{Name: "checks", AccountID: account.ID}
|
|
require.NoError(t, models.DB().Create(&group).Error)
|
|
monitor := models.Monitor{GroupID: group.ID, Host: "example.com", Enabled: true}
|
|
require.NoError(t, models.DB().Create(&monitor).Error)
|
|
enabled := true
|
|
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
|
require.NoError(t, models.DB().Create(&check).Error)
|
|
worker := &models.WorkerNode{
|
|
WorkerID: "generic-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
|
}
|
|
require.NoError(t, models.DB().Create(worker).Error)
|
|
|
|
require.NoError(t, models.EnqueueDueCheckTasks(worker, []string{"http"}, 1))
|
|
picked, err := models.TasksForWorker(worker, 1)
|
|
require.NoError(t, err)
|
|
require.Len(t, picked, 1)
|
|
assert.Equal(t, models.TaskKindCheck, picked[0].Kind)
|
|
assert.Equal(t, models.TaskStateLeased, picked[0].State)
|
|
assert.Equal(t, check.ID, *picked[0].CheckID)
|
|
}
|
|
|
|
func TestTasksForWorker_ChecksRespectPrivateAccountScope(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
seedRegion(t, "test")
|
|
accountA, _ := seedAccountUserPlan(t)
|
|
accountB, _ := seedAccountUserPlan(t)
|
|
accountID := accountA.ID
|
|
private := &models.WorkerNode{
|
|
WorkerID: "private-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
|
}
|
|
platform := &models.WorkerNode{
|
|
WorkerID: "platform-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(),
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
|
}
|
|
require.NoError(t, models.DB().Create(private).Error)
|
|
require.NoError(t, models.DB().Create(platform).Error)
|
|
task := models.Task{JobID: uuid.NewString(), Kind: models.TaskKindCheck, State: models.TaskStateQueued, AccountID: accountB.ID, Payload: datatypes.JSON([]byte(`{"kind":"http"}`)), NotBefore: time.Now().Add(-time.Second), MaxAttempts: 5, IdempotencyKey: "cross-account-" + uuid.NewString()}
|
|
require.NoError(t, models.DB().Create(&task).Error)
|
|
picked, err := models.TasksForWorker(private, 1)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, picked)
|
|
picked, err = models.TasksForWorker(platform, 1)
|
|
require.NoError(t, err)
|
|
require.Len(t, picked, 1)
|
|
assert.Equal(t, task.ID, picked[0].ID)
|
|
}
|
|
|
|
func TestChecksForWorker_RespectsPrivateAccountScope(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
seedRegion(t, "test")
|
|
accountA, _ := seedAccountUserPlan(t)
|
|
accountB, _ := seedAccountUserPlan(t)
|
|
makeCheck := func(accountID int64, host string) models.Check {
|
|
group := models.Group{Name: host, AccountID: accountID}
|
|
require.NoError(t, models.DB().Create(&group).Error)
|
|
monitor := models.Monitor{GroupID: group.ID, Host: host, Enabled: true}
|
|
require.NoError(t, models.DB().Create(&monitor).Error)
|
|
enabled := true
|
|
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
|
require.NoError(t, models.DB().Create(&check).Error)
|
|
return check
|
|
}
|
|
owned := makeCheck(accountA.ID, "owned.example")
|
|
_ = makeCheck(accountB.ID, "other.example")
|
|
accountID := accountA.ID
|
|
private := &models.WorkerNode{
|
|
WorkerID: "private-legacy-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
|
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
|
}
|
|
require.NoError(t, models.DB().Create(private).Error)
|
|
checks := models.ChecksForWorker(private, []string{"http"}, 10)
|
|
require.Len(t, checks, 1)
|
|
assert.Equal(t, owned.ID, checks[0].ID)
|
|
}
|
|
|
|
func TestTasksForWorker_SkipsExpiredDeadline(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
now := time.Now()
|
|
worker := &models.WorkerNode{
|
|
WorkerID: "worker-deadline-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Concurrency: 1,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
|
}
|
|
require.NoError(t, models.DB().Create(worker).Error)
|
|
|
|
expired := time.Now().Add(-time.Second)
|
|
task := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{91})
|
|
require.NoError(t, models.DB().Model(&task).Update("deadline", expired).Error)
|
|
picked, err := models.TasksForWorker(worker, 1)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, picked)
|
|
|
|
var stored models.Task
|
|
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
|
assert.Equal(t, models.TaskStateQueued, stored.State)
|
|
assert.Equal(t, 0, stored.Attempts)
|
|
}
|
|
|
|
func TestReapExpiredTasksTerminatesExpiredQueuedNotification(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
|
require.NoError(t, models.DB().Create(&message).Error)
|
|
deadline := time.Now().Add(-time.Minute)
|
|
// This reaper test intentionally has no eligible worker; insert directly
|
|
// so it tests deadline handling rather than producer capability validation.
|
|
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{92})
|
|
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{"message_id": message.ID, "deadline": deadline}).Error)
|
|
|
|
_, _, err := models.ReapExpiredTasks()
|
|
require.NoError(t, err)
|
|
var stored models.Task
|
|
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
|
assert.Equal(t, models.TaskStateDead, stored.State)
|
|
assert.Equal(t, "notification deadline expired", stored.LastError)
|
|
var storedMessage models.Message
|
|
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
|
assert.Equal(t, "error", storedMessage.State)
|
|
var auditCount int64
|
|
require.NoError(t, models.DB().Model(&models.NotificationDelivery{}).Where("task_id = ? AND status = ?", task.ID, "expired").Count(&auditCount).Error)
|
|
assert.EqualValues(t, 1, auditCount)
|
|
}
|
|
|
|
func TestReapExpiredTasks_RecyclesLeasesAndDeadsExhaustedRetries(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
|
|
now := time.Now()
|
|
w := &models.WorkerNode{
|
|
WorkerID: "worker-reap-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
LastSeen: &now,
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"task_envelope": true,
|
|
"notification_methods": []string{"email"},
|
|
"notification_accounts": []int64{},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w).Error)
|
|
|
|
// 1) A leased task whose lease expired — should go back to queued.
|
|
expiredLease := time.Now().Add(-time.Minute)
|
|
leased := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{10})
|
|
require.NoError(t, models.DB().Model(&leased).Updates(map[string]interface{}{
|
|
"state": models.TaskStateLeased,
|
|
"lease_owner": w.WorkerID,
|
|
"lease_expires_at": expiredLease,
|
|
"attempts": 1,
|
|
}).Error)
|
|
|
|
// 2) A failed_retry task past its not_before and at max_attempts — should move to dead.
|
|
failedRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{11})
|
|
require.NoError(t, models.DB().Model(&failedRetry).Updates(map[string]interface{}{
|
|
"state": models.TaskStateFailedRetry,
|
|
"attempts": 5,
|
|
"max_attempts": 5,
|
|
"not_before": time.Now().Add(-time.Minute),
|
|
}).Error)
|
|
|
|
// 3) A failed_retry task past not_before but attempts < max_attempts — must stay failed_retry.
|
|
pendingRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{12})
|
|
require.NoError(t, models.DB().Model(&pendingRetry).Updates(map[string]interface{}{
|
|
"state": models.TaskStateFailedRetry,
|
|
"attempts": 2,
|
|
"max_attempts": 5,
|
|
"not_before": time.Now().Add(-time.Minute),
|
|
}).Error)
|
|
|
|
reaped, deaded, err := models.ReapExpiredTasks()
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, reaped, "one expired lease should be returned to queue")
|
|
assert.Equal(t, 1, deaded, "one exhausted retry should move to dead")
|
|
|
|
var leasedRow models.Task
|
|
require.NoError(t, models.DB().First(&leasedRow, leased.ID).Error)
|
|
assert.Equal(t, models.TaskStateQueued, leasedRow.State)
|
|
assert.Empty(t, leasedRow.LeaseOwner)
|
|
assert.Nil(t, leasedRow.LeaseExpiresAt)
|
|
|
|
var deadRow models.Task
|
|
require.NoError(t, models.DB().First(&deadRow, failedRetry.ID).Error)
|
|
assert.Equal(t, models.TaskStateDead, deadRow.State)
|
|
|
|
var pendingRow models.Task
|
|
require.NoError(t, models.DB().First(&pendingRow, pendingRetry.ID).Error)
|
|
assert.Equal(t, models.TaskStateFailedRetry, pendingRow.State)
|
|
}
|
|
|
|
func TestReapExpiredTasks_ExpiredNotificationLeaseExhaustionFinalizesMessage(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
|
require.NoError(t, models.DB().Create(&message).Error)
|
|
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{99})
|
|
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{
|
|
"message_id": message.ID, "state": models.TaskStateLeased, "attempts": 5, "max_attempts": 5,
|
|
"lease_expires_at": time.Now().Add(-time.Minute),
|
|
}).Error)
|
|
_, deaded, err := models.ReapExpiredTasks()
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, deaded)
|
|
var storedTask models.Task
|
|
require.NoError(t, models.DB().First(&storedTask, task.ID).Error)
|
|
assert.Equal(t, models.TaskStateDead, storedTask.State)
|
|
assert.Equal(t, "lease expired after max attempts", storedTask.LastError)
|
|
var storedMessage models.Message
|
|
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
|
assert.Equal(t, "error", storedMessage.State)
|
|
require.NotNil(t, storedMessage.Error)
|
|
assert.Equal(t, "lease expired after max attempts", *storedMessage.Error)
|
|
var delivery models.NotificationDelivery
|
|
require.NoError(t, models.DB().Where("task_id = ?", task.ID).First(&delivery).Error)
|
|
assert.Equal(t, "dead", delivery.Status)
|
|
assert.Equal(t, "lease expired after max attempts", delivery.Error)
|
|
}
|
|
|
|
func TestNotificationTaskLeaseOutlivesExecutionTimeoutAndReapsAfterExpiry(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
account, _ := seedAccountUserPlan(t)
|
|
notification := seedNotification(t, account.ID)
|
|
contact := seedEmailContact(t, account.ID)
|
|
seedRegion(t, "test")
|
|
worker := &models.WorkerNode{
|
|
WorkerID: "notification-lease-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
|
}
|
|
require.NoError(t, models.DB().Create(worker).Error)
|
|
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{101})
|
|
leased, err := models.TasksForWorker(worker, 1)
|
|
require.NoError(t, err)
|
|
require.Len(t, leased, 1)
|
|
require.NotNil(t, leased[0].LeaseExpiresAt)
|
|
assert.GreaterOrEqual(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationTaskLeaseTTL-time.Second)
|
|
assert.Greater(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationExecutionTimeout)
|
|
|
|
withinExecution := time.Now().Add(models.DefaultNotificationExecutionTimeout)
|
|
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", withinExecution).Error)
|
|
reaped, _, err := models.ReapExpiredTasks()
|
|
require.NoError(t, err)
|
|
assert.Zero(t, reaped)
|
|
var stored models.Task
|
|
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
|
assert.Equal(t, models.TaskStateLeased, stored.State)
|
|
|
|
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error)
|
|
reaped, _, err = models.ReapExpiredTasks()
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, reaped)
|
|
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
|
assert.Equal(t, models.TaskStateQueued, stored.State)
|
|
}
|
|
|
|
func mustEnqueue(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
|
t.Helper()
|
|
payload := []byte(`{"method":"` + method + `"}`)
|
|
task, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
|
AccountID: accountID,
|
|
NotificationID: notificationID,
|
|
ContactID: contactID,
|
|
Method: method,
|
|
EventIDs: eventIDs,
|
|
Payload: payload,
|
|
NotBefore: time.Now().Add(-time.Second),
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotZero(t, task.ID)
|
|
return *task
|
|
}
|
|
|
|
// mustEnqueueRaw inserts a Task row directly without going through the producer
|
|
// precheck. The selector test deliberately mixes methods (email + telegram) on a
|
|
// worker that only handles email; the producer would refuse the telegram row,
|
|
// which is the wrong thing to assert about in a selector test.
|
|
func mustEnqueueRaw(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
|
t.Helper()
|
|
payload := datatypes.JSON([]byte(`{"method":"` + method + `"}`))
|
|
contact := contactID
|
|
task := models.Task{
|
|
JobID: uuid.New().String(),
|
|
Kind: models.TaskKindNotification,
|
|
State: models.TaskStateQueued,
|
|
AccountID: accountID,
|
|
ContactID: &contact,
|
|
Payload: payload,
|
|
NotBefore: time.Now().Add(-time.Second),
|
|
Attempts: 0,
|
|
MaxAttempts: 5,
|
|
IdempotencyKey: models.NotificationIdempotencyKey(notificationID, contactID, eventIDs[0]),
|
|
}
|
|
require.NoError(t, models.DB().Create(&task).Error)
|
|
require.NotZero(t, task.ID)
|
|
return task
|
|
}
|
|
|
|
func mustJSON(t *testing.T, v interface{}) []byte {
|
|
t.Helper()
|
|
b, err := json.Marshal(v)
|
|
require.NoError(t, err)
|
|
return b
|
|
}
|
|
|
|
// TestWorkerNodeCapabilities_NotificationFlags confirms that the JSON-backed
|
|
// capabilities blob correctly exposes the notification_methods and
|
|
// notification_accounts arrays that the selector and credential push depend on.
|
|
func TestWorkerNodeCapabilities_NotificationFlags(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
seedRegion(t, "test")
|
|
|
|
w := &models.WorkerNode{
|
|
WorkerID: "caps-worker-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"notification_methods": []string{"email", "telegram"},
|
|
"notification_accounts": []int64{7, 8},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w).Error)
|
|
|
|
got := models.WorkerNode{}
|
|
require.NoError(t, models.DB().First(&got, w.ID).Error)
|
|
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
|
assert.Equal(t, []int64{7, 8}, got.NotificationAccounts())
|
|
assert.True(t, got.CanDeliverNotification("email", 7))
|
|
assert.False(t, got.CanDeliverNotification("email", 9), "account 9 is not in the allowed list")
|
|
assert.False(t, got.CanDeliverNotification("mattermost", 7), "method not authorized")
|
|
|
|
// Operated-style worker: empty accounts list means "all accounts".
|
|
w2 := &models.WorkerNode{
|
|
WorkerID: "ops-worker-" + uuid.NewString(),
|
|
RegionCode: "test",
|
|
Status: "active",
|
|
AuthToken: uuid.NewString(),
|
|
Concurrency: 4,
|
|
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
|
"check_types": []string{"http"},
|
|
"notification_methods": []string{"email"},
|
|
"notification_accounts": []int64{},
|
|
})),
|
|
}
|
|
require.NoError(t, models.DB().Create(w2).Error)
|
|
|
|
got2 := models.WorkerNode{}
|
|
require.NoError(t, models.DB().First(&got2, w2.ID).Error)
|
|
assert.True(t, got2.CanDeliverNotification("email", 9999), "empty accounts list means all accounts")
|
|
}
|
|
|
|
func TestWorkerNodeReportedWorkloadUsesDisjointHeartbeatFields(t *testing.T) {
|
|
w := &models.WorkerNode{Capabilities: datatypes.JSON([]byte(`{"active_checks":2,"queue_depth":3,"active_notifications":5,"notification_queue_depth":7}`))}
|
|
assert.Equal(t, 17, w.ReportedWorkload())
|
|
}
|
|
|
|
// TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults verifies that
|
|
// EnsureConfiguredWorkerNode (the in-cluster worker provisioner) populates
|
|
// notification_methods / notification_accounts on the JSON blob so the new
|
|
// selector and the new credential push work out of the box for the bundled
|
|
// worker.
|
|
func TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults(t *testing.T) {
|
|
models.Drop()
|
|
models.Migrate()
|
|
|
|
t.Setenv("WORKER_AUTH_TOKEN", "secret-token-for-test-xyz")
|
|
t.Setenv("DEPLOY_ENV", "test-env")
|
|
t.Setenv("RSMON_WORKER_ID", "worker-test")
|
|
t.Setenv("NOTIFICATION_METHODS", "email,telegram")
|
|
t.Setenv("NOTIFICATION_ACCOUNTS", "11,22")
|
|
|
|
models.EnsureConfiguredWorkerNode()
|
|
|
|
var got models.WorkerNode
|
|
require.NoError(t, models.DB().Where("worker_id = ?", "worker-test").First(&got).Error)
|
|
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
|
assert.Equal(t, []int64{11, 22}, got.NotificationAccounts())
|
|
}
|
|
|
|
// TestNotificationIdempotencyKeyFormat pins the producer-side key shape so the
|
|
// result handler can re-derive it for matching without depending on internal
|
|
// package state.
|
|
func TestNotificationIdempotencyKeyFormat(t *testing.T) {
|
|
key := models.NotificationIdempotencyKey(7, 9, 13)
|
|
assert.Equal(t, "notif:7:contact:9:event:13", key)
|
|
}
|
|
|
|
// guard against uuid being accidentally dropped from the imports.
|
|
var _ = uuid.New
|