feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
2026-07-13 17:55:14 +03:00
Коммит 2c7a0236da
309 изменённых файлов: 44004 добавлений и 0 удалений

60
app/models/access.go Обычный файл
Просмотреть файл

@@ -0,0 +1,60 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Access represents membership of a User (or ApiKey) within a tenant
// Account, optionally scoped to a Group or Monitor.
//
// A User can hold many Access rows across many Accounts — the Access
// table is the source of truth for "who can see what". Each row answers:
//
// "Does user U have permission P on scope S of account A?"
//
// Where:
//
// - A = AccountID (tenant)
// - U = UserID (or ApiKeyID for service tokens)
// - P = Role ("owner" | "admin" | "manager" | "view" |
// "notify")
// - S = Kind + (GroupID | MonitorID) — defaults to account-wide when
// Kind = "account" and both ids are
// nil.
//
// One Access row may also reference the Invite that produced it via
// InviteID. The Invite is preserved after registration so the access
// history stays auditable — system-registered users and admin-added
// users have nil InviteID.
//
// See docs/plans/users-and-rbac.md for the full RBAC matrix.
type Access struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
// Kind access kind, account \ group \ monitor
Kind string `gorm:"not null;default:'account'" json:"kind"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
User *User `json:"-"`
ApiKeyID *int64 `gorm:"type:bigint REFERENCES api_keys(id)" json:"-"` //nolint:revive // accepted lint exception
ApiKey *ApiKey `json:"-"` //nolint:revive // accepted lint exception
InviteID *int64 `gorm:"type:bigint REFERENCES invites(id)" json:"-"`
Invite *Invite `json:"-"`
GroupID *int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty"`
MonitorID *int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Role string `json:"role"`
// SeatType is additive to Role: role remains the authorization decision,
// while seat type is the billing entitlement.
SeatType string `gorm:"size:16;not null;default:'login'" json:"seat_type"`
Status string `gorm:"size:16;not null;default:'active'" json:"status"`
NotifyOnly bool `gorm:"not null;default:false" json:"notify_only"`
SeatAddonID *int64 `json:"seat_addon_id,omitempty"`
concerns.Timestamped `json:"-"`
Audited
}

117
app/models/account.go Обычный файл
Просмотреть файл

@@ -0,0 +1,117 @@
package models
import (
"time"
"github.com/pkg/errors"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Account represents a tenant — an isolated billing/permissions boundary
// that groups users, contacts, groups, monitors and notifications.
//
// Access to an Account is granted via the Access join table (see Access).
// `Role` on Account is a gorm:"-":all" virtual column populated by the
// controllers for the current session — it is the role the calling user
// holds on THIS account, not a property of the account itself.
type Account struct {
concerns.Model
Name string `json:"name"`
Accesses []Access `json:"-"`
Contacts []Contact `json:"-"`
Groups []Group `json:"-"`
Notifications []Notification `json:"-"`
PlanID *int64 `gorm:"type:bigint REFERENCES plans(id)" json:"-"`
Plan *Plan `json:"plan"`
// Diagnostic overrides are available only to plans that include confirmations.
// Nil keeps the catalog value; bounds are enforced by DiagnosticSettings.
ConfirmTimeoutSec *int `json:"confirm_timeout_sec,omitempty"`
HealthWindowSec *int `json:"health_window_sec,omitempty"`
HealthRateThreshold *float64 `json:"health_rate_threshold,omitempty"`
HealthMinAttempts *int `json:"health_min_attempts,omitempty"`
Role string `gorm:"-:all" json:"role"`
Timezone string `json:"timezone"`
Language string `gorm:"default:'ru'" json:"language"`
Disabled bool `gorm:"not null;default:false" json:"disabled"`
Blocked bool `gorm:"not null;default:false" json:"blocked"`
PaidUntil *time.Time `json:"paid_until"`
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
Deleted bool `gorm:"not null;default:false"`
concerns.Timestamped
Audited
}
// Users provides functionality.
func (a Account) Users() []User { //nolint:gocritic // hugeParam: accepted for interface compatibility
users := make([]User, 0)
err := DB().Where("id IN (SELECT user_id FROM accesses WHERE account_id = ?)", a.ID).Find(&users).Error
if err != nil {
panic(err)
}
return users
}
// CreateAccountForUser provides functionality.
func CreateAccountForUser(name string, u *User) (*Account, error) {
trialPlan := Plan{}
if err := DB().Where("code = ? AND archived = FALSE", "team").First(&trialPlan).Error; err != nil {
return nil, errors.Wrap(err, "failed to find trial plan")
}
trialEndsAt := time.Now().UTC().AddDate(0, 0, 14)
account := Account{PlanID: &trialPlan.ID, TrialEndsAt: &trialEndsAt}
if name != "" {
account.Name = name
}
err := DB().Transaction(func(tx *gorm.DB) error {
if err := tx.Create(&account).Error; err != nil {
return errors.Wrap(err, "failed to create account")
}
if err := tx.Create(&Subscription{
AccountID: account.ID, PlanID: trialPlan.ID, Provider: "manual", Status: SubscriptionStatusTrialing,
BillingCycle: "monthly", Currency: trialPlan.Currency, AmountMinor: trialPlan.PriceMonthlyMinor, CurrentPeriodEnd: &trialEndsAt, TrialEndsAt: &trialEndsAt,
}).Error; err != nil {
return errors.Wrap(err, "failed to create subscription")
}
var subscription Subscription
if err := tx.Where("account_id = ?", account.ID).First(&subscription).Error; err != nil {
return errors.Wrap(err, "failed to load trial subscription")
}
if err := tx.Create(&SubscriptionEvent{SubscriptionID: subscription.ID, AccountID: account.ID, Provider: "manual", Kind: "trial_started", ToPlanID: &trialPlan.ID, ActorUserID: &u.ID, CreatedAt: time.Now().UTC()}).Error; err != nil {
return errors.Wrap(err, "failed to record trial")
}
access := Access{AccountID: account.ID, UserID: &u.ID, Role: "owner", SeatType: "admin"}
if err := tx.Create(&access).Error; err != nil {
return errors.Wrap(err, "failed to create access")
}
group := Group{AccountID: account.ID, Name: "Основные"}
if err := tx.Create(&group).Error; err != nil {
return errors.Wrap(err, "failed to create group")
}
notification := Notification{AccountID: account.ID, Name: "Основные", Enabled: true}
if err := tx.Create(&notification).Error; err != nil {
return errors.Wrap(err, "failed to create notification")
}
if u.Email != nil {
contact := Contact{AccountID: &account.ID, UserID: &u.ID, Kind: "email", Value: *u.Email}
if err := tx.Create(&contact).Error; err != nil {
return errors.Wrap(err, "failed to create contact")
}
if err := tx.Model(&notification).Association("Contacts").Append(&contact); err != nil {
return errors.Wrap(err, "failed to add contact to notification")
}
}
if err := tx.Model(&notification).Association("Groups").Append(&group); err != nil {
return errors.Wrap(err, "failed to add group to notification")
}
return nil
})
if err != nil {
return nil, err
}
return &account, nil
}

283
app/models/account_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,283 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestCreateAccountForUserStartsTeamTrial(t *testing.T) {
models.Drop()
models.Migrate()
email := "team-trial@example.test"
user := models.User{Name: "Trial User", Email: &email, Enabled: true, Confirmed: true}
require.NoError(t, models.DB().Create(&user).Error)
account, err := models.CreateAccountForUser("Trial Account", &user)
require.NoError(t, err)
require.NotNil(t, account.PlanID)
require.NotNil(t, account.TrialEndsAt)
var plan models.Plan
require.NoError(t, models.DB().First(&plan, *account.PlanID).Error)
assert.Equal(t, "team", plan.Code)
var subscription models.Subscription
require.NoError(t, models.DB().Where("account_id = ?", account.ID).First(&subscription).Error)
assert.Equal(t, models.SubscriptionStatusTrialing, subscription.Status)
assert.Equal(t, plan.ID, subscription.PlanID)
require.NotNil(t, subscription.TrialEndsAt)
assert.WithinDuration(t, *account.TrialEndsAt, *subscription.TrialEndsAt, time.Millisecond)
}
// TestAccountModel tests basic Account model functionality
func TestAccountModel(t *testing.T) {
// Test Account structure
account := models.Account{
Name: "Test Account",
Timezone: "UTC",
Language: "en",
Deleted: false,
}
assert.Equal(t, "Test Account", account.Name)
assert.Equal(t, "UTC", account.Timezone)
assert.Equal(t, "en", account.Language)
assert.False(t, account.Deleted)
}
// TestAccountDisplayName tests User.DisplayName method
func TestUserDisplayName(t *testing.T) {
tests := []struct {
name string
user models.User
expected string
}{
{
name: "User with email",
user: models.User{
Name: "John Doe",
Email: stringPtr("john@example.com"),
},
expected: "John Doe john@example.com",
},
{
name: "User without email",
user: models.User{
Name: "Jane Doe",
Email: nil,
},
expected: "Jane Doe",
},
{
name: "User with empty name and email",
user: models.User{
Name: "",
Email: stringPtr("test@example.com"),
},
expected: " test@example.com",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.user.DisplayName()
assert.Equal(t, tt.expected, result)
})
}
}
// TestUserGravatar tests User.Gravatar method
func TestUserGravatar(t *testing.T) {
tests := []struct {
name string
user models.User
size int
expected string
}{
{
name: "User with email",
user: models.User{
Email: stringPtr("test@example.com"),
},
size: 32,
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=32&d=blank",
},
{
name: "User without email",
user: models.User{Email: nil},
size: 32,
expected: "",
},
{
name: "Different size",
user: models.User{
Email: stringPtr("test@example.com"),
},
size: 64,
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=64&d=blank",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.user.Gravatar(tt.size)
assert.Equal(t, tt.expected, result)
})
}
}
// TestUserAsJSON tests User.AsJSON method
func TestUserAsJSON(t *testing.T) {
email := "test@example.com"
user := models.User{
ID: 123,
Email: &email,
Name: "Test User",
}
result := user.AsJSON()
assert.NotNil(t, result)
assert.Equal(t, int64(123), result["id"])
assert.Equal(t, &email, result["email"])
assert.Contains(t, result["avatar"], "gravatar.com")
}
// TestAccessModel tests Access model structure
func TestAccessModel(t *testing.T) {
access := models.Access{
AccountID: 1,
Kind: "account",
Role: "owner",
}
assert.Equal(t, int64(1), access.AccountID)
assert.Equal(t, "account", access.Kind)
assert.Equal(t, "owner", access.Role)
}
// TestGroupModel tests Group model structure
func TestGroupModel(t *testing.T) {
group := models.Group{
AccountID: 1,
Name: "Test Group",
MonitorsCount: 5,
}
assert.Equal(t, int64(1), group.AccountID)
assert.Equal(t, "Test Group", group.Name)
assert.Equal(t, 5, group.MonitorsCount)
}
// TestGroupIdsForAccountId tests GroupIdsForAccountId function
func TestGroupIdsForAccountId(t *testing.T) {
// This test would require a database connection
// For now, we test that it doesn't panic with invalid input
t.Run("handles zero account id", func(t *testing.T) {
// Note: This will panic without DB connection, which is expected behavior
// In a real test, we'd set up a test database
})
}
// TestAccountTableDrivenTests demonstrates table-driven testing pattern
func TestAccountValidationTableDriven(t *testing.T) {
tests := []struct {
name string
account models.Account
wantErr bool
}{
{
name: "Valid account",
account: models.Account{
Name: "Valid Account",
Timezone: "UTC",
Language: "en",
},
wantErr: false,
},
{
name: "Account with empty name",
account: models.Account{
Name: "",
Timezone: "UTC",
Language: "en",
},
wantErr: true, // Name should be required
},
{
name: "Account with invalid timezone",
account: models.Account{
Name: "Test Account",
Timezone: "Invalid/Timezone",
Language: "en",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Validation logic would go here
// For now, we just verify the test structure
assert.NotNil(t, tt.account)
})
}
}
// Helper function to create string pointer
func stringPtr(s string) *string {
return &s
}
// BenchmarkUserDisplayName benchmarks the DisplayName method
func BenchmarkUserDisplayName(b *testing.B) {
user := models.User{
Name: "Test User",
Email: stringPtr("test@example.com"),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = user.DisplayName()
}
}
// TestAccountConcurrentOperations tests concurrent access to account data
func TestAccountConcurrentOperations(t *testing.T) {
account := models.Account{
Name: "Concurrent Test",
Timezone: "UTC",
}
done := make(chan bool)
// Simulate concurrent reads
for i := 0; i < 10; i++ {
go func() {
_ = account.Name
done <- true
}()
}
// Wait for all goroutines
for i := 0; i < 10; i++ {
<-done
}
}
// ExampleAccountUsage provides an example of how to use Account model
func ExampleAccount() {
account := models.Account{
Name: "Example Account",
Timezone: "America/New_York",
Language: "en",
}
_ = account.Name
// Output:
}

38
app/models/api_key.go Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
package models
import (
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// ApiKey represents an API authentication key. //nolint:revive // accepted lint exception
type ApiKey struct { //nolint:revive // accepted lint exception
concerns.Model
Name string `json:"name" gorm:"not null"`
AccountID int64 `json:"account_id,omitempty"`
Account User `json:"-"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
User *User `json:"-"`
Accesses []Access `json:"accesses" gorm:"foreignkey:api_key_id"`
concerns.HasToken
concerns.Timestamped
Audited
}
func (n *ApiKey) BeforeCreate(tx *gorm.DB) error { //nolint:revive // accepted lint exception
n.SetToken()
return nil
}
// FillAccesses provides functionality.
func (n *ApiKey) FillAccesses() {
for k, a := range n.Accesses { //nolint:gocritic // range copy is acceptable here
if a.ID <= 0 {
n.Accesses[k].ID = 0
}
n.Accesses[k].AccountID = n.AccountID
}
}

63
app/models/audited.go Обычный файл
Просмотреть файл

@@ -0,0 +1,63 @@
// Package models provides GORM models and business logic.
// Audited models inspired by https://github.com/qor/audited
package models
import (
"gorm.io/gorm"
)
// AuditedCurrentUserKey is the GORM Set key for the current user.
const AuditedCurrentUserKey = "audited:current_user"
// Audited tracks creator and updater IDs.
type Audited struct {
CreatorID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
Creator *User `json:"-"`
UpdaterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
Updater *User `json:"-"`
}
func getCurrentUser(scope *gorm.DB) (int64, bool) {
var user interface{}
var hasUser bool
user, hasUser = scope.Get(AuditedCurrentUserKey)
// spew.Dump(user, hasUser)
if hasUser {
return user.(*User).ID, true
}
return 0, false
}
func assignCreatedBy(tx *gorm.DB) {
name := "CreatorID"
if field := tx.Statement.Schema.LookUpField(name); field != nil {
if user, ok := getCurrentUser(tx); ok {
tx.Statement.SetColumn(name, user)
}
}
}
func assignUpdatedBy(tx *gorm.DB) {
name := "UpdaterID"
if field := tx.Statement.Schema.LookUpField(name); field != nil {
if user, ok := getCurrentUser(tx); ok {
tx.Statement.SetColumn("UpdaterID", user, true)
}
}
}
// RegisterCallbacks register callback into GORM DB
func RegisterCallbacks(db *gorm.DB) {
callback := db.Callback()
if callback.Create().Get("audited:assign_created_by") == nil {
_ = callback.Create().After("gorm:before_create").Register("audited:assign_created_by", assignCreatedBy)
}
if callback.Update().Get("audited:assign_updated_by") == nil {
_ = callback.Update().After("gorm:before_update").Register("audited:assign_updated_by", assignUpdatedBy)
}
}

32
app/models/authidentity/auth_identity.go Обычный файл
Просмотреть файл

@@ -0,0 +1,32 @@
// Package authidentity provides the AuthIdentity and Basic types for QOR-style
// authentication identity management. Vendored from github.com/glebtv/auth/auth_identity
// to eliminate the rsgit.ru/rs/sessionmanager transitive dependency.
package authidentity
import "time"
// AuthIdentity combines Basic provider info with SignLogs for a full identity record.
type AuthIdentity struct {
Basic
SignLogs
}
// TableName returns the database table name for AuthIdentity.
func (AuthIdentity) TableName() string {
return "identities"
}
// Basic represents the core identity fields (provider, UID, encrypted password).
type Basic struct {
ID int64 `gorm:"primary_key" json:"id"`
Provider string
UID string `gorm:"column:uid"`
EncryptedPassword string
UserID *int64
ConfirmedAt *time.Time
}
// TableName returns the database table name for Basic.
func (Basic) TableName() string {
return "identities"
}

50
app/models/authidentity/sign_logs.go Обычный файл
Просмотреть файл

@@ -0,0 +1,50 @@
package authidentity
import (
"database/sql/driver"
"encoding/json"
"errors"
"time"
)
// SignLogs holds login history (log entries and sign-in count).
type SignLogs struct {
Log string `sql:"-"`
SignInCount uint
Logs []SignLog
}
// Scan implements sql.Scanner for deserializing SignLogs from JSON.
func (signLogs *SignLogs) Scan(data interface{}) (err error) {
switch values := data.(type) {
case []byte:
if len(values) != 0 {
return json.Unmarshal(values, signLogs)
}
case string:
return signLogs.Scan([]byte(values))
case []string:
for _, str := range values {
if err := signLogs.Scan(str); err != nil {
return err
}
}
default:
err = errors.New("unsupported driver -> Scan pair for SignLogs")
}
return
}
// Value implements driver.Valuer for serializing SignLogs to JSON.
func (signLogs SignLogs) Value() (driver.Value, error) {
results, err := json.Marshal(signLogs)
return string(results), err
}
// SignLog represents a single login event entry.
type SignLog struct {
UserAgent string
At *time.Time
IP string
}

31
app/models/bits.go Обычный файл
Просмотреть файл

@@ -0,0 +1,31 @@
package models
import "time"
// BeginningOfDay provides functionality.
func BeginningOfDay(t time.Time) time.Time {
year, month, day := t.Date()
return time.Date(year, month, day, 0, 0, 0, 0, t.Location())
}
// SetBit provides functionality.
// https://stackoverflow.com/questions/23192262/how-would-you-set-and-clear-a-single-bit-in-go
// Sets the bit at pos in the integer n.
func SetBit(n int, pos uint) int {
n |= (1 << pos)
return n
}
// ClearBit provides functionality.
// Clears the bit at pos in n.
func ClearBit(n int, pos uint) int {
mask := ^(1 << pos)
n &= mask
return n
}
// HasBit provides functionality.
func HasBit(n int, pos uint) bool {
val := n & (1 << pos)
return (val > 0)
}

171
app/models/check.go Обычный файл
Просмотреть файл

@@ -0,0 +1,171 @@
package models
import (
"encoding/json"
"strings"
"time"
"unicode"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
)
// Check provides functionality.
type Check struct {
ID int64 `gorm:"primarykey" json:"id"`
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
MonitorID int64 `gorm:"index;type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Monitor *Monitor `json:"-"`
Name *string `json:"name"`
Kind string `json:"kind"`
Interval int `json:"interval" validate:"required,gte=60"`
// URL to monitor
URL *string `json:"url,omitempty"`
// Other settings of the check
Settings datatypes.JSON `gorm:"not null;" json:"settings"`
State string `gorm:"not null;default:'UNK'" json:"state"`
LastStart *time.Time `json:"last_start"`
LastEnd *time.Time `json:"last_end"`
LastOk *time.Time `json:"last_ok"`
LastFail *time.Time `json:"last_fail"`
WasUp *time.Time `json:"was_up"`
Fails int `json:"fails"`
Expires *time.Time `json:"expires"`
Error *string `json:"error"`
Warnings pq.StringArray `gorm:"type:varchar(255)[]" json:"warnings"`
Infos pq.StringArray `gorm:"type:varchar(255)[]" json:"infos"`
// RequireQuorum enables multi-region result aggregation (Phase 3 of
// docs/todo.md): when >1 the check's State is NOT written directly by
// ApplyRemoteCheckResult — instead CheckRegionResult rows accumulate
// until app/models/check_aggregator.go decides OK/ERR/DEGRADED.
// Default 1 keeps the legacy single-region behavior unchanged.
RequireQuorum int `gorm:"not null;default:1" json:"require_quorum"`
// AggregationWindowSeconds is how long the aggregator waits for
// regional CheckRegionResult rows before deciding the check's State.
// Stored as int seconds (matching the existing GORM style — no
// time.Duration columns) and exposed via AggregationWindow(). Default
// 5s; ignored when RequireQuorum <= 1.
AggregationWindowSeconds int `gorm:"not null;default:5" json:"aggregation_window_seconds"`
IsNew bool `gorm:"-:all" sql:"-" json:"is_new,omitempty"`
Deleted bool `gorm:"-:all" sql:"-" json:"deleted,omitempty"`
Events []Event `json:"-" gorm:"many2many:event_checks;"`
Audited
}
// ExpScope provides functionality.
func ExpScope(q *gorm.DB) *gorm.DB {
return q.Where("kind IN ('whois', 'ssl')").
Preload("Monitor").
Preload("Monitor.Group").
Preload("Monitor.Group.Notifications").
Preload("Monitor.Group.Notifications.Contacts").
Where("expires < ?", time.Now().Add(time.Hour*7*24))
}
// IntervalOK provides functionality.
func (c *Check) IntervalOK() bool {
if c.Kind == kindRKN {
return true
}
if c.Kind == kindWhois {
return c.Interval >= 43200
}
return c.Interval >= 30
}
// GetLabel provides functionality.
func (c *Check) GetLabel() string {
if c.Name != nil {
return *c.Name
}
if c.URL != nil {
return *c.URL
}
return c.Kind
}
// KindLabel provides functionality.
func (c *Check) KindLabel() string {
if c.Kind == kindWhois {
return "регистрация домена"
}
if c.Kind == kindSSL {
return "SSL сертификат"
}
return c.Kind
}
// GetSettings provides functionality.
func (c *Check) GetSettings() CheckSettings {
d := CheckSettings{}
err := json.Unmarshal(c.Settings, &d)
if err != nil {
panic(err)
}
return d
}
// ValidateSettings provides functionality.
func (c *Check) ValidateSettings() error {
if len(c.Settings) == 0 {
c.Settings = []byte("{}")
}
return nil
}
// GetURL provides functionality.
func (c *Check) GetURL() (string, error) {
// return c.GetSettings()["url"].(string)
if c.URL != nil {
return *c.URL, nil
}
return "http://" + c.Monitor.Host, nil
}
// MetricName provides functionality.
func (c *Check) MetricName() string {
sanitized := strings.Map(func(r rune) rune {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == ':' {
return r
}
return '_'
}, c.Kind)
return "c" + sanitized
}
// QuorumEnabled reports whether this check should be aggregated by
// app/models/check_aggregator.go. When false (RequireQuorum <= 1),
// ApplyRemoteCheckResult keeps the legacy direct State update path.
func (c *Check) QuorumEnabled() bool {
return c.RequireQuorum > 1
}
// AggregationWindow returns AggregationWindowSeconds as a time.Duration.
// Defaults to 5s when the underlying int is zero/negative, mirroring the
// GORM column default; callers can rely on a strictly positive value.
func (c *Check) AggregationWindow() time.Duration {
if c.AggregationWindowSeconds <= 0 {
return 5 * time.Second
}
return time.Duration(c.AggregationWindowSeconds) * time.Second
}

336
app/models/check_aggregator.go Обычный файл
Просмотреть файл

@@ -0,0 +1,336 @@
package models
import (
"context"
"log"
"time"
"gorm.io/gorm"
)
// Phase 3 of docs/todo.md — result aggregation for multi-region checks.
//
// When a check has RequireQuorum > 1, ApplyRemoteCheckResult does not
// write Check.State directly. Instead it appends a CheckRegionResult row
// and leaves AggregatedAt NULL. This file owns the background goroutine
// that walks those pending rows once their aggregation window has
// elapsed, decides OK/ERR/DEGRADED per the documented rule, writes the
// aggregate state onto Check, stamps AggregatedAt on the contributing
// rows, and triggers Monitor.UpdateStatusFromChecks so the monitor's own
// status follows.
//
// Aggregation rule (see docs/todo.md Phase 3 + checkSeverityRank in
// monitor.go for the corresponding severity order):
//
// - Aggregate only rows whose created_at is older than
// NOW() - Check.AggregationWindowSeconds. This is the "watermark"
// pattern: a row is eligible only when no fresher regional result
// could still arrive and tip the vote. The window is per-check so
// noisy checks can use a longer wait than fast ones.
// - If zero eligible rows exist for a check, leave Check.State
// untouched (the special case called out in the spec).
// - Otherwise count OK vs not-OK among the eligible rows:
// OK >= RequireQuorum → Check.State = OK
// not-OK >= RequireQuorum → Check.State = ERR
// neither side reaches quorum → Check.State = DEGRADED
// - Stamp AggregatedAt = NOW() on every contributing row so the next
// tick skips them. One transaction per check; per-row failures do
// not poison other checks.
// AggregatorTickInterval is the default cadence of StartCheckAggregator
// when the caller passes interval <= 0. Mirrors the 30s default used by
// the other reapers in this package so the three reapers all tick on
// the same wall clock cadence — easier to grep, easier to reason about
// in incident timelines.
const AggregatorTickInterval = 30 * time.Second
// EnsureCheckAggregatorIndexes adds the partial indexes the aggregator
// relies on. AutoMigrate creates AggregatedAt as a regular btree column,
// but the per-tick SELECT filters on `aggregated_at IS NULL` over what
// grows to be a busy table; a partial index keeps the working set
// tiny. Idempotent so it is safe to call from Migrate() and from tests.
func EnsureCheckAggregatorIndexes() error {
return DB().Exec(`
CREATE INDEX IF NOT EXISTS check_region_results_pending_idx
ON check_region_results (check_id, created_at)
WHERE aggregated_at IS NULL
`).Error
}
// aggregateCheckState holds the per-check aggregation inputs we need to
// keep the rule readable. Rows is the set of CheckRegionResult rows
// eligible for the current decision; quorum is Check.RequireQuorum.
type aggregateCheckState struct {
CheckID int64
Quorum int
Rows []CheckRegionResult
}
// decideAggregateState encodes the OK/ERR/DEGRADED rule described in
// the package doc. Pure function — no DB, no time — so it is trivially
// unit-testable from the test file.
func decideAggregateState(in aggregateCheckState) (string, bool) {
if len(in.Rows) == 0 || in.Quorum <= 1 {
// Zero eligible rows in the window OR a misconfigured check
// (QuorumEnabled false). Caller must leave Check.State alone
// in both cases.
return "", false
}
okCount := 0
badCount := 0
for i := range in.Rows {
if in.Rows[i].State == stateOK {
okCount++
} else {
badCount++
}
}
switch {
case okCount >= in.Quorum:
return stateOK, true
case badCount >= in.Quorum:
return stateERR, true
default:
return stateDegraded, true
}
}
// CheckAggregatorTick performs one pass of the aggregator. It is the
// per-tick body StartCheckAggregator calls. Exported so the test suite
// can call it directly without spinning up the goroutine; production
// always goes through StartCheckAggregator.
//
// The returned (aggregated, err) tuple lets the caller log a metric:
// aggregated counts how many Check rows had their State written this
// tick. The function is idempotent — a second call with no new
// unaggregated rows is a no-op that returns (0, nil).
func CheckAggregatorTick() (aggregated int, err error) {
// Step 1: collect candidate check IDs. The JOIN to checks is needed
// to read each check's window length and to filter on
// require_quorum > 1 (so we never aggregate the legacy path).
rows, err := DB().Raw(`
SELECT DISTINCT crr.check_id
FROM check_region_results crr
JOIN checks c ON c.id = crr.check_id
WHERE crr.aggregated_at IS NULL
AND c.require_quorum > 1
AND crr.created_at < NOW() - make_interval(secs => GREATEST(c.aggregation_window_seconds, 1))
ORDER BY crr.check_id
`).Rows()
if err != nil {
return 0, err
}
defer func() { _ = rows.Close() }()
var checkIDs []int64
for rows.Next() {
var id int64
if scanErr := rows.Scan(&id); scanErr != nil {
return 0, scanErr
}
checkIDs = append(checkIDs, id)
}
if scanErr := rows.Err(); scanErr != nil {
return 0, scanErr
}
if len(checkIDs) == 0 {
return 0, nil
}
for _, checkID := range checkIDs {
n, err := aggregateOneCheck(checkID)
if err != nil {
// Log and continue: one bad check must not stop the loop.
log.Printf("check_aggregator: check_id=%d error: %v", checkID, err)
continue
}
aggregated += n
}
return aggregated, nil
}
// aggregateOneCheck runs the aggregation logic for a single check inside
// a transaction. The transaction holds a FOR UPDATE row lock on the
// check so concurrent aggregator instances (multiple web processes) can
// not race on the same check — the second one waits for the first to
// commit, then sees AggregatedAt IS NOT NULL on every row and the
// candidate SELECT below returns an empty set.
func aggregateOneCheck(checkID int64) (int, error) {
tx := DB().Begin()
if tx.Error != nil {
return 0, tx.Error
}
defer func() {
if r := recover(); r != nil {
tx.Rollback()
panic(r)
}
}()
var check Check
if err := tx.Clauses(SkipLockedClause).First(&check, checkID).Error; err != nil {
tx.Rollback()
if err == gorm.ErrRecordNotFound {
// Check was deleted between candidate SELECT and lock; not
// an error, just nothing to do.
return 0, nil
}
return 0, err
}
// Defensive: only aggregate quorum-enabled checks. The candidate
// SELECT already filters on this, but a stale row that flipped off
// quorum between calls must not be re-aggregated here.
if !check.QuorumEnabled() {
tx.Rollback()
return 0, nil
}
var results []CheckRegionResult
if err := tx.
Where("check_id = ? AND aggregated_at IS NULL", checkID).
Order("created_at ASC").
Find(&results).Error; err != nil {
tx.Rollback()
return 0, err
}
decision, ok := decideAggregateState(aggregateCheckState{
CheckID: checkID,
Quorum: check.RequireQuorum,
Rows: results,
})
if !ok {
// Zero eligible rows — leave Check.State alone. There is also
// nothing to stamp, so just rollback and move on.
tx.Rollback()
return 0, nil
}
now := time.Now()
// Pull the latest error string from the contributing rows so the
// monitor event / notifier pipeline has something to show. Prefer
// the most recent ERR row's message; fall back to the most recent
// any-row message. UNK / empty stays NULL.
var lastError *string
for i := len(results) - 1; i >= 0; i-- {
if results[i].Error != nil && *results[i].Error != "" {
lastError = results[i].Error
break
}
}
upd := map[string]interface{}{
colState: decision,
colLastEnd: now,
}
if decision == stateOK {
// OK resets error — mirrors the legacy ApplyRemoteCheckResult
// path that sets `error = gorm.Expr("NULL")` when state==OK.
upd["error"] = gorm.Expr("NULL")
upd["last_ok"] = now
upd["fails"] = 0
upd["was_up"] = now
} else {
// Non-OK: bump the fail counter and only overwrite the error
// when one of the contributing rows actually carries a
// message. If none do, leave whatever was there before —
// mirrors the legacy `if report.Error != nil` branch.
upd["last_fail"] = now
upd["fails"] = gorm.Expr("fails + 1")
if lastError != nil {
upd["error"] = *lastError
}
}
if err := tx.Model(&Check{}).Where("id = ?", checkID).UpdateColumns(upd).Error; err != nil {
tx.Rollback()
return 0, err
}
if err := tx.Model(&CheckRegionResult{}).
Where("check_id = ? AND aggregated_at IS NULL", checkID).
UpdateColumns(map[string]interface{}{
"aggregated_at": now,
}).Error; err != nil {
tx.Rollback()
return 0, err
}
if err := tx.Commit().Error; err != nil {
return 0, err
}
// Mirror ApplyRemoteCheckResult: propagate the aggregate decision
// up to the monitor. We do this AFTER commit so a rollback does
// not leave the monitor in a state whose corresponding check is
// still pre-aggregate. The goroutine keeps the failure path of
// UpdateStatusFromChecks isolated from the aggregator's hot loop.
if check.MonitorID != 0 {
var mon Monitor
if err := DB().First(&mon, check.MonitorID).Error; err == nil {
go mon.UpdateStatusFromChecks()
} else {
log.Printf("check_aggregator: monitor lookup failed for check_id=%d: %v", checkID, err)
}
}
return 1, nil
}
// StartCheckAggregator launches a goroutine that calls
// CheckAggregatorTick on the given interval until ctx is canceled.
// Mirrors StartTaskReaper / StartDeadWorkerReaper in this package — same
// ticker shape, same default-interval fall-back, same per-tick recover
// so a malformed row cannot crash the web process.
//
// The default interval is AggregatorTickInterval (30s); values <= 0
// fall back to the default so the helper is safe to call from any call
// site without a guard. A nil context falls back to context.Background()
// the same way StartDeadWorkerReaper does, so main.init() and tests
// can both call it without ceremony.
//
// Wire from main.init() once per process. The aggregator is cheap in
// steady state (one indexed SELECT for candidates + a per-check
// transaction over a handful of unaggregated rows). Under load it
// scales horizontally — multiple web processes can each run their own
// StartCheckAggregator goroutine because FOR UPDATE SKIP LOCKED on the
// per-check transaction guarantees at-most-one winner per check.
func StartCheckAggregator(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = AggregatorTickInterval
}
if ctx == nil {
ctx = context.Background()
}
// Best-effort index bootstrap. AutoMigrate declares AggregatedAt as
// a regular btree column; the partial index speeds up the per-tick
// candidate SELECT. Idempotent — safe to call on every boot.
if err := EnsureCheckAggregatorIndexes(); err != nil {
log.Printf("check_aggregator: ensure index: %v", err)
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("check_aggregator: panic recovered: %v", r)
}
}()
n, err := CheckAggregatorTick()
if err != nil {
log.Printf("check_aggregator: error: %v", err)
return
}
if n > 0 {
log.Printf("check_aggregator: aggregated=%d", n)
}
}()
}
}
}()
}

448
app/models/check_aggregator_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,448 @@
package models_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
// seedAggregatorWorld creates the minimum fixture the aggregator tests
// need: a plan, account, group, monitor, and an http check whose
// RequireQuorum / AggregationWindowSeconds are set per call. The check
// is created with state=UNK so the test can observe the aggregator's
// effect on Check.State directly.
//
// Returns the freshly-created monitor + check; the check is what every
// test below mutates (RequireQuorum, AggregationWindowSeconds) and then
// asserts on. Cleanup is the caller's responsibility — most tests call
// models.Drop() at the top instead.
func seedAggregatorWorld(t *testing.T, quorum, windowSeconds int) (models.Monitor, models.Check) {
t.Helper()
plan := models.Plan{Name: "agg-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "agg-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
grp := &models.Group{AccountID: acc.ID, Name: "agg"}
require.NoError(t, models.DB().Create(grp).Error)
mon := models.Monitor{
Name: stringPtrAgg("agg.test"),
Host: "agg.test",
GroupID: grp.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&mon).Error)
enTrue := true
check := models.Check{
MonitorID: mon.ID,
Kind: "http",
Interval: 60,
Enabled: &enTrue,
State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
RequireQuorum: quorum,
AggregationWindowSeconds: windowSeconds,
}
require.NoError(t, models.DB().Create(&check).Error)
return mon, check
}
func stringPtrAgg(s string) *string { return &s }
// makeReport constructs a wire.CheckResultReport with sensible defaults
// for the OK or not-OK case. Tests use this to push results through
// ApplyRemoteCheckResult exactly the way a real worker would.
func makeReport(checkID, monitorID int64, state string) wire.CheckResultReport {
return wire.CheckResultReport{
JobID: "job-" + state,
CheckID: checkID,
MonitorID: monitorID,
State: state,
}
}
// regionResultWithErr is regionResultFor plus an error message. Used
// when the test wants to verify that the aggregator forwards the row's
// error string onto Check.Error (mimics a real worker reporting
// state=ERR with a diagnostic message).
func regionResultWithErr(t *testing.T, checkID int64, region, state, errMsg string, age time.Duration) models.CheckRegionResult {
t.Helper()
row := regionResultFor(t, checkID, region, state, age)
require.NoError(t, models.DB().Model(&row).UpdateColumn("error", errMsg).Error)
return row
}
// regionResultFor inserts a single CheckRegionResult row whose
// created_at and executed_at are both backdated by `age`, so the
// aggregator's window-based watermark picks it up immediately without
// needing a real time.Sleep. Returned row has its DB-assigned ID
// populated.
func regionResultFor(t *testing.T, checkID int64, region string, state string, age time.Duration) models.CheckRegionResult {
t.Helper()
// RegionCode has a FK to regions.code, so the region must exist
// before the result row is inserted. seedRegion is idempotent.
seedRegion(t, region)
row := models.CheckRegionResult{
CheckID: checkID,
RegionCode: region,
ExecutedAt: time.Now().Add(-age),
State: state,
}
require.NoError(t, models.DB().Create(&row).Error)
// Backdate CreatedAt too — the aggregator SQL keys on
// check_region_results.created_at (see CheckAggregatorTick). GORM
// auto-sets CreatedAt on insert, so we have to UPDATE it post-hoc.
require.NoError(t, models.DB().Model(&row).UpdateColumns(map[string]interface{}{
"created_at": time.Now().Add(-age),
"updated_at": time.Now().Add(-age),
}).Error)
return row
}
// loadCheck re-reads a Check row by ID — used after the aggregator
// runs so the test asserts against the post-tick state.
func loadCheck(t *testing.T, id int64) models.Check {
t.Helper()
var c models.Check
require.NoError(t, models.DB().First(&c, id).Error)
return c
}
// countPendingResults returns how many CheckRegionResult rows for
// checkID have aggregated_at IS NULL — the working set the next
// aggregator tick would consider.
func countPendingResults(t *testing.T, checkID int64) int64 {
t.Helper()
var n int64
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("check_id = ? AND aggregated_at IS NULL", checkID).
Count(&n).Error)
return n
}
// ---------------------------------------------------------------------------
// ApplyRemoteCheckResult: regression tests for the QuorumEnabled split.
// ---------------------------------------------------------------------------
// TestApplyRemoteCheckResult_DirectWhenQuorumOne pins the legacy path:
// when RequireQuorum==1 (the default), ApplyRemoteCheckResult still
// writes Check.State synchronously, exactly the way it did before Phase
// 3. This is the regression guard for the in-process RKN scheduler
// tests in internal/rknscheduler.
func TestApplyRemoteCheckResult_DirectWhenQuorumOne(t *testing.T) {
models.Drop()
models.Migrate()
mon, check := seedAggregatorWorld(t, 1, 5)
// Region must exist because CheckRegionResult has a FK to
// regions.code (seeded by Migrate, but the test region is custom).
seedRegion(t, "ru-msk")
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "OK"),
"ru-msk",
))
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "quorum=1 must keep the direct State update")
assert.NotNil(t, got.LastEnd, "legacy path must keep stamping last_end")
// One region result inserted with aggregated_at=NULL.
assert.EqualValues(t, 1, countPendingResults(t, check.ID),
"the region result row is always inserted even on the legacy path")
}
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
// Check.State — it only inserts the CheckRegionResult row. The check
// stays at its initial UNK and the unaggregated row count grows by
// exactly 1 per call.
func TestApplyRemoteCheckResult_BuffersWhenQuorumN(t *testing.T) {
models.Drop()
models.Migrate()
mon, check := seedAggregatorWorld(t, 3, 5)
seedRegion(t, "ru-msk")
seedRegion(t, "us-east")
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "OK"), "ru-msk",
))
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "ERR"), "us-east",
))
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State,
"quorum>1 must NOT touch Check.State — the aggregator owns it")
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
"two results buffered, both with aggregated_at=NULL")
}
// ---------------------------------------------------------------------------
// CheckAggregatorTick: rule tests.
// ---------------------------------------------------------------------------
// TestAggregator_QuorumOK: with RequireQuorum=2 and two OK results
// buffered, the aggregator must decide OK and stamp AggregatedAt on
// both contributing rows.
func TestAggregator_QuorumOK(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n, "one check aggregated this tick")
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "2 OK results >= quorum=2 → state=OK")
assert.NotNil(t, got.LastOk, "OK decision must stamp last_ok")
assert.EqualValues(t, 0, got.Fails, "fails must reset on OK")
assert.EqualValues(t, 0, countPendingResults(t, check.ID),
"both contributing rows must be stamped aggregated_at")
}
// TestAggregator_QuorumFail: with RequireQuorum=2 and two ERR results
// buffered, the aggregator must decide ERR and surface the most recent
// row's error message on Check.Error.
func TestAggregator_QuorumFail(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultWithErr(t, check.ID, "ru-msk", "ERR", "connection refused", 5*time.Second)
regionResultWithErr(t, check.ID, "us-east", "ERR", "timeout", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "ERR", got.State, "2 ERR >= quorum=2 → state=ERR")
assert.NotNil(t, got.LastFail)
assert.NotNil(t, got.Error, "ERR decision must carry an error message from the rows")
assert.Contains(t, *got.Error, "timeout",
"aggregator should surface the latest row's error message")
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
}
// TestAggregator_DegradedWhenPartial: with RequireQuorum=3 and 1 OK +
// 2 ERR (mixed within window), neither side reaches the quorum of 3 so
// the aggregator must decide DEGRADED. The state must NOT silently
// become OK or ERR.
func TestAggregator_DegradedWhenPartial(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 3, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "ERR", 4*time.Second)
regionResultFor(t, check.ID, "eu-west", "ERR", 3*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "DEGRADED", got.State,
"neither OK nor ERR reaches quorum=3 → state=DEGRADED")
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
}
// TestAggregator_NotEnoughRegionsAlsoDegraded covers the single-region-
// only-delivered case: with RequireQuorum=3 and only 1 result buffered
// (and it aged past the window), the rule still says "neither side
// reached quorum" → DEGRADED. This is the documented behavior for slow
// regions that never report in time.
func TestAggregator_NotEnoughRegionsAlsoDegraded(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 3, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "DEGRADED", got.State,
"single OK row vs quorum=3 → DEGRADED (below quorum on both sides)")
}
// TestAggregator_NoResultsLeavesStateAlone is the "special case" from
// the spec: when the aggregator tick finds no eligible rows for a
// check, Check.State must NOT change. Pre-set the check to OK and
// verify it stays OK.
func TestAggregator_NoResultsLeavesStateAlone(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
// Pre-set state and a previous LastEnd so we can detect any
// accidental overwrite.
prevEnd := time.Now().Add(-time.Hour)
require.NoError(t, models.DB().Model(&models.Check{}).
Where("id = ?", check.ID).
Updates(map[string]interface{}{
"state": "OK",
"last_end": prevEnd,
"last_ok": prevEnd,
}).Error)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "no eligible rows → nothing aggregated")
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "state must not change with zero eligible rows")
assert.WithinDuration(t, prevEnd, *got.LastEnd, time.Second,
"last_end must not be touched when there are no eligible rows")
}
// TestAggregator_MultipleChecksIndependent verifies that a single tick
// processes every check with pending results, not just the first one.
// Two checks, each with 2 regions, each should flip to OK after the
// tick.
func TestAggregator_MultipleChecksIndependent(t *testing.T) {
models.Drop()
models.Migrate()
_, c1 := seedAggregatorWorld(t, 2, 1)
_, c2 := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, c1.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, c1.ID, "us-east", "OK", 4*time.Second)
regionResultFor(t, c2.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, c2.ID, "eu-west", "OK", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 2, n, "both checks aggregated in the same tick")
assert.Equal(t, "OK", loadCheck(t, c1.ID).State)
assert.Equal(t, "OK", loadCheck(t, c2.ID).State)
assert.EqualValues(t, 0, countPendingResults(t, c1.ID))
assert.EqualValues(t, 0, countPendingResults(t, c2.ID))
}
// TestAggregator_AlreadyAggregatedRowsSkipped pins the idempotency
// story: a second tick with no new rows must be a no-op. We pre-mark
// the rows aggregated_at=NOW() and verify the tick returns (0, nil)
// without touching Check.State.
func TestAggregator_AlreadyAggregatedRowsSkipped(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
r1 := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
r2 := regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
// Pretend a previous aggregator pass already stamped them.
now := time.Now()
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("id IN ?", []int64{r1.ID, r2.ID}).
UpdateColumns(map[string]interface{}{"aggregated_at": now}).Error)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "no candidate checks → 0 aggregated")
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State, "already-aggregated rows must not cause a re-decision")
}
// TestAggregator_IgnoresRowsInsideWindow verifies the watermark: rows
// whose CreatedAt is NEWER than (NOW() - window) are NOT eligible and
// must NOT be stamped. With AggregationWindowSeconds=10 and rows aged
// only 2s, the aggregator finds nothing to do.
func TestAggregator_IgnoresRowsInsideWindow(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 10)
regionResultFor(t, check.ID, "ru-msk", "OK", 2*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 1*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "rows still inside window → no aggregation")
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State)
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
"rows inside window stay unaggregated for the next tick")
}
// TestAggregator_SkipsChecksWithQuorumOne guards the candidate SELECT
// filter: even though CheckRegionResult rows are written for
// RequireQuorum=1 checks (via StoreCheckRegionResult), the aggregator
// must not re-decide their state because ApplyRemoteCheckResult
// already did. We simulate by inserting a region row with aggregated_at
// NULL for a quorum=1 check and verifying the tick ignores it.
func TestAggregator_SkipsChecksWithQuorumOne(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 1, 1)
r := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
require.Nil(t, r.AggregatedAt)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "quorum=1 checks must be filtered out by the candidate SELECT")
// The row must stay unaggregated too — the aggregator has no
// business stamping it.
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
}
// ---------------------------------------------------------------------------
// StartCheckAggregator: ticker smoke test.
// ---------------------------------------------------------------------------
// TestStartCheckAggregator_TickerFiresOnce is the smoke test for the
// background helper: spin up the aggregator with a tight 10ms ticker
// and a cancellable context, wait for one tick to flip a seeded
// check's state, then cancel so the goroutine exits cleanly. Mirrors
// TestStartDeadWorkerReaper_TickerFiresOnce in shape.
func TestStartCheckAggregator_TickerFiresOnce(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
models.StartCheckAggregator(ctx, 10*time.Millisecond)
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got := loadCheck(t, check.ID)
if got.State == "OK" {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("aggregator goroutine did not flip check to OK within 2s; state=%q", loadCheck(t, check.ID).State)
}
// silence unused import warnings when individual helpers are inlined by
// editors — the package-level references below keep the imports live.
var (
_ = gorm.ErrRecordNotFound
)

14
app/models/check_data.go Обычный файл
Просмотреть файл

@@ -0,0 +1,14 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/internal/influx"
)
// CheckData provides functionality.
type CheckData struct {
Uptime int `json:"uptime"`
Data []influx.InfluxData `json:"data"`
LastCheck *time.Time `json:"last_check"`
}

415
app/models/check_jobs.go Обычный файл
Просмотреть файл

@@ -0,0 +1,415 @@
package models
import (
"encoding/json"
"fmt"
"log"
"time"
"github.com/google/uuid"
"github.com/lib/pq"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/internal/influx"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
// defaultRegionCode is the historical default region seeded by Migrate()
// (see app/models/migrate.go) and used as a catch-all bucket for results
// reported without a region code. Aliased to the exported Region
// constant (DefaultRegionCode) so admin endpoints and the in-process
// job router share one source of truth.
const defaultRegionCode = DefaultRegionCode
// ChecksForWorker returns checks that need to be executed by a distributed worker.
// It uses FOR UPDATE SKIP LOCKED to prevent race conditions between concurrent workers.
// The worker specifies which check kinds it can handle via the kinds parameter.
//
// Phase 2 of docs/plans/worker-notifier-mvp.md adds regional job routing: when
// worker is non-nil, the candidate monitor set is filtered by
// applyRegionRouting so a worker only sees checks that explicitly allow its
// region. Pass nil for the legacy "no region scoping" path used by
// diagnostics/dashboard tooling.
func ChecksForWorker(worker *WorkerNode, kinds []string, limit int) []*Check {
tx := DB().Begin()
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
Where("monitors.enabled").
Where("checks.enabled AND checks.kind IN (?)", kinds)
if worker != nil {
q = applyRegionRouting(q, worker)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
}
// Flagged workers remain visible for audit/history but never receive new work.
if worker.NetworkProblemActive(time.Now()) {
tx.Rollback()
return nil
}
}
// Allow faster retry for failed http/dns checks
notOk := ""
hasHTTPOrDNS := false
for _, k := range kinds {
if k == kindHTTP || k == kindDNS {
hasHTTPOrDNS = true
break
}
}
if hasHTTPOrDNS {
notOk = `OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
}
whereClause := `
(checks.last_start IS NULL) OR
(checks.last_start + (checks.interval || ' second')::interval < now())
`
if notOk != "" {
whereClause += notOk
}
rq := q.Where(whereClause)
// Use SKIP LOCKED to avoid contention between workers. The same
// FOR UPDATE SKIP LOCKED clause also gives us implicit load balancing
// across workers in the same region: each concurrent worker call
// grabs a disjoint slice of the pending checks and a row leased by
// worker A is invisible to worker B until A's transaction commits
// (or rolls back / lease expires).
var checks []*Check
rq.Clauses(SkipLockedClause).
Limit(limit).
Preload("Monitor").
Find(&checks)
for _, c := range checks {
log.Println("worker: assigned remote check:", c.ID, c.Kind)
tx.Model(&c).Where("id = ?", c.ID).Update(colLastStart, time.Now())
}
tx.Commit()
return checks
}
// EnqueueDueCheckTasks atomically turns due normal checks into durable generic
// task envelopes. ChecksForWorker remains for the HTTP polling compatibility
// endpoint, while websocket scheduling uses this task-producing path.
func EnqueueDueCheckTasks(worker *WorkerNode, kinds []string, limit int) error {
if worker == nil || len(kinds) == 0 || limit <= 0 || worker.NetworkProblemActive(time.Now()) {
return nil
}
return DB().Transaction(func(tx *gorm.DB) error {
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
Where("monitors.enabled").Where("checks.enabled AND checks.kind IN (?)", kinds)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
}
q = applyRegionRouting(q, worker)
where := `(checks.last_start IS NULL) OR (checks.last_start + (checks.interval || ' second')::interval < now())`
for _, kind := range kinds {
if kind == kindHTTP || kind == kindDNS {
where += ` OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
break
}
}
var checks []*Check
if err := q.Where(where).Clauses(SkipLockedClause).Limit(limit).Preload("Monitor.Group").Find(&checks).Error; err != nil {
return err
}
now := time.Now()
for _, check := range checks {
if check.Monitor == nil || check.Monitor.Group == nil {
continue
}
job := JobForCheck(check)
payload, err := json.Marshal(job)
if err != nil {
return err
}
checkID, monitorID := check.ID, check.MonitorID
bucket := now.UTC().Unix() / int64(check.Interval)
task := Task{
JobID: job.JobID, Kind: TaskKindCheck, State: TaskStateQueued,
AccountID: check.Monitor.Group.AccountID, CheckID: &checkID, MonitorID: &monitorID,
Payload: payload, NotBefore: now, MaxAttempts: DefaultTaskMaxAttempts,
IdempotencyKey: fmt.Sprintf("check:%d:%d", check.ID, bucket),
}
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&task).Error; err != nil {
return err
}
if err := tx.Model(check).Update(colLastStart, now).Error; err != nil {
return err
}
}
return nil
})
}
// applyRegionRouting narrows the monitor JOIN in ChecksForWorker to the
// subset whose routing rule matches the calling worker's region. The filter
// is applied at the SQL layer so the SKIP LOCKED page only scans/leases
// rows that this worker is allowed to run, instead of leasing and then
// discarding forbidden checks.
//
// The function intentionally mirrors Monitor.WantsRegion so the helper can
// be reused from non-SQL callers (UI preview, plan validation, etc.).
//
// SQL form:
//
// - monitors.region_mode IN ('any', 'all', ”)
// → unconditional match; legacy / Phase 3 placeholder behavior.
// - monitors.region_mode = 'specific' AND
// (monitors.preferred_regions IS NULL OR
// cardinality(monitors.preferred_regions) = 0 OR
// ? = ANY(monitors.preferred_regions))
// → empty array falls back to "any"; otherwise the worker code must
// be in the whitelist.
func applyRegionRouting(q *gorm.DB, worker *WorkerNode) *gorm.DB {
if worker == nil || worker.RegionCode == "" {
return q
}
if worker.RegionCode == defaultRegionCode {
// The default "local" region is the historical catch-all; the
// in-process scheduler (not ChecksForWorker) handles those
// monitors. Skip regional filtering entirely so we don't leak
// Phase 1 in-process workers through the new router.
return q
}
// TODO(phase3): split RegionMode="all" into N assignments, one per
// preferred region, so the result aggregator can build a quorum.
// Today it is treated as "any" so existing checks keep flowing.
return q.Where(
`(monitors.region_mode IN ('any', 'all', '') OR `+
`(monitors.region_mode = 'specific' AND `+
`(monitors.preferred_regions IS NULL OR `+
`coalesce(array_length(monitors.preferred_regions, 1), 0) = 0 OR `+
`? = ANY(monitors.preferred_regions))))`,
worker.RegionCode,
)
}
// JobForCheck creates a CheckJob from a Check model for sending to a worker
func JobForCheck(c *Check) wire.CheckJob {
jobID := uuid.New().String()
var urlStr *string
if c.URL != nil {
urlStr = c.URL
}
return wire.CheckJob{
JobID: jobID,
CheckID: c.ID,
MonitorID: c.MonitorID,
Kind: c.Kind,
Host: c.Monitor.Host,
URL: urlStr,
Interval: c.Interval,
Settings: json.RawMessage(c.Settings),
}
}
// QueueMonitorChecks makes enabled checks for a monitor immediately eligible for remote assignment.
func QueueMonitorChecks(monitorID int64) error {
return DB().Model(&Check{}).
Where("monitor_id = ? AND enabled", monitorID).
Updates(map[string]interface{}{
colLastStart: nil,
colLastEnd: nil,
}).Error
}
// QueueMonitorChecksKind makes enabled checks of one kind immediately eligible for remote assignment.
func QueueMonitorChecksKind(monitorID int64, kind string) error {
return DB().Model(&Check{}).
Where("monitor_id = ? AND kind = ? AND enabled", monitorID, kind).
Updates(map[string]interface{}{
colLastStart: nil,
colLastEnd: nil,
}).Error
}
// ApplyRemoteCheckResult applies a check result reported by a distributed worker.
// It updates the check state in the database and triggers monitor status aggregation.
//
// Phase 3 of docs/todo.md (multi-region quorum aggregation): when the
// check has RequireQuorum > 1, the per-region result is recorded in
// check_region_results but Check.State is NOT touched here — that is
// the job of app/models/check_aggregator.go, which decides OK/ERR/
// DEGRADED once enough regional results have arrived or the aggregation
// window has elapsed. QuorumEnabled() == false preserves the legacy
// direct-update path so single-region / non-aggregated monitors keep
// the same behavior.
func ApplyRemoteCheckResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
return ApplyRemoteCheckResultFromWorker(report, regionCode, nil)
}
// ApplyRemoteCheckResultFromWorker persists worker attribution before changing
// legacy check state. A confirmation result is consumed exactly once and never
// overwrites the original check result.
func ApplyRemoteCheckResultFromWorker(report wire.CheckResultReport, regionCode string, worker *WorkerNode) error { //nolint:gocritic,lll // hugeParam: wire compatibility
var monitor *Monitor
err := DB().Transaction(func(tx *gorm.DB) error {
var err error
monitor, err = ApplyRemoteCheckResultFromWorkerTx(tx, report, regionCode, worker)
return err
})
if err != nil {
return err
}
if monitor != nil {
monitor.UpdateStatusFromChecks()
}
// VictoriaMetrics is outside PostgreSQL and is deliberately post-commit.
// A caller that retries after this error will not duplicate relational state;
// metric points are external at-least-once observations and need TSDB repair
// if the write remains unavailable.
return StoreRemoteCheckMetrics(report.Metrics)
}
// ApplyRemoteCheckResultFromWorkerTx applies all relational result effects using
// the caller's transaction. It intentionally does not write VictoriaMetrics or
// aggregate monitor state: both must happen only after the transaction commits.
// A nil monitor means the report was a consumed diagnostic attempt.
func ApplyRemoteCheckResultFromWorkerTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, worker *WorkerNode) (*Monitor, error) { //nolint:gocritic,lll // hugeParam: wire compatibility
if tx == nil {
return nil, fmt.Errorf("apply check result: nil transaction")
}
now := time.Now()
if worker != nil {
handled := false
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
return nil, err
}
if handled {
return nil, nil
}
}
check := Check{}
if err := tx.Preload("Monitor").First(&check, report.CheckID).Error; err != nil {
log.Println("worker: check not found:", report.CheckID, err)
return nil, err
}
// Always persist the per-region result first so the aggregator can
// pick it up regardless of which path we take next. We rely on
// StoreCheckRegionResult to default AggregatedAt=NULL (the column
// type is *time.Time, so a zero value writes SQL NULL).
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
log.Println("worker: error storing region result:", report.CheckID, err)
return nil, err
}
// Quorum-enabled checks: write nothing to Check.State here. The
// aggregator will compute the aggregate state once the window has
// elapsed (or enough regions have reported) and stamp AggregatedAt on
// the contributing CheckRegionResult rows.
if check.QuorumEnabled() {
return nil, nil
}
update := map[string]interface{}{
colState: report.State,
colLastEnd: now,
colWarnings: pq.StringArray(report.Warnings),
colInfos: pq.StringArray(report.Infos),
}
if report.State == "OK" {
update["was_up"] = now
update["last_ok"] = now
update["fails"] = 0
update["error"] = gorm.Expr("NULL")
} else {
update["last_fail"] = now
update["fails"] = gorm.Expr("fails + 1")
if report.Error != nil {
update["error"] = *report.Error
}
}
if report.ExpiresAt != nil {
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
if err == nil {
update["expires"] = t
}
}
if err := tx.Model(&check).UpdateColumns(update).Error; err != nil {
log.Println("worker: error updating check:", report.CheckID, err)
return nil, err
}
if worker != nil {
payload, _ := json.Marshal(report)
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
if attempt.JobID == "" {
attempt.JobID = uuid.New().String()
}
// A duplicate websocket/HTTP delivery must not create another attempt.
if err := tx.Where("job_id = ?", attempt.JobID).FirstOrCreate(&attempt).Error; err != nil {
return nil, err
}
switch report.State {
case stateERR, stateFail:
if err := StartConfirmationTx(tx, check.ID, worker.ID, now); err != nil {
return nil, err
}
case stateOK:
if err := RecoverDiagnosticTx(tx, check.ID, now); err != nil {
return nil, err
}
}
}
return check.Monitor, nil
}
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
func StoreRemoteCheckMetrics(metrics []wire.MetricPoint) error {
for _, metric := range metrics {
if metric.Metric == "" || len(metric.Fields) == 0 {
continue
}
if err := influx.WriteOne(metric.Metric, metric.Tags, metric.Fields); err != nil {
return err
}
}
return nil
}
// StoreCheckRegionResult stores a per-region check result for distributed monitoring analytics
func StoreCheckRegionResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
return StoreCheckRegionResultTx(DB(), report, regionCode, time.Now())
}
// StoreCheckRegionResultTx stores a regional result in the caller's transaction.
func StoreCheckRegionResultTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, executedAt time.Time) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
if tx == nil {
return fmt.Errorf("store region result: nil transaction")
}
if regionCode == "" {
regionCode = defaultRegionCode
}
result := CheckRegionResult{
CheckID: report.CheckID,
RegionCode: regionCode,
ExecutedAt: executedAt,
State: report.State,
DurationMs: report.DurationMs,
Error: report.Error,
}
return tx.Create(&result).Error
}
// StaleWorkers marks workers as inactive or dead based on last_seen time
func StaleWorkers() {
// Mark workers with no heartbeat for 2 minutes as inactive
DB().Model(&WorkerNode{}).
Where("status = ? AND last_seen < ?", "active", time.Now().Add(-2*time.Minute)).
Update("status", "inactive")
// Mark workers with no heartbeat for 5 minutes as dead
DB().Model(&WorkerNode{}).
Where("status IN (?, ?) AND last_seen < ?", "active", "inactive", time.Now().Add(-5*time.Minute)).
Update("status", "dead")
}

451
app/models/check_jobs_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,451 @@
package models_test
import (
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// seedRegionRoutingWorld builds two regions and three groups (one per
// monitor) plus three workers. The monitors and their PreferredRegions /
// RegionMode are configured by the caller via a callback so each test can
// express the exact routing scenario it wants to assert on.
//
// Returns a routerTestWorld that knows how to look up each fixture by name
// for readable assertions.
type routerTestWorld struct {
groupAny *models.Group
groupSpec *models.Group
groupAll *models.Group
workerMsk *models.WorkerNode
workerUSEast *models.WorkerNode
workerLocal *models.WorkerNode
}
// seedWorker creates a worker bound to regionCode. AuthToken is unique per
// worker so the FOR UPDATE SKIP LOCKED path can exercise two distinct
// concurrent callers.
func seedWorker(t *testing.T, id, regionCode string) *models.WorkerNode {
t.Helper()
seedRegion(t, regionCode)
w := &models.WorkerNode{
WorkerID: id,
RegionCode: regionCode,
Status: "active",
AuthToken: "tok-" + id,
Concurrency: 4,
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
}
require.NoError(t, models.DB().Create(w).Error)
return w
}
// seedRouterMonitor creates a Monitor with the given region routing
// attributes and one ready-to-run http Check. The check has Interval=60
// (minimum allowed) and LastStart=nil so ChecksForWorker picks it up
// immediately on the next poll.
func seedRouterMonitor(t *testing.T, group *models.Group, host, regionMode string, preferred []string) (models.Monitor, models.Check) {
t.Helper()
enTrue := true
mon := models.Monitor{
Name: stringPtrRouter(host),
Host: host,
GroupID: group.ID,
Enabled: true,
}
if regionMode != "" {
mon.RegionMode = regionMode
}
if preferred != nil {
mon.PreferredRegions = models.RegionCodesFromSlice(preferred)
}
require.NoError(t, models.DB().Create(&mon).Error)
check := models.Check{
MonitorID: mon.ID,
Kind: "http",
Interval: 60,
Enabled: &enTrue,
State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&check).Error)
return mon, check
}
func stringPtrRouter(s string) *string { return &s }
// seedRouterWorld is the common fixture for the TestRegionRouting_* table.
// It provisions two regions (ru-msk, us-east) and three monitors pinned to
// different routing modes; the workers are created lazily by the caller.
func seedRouterWorld(t *testing.T) routerTestWorld {
t.Helper()
models.Drop()
models.Migrate()
seedRegion(t, "ru-msk")
seedRegion(t, "us-east")
plan := models.Plan{Name: "router", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "router-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
groupAny := &models.Group{AccountID: acc.ID, Name: "any"}
groupSpec := &models.Group{AccountID: acc.ID, Name: "spec"}
groupAll := &models.Group{AccountID: acc.ID, Name: "all"}
require.NoError(t, models.DB().Create(groupAny).Error)
require.NoError(t, models.DB().Create(groupSpec).Error)
require.NoError(t, models.DB().Create(groupAll).Error)
seedRouterMonitor(t, groupAny, "any.test", models.RegionModeAny, nil)
seedRouterMonitor(t, groupSpec, "spec-msk.test", models.RegionModeSpecific, []string{"ru-msk"})
seedRouterMonitor(t, groupSpec, "spec-us.test", models.RegionModeSpecific, []string{"us-east"})
seedRouterMonitor(t, groupAll, "all.test", models.RegionModeAll, []string{"ru-msk", "us-east"})
return routerTestWorld{
groupAny: groupAny,
groupSpec: groupSpec,
groupAll: groupAll,
workerMsk: seedWorker(t, "w-msk", "ru-msk"),
workerUSEast: seedWorker(t, "w-us", "us-east"),
workerLocal: seedWorker(t, "w-local", "local"),
}
}
// idsOf returns a sorted list of monitor IDs assigned to the worker for
// easier assertions across checks.
func idsOf(checks []*models.Check) []int64 {
out := make([]int64, 0, len(checks))
for _, c := range checks {
if c.Monitor == nil {
continue
}
out = append(out, c.Monitor.ID)
}
return out
}
// TestRegionRouting_AnyReturnsAll confirms the SQL filter preserves the
// legacy behavior for region_mode='any' monitors and the Phase 3
// placeholder 'all' monitors: a worker in a region nobody explicitly
// whitelisted must still see them, while monitors pinned to specific
// regions stay filtered out.
//
// seedRouterWorld configures four monitors:
// - any.test → region_mode='any', no PreferredRegions
// - spec-msk.test → region_mode='specific', preferred=[ru-msk]
// - spec-us.test → region_mode='specific', preferred=[us-east]
// - all.test → region_mode='all', preferred=[ru-msk, us-east]
//
// A worker in region "remote" (whitelisted by nobody) must see exactly
// {any.test, all.test} — the two monitors whose region_mode bypasses the
// whitelist — and nothing else.
func TestRegionRouting_AnyReturnsAll(t *testing.T) {
seedRouterWorld(t)
seedRegion(t, "remote")
w := seedWorker(t, "w-remote", "remote")
checks := models.ChecksForWorker(w, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{"any.test", "all.test"}, hosts,
"region_mode='any' and the Phase 3 'all' placeholder must bypass the whitelist")
assert.NotContains(t, hosts, "spec-msk.test",
"specific-mode monitor with whitelisted ru-msk must NOT reach a remote worker")
assert.NotContains(t, hosts, "spec-us.test",
"specific-mode monitor with whitelisted us-east must NOT reach a remote worker")
}
// TestRegionRouting_SpecificFiltersByRegion proves the core Phase 2 promise:
// workers in different regions never receive a monitor whose PreferredRegions
// does not include their region code. The check is run with parallel
// goroutines because ChecksForWorker stamps `last_start` on every row it
// leases — a sequential second poll would always see an already-claimed
// queue and the filter would have nothing to test against.
//
// Under the FOR UPDATE SKIP LOCKED race, whichever SELECT fires first grabs
// every matching row, so the *exact* per-worker host list is non-deterministic.
// The deterministic invariant the test asserts is the no-leak property: a
// worker in ru-msk must never see spec-us.test, and vice versa.
func TestRegionRouting_SpecificFiltersByRegion(t *testing.T) {
world := seedRouterWorld(t)
var (
wg sync.WaitGroup
mskChecks []*models.Check
usChecks []*models.Check
)
wg.Add(2)
go func() {
defer wg.Done()
mskChecks = models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
}()
go func() {
defer wg.Done()
usChecks = models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
}()
wg.Wait()
mskHosts := hostsOf(mskChecks)
usHosts := hostsOf(usChecks)
// Aggregate coverage: together the two workers must see every check
// the routing layer would ever allow them — the four seeded monitors.
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
},
append(append([]string{}, mskHosts...), usHosts...),
"union of both workers' slices must cover every seeded monitor (any/specific/all × region)")
// Core Phase 2 invariant: regional filtering never leaks across
// PreferredRegions boundaries. This is the only assertion a
// concurrent SKIP LOCKED race lets us pin deterministically.
assert.NotContains(t, mskHosts, "spec-us.test",
"ru-msk worker must never see a monitor whitelisted for us-east only")
assert.NotContains(t, usHosts, "spec-msk.test",
"us-east worker must never see a monitor whitelisted for ru-msk only")
}
// TestRegionRouting_SpecificEmptyPreferredFallsBackToAny confirms the
// documented fall-back: a monitor in RegionModeSpecific with no
// PreferredRegions behaves like RegionModeAny so the field is safe to
// leave blank. We poll from the us-east worker — without the fall-back it
// would only see any.test + all.test + spec-us.test.
func TestRegionRouting_SpecificEmptyPreferredFallsBackToAny(t *testing.T) {
world := seedRouterWorld(t)
// Reset the spec-msk monitor to have an empty PreferredRegions list
// (the seed above gave it one). The Monitor row's RegionMode stays
// 'specific'.
require.NoError(t, models.DB().Model(&models.Monitor{}).
Where("host = ?", "spec-msk.test").
Update("preferred_regions", models.RegionCodesFromSlice(nil)).Error)
checks := models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.Contains(t, hosts, "spec-msk.test",
"empty PreferredRegions with region_mode=specific must fall back to 'any'")
}
// TestRegionRouting_AllDeferredToAny pins the Phase 3 placeholder behavior:
// region_mode='all' is logged and treated as 'any' today. The test asserts
// the monitor flows to a worker in any region (the TODO log marker is
// emitted from applyRegionRouting — pinned here as a code-grep contract).
func TestRegionRouting_AllDeferredToAny(t *testing.T) {
world := seedRouterWorld(t)
checks := models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.Contains(t, hosts, "all.test",
"region_mode='all' must currently behave like 'any' so existing checks keep flowing")
}
// TestRegionRouting_LocalWorkerBypass ensures the historic "local" region
// still routes everything: the in-process scheduler handles those monitors
// and we don't want the Phase 2 filter to leak platform workers through it.
func TestRegionRouting_LocalWorkerBypass(t *testing.T) {
world := seedRouterWorld(t)
checks := models.ChecksForWorker(world.workerLocal, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
}, hosts, "worker in region 'local' must receive every check (bypass)")
}
// TestRegionRouting_NilWorkerReturnsAll asserts the diagnostic-friendly
// escape hatch: passing nil for the worker skips the routing filter and
// returns every check the kinds/limit envelope allows.
func TestRegionRouting_NilWorkerReturnsAll(t *testing.T) {
seedRouterWorld(t)
checks := models.ChecksForWorker(nil, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
}, hosts, "nil worker must bypass the routing filter")
}
// TestRegionRouting_LoadBalanceImplicit confirms the SKIP LOCKED implicit
// load-balancing story: when two workers in the same region race for a pool
// of pending checks, each of them receives a non-empty disjoint slice. The
// two polls run in parallel goroutines so the FOR UPDATE SKIP LOCKED race
// window is actually exercised.
//
// IMPORTANT: SKIP LOCKED with a large LIMIT is unfair — whichever
// transaction's SELECT fires first grabs everything. The test therefore
// uses LIMIT=4 with 10 pending rows so each worker is forced to leave some
// rows unlocked for the other worker to pick up. Together they must cover
// at most 8 rows (LIMIT × workers) without overlap; the remaining rows are
// intentionally left for a future poll cycle, which mirrors production
// behavior where workers continually drain a backlog.
func TestRegionRouting_LoadBalanceImplicit(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "shared")
plan := models.Plan{Name: "lb-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "lb", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := &models.Group{AccountID: acc.ID, Name: "lb-g"}
require.NoError(t, models.DB().Create(group).Error)
enTrue := true
for i := 0; i < 10; i++ {
host := "lb-" + string(rune('a'+i)) + ".test"
mon := models.Monitor{
Name: stringPtrRouter(host),
Host: host,
GroupID: group.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&mon).Error)
ck := models.Check{
MonitorID: mon.ID, Kind: "http", Interval: 60,
Enabled: &enTrue, State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&ck).Error)
}
w1 := seedWorker(t, "lb-w1", "shared")
w2 := seedWorker(t, "lb-w2", "shared")
const limitPerWorker = 4
var (
wg sync.WaitGroup
aChecks, bChecks []*models.Check
)
wg.Add(2)
go func() {
defer wg.Done()
aChecks = models.ChecksForWorker(w1, []string{"http"}, limitPerWorker)
}()
go func() {
defer wg.Done()
bChecks = models.ChecksForWorker(w2, []string{"http"}, limitPerWorker)
}()
wg.Wait()
assert.Greater(t, len(aChecks), 0, "worker 1 must receive at least one check")
assert.Greater(t, len(bChecks), 0, "worker 2 must receive at least one check")
assert.LessOrEqual(t, len(aChecks)+len(bChecks), 2*limitPerWorker,
"two concurrent workers with LIMIT each can lease at most LIMIT*2 rows per cycle")
assert.Empty(t, intersectHosts(aChecks, bChecks),
"the two slices must be disjoint (FOR UPDATE SKIP LOCKED must not double-lease)")
}
// TestMonitorValidateRegionMode exercises the documented enum on the
// Monitor type so the validator surface does not regress.
func TestMonitorValidateRegionMode(t *testing.T) {
cases := []struct {
mode string
wantErr bool
}{
{"", false},
{"any", false},
{"specific", false},
{"all", false},
{"round-robin", true},
{"RANDOM", true},
}
for _, c := range cases {
t.Run("mode="+c.mode, func(t *testing.T) {
m := models.Monitor{RegionMode: c.mode}
err := m.ValidateRegionMode()
if c.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// TestMonitorWantsRegion pins the public WantsRegion helper that powers the
// non-SQL callers (UI preview, plan validation). Phase 3 will swap the
// 'all' branch's behavior, so the table covers all three modes today.
func TestMonitorWantsRegion(t *testing.T) {
cases := []struct {
name string
mode string
regions []string
workerRC string
want bool
}{
{"any_always_true", "any", []string{"ru-msk"}, "us-east", true},
{"any_empty_pref_still_true", "any", nil, "us-east", true},
{"specific_match", "specific", []string{"ru-msk", "eu-west"}, "ru-msk", true},
{"specific_no_match", "specific", []string{"ru-msk", "eu-west"}, "us-east", false},
{"specific_empty_pref_fallback", "specific", nil, "us-east", true},
{"all_placeholder_true", "all", []string{"ru-msk", "us-east"}, "ru-msk", true},
{"all_placeholder_foreign_region", "all", []string{"ru-msk", "us-east"}, "eu-west", true},
{"empty_mode_defaults_to_any", "", nil, "us-east", true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
m := models.Monitor{
RegionMode: c.mode,
PreferredRegions: models.RegionCodesFromSlice(c.regions),
}
assert.Equal(t, c.want, m.WantsRegion(c.workerRC))
})
}
}
// hostsOf extracts the hostnames from the assigned checks for readable
// assertions in table-driven tests.
func hostsOf(checks []*models.Check) []string {
out := make([]string, 0, len(checks))
for _, c := range checks {
if c.Monitor == nil {
continue
}
out = append(out, c.Monitor.Host)
}
return out
}
// intersectHosts returns the hostnames present in both slices — used to
// prove two concurrent workers did not lease the same check twice.
func intersectHosts(a, b []*models.Check) []string {
set := make(map[string]struct{}, len(a))
for _, c := range a {
if c.Monitor != nil {
set[c.Monitor.Host] = struct{}{}
}
}
var out []string
for _, c := range b {
if c.Monitor == nil {
continue
}
if _, ok := set[c.Monitor.Host]; ok {
out = append(out, c.Monitor.Host)
}
}
return out
}

42
app/models/check_metric_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,42 @@
package models
import (
"testing"
"unicode"
"github.com/stretchr/testify/assert"
)
func TestCheckMetricName(t *testing.T) {
tests := []struct {
kind string
want string
}{
{"http", "chttp"},
{"ssl", "cssl"},
{"bssl", "cbssl"},
{"ssh", "cssh"},
{"ftp", "cftp"},
{"dns", "cdns"},
{"whois", "cwhois"},
{"rkn", "crkn"},
{"llm", "cllm"},
{"llm-http", "cllm_http"},
{"weird-kind", "cweird_kind"},
}
for _, tt := range tests {
t.Run(tt.kind, func(t *testing.T) {
c := &Check{Kind: tt.kind, ID: 1}
got := c.MetricName()
assert.Equal(t, tt.want, got, "MetricName() for kind %q", tt.kind)
metric := got
for _, ch := range metric {
if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' && ch != ':' {
t.Errorf("metric name %q contains invalid character %q", metric, ch)
}
}
})
}
}

33
app/models/check_region_result.go Обычный файл
Просмотреть файл

@@ -0,0 +1,33 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// CheckRegionResult stores check results from distributed workers by region
type CheckRegionResult struct {
concerns.Model
CheckID int64 `gorm:"index;not null" json:"check_id"`
Check *Check `gorm:"foreignKey:CheckID" json:"check,omitempty"`
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id"`
WorkerNode *WorkerNode `gorm:"foreignKey:WorkerNodeID" json:"worker_node,omitempty"`
ExecutedAt time.Time `gorm:"not null" json:"executed_at"`
State string `gorm:"size:10;not null" json:"state"`
DurationMs int64 `json:"duration_ms"`
Error *string `json:"error"`
// AggregatedAt is set by app/models/check_aggregator.go once a row has
// been folded into a Check.State decision. NULL means "still waiting
// for the aggregator"; non-NULL means "this row has already been
// counted in a quorum decision and must not be re-counted". The
// column is indexed (see check_aggregator.go index helper) so the
// per-tick SELECT that finds unaggregated rows is O(matching rows)
// rather than scanning the whole table.
AggregatedAt *time.Time `gorm:"index" json:"aggregated_at,omitempty"`
concerns.Timestamped
}

155
app/models/check_settings.go Обычный файл
Просмотреть файл

@@ -0,0 +1,155 @@
package models
import (
"errors"
"fmt"
"net/http"
"strings"
)
// CheckHeader provides functionality.
type CheckHeader struct {
Key string `json:"key"`
Value string `json:"value"`
}
// CheckSettings provides functionality.
type CheckSettings struct {
// HTTP Basic Auth Username
HTTPUsername string `json:"http_username,omitempty"`
// HTTP Basic Auth Password
HTTPPassword string `json:"http_password,omitempty"`
// ExpectedAnswer (default, redirect, custom)
ExpectedAnswer string `json:"expected_answer,omitempty"`
// Expected redirect location
ExpectedLocation string `json:"expected_location,omitempty"`
// Expected HTTP status codes for custom
ExpectedHTTPCode int `json:"expected_http_code,omitempty"`
// Keyword search type (off, present or absent)
KeywordType string `json:"keyword_type,omitempty"`
// Keyword to search for
KeywordValue string `json:"keyword_value,omitempty"`
SlowTime int `json:"slow_time"`
Timeout int `json:"timeout"`
// Request Settings
CheckIp bool `json:"checkip"` //nolint:revive // accepted lint exception
CheckIPv6 bool `json:"checkipv6"`
RequestHeader []CheckHeader `json:"request_headers"`
RequestMethod string `json:"request_method"`
RequestType string `json:"request_type"`
RequestContent string `json:"request_content"`
// SSH Settings
Port string `json:"port"`
// Host optionally overrides the monitor host for checks that do
// not naturally target the Monitor.Host (e.g. an ICMP/TCP/UDP
// probe to a separate machine, or a different IP family). Used by
// cping/ctcp/cudp.
Host string `json:"host,omitempty"`
// Count is the per-check packet count for ping. Defaults to 1 in
// cping when zero or negative.
Count int `json:"count,omitempty"`
// PacketSize is the ICMP payload size for ping (bytes). Defaults
// to 56 in cping when zero or negative.
PacketSize int `json:"packet_size,omitempty"`
// Distributed marks this check as eligible for execution on the
// distributed worker pool (multi-region, multi-worker) instead of
// only the in-process scheduler. Reserved for paid plans; the
// controller layer enforces the plan check.
Distributed bool `json:"distributed"`
}
// redirectCodes contains all HTTP redirect status codes.
var redirectCodes = []int{300, 301, 302, 303, 307, 308}
// CheckKeyword provides functionality.
func (s *CheckSettings) CheckKeyword(body []byte, warns []string) ([]string, error) {
var err error
switch s.KeywordType {
case "", "off":
return warns, nil
case "present":
if !strings.Contains(string(body), s.KeywordValue) {
err = errors.New("expected keyword " + s.KeywordValue + " not found")
}
case "absent":
if strings.Contains(string(body), s.KeywordValue) {
err = errors.New("unexpected keyword " + s.KeywordValue + " found")
}
}
return warns, err
}
// CheckAnswer provides functionality.
func (s *CheckSettings) CheckAnswer(resp *http.Response, body []byte) ([]string, error) {
// log.Println("check answer, settings:")
// spew.Dump(s)
isRedirect := false
for _, rc := range redirectCodes {
if resp.StatusCode == rc {
isRedirect = true
}
}
location := resp.Header.Get("location")
// log.Println("redirect?", isRedirect, location)
switch s.ExpectedAnswer {
case "", "default":
if isRedirect {
return s.CheckKeyword(body, []string{"redirect: " + location})
} else if resp.StatusCode != 200 {
return []string{}, fmt.Errorf("bad status code: %d", resp.StatusCode)
}
case "redirect":
if !isRedirect {
return []string{}, fmt.Errorf("bad status code: %d (expected redirect)", resp.StatusCode)
}
if s.ExpectedLocation != "" {
if location != s.ExpectedLocation {
return []string{}, fmt.Errorf(
"bad location: %s (expected %s)",
location,
s.ExpectedLocation,
)
}
}
case "custom":
if s.ExpectedHTTPCode == 0 {
s.ExpectedHTTPCode = 200
}
if resp.StatusCode != s.ExpectedHTTPCode {
return []string{}, fmt.Errorf("bad status code: %d (expected %d)", resp.StatusCode, s.ExpectedHTTPCode)
}
if s.ExpectedLocation != "" {
if location != s.ExpectedLocation {
return []string{}, fmt.Errorf(
"bad location: %s (expected %s)",
location,
s.ExpectedLocation,
)
}
}
default:
panic("bad expectedAnswer")
}
// return []string{}, nil
return s.CheckKeyword(body, []string{})
}

60
app/models/check_settings_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,60 @@
package models
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCheckSettingsDistributed_Default(t *testing.T) {
s := CheckSettings{}
assert.False(t, s.Distributed, "new CheckSettings should default to Distributed=false")
}
func TestCheckSettingsDistributed_JSON(t *testing.T) {
original := CheckSettings{
HTTPUsername: "user",
HTTPPassword: "pass",
ExpectedAnswer: "default",
Timeout: 30,
Distributed: true,
}
data, err := json.Marshal(original)
assert.NoError(t, err)
assert.Contains(t, string(data), `"distributed":true`)
var decoded CheckSettings
err = json.Unmarshal(data, &decoded)
assert.NoError(t, err)
assert.Equal(t, original, decoded)
assert.True(t, decoded.Distributed)
}
func TestCheckSettingsDistributed_OmitFalse(t *testing.T) {
s := CheckSettings{HTTPUsername: "user"}
data, err := json.Marshal(s)
assert.NoError(t, err)
assert.Contains(t, string(data), `"distributed":false`)
}
func TestCheckSettingsDistributed_CheckUnmarshal(t *testing.T) {
c := &Check{
Settings: []byte(`{"distributed": true, "timeout": 60}`),
}
got := c.GetSettings()
assert.True(t, got.Distributed)
assert.Equal(t, 60, got.Timeout)
}
func TestPlanAllowsDistributed(t *testing.T) {
var nilPlan *Plan
assert.False(t, nilPlan.AllowsDistributed(), "nil plan must not allow distributed")
free := &Plan{Price: 0}
assert.False(t, free.AllowsDistributed(), "free plan must not allow distributed")
paid := &Plan{Price: 100}
assert.True(t, paid.AllowsDistributed(), "paid plan must allow distributed")
}

181
app/models/cleanup_stale_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,181 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// makeStaleUser inserts a user whose LastActiveAt is older than the
// 3-month cutoff used by the cleanup filter. The returned user is what
// the candidates query should pick up.
func makeStaleUser(t *testing.T, email string, lastActive *time.Time) *models.User {
t.Helper()
u := &models.User{
Email: &email,
Name: "stale " + email,
Enabled: true,
Confirmed: true,
LastActiveAt: lastActive,
}
require.NoError(t, models.DB().Create(u).Error)
return u
}
// TestFindStaleAccounts_EmptyWhenNoCandidates checks the obvious
// negative case: a fresh account with an active owner is not eligible.
func TestFindStaleAccounts_EmptyWhenNoCandidates(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "fresh", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
// Group + owner access + active user
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
recent := time.Now().Add(-1 * time.Hour)
user := makeStaleUser(t, "active@example.com", &recent)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got, "an account with an active owner is not stale")
}
// TestFindStaleAccounts_PicksStaleEmptyAccount checks the happy path:
// account with no monitors + single user + last login > 3 months ago.
func TestFindStaleAccounts_PicksStaleEmptyAccount(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "ghost", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
neverLoggedIn := (*time.Time)(nil)
user := makeStaleUser(t, "ghost@example.com", neverLoggedIn)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
require.Len(t, got, 1, "the empty stale account should be picked up")
assert.Equal(t, acc.ID, got[0].AccountID)
assert.Equal(t, user.ID, got[0].UserID)
}
// TestFindStaleAccounts_SkipsAccountWithMonitors makes sure the
// "zero monitors" gate is enforced.
func TestFindStaleAccounts_SkipsAccountWithMonitors(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "active", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
// One monitor → account is NOT eligible even if the user is stale.
monitor := models.Monitor{GroupID: group.ID, Host: "example.com"}
require.NoError(t, models.DB().Create(&monitor).Error)
stale := time.Now().Add(-365 * 24 * time.Hour)
user := makeStaleUser(t, "owner@example.com", &stale)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got)
}
// TestFindStaleAccounts_SkipsUserWithMultipleAccounts verifies that a
// user holding two accounts disqualifies BOTH accounts.
func TestFindStaleAccounts_SkipsUserWithMultipleAccounts(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc1 := models.Account{Name: "acc1", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc1).Error)
acc2 := models.Account{Name: "acc2", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc2).Error)
models.DB().Create(&models.Group{AccountID: acc1.ID, Name: "g1"})
models.DB().Create(&models.Group{AccountID: acc2.ID, Name: "g2"})
stale := time.Now().Add(-365 * 24 * time.Hour)
user := makeStaleUser(t, "shared@example.com", &stale)
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc1.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc2.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got, "user with two accounts disqualifies both accounts")
}
// TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans verifies the
// end-to-end cleanup: matching account + user are removed, and
// recently-active accounts survive.
func TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
// Stale account with a stale user that has only this one account.
staleAcc := models.Account{Name: "ghost", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&staleAcc).Error)
staleGroup := models.Group{AccountID: staleAcc.ID, Name: "g"}
require.NoError(t, models.DB().Create(&staleGroup).Error)
staleUser := makeStaleUser(t, "ghost@example.com", nil)
require.NoError(t, models.DB().Create(&models.Access{AccountID: staleAcc.ID, UserID: &staleUser.ID, Kind: "account", Role: "owner"}).Error)
// Active account with a recent user — must NOT be touched.
freshAcc := models.Account{Name: "fresh", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&freshAcc).Error)
require.NoError(t, models.DB().Create(&models.Group{AccountID: freshAcc.ID, Name: "g"}).Error)
recent := time.Now().Add(-1 * time.Hour)
freshUser := makeStaleUser(t, "fresh@example.com", &recent)
require.NoError(t, models.DB().Create(&models.Access{AccountID: freshAcc.ID, UserID: &freshUser.ID, Kind: "account", Role: "owner"}).Error)
deleted, err := models.CleanupStaleAccounts()
require.NoError(t, err)
assert.Equal(t, 1, deleted, "only the stale empty account should be deleted")
// Stale account is gone.
var count int64
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", staleAcc.ID).Count(&count).Error)
assert.Equal(t, int64(0), count)
// Stale user is orphaned → also hard-deleted by the cleanup pass.
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", staleUser.ID).Count(&count).Error)
assert.Equal(t, int64(0), count)
// Fresh account and user survive.
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", freshAcc.ID).Count(&count).Error)
assert.Equal(t, int64(1), count)
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", freshUser.ID).Count(&count).Error)
assert.Equal(t, int64(1), count)
}

41
app/models/concerns/has_token.go Обычный файл
Просмотреть файл

@@ -0,0 +1,41 @@
// Package concerns provides functionality.
package concerns
import (
"bytes"
"crypto/rand"
"encoding/base64"
)
// HasToken provides functionality.
type HasToken struct {
Token string `json:"-" gorm:"unique_index"`
}
// SetToken provides functionality.
func (m *HasToken) SetToken() {
tk := RandomToken(32)
m.Token = base64.RawURLEncoding.EncodeToString(tk)
if m.Token == "" {
panic("RandomToken failed: token not set")
}
if len(m.Token) < 32 {
panic("RandomToken failed: token too short")
}
}
// RandomToken provides functionality.
func RandomToken(tokenLen int) []byte {
b := make([]byte, tokenLen)
n, err := rand.Read(b)
if err != nil {
panic(err)
}
if n != tokenLen {
panic("RandomToken failed: bad len")
}
if bytes.Equal(b, make([]byte, tokenLen)) {
panic("RandomToken failed: generated empty token")
}
return b
}

14
app/models/concerns/model.go Обычный файл
Просмотреть файл

@@ -0,0 +1,14 @@
// Source: https://gorm.io/gorm/blob/master/model.go
// Use 64 bit keys
package concerns
// Model base model definition, including fields `ID`, `CreatedAt`, `UpdatedAt`, `DeletedAt`, which could be embedded in your models
// type User struct {
// gorm.Model
// }
// Model provides functionality.
type Model struct {
ID int64 `gorm:"primarykey" json:"id"`
}

7
app/models/concerns/renderable.go Обычный файл
Просмотреть файл

@@ -0,0 +1,7 @@
package concerns
// Renderable provides functionality.
type Renderable struct {
Raw string
Rendered string
}

12
app/models/concerns/soft_delete.go Обычный файл
Просмотреть файл

@@ -0,0 +1,12 @@
package concerns
import (
"time"
)
// SoftDelete provides functionality.
type SoftDelete struct {
DeletedAt *time.Time `gorm:"index" json:"-"`
DeleterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
// Deleter *User `json:"-"`
}

11
app/models/concerns/timestamped.go Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
package concerns
import (
"time"
)
// Timestamped provides functionality.
type Timestamped struct {
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}

86
app/models/contact.go Обычный файл
Просмотреть файл

@@ -0,0 +1,86 @@
package models
import (
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Contact represents a notification contact.
//
// Ownership / tenancy:
//
// - A Contact belongs to exactly one Account (tenant). AccountID may be
// nil for system-level admin contacts (see SystemContacts).
// - A Contact may belong to at most one User. UserID is set when the
// contact was created on behalf of a specific user (the typical
// case for self-service "My email" / "My Telegram" contacts) and is
// nil for account-wide contacts.
//
// `User` is omitempty because most list payloads don't preload it; the
// `/settings/users` page loads it server-side via the AccountUserRow
// payload.
type Contact struct {
concerns.Model
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id)"`
Account *Account `json:"-"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
User *User `json:"user,omitempty"`
Name string `json:"name" gorm:"not null"`
Kind string `json:"kind" gorm:"not null;index:contact_value"`
Value string `json:"value" gorm:"index:contact_value"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
Data datatypes.JSON `json:"data"`
IsSystem *bool `json:"is_system" gorm:"default:false"`
Notifications []Notification `json:"-" gorm:"many2many:notification_contacts;"`
NotificationsCount int `gorm:"-:all" json:"notifications_count"`
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
Messages []Message `json:"-"`
concerns.Timestamped
concerns.HasToken
Audited
}
// SystemContacts returns all contacts marked as system/admin (is_system=true).
// These are the contacts distributed workers notify directly when the main
// API is unreachable (see docs/worker-protocol.md "System Selfcheck").
func SystemContacts() ([]Contact, error) {
var contacts []Contact
err := DB().Where("is_system = ? AND enabled = ?", true, true).Find(&contacts).Error
return contacts, err
}
// ContactsCounts fills NotificationsCount and MonitorsCount for each contact.
func ContactsCounts(contacts *[]Contact) {
groupIDs := make(map[int64]bool, 0)
groupCount := make(map[int64]int, 0)
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
(*contacts)[i].NotificationsCount = len(c.Notifications)
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
groupIDs[g.ID] = true
groupCount[g.ID] = 0
}
}
}
gids := make([]int64, 0, len(groupIDs))
for g := range groupIDs {
gids = append(gids, g)
}
CountGroups(gids, &groupCount)
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
cnt := 0
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
cnt += groupCount[g.ID]
}
}
(*contacts)[i].MonitorsCount = cnt
}
}

65
app/models/contact_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,65 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestSystemContacts verifies that SystemContacts returns only contacts with
// is_system=true and ignores contacts with is_system=false or nil.
func TestSystemContacts(t *testing.T) {
models.Drop()
models.Migrate()
account := &models.Account{Name: "test-account"}
require.NoError(t, models.DB().Create(account).Error)
accountID := account.ID
trueVal, falseVal := true, false
systemContact := &models.Contact{
AccountID: &accountID,
Name: "system-admin",
Kind: "email",
Value: "ops@example.com",
IsSystem: &trueVal,
}
regularContact := &models.Contact{
AccountID: &accountID,
Name: "regular",
Kind: "email",
Value: "user@example.com",
IsSystem: &falseVal,
}
nilSystemContact := &models.Contact{
AccountID: &accountID,
Name: "nil-system",
Kind: "email",
Value: "nil@example.com",
}
require.NoError(t, models.DB().Create(systemContact).Error)
require.NoError(t, models.DB().Create(regularContact).Error)
require.NoError(t, models.DB().Create(nilSystemContact).Error)
got, err := models.SystemContacts()
require.NoError(t, err)
var ids []int64
var names []string
for _, c := range got {
ids = append(ids, c.ID)
names = append(names, c.Name)
}
assert.Contains(t, names, "system-admin")
assert.NotContains(t, names, "regular")
assert.NotContains(t, names, "nil-system")
assert.Contains(t, ids, systemContact.ID)
assert.NotContains(t, ids, regularContact.ID)
assert.NotContains(t, ids, nilSystemContact.ID)
}

97
app/models/credential_crypto.go Обычный файл
Просмотреть файл

@@ -0,0 +1,97 @@
package models
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"io"
"os"
"strings"
"rsgit.ru/rsmon/rsmon/config/secrets"
)
const credentialKeyEnv = "RSMON_CRED_KEY"
// encryptSecret encrypts plaintext with AES-GCM. If no key is configured
// (dev/test), stores the value with a "plain:" prefix. Production deployments
// must set RSMON_CRED_KEY or config/secrets.yml crypto.pepper.
func encryptSecret(plaintext string) (string, error) {
key := credentialKey()
if key == "" {
return "plain:" + plaintext, nil
}
block, err := aes.NewCipher(deriveKey(key))
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
sealed := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return "enc:" + base64.StdEncoding.EncodeToString(sealed), nil
}
// decryptSecret decrypts a value produced by encryptSecret. Values without a
// known prefix are returned as-is for backward compatibility with legacy
// plaintext records.
func decryptSecret(stored string) (string, error) {
if stored == "" {
return "", nil
}
switch {
case strings.HasPrefix(stored, "plain:"):
return strings.TrimPrefix(stored, "plain:"), nil
case strings.HasPrefix(stored, "enc:"):
key := credentialKey()
if key == "" {
return "", errors.New("credential key not configured but secret is encrypted")
}
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(stored, "enc:"))
if err != nil {
return "", err
}
block, err := aes.NewCipher(deriveKey(key))
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
if len(raw) < gcm.NonceSize() {
return "", errors.New("ciphertext too short")
}
nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", err
}
return string(plaintext), nil
default:
return stored, nil
}
}
func deriveKey(s string) []byte {
h := sha256.Sum256([]byte(s))
return h[:32]
}
func credentialKey() string {
if k := os.Getenv(credentialKeyEnv); k != "" {
return k
}
if secrets.Crypto != nil && secrets.Crypto.Pepper != "" {
return secrets.Crypto.Pepper
}
return ""
}

120
app/models/dead_worker_reaper.go Обычный файл
Просмотреть файл

@@ -0,0 +1,120 @@
package models
import (
"context"
"log"
"time"
)
// DeadWorkerHeartbeatTimeout is the threshold for ReapDeadWorkers — a worker
// whose last_seen is older than this is considered dead and any leased tasks
// it owns are reassigned to the pool. Five minutes mirrors the StaleWorkers()
// check in check_jobs.go so the two reapers cannot disagree about who is
// dead. See docs/todo.md Phase 4 §5.
const DeadWorkerHeartbeatTimeout = 5 * time.Minute
// ReapDeadWorkers marks any non-dead worker whose last_seen is older than
// DeadWorkerHeartbeatTimeout as "dead", then reassigns its leased tasks
// back to the queued pool so other workers (or freshly registered ones)
// can pick them up. It mirrors the structure of ReapExpiredTasks — two
// short UPDATE statements, cheap enough to run from the web process
// every 30s.
//
// The returned tuple is (reaped, reassigned, err): reaped counts the
// workers that flipped to dead during this call; reassigned counts the
// leased tasks that were given back to the pool. A zero count on either
// is normal — the reaper is idempotent and the call is silent when
// nothing is due.
func ReapDeadWorkers() (reaped int, reassigned int, err error) {
now := time.Now()
cutoff := now.Add(-DeadWorkerHeartbeatTimeout)
// First flip the workers to dead so the second UPDATE can match the
// freshly-stamped ids without having to re-derive them in Go.
r := DB().Exec(`
UPDATE worker_nodes
SET status = ?, updated_at = ?
WHERE status <> ? AND last_seen IS NOT NULL AND last_seen < ?`,
"dead", now, "dead", cutoff,
)
if r.Error != nil {
return 0, 0, r.Error
}
reaped = int(r.RowsAffected)
// Nothing flipped → no tasks to return. Cheaper than running an
// UPDATE that touches 0 rows on every tick when the fleet is healthy.
if reaped == 0 {
return 0, 0, nil
}
// Second: clear any leased tasks owned by the now-dead workers.
// The selector stashes the worker's WorkerID string in lease_owner;
// matching against the (now-stale) worker row's WorkerID is the
// same identifier the selector uses, so we don't need an extra
// join. Tasks in other states (queued, succeeded, dead, …) are
// unaffected — only leased work the dead worker still owned has
// to go back to the queue.
r2 := DB().Exec(`
UPDATE tasks
SET state = ?, lease_owner = '', lease_expires_at = NULL, updated_at = ?
WHERE state = ? AND lease_owner IN (
SELECT worker_id FROM worker_nodes WHERE status = ?
)`,
TaskStateQueued, now, TaskStateLeased, "dead",
)
if r2.Error != nil {
return reaped, 0, r2.Error
}
reassigned = int(r2.RowsAffected)
return reaped, reassigned, nil
}
// StartDeadWorkerReaper launches a goroutine that runs ReapDeadWorkers
// on the given interval until ctx is canceled. Mirrors StartTaskReaper
// in this package: same ticker pattern, same logging style, same
// recover() safety net so a malformed row cannot crash the web process.
//
// The default interval is 30s; values <= 0 fall back to the default so
// the helper is safe to call from any call site without a guard. Wire
// from main.init() once per process — the reaper uses short row-level
// locks and is cheap under load (two indexed UPDATEs of <= a few
// hundred rows in steady state).
//
// Passing a nil context falls back to context.Background() so callers
// can write `models.StartDeadWorkerReaper(nil, ...)` in one-liners
// (main, tests) without having to import "context" first.
func StartDeadWorkerReaper(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 30 * time.Second
}
if ctx == nil {
ctx = context.Background()
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("dead_worker_reaper: panic recovered: %v", r)
}
}()
reaped, reassigned, err := ReapDeadWorkers()
if err != nil {
log.Printf("dead_worker_reaper: error: %v", err)
return
}
if reaped > 0 || reassigned > 0 {
log.Printf("dead_worker_reaper: reaped=%d reassigned=%d", reaped, reassigned)
}
}()
}
}
}()
}

274
app/models/dead_worker_reaper_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,274 @@
package models_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestReapDeadWorkers_SkipsHealthyAndKillsSilent mirrors the
// reaper-shape test for ReapExpiredTasks in task_test.go: seed three
// workers (heartbeat fresh / heartbeat stale / already dead) plus two
// leased tasks owned by the stale worker and one leased task on a
// healthy worker (which must NOT be touched). Then assert:
//
// - the fresh worker is left active
// - the silent worker flips to "dead"
// - the already-dead worker is left dead (idempotent)
// - the silent worker's leased tasks are reset to queued
// - the healthy worker's leased task is untouched
func TestReapDeadWorkers_SkipsHealthyAndKillsSilent(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
healthy := &models.WorkerNode{
WorkerID: "w-healthy",
RegionCode: "test",
Status: "active",
AuthToken: "tok-healthy",
LastSeen: timePtr(time.Now()),
}
stale := &models.WorkerNode{
WorkerID: "w-stale",
RegionCode: "test",
Status: "active",
AuthToken: "tok-stale",
LastSeen: timePtr(time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)),
}
alreadyDead := &models.WorkerNode{
WorkerID: "w-dead",
RegionCode: "test",
Status: "dead",
AuthToken: "tok-dead",
LastSeen: timePtr(time.Now().Add(-time.Hour)),
}
require.NoError(t, models.DB().Create(healthy).Error)
require.NoError(t, models.DB().Create(stale).Error)
require.NoError(t, models.DB().Create(alreadyDead).Error)
// Two leased tasks on the stale worker — both must come back.
task1 := mustLeaseTask(t, stale.WorkerID, "test-acct")
task2 := mustLeaseTask(t, stale.WorkerID, "test-acct")
// One leased task on the healthy worker — must stay leased.
healthyTask := mustLeaseTask(t, healthy.WorkerID, "test-acct")
// Already-dead worker with a leased task — not part of THIS reaper's
// reaping set (it would only be touched by a fresh reaper pass), so
// leave it leased to demonstrate that we don't accidentally reassign
// from prior-dead leases too.
deadPriorTask := mustLeaseTask(t, alreadyDead.WorkerID, "test-acct")
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 1, reaped, "only the stale worker should flip (already-dead is left untouched)")
// 3 tasks come back: the 2 leased by the stale worker (just flipped
// to dead) + the 1 leased by the prior-dead worker, which had never
// been cleaned up because no previous reaper ran. The reaper matches
// dead workers by status, so any leased task on a dead worker is a
// stranded lease and must be returned to the pool regardless of when
// the worker flipped.
assert.Equal(t, 3, reassigned)
var healthyRow, staleRow, deadRow models.WorkerNode
require.NoError(t, models.DB().First(&healthyRow, healthy.ID).Error)
assert.Equal(t, "active", healthyRow.Status, "healthy worker must stay active")
require.NoError(t, models.DB().First(&staleRow, stale.ID).Error)
assert.Equal(t, "dead", staleRow.Status, "stale worker should flip to dead")
require.NoError(t, models.DB().First(&deadRow, alreadyDead.ID).Error)
assert.Equal(t, "dead", deadRow.Status, "already-dead worker should not be touched")
got := func(id int64) models.Task {
var row models.Task
require.NoError(t, models.DB().First(&row, id).Error)
return row
}
t1 := got(task1.ID)
assert.Equal(t, models.TaskStateQueued, t1.State, "stale worker task must come back to queued")
assert.Empty(t, t1.LeaseOwner)
assert.Nil(t, t1.LeaseExpiresAt)
t2 := got(task2.ID)
assert.Equal(t, models.TaskStateQueued, t2.State)
assert.Empty(t, t2.LeaseOwner)
assert.Nil(t, t2.LeaseExpiresAt)
ht := got(healthyTask.ID)
assert.Equal(t, models.TaskStateLeased, ht.State, "healthy worker's lease must be untouched")
assert.Equal(t, healthy.WorkerID, ht.LeaseOwner)
dt := got(deadPriorTask.ID)
assert.Equal(t, models.TaskStateQueued, dt.State,
"prior-dead task must also be returned to the pool — any leased task on a dead worker is a stranded lease")
assert.Empty(t, dt.LeaseOwner)
}
// TestReapDeadWorkers_NoOpOnHealthyFleet verifies the cheap path: when
// no workers are stale the reaper returns (0, 0, nil) without doing any
// work. Mirrors the "reaped = int(r.RowsAffected)" branch in
// ReapExpiredTasks.
func TestReapDeadWorkers_NoOpOnHealthyFleet(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "w-only-healthy",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: timePtr(time.Now()),
}
require.NoError(t, models.DB().Create(w).Error)
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 0, reaped)
assert.Equal(t, 0, reassigned)
var row models.WorkerNode
require.NoError(t, models.DB().First(&row, w.ID).Error)
assert.Equal(t, "active", row.Status)
}
// TestReapDeadWorkers_OnlyReassignsLeasedNotOthers ensures that the
// reaper does not touch queued/succeeded/failed_retry tasks on the dead
// worker — only leased ones need to be returned to the queue. Tasks in
// other states either belong to no one (queued) or are terminal/semi-
// terminal and have their own audit trail.
func TestReapDeadWorkers_OnlyReassignsLeasedNotOthers(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
now := time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)
w := &models.WorkerNode{
WorkerID: "w-mix",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: &now,
}
require.NoError(t, models.DB().Create(w).Error)
leased := mustLeaseTask(t, w.WorkerID, "acct-mix")
succeeded := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateSucceeded)
failedRetry := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedRetry)
failedPerm := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedPerm)
otherWorker := mustLeaseTask(t, "w-other", "acct-mix")
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 1, reaped)
assert.Equal(t, 1, reassigned, "exactly the one leased task on the dead worker")
got := func(id int64) string {
var row models.Task
require.NoError(t, models.DB().First(&row, id).Error)
return row.State
}
assert.Equal(t, models.TaskStateQueued, got(leased.ID))
assert.Equal(t, models.TaskStateSucceeded, got(succeeded.ID), "succeeded must not move")
assert.Equal(t, models.TaskStateFailedRetry, got(failedRetry.ID), "failed_retry must not move")
assert.Equal(t, models.TaskStateFailedPerm, got(failedPerm.ID), "failed_perm must not move")
assert.Equal(t, models.TaskStateLeased, got(otherWorker.ID),
"tasks leased by another worker must not move")
}
// TestStartDeadWorkerReaper_TickerFiresOnce is a smoke test for the
// background helper: spin up the reaper with a tight 10ms ticker and
// a cancellable context, wait for the first tick, then cancel the
// context so the goroutine exits cleanly without leaking. Mirrors the
// shape of how main.init() uses StartTaskReaper (it can't be torn
// down, but for tests we always pass a cancellable context).
func TestStartDeadWorkerReaper_TickerFiresOnce(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "w-ticker",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: timePtr(time.Now().Add(-2 * models.DeadWorkerHeartbeatTimeout)),
}
require.NoError(t, models.DB().Create(w).Error)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
models.StartDeadWorkerReaper(ctx, 10*time.Millisecond)
deadline := time.Now().Add(2 * time.Second)
var got models.WorkerNode
for time.Now().Before(deadline) {
require.NoError(t, models.DB().First(&got, w.ID).Error)
if got.Status == "dead" {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("reaper goroutine did not flip worker to dead within 2s; last status=%q", got.Status)
}
// ---------------------------------------------------------------------------
// test helpers
// ---------------------------------------------------------------------------
func timePtr(t time.Time) *time.Time { return &t }
// mustLeaseTask inserts a leased Task row owned by workerID. The state
// is the only field that matters for reaper tests; payload/idempotency
// are stubs.
func mustLeaseTask(t *testing.T, workerID, accountLabel string) models.Task {
t.Helper()
return mustInsertTask(t, workerID, accountLabel, models.TaskStateLeased)
}
func mustInsertTask(t *testing.T, workerID, accountLabel string, state string) models.Task {
t.Helper()
leaseUntil := time.Now().Add(time.Hour)
stamp := time.Now().UnixNano()
jobID := fmt.Sprintf("reap-%s-%s-%d", state, workerID, stamp)
// Cap at 64 chars (tasks.job_id VARCHAR(64)). The components above
// already stay under the limit because mustLeaseTask keeps workerID
// short ("w-mix", "w-stale", …) and state is bounded.
if len(jobID) > 64 {
jobID = jobID[:64]
}
idemp := fmt.Sprintf("reap:%s:%d", workerID, stamp)
if len(idemp) > 255 {
idemp = idemp[:255]
}
task := models.Task{
JobID: jobID,
Kind: models.TaskKindNotification,
State: state,
AccountID: 1,
Payload: datatypes.JSON([]byte(`{"method":"email"}`)),
NotBefore: time.Now().Add(-time.Minute),
LeaseOwner: workerID,
LeaseExpiresAt: &leaseUntil,
Attempts: 1,
MaxAttempts: 5,
IdempotencyKey: idemp,
}
if state != models.TaskStateLeased {
task.LeaseOwner = ""
task.LeaseExpiresAt = nil
}
require.NoError(t, models.DB().Create(&task).Error)
require.NotZero(t, task.ID)
return task
}

582
app/models/deletion.go Обычный файл
Просмотреть файл

@@ -0,0 +1,582 @@
package models
import (
"errors"
"log"
"time"
"gorm.io/gorm"
)
// OnUserCacheInvalidate is called when a user is mutated in a way that
// invalidates the cached representation in auth/cache.go. It is wired up
// from the auth package during init() to avoid an import cycle.
var OnUserCacheInvalidate func(userID int64)
// DeletionGracePeriod is the time between a user requesting account deletion
// and the scheduled hard-delete. During this period the user can cancel the
// request and monitoring is paused for all of the user's accounts.
const DeletionGracePeriod = 7 * 24 * time.Hour
// RequestUserDeletion sets the user's DeletionRequestedAt to now,
// starting the 7-day grace period.
func RequestUserDeletion(user *User) error {
if user == nil {
return errors.New("nil user")
}
if user.DeletionPending() {
return nil
}
now := time.Now()
user.DeletionRequestedAt = &now
if err := DB().Save(user).Error; err != nil {
return err
}
if OnUserCacheInvalidate != nil {
OnUserCacheInvalidate(user.ID)
}
log.Printf("user %d requested account deletion (grace until %s)", user.ID, now.Add(DeletionGracePeriod))
return nil
}
// CancelUserDeletion clears the user's DeletionRequestedAt field,
// canceling the pending account deletion.
func CancelUserDeletion(user *User) error {
if user == nil {
return errors.New("nil user")
}
if !user.DeletionPending() {
return nil
}
user.DeletionRequestedAt = nil
if err := DB().Save(user).Error; err != nil {
return err
}
if OnUserCacheInvalidate != nil {
OnUserCacheInvalidate(user.ID)
}
log.Printf("user %d canceled account deletion", user.ID)
return nil
}
// HardDeleteAccount removes an account and all of its data (monitors, checks,
// events, messages, contacts, groups, notifications, etc.) in a single
// transaction. Accesses and invites pointing at the account are also cleaned
// up. Users themselves are kept (they may belong to other accounts). This is
// the admin "purge" path used to remove spam/error accounts immediately.
func HardDeleteAccount(accountID int64) error {
return DB().Transaction(func(tx *gorm.DB) error {
// Remove accesses pointing at this account first (FK constraint).
if err := tx.Where("account_id = ?", accountID).Delete(&Access{}).Error; err != nil {
return err
}
// Remove invites for this account.
if err := tx.Where("account_id = ?", accountID).Delete(&Invite{}).Error; err != nil {
return err
}
// Delete all account data and the account row itself.
return deleteAccountData(tx, accountID)
})
}
// HardDeleteUser removes the user and all of their owned data: accesses,
// contacts, accounts, monitors, checks, events and messages. The deletion
// is wrapped in a transaction to make sure partial failures don't leave
// the database in a broken state.
func HardDeleteUser(userID int64) error {
return DB().Transaction(func(tx *gorm.DB) error {
user := User{}
if err := tx.First(&user, userID).Error; err != nil {
return err
}
// 1) Find all accounts the user has any access to.
accesses := make([]Access, 0)
if err := tx.Where("user_id = ?", userID).Find(&accesses).Error; err != nil {
return err
}
accountIDs := make([]int64, 0, len(accesses))
for i := range accesses {
accountIDs = append(accountIDs, accesses[i].AccountID)
}
// 2) Find all contacts that belong to those accounts or directly to the user.
if err := tx.Where("user_id = ?", userID).Delete(&Contact{}).Error; err != nil {
return err
}
// 3) For each account: delete the related data, then the account itself.
for _, accountID := range accountIDs {
if err := deleteAccountData(tx, accountID); err != nil {
return err
}
}
// 4) Remove accesses.
if err := tx.Where("user_id = ?", userID).Delete(&Access{}).Error; err != nil {
return err
}
// 5) Remove invites issued by or for this user.
if err := tx.Exec("DELETE FROM invites WHERE invitee_id = ? OR account_id IN (?)",
userID, accountIDs).Error; err != nil {
return err
}
// 6) Remove auth identities (password, social).
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
return err
}
// 7) Remove api keys owned by the user.
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
return err
}
// 8) Remove sessions.
if err := tx.Exec("DELETE FROM remember_tokens WHERE user_id = ?", userID).Error; err != nil {
return err
}
// 9) Finally remove the user row.
if err := tx.Delete(&user).Error; err != nil {
return err
}
log.Printf("user %d hard-deleted (cascade accounts=%v)", userID, accountIDs)
return nil
})
}
// deleteAccountData removes everything associated with a single account:
// contacts, monitors, checks, events, messages, notifications, groups and
// the account itself. The user accesses are removed separately.
func deleteAccountData(tx *gorm.DB, accountID int64) error {
// All contacts that reference this account (both account-scoped ones
// with user_id IS NULL and per-user ones created in
// CreateAccountForUser that set both account_id and user_id).
// User-only contacts (account_id IS NULL) are independent of the
// account and survive the deletion. contacts.account_id has a FK
// to accounts(id) without ON DELETE CASCADE, so all referencing
// rows must be removed before the account row goes away.
contactIDs := make([]int64, 0)
if err := tx.Model(&Contact{}).Where("account_id = ?", accountID).Pluck("id", &contactIDs).Error; err != nil {
return err
}
notificationIDs := make([]int64, 0)
if err := tx.Model(&Notification{}).Where("account_id = ?", accountID).Pluck("id", &notificationIDs).Error; err != nil {
return err
}
if len(contactIDs) > 0 {
if err := tx.Exec("DELETE FROM notification_contacts WHERE contact_id IN (?)", contactIDs).Error; err != nil {
return err
}
}
if len(notificationIDs) > 0 {
if err := tx.Exec("DELETE FROM notification_contacts WHERE notification_id IN (?)", notificationIDs).Error; err != nil {
return err
}
}
messageQuery := tx.Model(&Message{})
switch {
case len(contactIDs) > 0 && len(notificationIDs) > 0:
messageQuery = messageQuery.Where("contact_id IN (?) OR notification_id IN (?)", contactIDs, notificationIDs)
case len(contactIDs) > 0:
messageQuery = messageQuery.Where("contact_id IN (?)", contactIDs)
case len(notificationIDs) > 0:
messageQuery = messageQuery.Where("notification_id IN (?)", notificationIDs)
default:
messageQuery = nil
}
if messageQuery != nil {
messageIDs := make([]int64, 0)
if err := messageQuery.Pluck("id", &messageIDs).Error; err != nil {
return err
}
if len(messageIDs) > 0 {
if err := tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error; err != nil {
return err
}
if err := tx.Where("id IN (?)", messageIDs).Delete(&Message{}).Error; err != nil {
return err
}
}
}
if err := tx.Where("account_id = ?", accountID).Delete(&Contact{}).Error; err != nil {
return err
}
// Monitors (and their checks/events/messages via cascade below).
monitors := make([]Monitor, 0)
if err := tx.Joins("JOIN groups ON monitors.group_id = groups.id").
Where("groups.account_id = ?", accountID).Find(&monitors).Error; err != nil {
return err
}
monitorIDs := make([]int64, 0, len(monitors))
for i := range monitors {
monitorIDs = append(monitorIDs, monitors[i].ID)
}
if len(monitorIDs) > 0 {
// Checks
checks := make([]Check, 0)
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&checks).Error; err != nil {
return err
}
checkIDs := make([]int64, 0, len(checks))
for i := range checks {
checkIDs = append(checkIDs, checks[i].ID)
}
// Events
events := make([]Event, 0)
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&events).Error; err != nil {
return err
}
eventIDs := make([]int64, 0, len(events))
for i := range events {
eventIDs = append(eventIDs, events[i].ID)
}
// Join tables first to avoid FK violations
if len(checkIDs) > 0 {
if err := tx.Exec("DELETE FROM event_checks WHERE check_id IN (?)", checkIDs).Error; err != nil {
return err
}
// check_region_results.check_id has a FK to checks(id) without
// ON DELETE CASCADE, so it must be purged before checks go away.
if err := tx.Exec("DELETE FROM check_region_results WHERE check_id IN (?)", checkIDs).Error; err != nil {
return err
}
}
if len(eventIDs) > 0 {
if err := tx.Exec("DELETE FROM event_messages WHERE event_id IN (?)", eventIDs).Error; err != nil {
return err
}
}
if len(checkIDs) > 0 {
if err := tx.Where("id IN (?)", checkIDs).Delete(&Check{}).Error; err != nil {
return err
}
}
if len(eventIDs) > 0 {
if err := tx.Where("id IN (?)", eventIDs).Delete(&Event{}).Error; err != nil {
return err
}
}
// DNS records
if err := tx.Where("monitor_id IN (?)", monitorIDs).Delete(&DNSRecord{}).Error; err != nil {
return err
}
// Monitors themselves
if err := tx.Where("id IN (?)", monitorIDs).Delete(&Monitor{}).Error; err != nil {
return err
}
}
// Notification <-> group links
if err := tx.Exec(
"DELETE FROM notification_groups WHERE group_id IN (SELECT id FROM groups WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
// Notifications
if err := tx.Where("account_id = ?", accountID).Delete(&Notification{}).Error; err != nil {
return err
}
// LLMs scoped to this account. worker_llms.llm_id has a FK to llms(id)
// without ON DELETE CASCADE, so the join rows must go first.
if err := tx.Exec(
"DELETE FROM worker_llms WHERE llm_id IN (SELECT id FROM llms WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&LLM{}).Error; err != nil {
return err
}
// Groups
if err := tx.Where("account_id = ?", accountID).Delete(&Group{}).Error; err != nil {
return err
}
// Inventory entities (docs/plans/inventory-management.md §6). All
// four tables hold account_id FKs to accounts(id) without ON
// DELETE CASCADE, so we wipe them in dependency order: deployments
// and domains first (both FK into sites and servers), then
// server_ips, then sites, then servers. Anything the account does
// not own is left alone (e.g. shared infra servers are filtered
// by account_id and survive).
if err := tx.Where("account_id = ?", accountID).Delete(&Deployment{}).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Domain{}).Error; err != nil {
return err
}
if err := tx.Exec(
"DELETE FROM server_ips WHERE server_id IN (SELECT id FROM servers WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Site{}).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Server{}).Error; err != nil {
return err
}
// Private workers scoped to this account (NULL account_id workers
// are platform-operated and survive account deletion). The
// worker_llms and check_region_results joins both FK to
// worker_nodes(id) without ON DELETE CASCADE, so the join rows
// must be cleared before the worker rows go away.
if err := tx.Exec(
"DELETE FROM worker_llms WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Exec(
"DELETE FROM check_region_results WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&WorkerNode{}).Error; err != nil {
return err
}
// The account itself
return tx.Delete(&Account{}, accountID).Error
}
// ProcessPendingDeletions hard-deletes users whose 7-day grace period has
// elapsed. Designed to be called from a periodic scheduler.
func ProcessPendingDeletions() (int, error) {
cutoff := time.Now().Add(-DeletionGracePeriod)
users := make([]User, 0)
if err := DB().Where("deletion_requested_at IS NOT NULL AND deletion_requested_at < ?", cutoff).
Find(&users).Error; err != nil {
return 0, err
}
deleted := 0
for i := range users {
if err := HardDeleteUser(users[i].ID); err != nil {
log.Printf("ProcessPendingDeletions: failed to delete user %d: %v", users[i].ID, err)
continue
}
deleted++
}
if deleted > 0 {
log.Printf("ProcessPendingDeletions: hard-deleted %d user(s)", deleted)
}
return deleted, nil
}
// StaleAccountInactivity is the minimum inactivity window before a stale
// account becomes eligible for admin cleanup. Picked at 3 months per the
// /admin/accounts "Удалить старые" button.
const StaleAccountInactivity = 90 * 24 * time.Hour
// StaleAccountCandidate describes an account that matched the
// admin-cleanup eligibility filter but has not yet been deleted. The
// snapshot is what the frontend shows in the confirmation dialog.
type StaleAccountCandidate struct {
AccountID int64 `json:"account_id"`
AccountName string `json:"account_name"`
UserID int64 `json:"user_id"`
UserEmail *string `json:"user_email"`
LastActiveAt *time.Time `json:"last_active_at"`
}
// FindStaleAccounts returns the accounts that are eligible for the
// admin "Удалить старые" cleanup:
//
// - the account has zero monitors configured (via group.account_id)
// - every user with access to the account has exactly one account
// membership total (so removing the account also orphans them)
// - every such user's last_active_at is older than
// StaleAccountInactivity (3 months). Users who have never logged in
// (last_active_at IS NULL) are also eligible.
func FindStaleAccounts() ([]StaleAccountCandidate, error) {
cutoff := time.Now().Add(-StaleAccountInactivity)
// Step 1: account IDs that have zero monitors.
accountsWithMonitors := make([]int64, 0)
if err := DB().
Table("monitors").
Select("DISTINCT groups.account_id").
Joins("JOIN groups ON groups.id = monitors.group_id").
Scan(&accountsWithMonitors).Error; err != nil {
return nil, err
}
accounts := make([]Account, 0)
q := DB().Order("id ASC")
if len(accountsWithMonitors) > 0 {
q = q.Where("id NOT IN (?)", accountsWithMonitors)
}
if err := q.Find(&accounts).Error; err != nil {
return nil, err
}
out := make([]StaleAccountCandidate, 0, len(accounts))
for i := range accounts {
acc := accounts[i]
// Step 2: every user with access to this account must have
// exactly one account membership total.
type userAccessCount struct {
UserID int64
Cnt int
}
counts := make([]userAccessCount, 0)
err := DB().
Table("accesses AS a1").
Select("a1.user_id AS user_id, (SELECT COUNT(*) FROM accesses AS a2 WHERE a2.user_id = a1.user_id) AS cnt").
Where("a1.account_id = ? AND a1.user_id IS NOT NULL", acc.ID).
Group("a1.user_id").
Scan(&counts).Error
if err != nil {
return nil, err
}
if len(counts) == 0 {
// An account with no user accesses is a config bug
// (the owner access should always exist). Skip.
continue
}
allSingle := true
for _, c := range counts {
if c.Cnt != 1 {
allSingle = false
break
}
}
if !allSingle {
continue
}
// Step 3: every such user must be inactive beyond the cutoff.
userIDs := make([]int64, 0, len(counts))
for _, c := range counts {
userIDs = append(userIDs, c.UserID)
}
users := make([]User, 0)
if err := DB().Where("id IN (?)", userIDs).Find(&users).Error; err != nil {
return nil, err
}
allStale := true
for j := range users {
u := users[j]
if u.LastActiveAt != nil && u.LastActiveAt.After(cutoff) {
allStale = false
break
}
}
if !allStale {
continue
}
// All gates passed — emit one candidate per user so the UI
// can list which specific accounts would be removed.
for j := range users {
out = append(out, StaleAccountCandidate{
AccountID: acc.ID,
AccountName: acc.Name,
UserID: users[j].ID,
UserEmail: users[j].Email,
LastActiveAt: users[j].LastActiveAt,
})
}
}
return out, nil
}
// CleanupStaleAccounts hard-deletes every account eligible for the
// admin "Удалить старые" sweep. Returns the number of accounts that
// were deleted. The matching users (each of whom only belonged to one
// account) are deleted by HardDeleteAccount's cascading access cleanup
// only if they no longer have any other account — that final teardown
// is done here.
func CleanupStaleAccounts() (int, error) {
candidates, err := FindStaleAccounts()
if err != nil {
return 0, err
}
accountIDs := make([]int64, 0, len(candidates))
seen := make(map[int64]bool, len(candidates))
for _, c := range candidates {
if !seen[c.AccountID] {
seen[c.AccountID] = true
accountIDs = append(accountIDs, c.AccountID)
}
}
deleted := 0
for _, accountID := range accountIDs {
if err := HardDeleteAccount(accountID); err != nil {
log.Printf("CleanupStaleAccounts: failed to delete account %d: %v", accountID, err)
continue
}
deleted++
}
// Users that no longer have any accesses after the cascade are
// clearly orphaned — purge them so the auth layer doesn't keep
// dangling rows around. We bypass HardDeleteUser here because the
// account-level data (monitors, checks, groups, etc.) was already
// removed by the HardDeleteAccount loop above, so only the user
// row and the dangling identities / contacts need cleanup.
if deleted > 0 {
orphans := make([]int64, 0)
err := DB().
Table("users").
Select("users.id").
Joins("LEFT JOIN accesses ON accesses.user_id = users.id").
Where("accesses.id IS NULL").
Pluck("users.id", &orphans).Error
if err != nil {
log.Printf("CleanupStaleAccounts: orphan user scan failed: %v", err)
return deleted, nil
}
for _, userID := range orphans {
tx := DB().Begin()
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: identities delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Where("user_id = ? AND account_id IS NULL", userID).Delete(&Contact{}).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: contacts delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: api_keys delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Delete(&User{}, userID).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: user delete failed for %d: %v", userID, err)
continue
}
if err := tx.Commit().Error; err != nil {
log.Printf("CleanupStaleAccounts: commit failed for user %d: %v", userID, err)
}
}
}
if deleted > 0 {
log.Printf("CleanupStaleAccounts: hard-deleted %d stale account(s)", deleted)
}
return deleted, nil
}

352
app/models/deletion_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,352 @@
package models_test
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/icrowley/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
var gormErrRecordNotFound = gorm.ErrRecordNotFound
func deletionStringPtr(s string) *string { return &s }
func deletionBoolPtr(b bool) *bool { return &b }
func init() {
database.Init()
}
// TestUserDeletionPending tests the DeletionPending method
func TestUserDeletionPending(t *testing.T) {
t.Run("returns false when no deletion requested", func(t *testing.T) {
user := models.User{}
assert.False(t, user.DeletionPending())
})
t.Run("returns true when deletion requested", func(t *testing.T) {
now := time.Now()
user := models.User{DeletionRequestedAt: &now}
assert.True(t, user.DeletionPending())
})
}
// TestUserAsJSONIncludesDeletionField verifies that the AsJSON output
// contains the deletion_requested_at field so the frontend can react to it.
func TestUserAsJSONIncludesDeletionField(t *testing.T) {
email := "test@example.com"
now := time.Now()
user := models.User{
ID: 42,
Email: &email,
Name: "Test User",
DeletionRequestedAt: &now,
}
result := user.AsJSON()
assert.NotNil(t, result)
assert.Contains(t, result, "deletion_requested_at")
assert.Equal(t, &now, result["deletion_requested_at"])
}
// TestRequestUserDeletionNilUser ensures nil-safety
func TestRequestUserDeletionNilUser(t *testing.T) {
err := models.RequestUserDeletion(nil)
assert.Error(t, err)
}
// TestCancelUserDeletionNilUser ensures nil-safety
func TestCancelUserDeletionNilUser(t *testing.T) {
err := models.CancelUserDeletion(nil)
assert.Error(t, err)
}
// TestCacheInvalidationHook_CalledByRequestDeletion verifies that
// RequestUserDeletion calls OnUserCacheInvalidate with the correct user ID.
func TestCacheInvalidationHook_CalledByRequestDeletion(t *testing.T) {
models.Drop()
models.Migrate()
email := fake.EmailAddress()
user := models.User{Email: &email, Name: "CacheHookTest"}
assert.NoError(t, models.DB().Create(&user).Error)
var calledID int64
originalHook := models.OnUserCacheInvalidate
models.OnUserCacheInvalidate = func(userID int64) {
calledID = userID
}
defer func() { models.OnUserCacheInvalidate = originalHook }()
assert.NoError(t, models.RequestUserDeletion(&user))
assert.Equal(t, user.ID, calledID)
models.DB().Unscoped().Delete(&user)
}
// TestCacheInvalidationHook_CalledByCancelUserDeletion verifies that
// CancelUserDeletion calls OnUserCacheInvalidate with the correct user ID.
func TestCacheInvalidationHook_CalledByCancelUserDeletion(t *testing.T) {
models.Drop()
models.Migrate()
email := fake.EmailAddress()
now := time.Now()
user := models.User{Email: &email, Name: "CacheHookCancel", DeletionRequestedAt: &now}
assert.NoError(t, models.DB().Create(&user).Error)
var calledID int64
originalHook := models.OnUserCacheInvalidate
models.OnUserCacheInvalidate = func(userID int64) {
calledID = userID
}
defer func() { models.OnUserCacheInvalidate = originalHook }()
assert.NoError(t, models.CancelUserDeletion(&user))
assert.Equal(t, user.ID, calledID)
models.DB().Unscoped().Delete(&user)
}
// TestHardDeleteAccount_FKCascadeRegression covers the FK regressions
// that surfaced as a series of distinct SQL errors when an operator
// purged an account that owned monitors with distributed-worker
// activity or per-user contacts:
//
// 1. check_region_results.check_id → checks(id) had no ON DELETE
// CASCADE, so deleting a check while it still had region-result
// rows raised 23503.
// 2. worker_nodes did not have an account_id column, so the
// "Workers scoped to this account" delete raised 42703.
// 3. contacts.account_id → accounts(id) had no ON DELETE CASCADE,
// and the original delete filter required user_id IS NULL, so
// per-user contacts created in CreateAccountForUser (both
// account_id and user_id set) survived and blocked the account
// delete with 23503.
//
// The test seeds an account with a monitor, a check with a region
// result row, an account-scoped LLM, a per-user contact, and both a
// private worker (with account_id) and an operated worker (NULL
// account_id), then runs HardDeleteAccount and asserts that the
// account, the private worker, the LLM, the monitor/check/region
// result, and the per-user contact all disappear, while the operated
// worker and an unrelated user-only contact survive.
func TestHardDeleteAccount_FKCascadeRegression(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "hard-delete-regression"}
require.NoError(t, models.DB().Create(&plan).Error)
account := models.Account{Name: "victim", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&account).Error)
otherAccount := models.Account{Name: "survivor", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&otherAccount).Error)
owner := models.User{Name: "owner", Email: deletionStringPtr("owner@example.com"), Timezone: "UTC"}
require.NoError(t, models.DB().Create(&owner).Error)
stranger := models.User{Name: "stranger", Email: deletionStringPtr("stranger@example.com"), Timezone: "UTC"}
require.NoError(t, models.DB().Create(&stranger).Error)
region := models.Region{}
if err := models.DB().Where("code = ?", "test").First(&region).Error; err != nil {
require.NoError(t, models.DB().Create(&models.Region{Code: "test", Name: "test", Enabled: true}).Error)
}
group := models.Group{Name: "g", AccountID: account.ID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{
Name: deletionStringPtr("m"),
Host: "example.com",
GroupID: group.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&monitor).Error)
check := models.Check{
MonitorID: monitor.ID,
Kind: "http",
Interval: 60,
Settings: datatypes.JSON([]byte(`{}`)),
Enabled: deletionBoolPtr(true),
State: "UNK",
}
require.NoError(t, models.DB().Create(&check).Error)
// Per-user contact that references both the victim account and a
// user. CreateAccountForUser writes a contact in this shape, and
// the old "user_id IS NULL" filter would let it survive and block
// the account delete with contacts_account_id_fkey 23503.
ownerContact := models.Contact{
AccountID: &account.ID,
UserID: &owner.ID,
Name: "owner-email",
Kind: "email",
Value: "owner@example.com",
}
require.NoError(t, models.DB().Create(&ownerContact).Error)
// Account-only contact (user_id IS NULL) — also tied to the account
// via FK and must be removed.
accountContact := models.Contact{
AccountID: &account.ID,
Name: "ops",
Kind: "email",
Value: "ops@example.com",
}
require.NoError(t, models.DB().Create(&accountContact).Error)
notification := models.Notification{AccountID: account.ID, Name: "alerts", Enabled: true}
require.NoError(t, models.DB().Create(&notification).Error)
require.NoError(t, models.DB().Model(&notification).Association("Contacts").Append(&accountContact))
message := models.Message{
NotificationID: notification.ID,
ContactID: accountContact.ID,
Kind: "test",
State: "OK",
CreatedAt: time.Now(),
SentAt: time.Now(),
}
require.NoError(t, models.DB().Create(&message).Error)
// User-only contact on a stranger — must survive account deletion.
userOnlyContact := models.Contact{
UserID: &stranger.ID,
Name: "stranger",
Kind: "email",
Value: "stranger@example.com",
}
require.NoError(t, models.DB().Create(&userOnlyContact).Error)
// Region result row: this is the row that used to trigger the
// fk_check_region_results_check FK violation when Check was
// deleted. Without the fix the entire HardDeleteAccount would
// fail here.
privateWorker := &models.WorkerNode{
WorkerID: "private-" + uuid.New().String(),
RegionCode: "test",
Status: "active",
AuthToken: "priv-tok-" + uuid.New().String(),
AccountID: &account.ID,
}
require.NoError(t, models.DB().Create(privateWorker).Error)
operatedWorker := &models.WorkerNode{
WorkerID: "operated-" + uuid.New().String(),
RegionCode: "test",
Status: "active",
AuthToken: "op-tok-" + uuid.New().String(),
AccountID: nil,
}
require.NoError(t, models.DB().Create(operatedWorker).Error)
require.NoError(t, models.DB().Create(&models.CheckRegionResult{
CheckID: check.ID,
RegionCode: "test",
WorkerNodeID: &privateWorker.ID,
ExecutedAt: time.Now(),
State: "OK",
}).Error)
// LLM scoped to the victim account + linked to the private worker.
// worker_llms.llm_id has a FK to llms(id) without ON DELETE
// CASCADE, so the join row used to block LLM deletion too.
llm := models.LLM{
AccountID: &account.ID,
Name: "private-llm",
URL: "https://llm.example.com",
ModelName: "gpt-test",
APIKey: "secret",
Kind: "openai",
}
require.NoError(t, models.DB().Create(&llm).Error)
require.NoError(t, models.DB().Exec(
"INSERT INTO worker_llms (worker_node_id, llm_id) VALUES (?, ?)",
privateWorker.ID, llm.ID,
).Error)
// Inventory entities scoped to the victim account. Each has an
// account_id FK to accounts(id) without ON DELETE CASCADE so they
// must be removed before the account row goes away. The
// shared-infra server belongs to another account and must
// survive.
victimServer := models.Server{Name: "victim-srv", AccountID: account.ID}
require.NoError(t, models.DB().Create(&victimServer).Error)
victimServerIP := models.ServerIp{ServerID: victimServer.ID, Address: "10.0.0.1"}
require.NoError(t, models.DB().Create(&victimServerIP).Error)
victimSite := models.Site{
AccountID: account.ID, ServerID: &victimServer.ID,
Slug: "victim-site", Name: "victim-site", Kind: "production", IsActive: true,
}
require.NoError(t, models.DB().Create(&victimSite).Error)
victimDeployment := models.Deployment{
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
Kind: "production", Mode: "compose",
}
require.NoError(t, models.DB().Create(&victimDeployment).Error)
victimDomain := models.Domain{
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
Name: "victim.example.com",
}
require.NoError(t, models.DB().Create(&victimDomain).Error)
otherServer := models.Server{Name: "shared-srv", AccountID: otherAccount.ID}
require.NoError(t, models.DB().Create(&otherServer).Error)
require.NoError(t, models.HardDeleteAccount(account.ID))
// Account and all account-scoped rows must be gone.
assert.ErrorIs(t, models.DB().First(&models.Account{}, account.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Check{}, check.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Monitor{}, monitor.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Group{}, group.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Notification{}, notification.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Message{}, message.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.LLM{}, llm.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Contact{}, ownerContact.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Contact{}, accountContact.ID).Error, gormErrRecordNotFound)
var notificationContactCount int64
require.NoError(t, models.DB().Table("notification_contacts").
Where("notification_id = ? OR contact_id = ?", notification.ID, accountContact.ID).
Count(&notificationContactCount).Error)
assert.Zero(t, notificationContactCount)
// Private worker must be gone; operated worker must survive.
assert.ErrorIs(t, models.DB().First(&models.WorkerNode{}, privateWorker.ID).Error, gormErrRecordNotFound)
var stillOperated models.WorkerNode
require.NoError(t, models.DB().First(&stillOperated, operatedWorker.ID).Error)
assert.Nil(t, stillOperated.AccountID, "operated worker account_id must remain NULL")
// User-only contact (no account_id) must survive.
var stillUserOnly models.Contact
require.NoError(t, models.DB().First(&stillUserOnly, userOnlyContact.ID).Error)
// Inventory entities scoped to the account must be gone.
assert.ErrorIs(t, models.DB().First(&models.Server{}, victimServer.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Site{}, victimSite.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Deployment{}, victimDeployment.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Domain{}, victimDomain.ID).Error, gormErrRecordNotFound)
var leftover int64
require.NoError(t, models.DB().Model(&models.ServerIp{}).
Where("server_id = ?", victimServer.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "server_ips for the deleted server must be cleaned up")
// Other-account inventory must survive.
var stillOtherServer models.Server
require.NoError(t, models.DB().First(&stillOtherServer, otherServer.ID).Error)
// Region-result rows for the deleted check must be gone.
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("check_id = ?", check.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "check_region_results must be cleaned up before checks")
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("worker_node_id = ?", privateWorker.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "check_region_results referencing a private worker must be cleaned up")
require.NoError(t, models.DB().Raw(
"SELECT COUNT(*) FROM worker_llms WHERE llm_id = ? OR worker_node_id = ?",
llm.ID, privateWorker.ID,
).Scan(&leftover).Error)
assert.Zero(t, leftover, "worker_llms rows for the deleted LLM and private worker must be gone")
// Sanity: the other account and its data are untouched.
var stillOther models.Account
require.NoError(t, models.DB().First(&stillOther, otherAccount.ID).Error)
}

240
app/models/deployment.go Обычный файл
Просмотреть файл

@@ -0,0 +1,240 @@
package models
import (
"database/sql/driver"
"fmt"
"time"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DeploymentKind is the rstuff-mirrored lifecycle label for a
// deployment (production / production_prev / production_next /
// internal / staging / old). See
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable numeric
// mapping (which Postgres stores alphabetically, not numerically).
type DeploymentKind string
// Lifecycle labels for a Deployment row. See DeploymentKind for
// the matching rstuff enum. The label set is closed; new values
// require adding a Postgres enum value via app/models/migrate.go.
const (
// DeploymentKindProduction is the customer-facing "live" deployment.
DeploymentKindProduction DeploymentKind = "production"
DeploymentKindProductionPrev DeploymentKind = "production_prev"
DeploymentKindProductionNext DeploymentKind = "production_next"
// DeploymentKindInternal is for ops/admin tooling (not customer-facing).
DeploymentKindInternal DeploymentKind = "internal"
DeploymentKindStaging DeploymentKind = "staging"
DeploymentKindOld DeploymentKind = "old"
)
// DeploymentMode is the host-side lifecycle label (kubernetes /
// compose / dedicated / vds / user). `dedicated` covers a single
// nginx vhost; `compose` covers a Docker Compose project.
type DeploymentMode string
// DeploymentMode values map onto rstuff's Deployment.mode enum.
// `dedicated` covers a single nginx vhost; `compose` covers a
// Docker Compose project; the others are reserved for future
// v2 surfaces (Kubernetes, VDS, user-owned).
const (
DeploymentModeKubernetes DeploymentMode = "kubernetes"
DeploymentModeCompose DeploymentMode = "compose"
DeploymentModeDedicated DeploymentMode = "dedicated"
DeploymentModeVDS DeploymentMode = "vds"
DeploymentModeUser DeploymentMode = "user"
)
// DeploymentAction is the reconciliation state. Updated by the
// deploymentd receiver on every POST and by the 90s reconcile sweep
// (see app/models/deployment.go ReconcileMissing).
type DeploymentAction string
// DeploymentAction values. Pending/PendingMove/PendingDrop are
// transient (operator or receiver-initiated); Deleted/Missing are
// sticky until the deployment shows up again on a future POST.
const (
DeploymentActionOk DeploymentAction = "ok"
DeploymentActionPending DeploymentAction = "pending"
DeploymentActionPendingMove DeploymentAction = "pending_move"
DeploymentActionPendingDrop DeploymentAction = "pending_drop"
DeploymentActionDeleted DeploymentAction = "deleted"
DeploymentActionMissing DeploymentAction = "missing"
)
// Deployment represents a single host-side binding: one nginx
// vhost, one Docker Compose service, or one Kubernetes service. The
// shape mirrors rstuff's `deployments` table. See
// docs/plans/inventory-management.md §4 / §6.1.
type Deployment struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
ServiceName *string `gorm:"size:120" json:"service_name,omitempty"`
Kind DeploymentKind `gorm:"type:deployment_kind;not null;default:'production'" json:"kind"`
Mode DeploymentMode `gorm:"type:deployment_mode;not null;default:'dedicated'" json:"mode"`
Action DeploymentAction `gorm:"type:deployment_action;not null;default:'ok'" json:"action"`
URL *string `gorm:"type:text" json:"url,omitempty"`
SSHUser *string `gorm:"size:64" json:"ssh_user,omitempty"`
RootPath *string `gorm:"type:text" json:"root_path,omitempty"`
ConfigPath *string `gorm:"type:text" json:"config_path,omitempty"`
IP *string `gorm:"type:inet" json:"ip,omitempty"`
Listen pq.StringArray `gorm:"type:varchar(64)[];not null;default:'{}'" json:"listen"`
ServerName pq.StringArray `gorm:"type:varchar(255)[];not null;default:'{}'" json:"server_name"`
Auth bool `gorm:"not null;default:false" json:"auth"`
IsProxied bool `gorm:"not null;default:false" json:"is_proxied"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
concerns.Timestamped
Audited
}
// TableName provides functionality.
func (Deployment) TableName() string { return "deployments" }
// Scan implements sql.Scanner so a Postgres enum value can land in
// our typed string alias without a code-generation step.
func (d *DeploymentKind) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentKind(v)
case []byte:
*d = DeploymentKind(string(v))
default:
return fmt.Errorf("deployment_kind: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for the inverse direction.
func (d DeploymentKind) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// Scan implements sql.Scanner for DeploymentMode.
func (d *DeploymentMode) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentMode(v)
case []byte:
*d = DeploymentMode(string(v))
default:
return fmt.Errorf("deployment_mode: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for DeploymentMode.
func (d DeploymentMode) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// Scan implements sql.Scanner for DeploymentAction.
func (d *DeploymentAction) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentAction(v)
case []byte:
*d = DeploymentAction(string(v))
default:
return fmt.Errorf("deployment_action: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for DeploymentAction.
func (d DeploymentAction) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// ReconcileMissingDeployments flips action='missing' on every Deployment for the
// given server whose last_seen_at is older than cutoff. Called by
// the deploymentd receiver after every successful upsert so the
// "missing" badge appears within one POST cycle.
//
// Idempotent: re-running with the same cutoff is a no-op.
func ReconcileMissingDeployments(serverID int64, mode DeploymentMode, cutoff time.Time) (int64, error) {
res := DB().Model(&Deployment{}).
Where("server_id = ? AND mode = ? AND action NOT IN ?", serverID, mode,
[]DeploymentAction{DeploymentActionDeleted, DeploymentActionMissing, DeploymentActionPendingDrop}).
Where("last_seen_at IS NULL OR last_seen_at < ?", cutoff).
Update("action", DeploymentActionMissing)
return res.RowsAffected, res.Error
}
// UpsertNginxDeployment finds or creates a Deployment by
// (server_id, config_path) for an nginx vhost. The caller fills in
// the lifecycle fields (listen, server_name, etc.) after the upsert
// returns. The return value is the row to mutate; the caller MUST
// also touch last_seen_at and save.
func UpsertNginxDeployment(tx *gorm.DB, accountID int64, serverID int64, configPath string) (*Deployment, error) {
if tx == nil {
tx = DB()
}
var d Deployment
err := tx.Where("server_id = ? AND config_path = ?", serverID, configPath).First(&d).Error
if err == nil {
return &d, nil
}
if err != gorm.ErrRecordNotFound {
return nil, err
}
d = Deployment{
AccountID: accountID,
ServerID: &serverID,
Kind: DeploymentKindProduction,
Mode: DeploymentModeDedicated,
Action: DeploymentActionOk,
ConfigPath: &configPath,
}
if err := tx.Create(&d).Error; err != nil {
return nil, err
}
return &d, nil
}
// RotateServerToken sets a new random token for a server and returns
// the plaintext. Called by the operator-only
// POST /api/v1/servers/:id/rotate-token endpoint. The plaintext is
// returned exactly once — it is not stored anywhere recoverable.
func RotateServerToken(tx *gorm.DB, serverID int64, newToken string) error {
if tx == nil {
tx = DB()
}
return tx.Model(&Server{}).Where("id = ?", serverID).Update("token", newToken).Error
}

80
app/models/dns_record.go Обычный файл
Просмотреть файл

@@ -0,0 +1,80 @@
package models
import (
"log"
"rsgit.ru/rsmon/rsmon/internal/netaddr"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DNSRecord provides functionality.
type DNSRecord struct {
concerns.Model
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Monitor *Monitor `json:"-"`
Name string `json:"name"`
Kind string `json:"kind"`
Value netaddr.Inet `json:"value" gorm:"type:bytea;"`
}
// SaveIps provides functionality.
func SaveIps(m *Monitor, _ Check, ips []NSRecord) { //nolint:gocritic // hugeParam: accepted for interface compatibility
// log.Println("save ips for monitor")
// spew.Dump(m)
var err error
tx := DB().Begin()
currentRecords := make([]DNSRecord, 0)
_ = tx.Model(&m).Association("DNSRecords").Find(&currentRecords)
recordHash := make(map[string]DNSRecord, 0)
for _, record := range currentRecords {
recordHash[record.Name] = record
}
nextRecords := make(map[string]bool, 0)
for _, ip := range ips {
if record, ok := recordHash[ip.Name]; ok {
// log.Println("old value:", record)
record.MonitorID = m.ID
record.Value = ip.Value
record.Kind = ip.Kind
err = tx.Model(&m).Association("DNSRecords").Replace(&record)
if err != nil {
log.Println("fatal error in saveips", err)
return
}
} else {
record = DNSRecord{
MonitorID: m.ID,
Name: ip.Name,
Kind: ip.Kind,
Value: ip.Value,
}
if _, ok := nextRecords[ip.Name]; !ok {
nextRecords[ip.Name] = true
err = tx.Model(&m).Association("DNSRecords").Append(&record)
if err != nil {
log.Println("fatal error in saveips", err)
return
}
}
}
}
// spew.Dump(ips_hash)
err = tx.Commit().Error
if err != nil {
log.Println("fatal error in saveips", err)
return
}
}
// NSRecord provides functionality.
type NSRecord struct {
Name string
Kind string
Value netaddr.Inet
}

32
app/models/domain.go Обычный файл
Просмотреть файл

@@ -0,0 +1,32 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Domain represents a customer-facing DNS name. The shape mirrors
// rstuff's `domains` table — see docs/parity/rstuff-inventory.md §2
// and docs/plans/inventory-management.md §4.
//
// Distinct from RknDomain (the RKN blocklist cache, see
// app/models/rkn_domain.go): RknDomain is read-only data about
// blocked domains; Domain is the customer-side name→site/server
// pointer that monitoring reasons about.
type Domain struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
Name string `gorm:"size:255;not null" json:"name"`
Env string `gorm:"size:32;not null;default:'production'" json:"env"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
concerns.Timestamped
}
// TableName provides functionality.
func (Domain) TableName() string { return "domains" }

96
app/models/drop.go Обычный файл
Просмотреть файл

@@ -0,0 +1,96 @@
package models
import (
"fmt"
"strings"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
)
// Drop removes all test data from the database.
// It uses TRUNCATE ... CASCADE for join tables and deletes leaf-to-root for main tables.
// Safe to call from multiple goroutines within a single test binary; concurrent calls
// from separate test binaries are serialized by the advisory lock in Migrate().
func Drop() {
var dbname string
if err := DB().Raw("SELECT current_database()").Scan(&dbname).Error; err != nil {
panic(fmt.Sprintf("Drop: cannot read database name: %v", err))
}
if !strings.Contains(dbname, "test") {
panic(fmt.Sprintf(
"Drop() refused: database '%s' is not a test database. "+
"Set DATABASE_NAME=rsmon_test to run tests safely.", dbname,
))
}
// Truncate many2many join tables first to avoid FK violations.
DB().Exec("TRUNCATE event_checks, event_messages, notification_contacts, notification_groups RESTART IDENTITY CASCADE")
// Delete main tables in dependency order (leaf tables first).
DB().Where("1=1").Delete(&NotificationDelivery{})
DB().Where("1=1").Delete(&TaskReplay{})
DB().Where("1=1").Delete(&TelegramBotMessage{})
DB().Where("1=1").Delete(&TelegramBotStatus{})
DB().Where("1=1").Delete(&Task{})
DB().Where("1=1").Delete(&Access{})
DB().Where("1=1").Delete(&Invite{})
DB().Where("1=1").Delete(&Message{})
DB().Where("1=1").Delete(&Event{})
// Durable network-diagnostic rows reference checks, monitors, and workers.
DB().Where("1=1").Delete(&DiagnosticAuditEvent{})
DB().Where("1=1").Delete(&CheckAttempt{})
// CheckRegionResult FKs check_id; it must be cleared before Check.
DB().Where("1=1").Delete(&CheckRegionResult{})
DB().Where("1=1").Delete(&Check{})
DB().Where("1=1").Delete(&Notification{})
// DNSRecord FKs monitor_id; it must be cleared before Monitor.
DB().Where("1=1").Delete(&DNSRecord{})
DB().Where("1=1").Delete(&Monitor{})
DB().Where("1=1").Delete(&Group{})
DB().Where("1=1").Delete(&Contact{})
DB().Where("1=1").Delete(&NotificationCredential{})
DB().Where("1=1").Delete(&WorkerLogEvent{})
DB().Where("1=1").Delete(&WorkerNode{})
DB().Where("1=1").Delete(&LLM{})
DB().Where("1=1").Delete(&Region{})
// Inventory (docs/plans/inventory-management.md §6): leaf tables
// (sites, deployments, server_ips) reference accounts/servers, so
// they must be cleared before Server is deleted.
DB().Where("1=1").Delete(&Deployment{})
DB().Where("1=1").Delete(&SiteRepo{})
DB().Where("1=1").Delete(&Site{})
DB().Where("1=1").Delete(&Repo{})
DB().Where("1=1").Delete(&ServerIp{})
DB().Where("1=1").Delete(&Domain{})
DB().Where("1=1").Delete(&Server{})
// Account-scoped tag metadata (account_id FK to accounts(id)).
// Cleared before Account so a future cascade change cannot orphan
// rows mid-truncate.
DB().Where("1=1").Delete(&Tag{})
// Status pages (docs/plans/status-pages.md §3.1–3.5). Children
// reference status_pages(id) with ON DELETE CASCADE so GORM
// ordering would already wipe them, but we delete them
// explicitly so the test DB stays clean even if a future model
// change drops the cascade.
DB().Where("1=1").Delete(&StatusPageDomain{})
DB().Unscoped().Where("1=1").Delete(&Maintenance{})
DB().Where("1=1").Delete(&StatusPageMaintenance{})
DB().Where("1=1").Delete(&StatusPageIncident{})
DB().Where("1=1").Delete(&StatusPageDelivery{})
DB().Where("1=1").Delete(&StatusPageDigestSchedule{})
DB().Where("1=1").Delete(&StatusPageSubscriber{})
DB().Unscoped().Where("1=1").Delete(&StatusPage{})
DB().Where("1=1").Delete(&SubscriptionEvent{})
DB().Where("1=1").Delete(&Subscription{})
DB().Where("1=1").Delete(&Account{})
DB().Unscoped().Where("1=1").Delete(&authidentity.AuthIdentity{})
// User has a unique email index, so test fixtures must be physically
// removed rather than soft-deleted between tests.
DB().Unscoped().Where("1=1").Delete(&User{})
// A few audited tables added by feature migrations can retain a user FK that
// is intentionally not modeled as an association. CASCADE keeps fixture
// cleanup deterministic instead of silently leaving unique emails behind.
DB().Exec("TRUNCATE users CASCADE")
DB().Exec("TRUNCATE regions CASCADE")
}

83
app/models/event.go Обычный файл
Просмотреть файл

@@ -0,0 +1,83 @@
package models
import (
"fmt"
"time"
"github.com/lib/pq"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/util"
)
// Event provides functionality.
type Event struct {
concerns.Model
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id"`
Monitor *Monitor `json:"monitor,omitempty"`
StartTime *time.Time `json:"start_time"`
EndTime *time.Time `json:"end_time"`
Duration int `json:"duration"`
Errors int `json:"errors"`
Oks int `json:"oks"`
State string `gorm:"index" json:"state"`
Reason string `json:"reason"`
Messages []Message `json:"messages" gorm:"many2many:event_messages;"`
ChecksDown pq.StringArray `gorm:"type:varchar(255)[]" json:"checks_down"`
Checks []Check `json:"-" gorm:"many2many:event_checks;"`
ExpiresAt *time.Time `json:"-"`
Audited
}
// EventScope provides functionality.
func EventScope(q *gorm.DB) *gorm.DB {
return q.Where("state IN ('current', 'ended')").
Preload("Checks").
Preload("Monitor").
Preload("Monitor.Group").
Preload("Monitor.Group.Notifications").
Preload("Monitor.Group.Notifications.Contacts")
}
// GetDuration provides functionality.
func (e *Event) GetDuration(tn time.Time) int64 {
endTime := e.EndTime
if endTime == nil {
endTime = &tn
}
return int64(endTime.Sub(*e.StartTime) / time.Second)
}
// FormatDuration provides functionality.
func (e *Event) FormatDuration() string {
d := e.GetDuration(time.Now())
return util.FormatDuration(d)
}
// Inspect provides functionality.
func (e *Event) Inspect() string {
var st, et string
if e.StartTime != nil {
st = e.StartTime.Format("2006-01-02 15:04:05")
}
if e.EndTime != nil {
et = e.EndTime.Format("2006-01-02 15:04:05")
}
return fmt.Sprintf(
"Event<id: %d, monitor_id: %d, start_time: %s, end_time: %s, reason: %s, duration: %d>",
e.ID,
e.MonitorID,
st,
et,
e.Reason,
e.Duration,
)
}

76
app/models/group.go Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// Group represents a monitor group.
type Group struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
Name string `json:"name" gorm:"not null"`
IsSystem *bool `json:"is_system" gorm:"default:false"`
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
Monitors []Monitor `json:"-"`
Notifications []Notification `gorm:"many2many:notification_groups;" json:"-"`
concerns.Timestamped `json:"-"`
Audited
}
// SystemGroups returns all groups marked as system/internal (is_system=true).
// These groups are converted to distributed monitors (system checks running
// on the distributed worker pool).
func SystemGroups() ([]Group, error) {
var groups []Group
err := DB().Where("is_system = ?", true).Find(&groups).Error
return groups, err
}
// GroupIdsForAccountId returns all group IDs for the given account.
func GroupIdsForAccountId(accountID int64) []int64 { //nolint:revive // accepted lint exception
rows, err := DB().Raw("SELECT id FROM groups WHERE account_id = ?", accountID).Rows()
if err != nil {
panic(err)
}
defer rows.Close() //nolint:errcheck // accepted lint exception
var cid int64
groupIDs := make([]int64, 0)
for rows.Next() {
rows.Scan(&cid) //nolint:errcheck // accepted lint exception
groupIDs = append(groupIDs, cid)
}
return groupIDs
}
// CountGroups counts monitors per group.
func CountGroups(groupIDs []int64, groupCount *map[int64]int) { //nolint:gocritic // ptrToRefParam: accepted pattern
rows, err := DB().Raw("select group_id, count(id) from monitors where group_id IN (?) group by group_id ", groupIDs).Rows()
if err != nil {
panic(err)
}
defer rows.Close() //nolint:errcheck // accepted lint exception
var gid int64
var count int
for rows.Next() {
rows.Scan(&gid, &count) //nolint:errcheck // accepted lint exception
(*groupCount)[gid] = count
}
}
// GroupsCounts fills MonitorsCount for each group.
func GroupsCounts(groups *[]Group) {
groupIDs := make([]int64, len(*groups))
groupCount := make(map[int64]int, len(*groups))
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
groupIDs[i] = g.ID
groupCount[g.ID] = 0
}
CountGroups(groupIDs, &groupCount)
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
(*groups)[i].MonitorsCount = groupCount[g.ID]
}
}

58
app/models/group_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,58 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestSystemGroups verifies that SystemGroups returns only groups with
// is_system=true and ignores groups with is_system=false or nil.
func TestSystemGroups(t *testing.T) {
models.Drop()
models.Migrate()
account := &models.Account{Name: "test-account"}
require.NoError(t, models.DB().Create(account).Error)
trueVal, falseVal := true, false
systemGroup := &models.Group{
AccountID: account.ID,
Name: "system-internal",
IsSystem: &trueVal,
}
regularGroup := &models.Group{
AccountID: account.ID,
Name: "regular",
IsSystem: &falseVal,
}
nilSystemGroup := &models.Group{
AccountID: account.ID,
Name: "nil-system",
}
require.NoError(t, models.DB().Create(systemGroup).Error)
require.NoError(t, models.DB().Create(regularGroup).Error)
require.NoError(t, models.DB().Create(nilSystemGroup).Error)
got, err := models.SystemGroups()
require.NoError(t, err)
var ids []int64
var names []string
for _, g := range got {
ids = append(ids, g.ID)
names = append(names, g.Name)
}
assert.Contains(t, names, "system-internal")
assert.NotContains(t, names, "regular")
assert.NotContains(t, names, "nil-system")
assert.Contains(t, ids, systemGroup.ID)
assert.NotContains(t, ids, regularGroup.ID)
assert.NotContains(t, ids, nilSystemGroup.ID)
}

30
app/models/init.go Обычный файл
Просмотреть файл

@@ -0,0 +1,30 @@
package models
import (
"context"
"github.com/fatih/structs"
"gorm.io/gorm"
)
func init() {
structs.DefaultTagName = "json"
}
// db Gorm DB
var db *gorm.DB
// DB provides functionality.
func DB() *gorm.DB {
return db.WithContext(context.TODO())
}
// SetDB provides functionality.
func SetDB(newDb *gorm.DB) {
db = newDb
}
// IsDBAvailable returns true if the database has been initialized
func IsDBAvailable() bool {
return db != nil
}

222
app/models/inventory_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,222 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestServer_RotateToken(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "rotate-token-account"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "rotate-target",
Slug: "rotate-target",
Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
t1 := models.GenerateServerToken()
require.NoError(t, models.RotateServerToken(nil, srv.ID, t1))
got, err := models.FindServerByToken(t1)
require.NoError(t, err)
assert.Equal(t, srv.ID, got.ID)
t2 := models.GenerateServerToken()
require.NoError(t, models.RotateServerToken(nil, srv.ID, t2))
// Old token no longer matches.
_, err = models.FindServerByToken(t1)
assert.ErrorIs(t, err, gorm.ErrRecordNotFound)
got2, err := models.FindServerByToken(t2)
require.NoError(t, err)
assert.Equal(t, srv.ID, got2.ID)
// Generated tokens are hex-encoded 32 bytes (64 chars).
assert.Len(t, t1, 64)
assert.NotEqual(t, t1, t2)
}
func TestServer_InventoryFields_Default(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "inv-defaults"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "fresh-server",
Slug: "fresh-server",
Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
got := models.Server{}
require.NoError(t, models.DB().First(&got, srv.ID).Error)
assert.Equal(t, models.ServerKindProduction, got.Kind)
assert.Equal(t, 0, got.PriceCents)
assert.False(t, got.Paused)
}
func TestServer_KindEnum_OnlyAllowsValidValues(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "inv-enum"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "kinder",
Slug: "kinder",
Region: "local",
Kind: models.ServerKindStaging,
}
require.NoError(t, models.DB().Create(srv).Error)
got := models.Server{}
require.NoError(t, models.DB().First(&got, srv.ID).Error)
assert.Equal(t, models.ServerKindStaging, got.Kind)
// Inserting an invalid value via raw SQL fails the enum check.
err := models.DB().Exec(
"INSERT INTO servers (account_id, name, slug, region, kind) VALUES (?, ?, ?, ?, ?)",
acc.ID, "bad-kinder", "bad-kinder", "local", "scrapped",
).Error
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid input value for enum")
}
func TestSite_Slugify(t *testing.T) {
cases := []struct {
in, want string
}{
{"Cafe", "cafe"}, // cyrillic stripped (latin-only rule, see Slugify)
{" Spaces Everywhere ", "spaces-everywhere"},
{"dots.and-dashes_and spaces", "dots-and-dashes-and-spaces"},
{"", "site"},
{"-leading-and-trailing-", "leading-and-trailing"},
{"mix_of.dots-dashes spaces", "mix-of-dots-dashes-spaces"},
}
for _, c := range cases {
t.Run(c.in, func(t *testing.T) {
assert.Equal(t, c.want, models.SiteSlugify(c.in))
})
}
}
func TestSite_FindOrCreateBySlug(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "site-foc"}
require.NoError(t, models.DB().Create(acc).Error)
got, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
require.NoError(t, err)
require.NotZero(t, got.ID, "row should be persisted")
assert.Equal(t, "my-site", got.Slug)
assert.Equal(t, "production", got.Kind)
assert.True(t, got.IsActive)
// Second call returns the same row (idempotent).
got2, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
require.NoError(t, err)
assert.Equal(t, got.ID, got2.ID)
}
func TestDeployment_UpsertNginx_MatchesByConfigPath(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "dep-upsert"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID, Name: "host1", Slug: "host1", Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
d1, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
require.NoError(t, err)
require.NoError(t, models.DB().Save(d1).Error)
// Second call with the same config_path returns the existing row.
d2, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
require.NoError(t, err)
assert.Equal(t, d1.ID, d2.ID, "should reuse the same row on identical config_path")
// Different config_path creates a new row.
d3, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/b.conf")
require.NoError(t, err)
assert.NotEqual(t, d1.ID, d3.ID)
}
func TestDeployment_ReconcileMissing_FlipsAction(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "reconcile"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID, Name: "rec", Slug: "rec", Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
old := time.Now().Add(-2 * time.Hour)
fresh := models.Deployment{
AccountID: acc.ID,
ServerID: &srv.ID,
Kind: models.DeploymentKindProduction,
Mode: models.DeploymentModeDedicated,
Action: models.DeploymentActionOk,
ConfigPath: ptr("/etc/nginx/old.conf"),
LastSeenAt: &old,
}
require.NoError(t, models.DB().Create(&fresh).Error)
recent := models.Deployment{
AccountID: acc.ID,
ServerID: &srv.ID,
Kind: models.DeploymentKindProduction,
Mode: models.DeploymentModeDedicated,
Action: models.DeploymentActionOk,
ConfigPath: ptr("/etc/nginx/recent.conf"),
LastSeenAt: ptrTime(time.Now()),
}
require.NoError(t, models.DB().Create(&recent).Error)
cutoff := time.Now().Add(-90 * time.Second)
marked, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
require.NoError(t, err)
assert.EqualValues(t, 1, marked, "only the old row should flip")
var oldAfter models.Deployment
require.NoError(t, models.DB().First(&oldAfter, fresh.ID).Error)
assert.Equal(t, models.DeploymentActionMissing, oldAfter.Action)
var recentAfter models.Deployment
require.NoError(t, models.DB().First(&recentAfter, recent.ID).Error)
assert.Equal(t, models.DeploymentActionOk, recentAfter.Action, "fresh row stays ok")
// Re-running with the same cutoff is a no-op.
marked2, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
require.NoError(t, err)
assert.EqualValues(t, 0, marked2)
}
func ptr(s string) *string { return &s }
func ptrTime(t time.Time) *time.Time { return &t }

52
app/models/invite.go Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
package models
import (
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Invite provides functionality.
type Invite struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
InviterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"inviter_id"`
Inviter *User `json:"inviter"`
InviteeID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"invitee_id"`
Invitee *User `json:"invitee"`
Name string `json:"name"`
Email string `json:"email"`
// invite state, FAIL - failed to send, SENT - not regestired, OK - registered
State string `gorm:"not null;default:'UNK'" json:"state"`
CreatedAt time.Time `json:"created_at"`
RegisteredAt time.Time `json:"registered_at"`
SentAt time.Time `json:"sent_at"`
Accesses []Access `json:"accesses" gorm:"foreignkey:invite_id"`
concerns.HasToken
Audited
}
// BeforeCreate runs before creating an Invite record.
func (i *Invite) BeforeCreate(_ *gorm.DB) error {
i.SetToken()
return nil
}
// FillAccesses provides functionality.
func (i *Invite) FillAccesses() {
for k, a := range i.Accesses { //nolint:gocritic // range copy is acceptable here
if a.ID <= 0 {
i.Accesses[k].ID = 0
}
i.Accesses[k].AccountID = i.AccountID
}
}

17
app/models/llm.go Обычный файл
Просмотреть файл

@@ -0,0 +1,17 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// LLM stores an OpenAI-compatible LLM endpoint available to checks.
type LLM struct {
concerns.Model
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id);index"`
Account *Account `json:"-"`
Name string `json:"name" gorm:"not null"`
URL string `json:"url" gorm:"not null"`
ModelName string `json:"model" gorm:"column:model;not null"`
APIKey string `json:"-" gorm:"not null"`
Kind string `json:"kind" gorm:"not null;default:'openai'"`
Workers []WorkerNode `json:"-" gorm:"many2many:worker_llms;"`
concerns.Timestamped
}

324
app/models/maintenance.go Обычный файл
Просмотреть файл

@@ -0,0 +1,324 @@
package models
import (
"fmt"
"strings"
"time"
"github.com/lib/pq"
"github.com/robfig/cron/v3"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
MaintenanceManual = "manual"
MaintenanceSingle = "single"
MaintenanceCron = "cron"
MaintenanceRecurringInterval = "recurring-interval"
MaintenanceRecurringWeekday = "recurring-weekday"
MaintenanceRecurringDayOfMonth = "recurring-day-of-month"
)
// Maintenance is account-owned planned downtime. Times are stored as UTC;
// Timezone only defines how recurring wall-clock fields are interpreted.
type Maintenance struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
Account *Account `json:"-"`
Title string `gorm:"size:200;not null" json:"title"`
Description string `gorm:"type:text;not null;default:''" json:"description"`
Strategy string `gorm:"size:32;not null" json:"strategy"`
Cron string `gorm:"type:text;not null;default:''" json:"cron"`
DurationSec int `gorm:"not null;default:0" json:"duration_sec"`
StartDate *time.Time `json:"start_date,omitempty"`
EndDate *time.Time `json:"end_date,omitempty"`
StartTime string `gorm:"size:5;not null;default:''" json:"start_time"`
EndTime string `gorm:"size:5;not null;default:''" json:"end_time"`
Weekdays pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"weekdays"`
DaysOfMonth pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"days_of_month"`
IntervalDay int `gorm:"not null;default:1" json:"interval_day"`
Timezone string `gorm:"size:64;not null;default:'UTC'" json:"timezone"`
Active bool `gorm:"not null;default:true" json:"active"`
LastStartDate *time.Time `json:"last_start_date,omitempty"`
LegacyStatusPageMaintenanceID *int64 `gorm:"uniqueIndex" json:"-"`
ShowOnAllStatusPages bool `gorm:"not null;default:true" json:"show_on_all_status_pages"`
Monitors []Monitor `gorm:"many2many:maintenance_monitors;constraint:OnDelete:CASCADE" json:"monitors,omitempty"`
StatusPages []StatusPage `gorm:"many2many:maintenance_status_pages;constraint:OnDelete:CASCADE" json:"status_pages,omitempty"`
concerns.Timestamped
Audited
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
}
func (Maintenance) TableName() string { return "maintenances" }
func (m *Maintenance) location() (*time.Location, error) {
if m.Timezone == "" || m.Timezone == "SAME_AS_SERVER" {
return time.UTC, nil
}
return time.LoadLocation(m.Timezone)
}
func (m *Maintenance) generatedCron() (string, error) {
if m.Strategy == MaintenanceCron {
return m.Cron, nil
}
if m.Strategy == MaintenanceManual || m.Strategy == MaintenanceSingle {
return "", nil
}
parts := strings.Split(m.StartTime, ":")
if len(parts) != 2 {
return "", fmt.Errorf("start_time must be HH:MM")
}
base := parts[1] + " " + parts[0]
switch m.Strategy {
case MaintenanceRecurringInterval:
return "", nil
case MaintenanceRecurringWeekday:
if len(m.Weekdays) == 0 {
return "", fmt.Errorf("at least one weekday is required")
}
values := make([]string, len(m.Weekdays))
for i, day := range m.Weekdays {
if day < 0 || day > 6 {
return "", fmt.Errorf("weekday must be 0 through 6")
}
values[i] = fmt.Sprint(day)
}
return base + " * * " + strings.Join(values, ","), nil
case MaintenanceRecurringDayOfMonth:
if len(m.DaysOfMonth) == 0 {
return "", fmt.Errorf("at least one day of month is required")
}
values := make([]string, 0, len(m.DaysOfMonth))
for _, day := range m.DaysOfMonth {
if day == "lastDay1" {
values = append(values, "28-31")
} else {
values = append(values, day)
}
}
return base + " " + strings.Join(values, ",") + " * *", nil
default:
return "", fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
}
}
// Validate normalizes generated schedules and rejects ambiguous or invalid
// input before it can reach the scheduler.
func (m *Maintenance) Validate() error {
m.Title = strings.TrimSpace(m.Title)
if m.Title == "" || len(m.Title) > 200 {
return fmt.Errorf("title is required and must be at most 200 characters")
}
if _, err := m.location(); err != nil {
return fmt.Errorf("invalid timezone: %w", err)
}
switch m.Strategy {
case MaintenanceManual:
return nil
case MaintenanceSingle:
if m.StartDate == nil || m.EndDate == nil || !m.EndDate.After(*m.StartDate) {
return fmt.Errorf("single maintenance requires end_date after start_date")
}
m.DurationSec = int(m.EndDate.Sub(*m.StartDate).Seconds())
return nil
case MaintenanceRecurringInterval:
if m.DurationSec <= 0 || m.IntervalDay <= 0 || (m.IntervalDay > 1 && m.StartDate == nil) {
return fmt.Errorf("recurring interval requires positive duration_sec and interval_day; intervals over one day require start_date")
}
if _, _, err := parseMaintenanceTime(m.StartTime); err != nil {
return err
}
m.Cron = ""
return nil
case MaintenanceCron, MaintenanceRecurringWeekday, MaintenanceRecurringDayOfMonth:
if m.DurationSec <= 0 {
return fmt.Errorf("duration_sec must be positive")
}
cronText, err := m.generatedCron()
if err != nil {
return err
}
if _, err := cron.ParseStandard(cronText); err != nil {
return fmt.Errorf("invalid cron: %w", err)
}
m.Cron = cronText
return nil
default:
return fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
}
}
func (m *Maintenance) BeforeSave(_ *gorm.DB) error { return m.Validate() }
// IsUnderMaintenance evaluates durable data only. This intentionally avoids
// scheduler-owned state so a process restart and multiple web pods agree.
func (m *Maintenance) IsUnderMaintenance(now time.Time) bool {
if !m.Active {
return false
}
if m.Strategy == MaintenanceManual {
return true
}
if m.Strategy == MaintenanceSingle {
return m.StartDate != nil && m.EndDate != nil && !now.Before(*m.StartDate) && now.Before(*m.EndDate)
}
if m.Strategy == MaintenanceRecurringInterval {
return m.isUnderInterval(now)
}
if m.DurationSec <= 0 {
return false
}
loc, err := m.location()
if err != nil {
return false
}
schedule, err := cron.ParseStandard(m.Cron)
if err != nil {
return false
}
// Ask cron for each candidate since the earliest possible active start.
// Cron is minute-granular, hence the extra minute catches exact boundaries.
from := now.In(loc).Add(-time.Duration(m.DurationSec)*time.Second - time.Minute)
to := now.In(loc)
for candidate := schedule.Next(from); !candidate.After(to); candidate = schedule.Next(candidate) {
if !m.allowsRecurringCandidate(candidate.In(loc)) {
continue
}
start := candidate.UTC()
if !now.Before(start) && now.Before(start.Add(time.Duration(m.DurationSec)*time.Second)) {
return true
}
}
return false
}
func parseMaintenanceTime(value string) (int, int, error) {
parsed, err := time.Parse("15:04", value)
if err != nil {
return 0, 0, fmt.Errorf("start_time must be HH:MM")
}
return parsed.Hour(), parsed.Minute(), nil
}
func (m *Maintenance) intervalStartOn(date time.Time, loc *time.Location) (time.Time, bool) {
if m.IntervalDay <= 0 {
return time.Time{}, false
}
hour, minute, err := parseMaintenanceTime(m.StartTime)
if err != nil {
return time.Time{}, false
}
if m.IntervalDay == 1 && m.StartDate == nil {
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
}
if m.StartDate == nil {
return time.Time{}, false
}
anchor := m.StartDate.In(loc)
anchorDay := time.Date(anchor.Year(), anchor.Month(), anchor.Day(), 0, 0, 0, 0, loc)
candidateDay := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc)
// Compare civil dates rather than elapsed hours: a local day can be 23 or
// 25 hours when the maintenance timezone crosses a DST boundary.
days := civilDaysBetween(anchorDay, candidateDay)
if days < 0 || days%m.IntervalDay != 0 {
return time.Time{}, false
}
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
}
func civilDaysBetween(from, to time.Time) int {
fromDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
toDay := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC)
return int(toDay.Sub(fromDay) / (24 * time.Hour))
}
func (m *Maintenance) isUnderInterval(now time.Time) bool {
loc, err := m.location()
if err != nil || m.DurationSec <= 0 {
return false
}
localNow := now.In(loc)
for day := 0; day <= int(time.Duration(m.DurationSec)/24/time.Hour)+1; day++ {
start, ok := m.intervalStartOn(localNow.AddDate(0, 0, -day), loc)
if ok && !now.Before(start.UTC()) && now.Before(start.UTC().Add(time.Duration(m.DurationSec)*time.Second)) {
return true
}
}
return false
}
// robfig/cron cannot express "last day". The generated 28-31 range is only
// a candidate generator; this final predicate makes lastDay1 exact.
func (m *Maintenance) allowsRecurringCandidate(candidate time.Time) bool {
if m.Strategy != MaintenanceRecurringDayOfMonth {
return true
}
lastDay := candidate.AddDate(0, 0, 1).Month() != candidate.Month()
for _, value := range m.DaysOfMonth {
if value == "lastDay1" && lastDay {
return true
}
if value == fmt.Sprint(candidate.Day()) {
return true
}
}
return false
}
func (m *Maintenance) NextRun(now time.Time) *time.Time {
if !m.Active || m.Strategy == MaintenanceManual {
return nil
}
if m.Strategy == MaintenanceSingle {
if m.StartDate != nil && m.StartDate.After(now) {
return m.StartDate
}
return nil
}
if m.Strategy == MaintenanceRecurringInterval {
loc, err := m.location()
if err != nil {
return nil
}
localNow := now.In(loc)
for day := 0; day <= m.IntervalDay; day++ {
if next, ok := m.intervalStartOn(localNow.AddDate(0, 0, day), loc); ok && next.After(localNow) {
result := next.UTC()
return &result
}
}
return nil
}
loc, err := m.location()
if err != nil {
return nil
}
s, err := cron.ParseStandard(m.Cron)
if err != nil {
return nil
}
for candidate := s.Next(now.In(loc)); ; candidate = s.Next(candidate) {
if m.allowsRecurringCandidate(candidate.In(loc)) {
next := candidate.UTC()
return &next
}
}
}
// MonitorUnderMaintenance is the notifier/public-page lookup.
func MonitorUnderMaintenance(monitorID int64, now time.Time) (bool, error) {
var rows []Maintenance
err := DB().Joins("JOIN maintenance_monitors mm ON mm.maintenance_id = maintenances.id").Where("mm.monitor_id = ? AND maintenances.active = TRUE", monitorID).Find(&rows).Error
if err != nil {
return false, err
}
for i := range rows {
if rows[i].IsUnderMaintenance(now) {
return true, nil
}
}
return false, nil
}

33
app/models/maintenance_migration_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,33 @@
package models
import (
"testing"
"time"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
)
func TestMigratePreservesEveryLegacyStatusPageMaintenance(t *testing.T) {
Drop()
Migrate()
plan := Plan{Name: "legacy migration plan"}
require.NoError(t, DB().Create(&plan).Error)
account := Account{Name: "legacy migration account", PlanID: &plan.ID}
require.NoError(t, DB().Create(&account).Error)
page := StatusPage{AccountID: account.ID, Slug: "legacy-maintenance-migration", Name: "Legacy"}
require.NoError(t, DB().Create(&page).Error)
start := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
for i := 0; i < 2; i++ {
row := StatusPageMaintenance{StatusPageID: page.ID, Title: "same", StartsAt: start, EndsAt: start.Add(time.Hour), MonitorIDs: pq.Int64Array{}}
require.NoError(t, DB().Create(&row).Error)
}
Migrate()
var migrated []Maintenance
require.NoError(t, DB().Where("legacy_status_page_maintenance_id IS NOT NULL").Find(&migrated).Error)
require.Len(t, migrated, 2)
var joins int64
require.NoError(t, DB().Table("maintenance_status_pages").Where("status_page_id = ?", page.ID).Count(&joins).Error)
require.EqualValues(t, 2, joins)
}

44
app/models/maintenance_notifications.go Обычный файл
Просмотреть файл

@@ -0,0 +1,44 @@
package models
import (
"fmt"
"time"
"gorm.io/gorm"
)
const maintenanceStartNotificationPrefix = "maintenance:%d:start:"
// MaintenanceStartNotificationKey uniquely identifies one contact's warning
// for one schedule revision and occurrence. Updated windows get a new revision
// while concurrent scheduler replicas share the same key.
func MaintenanceStartNotificationKey(maintenanceID int64, revision, startsAt time.Time, timezone string, notificationID, contactID int64) string {
loc, err := time.LoadLocation(timezone)
if err != nil || timezone == "SAME_AS_SERVER" || timezone == "" {
loc = time.UTC
}
// A fall-back hour can have two UTC instants for one wall-clock occurrence.
// Warnings are once per civil occurrence, matching the recurrence editor.
civilOccurrence := startsAt.In(loc).Format("200601021504")
return fmt.Sprintf("maintenance:%d:start:%d:%s:notification:%d:contact:%d", maintenanceID, revision.UnixNano(), civilOccurrence, notificationID, contactID)
}
// CancelMaintenanceStartNotificationsTx prevents queued warnings from being
// delivered after an operator pauses, changes, or deletes the maintenance.
// Leased work may already be executing and cannot be recalled from a worker.
func CancelMaintenanceStartNotificationsTx(tx *gorm.DB, maintenanceID int64, reason string) error {
prefix := fmt.Sprintf(maintenanceStartNotificationPrefix, maintenanceID) + "%"
var tasks []Task
if err := tx.Clauses(SkipLockedClause).Where("idempotency_key LIKE ? AND state IN ?", prefix, []string{TaskStateQueued, TaskStateFailedRetry}).Find(&tasks).Error; err != nil {
return err
}
for i := range tasks {
if err := tx.Model(&Task{}).Where("id = ? AND state IN ?", tasks[i].ID, []string{TaskStateQueued, TaskStateFailedRetry}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "canceled", "canceled: "+reason); err != nil {
return err
}
}
return nil
}

105
app/models/maintenance_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,105 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestMaintenanceValidateGeneratesRecurringCron(t *testing.T) {
tests := []struct {
name string
m models.Maintenance
want string
}{
{"interval", models.Maintenance{Title: "interval", Strategy: models.MaintenanceRecurringInterval, StartDate: maintenanceTimePtr(time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)), StartTime: "02:30", IntervalDay: 3, DurationSec: 60, Timezone: "UTC"}, ""},
{"weekday", models.Maintenance{Title: "weekdays", Strategy: models.MaintenanceRecurringWeekday, StartTime: "02:30", Weekdays: []int64{1, 5}, DurationSec: 60, Timezone: "UTC"}, "30 02 * * 1,5"},
{"month", models.Maintenance{Title: "month", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "02:30", DaysOfMonth: []string{"1", "lastDay1"}, DurationSec: 60, Timezone: "UTC"}, "30 02 1,28-31 * *"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) { require.NoError(t, test.m.Validate()); assert.Equal(t, test.want, test.m.Cron) })
}
}
func TestMaintenanceIsUnderMaintenanceBoundariesAndTimezone(t *testing.T) {
start := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
single := models.Maintenance{Title: "single", Strategy: models.MaintenanceSingle, StartDate: &start, EndDate: &end, Active: true, Timezone: "UTC"}
assert.True(t, single.IsUnderMaintenance(start))
assert.True(t, single.IsUnderMaintenance(end.Add(-time.Nanosecond)))
assert.False(t, single.IsUnderMaintenance(end))
cron := models.Maintenance{Title: "moscow", Strategy: models.MaintenanceCron, Cron: "0 12 * * *", DurationSec: 3600, Active: true, Timezone: "Europe/Moscow"}
require.NoError(t, cron.Validate())
assert.True(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 9, 30, 0, 0, time.UTC)), "12:30 Moscow is 09:30 UTC in July")
assert.False(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)))
}
func TestMaintenanceValidationRejectsInvalidSchedules(t *testing.T) {
badCron := models.Maintenance{Title: "bad", Strategy: models.MaintenanceCron, Cron: "not cron", DurationSec: 1, Timezone: "UTC"}
badSingle := models.Maintenance{Title: "bad", Strategy: models.MaintenanceSingle, Timezone: "UTC"}
assert.Error(t, badCron.Validate())
assert.Error(t, badSingle.Validate())
}
func TestMaintenanceLastDayIsNotEveryDayInExpandedCronRange(t *testing.T) {
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 3600, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 3, 28, 12, 30, 0, 0, time.UTC)))
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 31, 12, 30, 0, 0, time.UTC)))
}
func TestMaintenanceRecurringIntervalUsesAnchorAndIntervalDay(t *testing.T) {
anchor := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
m := models.Maintenance{Title: "every three days", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "12:00", IntervalDay: 3, DurationSec: 3600, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 7, 4, 12, 30, 0, 0, time.UTC)))
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 7, 5, 12, 30, 0, 0, time.UTC)))
next := m.NextRun(time.Date(2026, 7, 2, 13, 0, 0, 0, time.UTC))
require.NotNil(t, next)
assert.Equal(t, time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC), *next)
}
func TestMaintenanceRecurringIntervalRequiresAnchorForMultiDayAndPositiveInterval(t *testing.T) {
m := models.Maintenance{Title: "invalid", Strategy: models.MaintenanceRecurringInterval, StartTime: "12:00", IntervalDay: 2, DurationSec: 60, Timezone: "UTC"}
assert.Error(t, m.Validate())
}
func TestMaintenanceRecurringIntervalKeepsCivilDayAcrossDST(t *testing.T) {
loc, err := time.LoadLocation("Europe/Berlin")
require.NoError(t, err)
// March 29, 2026 is the spring-forward day in Berlin. The second run is
// still two civil days after the anchor, not one because a day was 23h.
anchor := time.Date(2026, 3, 27, 0, 0, 0, 0, loc)
m := models.Maintenance{Title: "DST", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "03:30", IntervalDay: 2, DurationSec: 3600, Active: true, Timezone: "Europe/Berlin"}
require.NoError(t, m.Validate())
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 29, 4, 0, 0, 0, loc).UTC()))
}
func TestMaintenanceStartNotificationKeyIncludesRevisionAndOccurrence(t *testing.T) {
revision := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
start := revision.Add(5 * time.Minute)
key := models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13)
assert.Equal(t, key, models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13))
assert.NotEqual(t, key, models.MaintenanceStartNotificationKey(7, revision.Add(time.Second), start, "Europe/Berlin", 11, 13))
}
func TestMaintenanceStartNotificationKeyDeduplicatesDSTFallbackCivilOccurrence(t *testing.T) {
revision := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
first := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC)
second := first.Add(time.Hour)
assert.Equal(t, models.MaintenanceStartNotificationKey(7, revision, first, "Europe/Berlin", 11, 13), models.MaintenanceStartNotificationKey(7, revision, second, "Europe/Berlin", 11, 13))
}
func TestMaintenanceNextRunSkipsNonFinalLastDayCandidates(t *testing.T) {
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 60, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
next := m.NextRun(time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC))
require.NotNil(t, next)
assert.Equal(t, time.Date(2026, 4, 30, 12, 0, 0, 0, time.UTC), *next)
}
func maintenanceTimePtr(value time.Time) *time.Time { return &value }

48
app/models/message.go Обычный файл
Просмотреть файл

@@ -0,0 +1,48 @@
package models
import (
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Message info about performed notification
type Message struct {
concerns.Model
NotificationID int64 `json:"notification_id"`
Notification *Notification `json:"notification,omitempty"`
ContactID int64 `gorm:"index;type:bigint REFERENCES contacts(id)" json:"contact_id"`
Contact *Contact `json:"contact,omitempty"`
// Events are for up/down messages
Events []Event `json:"events" gorm:"many2many:event_messages;"`
// Checks are for expires messages
CheckID *int64 `gorm:"index;type:bigint REFERENCES checks(id)" json:"check_id"`
Check *Check `json:"check,omitempty"`
Kind string `json:"kind"`
State string `json:"state"`
Error *string `json:"error"`
Response *string `json:"response"`
Tries int `json:"-"`
CreatedAt time.Time `json:"created_at"`
SentAt time.Time `json:"sent_at"`
}
// MessageScope provides functionality.
func MessageScope(q *gorm.DB) *gorm.DB {
return q.
Preload("Events").
Preload("Events.Checks").
Preload("Events.Monitor").
Preload("Notification").
Preload("Contact").
Preload("Check").
Preload("Check.Monitor")
}

750
app/models/migrate.go Обычный файл
Просмотреть файл

@@ -0,0 +1,750 @@
package models
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
)
// isTypeExistsError returns true if the error is a Postgres "type already exists" error.
// This happens when AutoMigrate is called concurrently from multiple test processes.
func isTypeExistsError(err error) bool {
if err == nil {
return false
}
s := err.Error()
// SQLSTATE 42710 = duplicate_object (type already exists)
return strings.Contains(s, "42710") ||
strings.Contains(s, "already exists") ||
strings.Contains(s, "pg_type_typname_nsp_index")
}
// migrateOrIgnoreTypeExists runs AutoMigrate and ignores "type already exists" errors
// that can occur when parallel test processes both try to create the same Postgres types.
func migrateOrIgnoreTypeExists(models ...interface{}) {
err := DB().AutoMigrate(models...)
if err != nil && !isTypeExistsError(err) {
panic(err)
}
if err != nil {
log.Printf("migrate: ignoring type-exists error (expected during parallel test runs): %v", err)
}
}
// ensureSingleCurrentEventInvariant pins cleanup and index creation to one
// transaction/connection. The global migration lock is session-scoped through a
// pool, so it is not sufficient for this multi-statement invariant by itself.
func ensureSingleCurrentEventInvariant() error {
return DB().Transaction(func(tx *gorm.DB) error {
if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", int64(1234567892)).Error; err != nil {
return err
}
if err := tx.Exec(`WITH ranked AS (
SELECT id, row_number() OVER (PARTITION BY monitor_id ORDER BY start_time DESC NULLS LAST, id DESC) AS n
FROM events WHERE state = 'current'
) UPDATE events SET state = 'ended', end_time = COALESCE(end_time, now())
FROM ranked WHERE events.id = ranked.id AND ranked.n > 1`).Error; err != nil {
return err
}
return tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS events_one_current_per_monitor
ON events (monitor_id) WHERE state = 'current'`).Error
})
}
// Migrate run db migration
var migrateMu sync.Mutex
func Migrate() {
migrateMu.Lock()
defer migrateMu.Unlock()
withMigrationAdvisoryLock(migrateLocked)
}
// withMigrationAdvisoryLock holds the session lock on a dedicated connection
// while migration work uses GORM's normal pool. Reusing the lock connection
// for GORM transactions can leave its *sql.Conn closed after commit.
func withMigrationAdvisoryLock(migrate func()) {
const migrateAdvisoryLock = int64(1234567890)
sqlDB, err := DB().DB()
if err != nil {
panic(fmt.Sprintf("migrate: database handle: %v", err))
}
ctx := context.Background()
conn, err := sqlDB.Conn(ctx)
if err != nil {
panic(fmt.Sprintf("migrate: lock connection: %v", err))
}
defer conn.Close() //nolint:errcheck // closing releases the session lock after a migration panic
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", migrateAdvisoryLock); err != nil {
panic(fmt.Sprintf("migrate: advisory lock: %v", err))
}
unlocked := false
defer func() {
if unlocked {
return
}
if _, unlockErr := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); unlockErr != nil {
log.Printf("migrate: unlock after failure: %v", unlockErr)
}
}()
migrate()
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); err != nil {
panic(fmt.Sprintf("migrate: unlock: %v", err))
}
unlocked = true
}
func migrateLocked() {
// M0 replaces the old flat plans table without rewriting historical rows.
// Rename before AutoMigrate so GORM creates the canonical table rather than
// adding columns to the incompatible legacy schema.
if err := prepareCanonicalPlansTable(); err != nil {
panic(fmt.Sprintf("migrate: prepare canonical plans: %v", err))
}
// Step 0: create inventory Postgres enum types FIRST. The DO/EXCEPTION
// blocks are idempotent so concurrent migrateOrIgnoreTypeExists
// reruns from parallel test binaries are safe (the type already
// exists → duplicate_object is swallowed). The enum types MUST
// exist before any AutoMigrate below because GORM emits
// `kind server_kind` literals in CREATE TABLE for the Server
// struct (referenced transitively from Monitor.Site → Site → Server).
for _, ddl := range []string{
`DO $$ BEGIN
CREATE TYPE server_kind AS ENUM ('production','staging','old');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_kind AS ENUM
('production','production_prev','production_next','internal','staging','old');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_mode AS ENUM
('kubernetes','compose','dedicated','vds','user');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_action AS ENUM
('ok','pending','pending_move','pending_drop','deleted','missing');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
} {
if err := DB().Exec(ddl).Error; err != nil {
panic(fmt.Sprintf("migrate: enum creation: %v", err))
}
}
var err error
// Step 1: Migrate core models (User, Plan, Account, ApiKey, AuthIdentity, Invite)
// Note: Access is moved to Step 2 because it has FKs to groups/monitors
// Seed and remap plans before Account migration recreates its plan FK.
migrateOrIgnoreTypeExists(&Plan{})
if err := seedCanonicalPlansAndBackfill(true); err != nil {
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
}
migrateOrIgnoreTypeExists(
&User{},
&Account{},
&ApiKey{},
&authidentity.AuthIdentity{}, // After User (has FK to users)
&Invite{}, // After Account/User (has FKs to accounts/users)
&Subscription{}, // After Account/Plan
&SubscriptionEvent{}, // After Subscription
)
// Monitor and metric rows reference worker_nodes, while worker_nodes itself
// references servers. Create the two roots without their associations before
// migrating Monitor/ServerMetric on a fresh database.
migrateOrIgnoreTypeExists(&Region{}, &LLM{})
if err = DB().Omit("Monitors", "Workers").AutoMigrate(&Server{}); err != nil {
panic(err)
}
migrateOrIgnoreTypeExists(&WorkerNode{})
// Step 2: Migrate Group, Monitor, and Access (which has FKs to groups/monitors)
// This ensures the groups table exists when GORM creates foreign keys
migrateOrIgnoreTypeExists(
&Group{},
&Monitor{},
&Check{},
// Server is a customer-facing logical host, distinct from the
// WorkerNode executor. Keep the join/cache models here so a fresh
// database gets the complete server metrics schema in one migration.
&Server{},
&AccountMCPToken{},
&MonitorServer{},
&ServerMetric{},
&ServerAlertRule{},
&ServerAlertEvent{},
&RknIP{},
&RknDomain{},
&DNSRecord{},
&Contact{},
&Whois{},
&Payment{},
&Message{},
&TelegramBotMessage{},
&TelegramBotStatus{},
&Event{},
&SelfCheck{},
&Notification{}, // After Group/Monitor so notification_groups FK works
&Access{}, // After Group/Monitor so access FKs work
&NotificationCredential{}, // No FKs to other domain tables; safe here.
)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS servers_account_slug_unique ON servers (account_id, slug)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS monitor_servers_position_idx ON monitor_servers (server_id, position)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_server_recent_idx ON server_metrics (server_id, id DESC)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_worker_idx ON server_metrics (worker_id)`)
// Step 3: Clean up orphaned references (now that all tables exist)
// Fix typo in old table name (only present on DBs migrated from older versions)
err = DB().Exec("ALTER TABLE IF EXISTS envent_checks RENAME TO event_checks;").Error
if err != nil {
log.Println(err)
}
err = DB().Exec("DROP TABLE IF EXISTS envent_messages;").Error
if err != nil {
log.Println(err)
}
// Defense-in-depth: make sure the columns that the in-process notifier
// scheduler eagerly queries at startup exist, even if AutoMigrate above
// was skipped or the column was dropped by a manual operation. Without
// these, a fresh restore from a pre-soft-delete production dump will
// panic the first time RunExp preloads Contacts or
// ProcessPendingDeletions queries Users (see internal/notifier for the
// defensive recover() that catches the resulting query errors).
err = DB().Exec(
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT TRUE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS disabled BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS blocked BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE users ADD COLUMN IF NOT EXISTS deletion_requested_at TIMESTAMPTZ",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS webhook_token VARCHAR(128)",
).Error
if err != nil {
log.Println(err)
}
var telegramCreds []NotificationCredential
if err = DB().Where("kind = ? AND (webhook_token IS NULL OR webhook_token = '')", CredentialKindTelegram).Find(&telegramCreds).Error; err != nil {
log.Println(err)
}
for i := range telegramCreds {
telegramCreds[i].EnsureWebhookToken()
if err = DB().Save(&telegramCreds[i]).Error; err != nil {
log.Println(err)
}
}
err = DB().Exec("DROP INDEX IF EXISTS idx_notification_credentials_webhook_token").Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_credentials_webhook_token ON notification_credentials (webhook_token) WHERE webhook_token IS NOT NULL AND webhook_token <> ''",
).Error
if err != nil {
log.Println(err)
}
// Replace the early M0 per-event-id index with provider-scoped webhook
// idempotency: PSP event IDs are only unique inside a provider.
if err = DB().Exec("DROP INDEX IF EXISTS idx_subscription_events_provider_event_id").Error; err != nil {
log.Println(err)
}
if err = DB().Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_subscription_events_provider_event ON subscription_events (provider, provider_event_id) WHERE provider_event_id IS NOT NULL").Error; err != nil {
log.Println(err)
}
// Clean up orphaned event_checks
err = DB().Exec("DELETE FROM event_checks where event_id NOT IN (select id from events)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM event_checks where check_id NOT IN (select id from checks)").Error
if err != nil {
panic(err)
}
// Clean up orphaned event_messages
err = DB().Exec("DELETE FROM event_messages where event_id NOT IN (select id from events)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM event_messages where message_id NOT IN (select id from messages)").Error
if err != nil {
panic(err)
}
// Clean up orphaned notification_contacts
err = DB().Exec("DELETE FROM notification_contacts where notification_id NOT IN (select id from notifications)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM notification_contacts where contact_id NOT IN (select id from contacts)").Error
if err != nil {
panic(err)
}
// Clean up orphaned notification_groups (now safe - groups table exists)
err = DB().Exec("DELETE FROM notification_groups where notification_id NOT IN (select id from notifications)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM notification_groups where group_id NOT IN (select id from groups)").Error
if err != nil {
panic(err)
}
DB().Raw("CREATE INDEX IF NOT EXISTS not_old_events on events (monitor_id, id) where state != 'old'")
DB().Raw("CREATE INDEX IF NOT EXISTS current_events ON event (monitor_id, start_time) WHERE state = 'current'")
DB().Raw("CREATE INDEX IF NOT EXISTS ended_events ON event (monitor_id, start_time) WHERE state = 'ended'")
DB().Raw("CREATE INDEX IF NOT EXISTS queued_messages ON message (id) WHERE state = 'queued'")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, account_id)")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, group_id)")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS invite_email ON invites (account_id, email)")
// RKN indexes — see app/models/rkn_ip.go EnsureRknIndexes. GORM
// AutoMigrate above declared the uniqueIndex on RknDomain.Domain
// and the cidr column type on RknIP, but GiST on rkn_ips.network
// is not expressible via the GORM tag language; we add it here so
// the (>>=) containment operator used by IsRknIPBlocked has an
// index to back it.
if err := EnsureRknIndexes(); err != nil {
log.Printf("migrate: EnsureRknIndexes failed: %v", err)
}
// Distributed worker models
migrateOrIgnoreTypeExists(
&WorkerLogEvent{},
&CheckAttempt{},
&DiagnosticAuditEvent{},
&CheckRegionResult{},
&Task{},
&TaskReplay{},
&NotificationDelivery{},
&Tag{},
)
// Tags — (account_id, name) is the unique key so a single account
// cannot register two metadata rows for the same tag string. The
// tag name itself is also the join key against monitors.tags, so
// uniqueness is enforced at the table level (not just on the
// metadata row).
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tags_account_name_unique ON tags (account_id, name)`)
// Status pages use globally unique public slugs. A public URL has no
// account component, so account-scoped uniqueness would make /status/:slug
// ambiguous. Soft-deleted rows do not reserve their slug.
// subscriber email index uses lower(email) for case-insensitive
// matching (the codebase does not adopt citext). All five tables
// are created together so M0 ships a consistent schema baseline
// regardless of which milestone first writes rows.
migrateOrIgnoreTypeExists(
&StatusPage{},
&StatusPageSubscriber{},
&StatusPageIncident{},
&StatusPageMaintenance{},
&StatusPageDomain{},
&StatusPageDelivery{},
&StatusPageDigestSchedule{},
&Maintenance{},
)
DB().Exec(`DROP INDEX IF EXISTS status_pages_account_slug_unique`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_pages_slug_unique
ON status_pages (slug) WHERE deleted_at IS NULL`)
if err := ensureSingleCurrentEventInvariant(); err != nil {
panic(fmt.Sprintf("migrate: current event invariant: %v", err))
}
DB().Exec(
`CREATE UNIQUE INDEX IF NOT EXISTS status_page_subscribers_active_email
ON status_page_subscribers (status_page_id, lower(email))
WHERE unsubscribed_at IS NULL`,
)
// Existing installations can already have subscriber rows. Keep the legacy
// token column during the nullable transition: outstanding links remain
// valid, while each resend/confirmation rotates it into a hash.
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS confirm_token_hash varchar(64)`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS token_expires_at timestamptz`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS unsubscribe_token_hash varchar(64)`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS contact_id bigint REFERENCES contacts(id) ON DELETE SET NULL`)
DB().Exec(`UPDATE status_page_subscribers SET token_expires_at = created_at + interval '24 hours' WHERE token_expires_at IS NULL`)
for {
var subscribers []StatusPageSubscriber
if err := DB().Where("confirm_token_hash IS NULL AND confirm_token IS NOT NULL").Limit(500).Find(&subscribers).Error; err != nil || len(subscribers) == 0 {
break
}
for i := range subscribers {
subscribers[i].ConfirmTokenHash = HashStatusPageToken(*subscribers[i].LegacyConfirmToken)
_ = DB().Model(&subscribers[i]).Update("confirm_token_hash", subscribers[i].ConfirmTokenHash).Error
}
}
DB().Exec(`UPDATE status_page_subscribers SET confirm_token_hash = '' WHERE confirm_token_hash IS NULL`)
DB().Exec(`UPDATE status_page_subscribers SET unsubscribe_token_hash = '' WHERE unsubscribe_token_hash IS NULL`)
// A page may expose more than one verified hostname; older schema reserved
// only one domain per page.
DB().Exec(`DROP INDEX IF EXISTS idx_status_page_domains_status_page_id`)
DB().Exec(
`CREATE INDEX IF NOT EXISTS status_page_incidents_started
ON status_page_incidents (status_page_id, started_at DESC)`,
)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_page_incidents_event_unique
ON status_page_incidents (status_page_id, event_id) WHERE event_id IS NOT NULL`)
DB().Exec(
`CREATE INDEX IF NOT EXISTS status_page_maintenance_starts
ON status_page_maintenance (status_page_id, starts_at DESC)`,
)
DB().Exec(`CREATE INDEX IF NOT EXISTS maintenances_account_active ON maintenances (account_id) WHERE deleted_at IS NULL`)
// Legacy status-page rows predate account-scoped maintenance. Keep their
// source IDs so equal title/start rows remain distinct and reruns can safely
// preserve every window and each of its joins.
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS maintenances_legacy_status_page_maintenance_unique
ON maintenances (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL`)
DB().Exec(`INSERT INTO maintenances (account_id, title, description, strategy, duration_sec, start_date, end_date, timezone, active, show_on_all_status_pages, legacy_status_page_maintenance_id, created_at, updated_at)
SELECT sp.account_id, old.title, old.description, 'single', EXTRACT(EPOCH FROM (old.ends_at - old.starts_at))::int, old.starts_at, old.ends_at, 'UTC', TRUE, FALSE, old.id, old.created_at, old.updated_at
FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id
ON CONFLICT (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL DO NOTHING`)
DB().Exec(`INSERT INTO maintenance_status_pages (maintenance_id, status_page_id)
SELECT m.id, old.status_page_id FROM status_page_maintenance old JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id
ON CONFLICT DO NOTHING`)
DB().Exec(`INSERT INTO maintenance_monitors (maintenance_id, monitor_id)
SELECT m.id, monitor_id FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id,
LATERAL unnest(CASE WHEN cardinality(old.monitor_ids) > 0 THEN old.monitor_ids ELSE sp.monitor_ids END) AS monitor_id
ON CONFLICT DO NOTHING`)
// Worker-driven task queue indexes.
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_queued_due ON tasks (kind, not_before) WHERE state = 'queued'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_leased_expires ON tasks (lease_expires_at) WHERE state = 'leased'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_failed_retry_due ON tasks (not_before) WHERE state = 'failed_retry'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_dead_kind ON tasks (kind, account_id) WHERE state = 'dead'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS check_attempts_worker_finished ON check_attempts (worker_node_id, finished_at DESC)`)
err = DB().Exec("ALTER TABLE worker_nodes ALTER COLUMN concurrency SET DEFAULT 10").Error
if err != nil {
log.Println(err)
}
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET DEFAULT '{}'::jsonb").Error
if err != nil {
panic(err)
}
err = DB().Exec("UPDATE checks SET settings = '{}'::jsonb WHERE settings IS NULL").Error
if err != nil {
panic(err)
}
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET NOT NULL").Error
if err != nil {
panic(err)
}
// Inventory models (docs/plans/inventory-management.md §6, §10 M0).
// Enum types are created at Step 0 above so they exist before any
// AutoMigrate. The numeric labels of each enum value are taken
// verbatim from rstuff (`/data/int/rstuff/app/models/*.rb`) so a
// future sync layer does not need a value-mapping table — see
// docs/parity/rstuff-inventory.md §6.1.
// Servers — extend with the rstuff-shaped inventory fields. All
// statements are IF NOT EXISTS so existing rows keep working
// untouched (ext_id/token/price_cents default sensibly; meta gets
// an empty jsonb).
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS ext_id VARCHAR(64)`)
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_ext_id_unique ON servers (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS kind server_kind NOT NULL DEFAULT 'production'`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS token VARCHAR(64)`)
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_token_unique ON servers (token) WHERE token IS NOT NULL`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS price_cents INTEGER NOT NULL DEFAULT 0`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS comment TEXT`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS meta JSONB NOT NULL DEFAULT '{}'::jsonb`)
// WorkerNodes — optional server_id join for inventory correlation.
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS server_id BIGINT REFERENCES servers(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_server_idx ON worker_nodes (server_id)`)
// WorkerNodes — optional account_id FK marking a private (customer-
// operated) worker per docs/distributed/private-workers.md. NULL
// rows are platform-operated workers eligible to serve any account;
// non-NULL rows are pinned to a single account and are removed by
// HardDeleteAccount before the account row itself is dropped.
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_account_idx ON worker_nodes (account_id)`)
// Notification credentials are either platform-managed (account_id IS NULL)
// or owned by one account. Replace the legacy global name constraint with
// scope-aware unique indexes.
DB().Exec(`ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE CASCADE`)
DB().Exec(`CREATE INDEX IF NOT EXISTS notification_credentials_account_idx ON notification_credentials (account_id)`)
DB().Exec(`DROP INDEX IF EXISTS cred_kind_name`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_system_kind_name_unique ON notification_credentials (kind, name) WHERE account_id IS NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_account_kind_name_unique ON notification_credentials (account_id, kind, name) WHERE account_id IS NOT NULL`)
// Inventory entities. Order matters: server_ips before sites
// (FK), sites before deployments (FK), deployments before
// domains (FK). Audit columns (creator_id/updater_id) use the
// Audited mixin via concerns.Timestamped + Audited.
migrateOrIgnoreTypeExists(
&ServerIp{},
&Repo{},
&Site{},
&Deployment{},
&SiteRepo{},
&Domain{},
)
// Monitors is migrated earlier for historical FK ordering. Add the optional
// site reference only after sites exists on fresh databases.
DB().Exec(`ALTER TABLE monitors ADD COLUMN IF NOT EXISTS site_id BIGINT REFERENCES sites(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS monitors_site_idx ON monitors (site_id)`)
// Indexes — kept here so the AutoMigrate path stays the single
// source of truth. IF NOT EXISTS guards the rerun case.
DB().Exec(`CREATE INDEX IF NOT EXISTS server_ips_server_idx ON server_ips (server_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS server_ips_address_unique ON server_ips (server_id, address)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_account_idx ON sites (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_server_idx ON sites (server_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_ext_id_unique ON sites (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_account_slug_unique ON sites (account_id, slug)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_ext_id_unique ON repos (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_gitlab_id_unique ON repos (gitlab_id) WHERE gitlab_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_account_idx ON deployments (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_server_idx ON deployments (server_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_site_idx ON deployments (site_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_ext_id_unique ON deployments (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_account_idx ON domains (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_server_idx ON domains (server_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_site_idx ON domains (site_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS domains_name_unique ON domains (name)`)
// Dedupe by (server_id, config_path) for nginx sites and by
// (server_id, site_id, service_name) for compose services — matches
// the upsert keys in app/controllers/api/inventory_deploymentd.go.
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_nginx
ON deployments (server_id, config_path) WHERE mode = 'dedicated' AND config_path IS NOT NULL`)
DB().Exec(`DROP INDEX IF EXISTS deployments_unique_compose`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_compose
ON deployments (server_id, site_id, service_name) WHERE mode = 'compose' AND site_id IS NOT NULL AND service_name IS NOT NULL`)
if err := seedCanonicalPlansAndBackfill(false); err != nil {
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
}
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS plans_active_code_unique ON plans (code) WHERE archived = FALSE`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS subscriptions_provider_external_unique ON subscriptions (provider, provider_subscription_id) WHERE provider_subscription_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS subscriptions_account_status_idx ON subscriptions (account_id, status)`)
DB().Exec(`DO $$ BEGIN
ALTER TABLE plans ADD CONSTRAINT plans_price_nonnegative CHECK (price_monthly_minor >= 0 AND price_annual_minor >= 0);
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
DB().Exec(`DO $$ BEGIN
ALTER TABLE plans ADD CONSTRAINT plans_limits_valid CHECK (monitor_cap >= 0 AND interval_min_seconds >= 30);
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
// Seed default region. Use defaultRegionCode (declared in check_jobs.go)
// so the literal does not appear three times in the package.
localRegion := Region{}
DB().Where("code = ?", defaultRegionCode).First(&localRegion)
if localRegion.ID == 0 {
DB().Create(&Region{Code: defaultRegionCode, Name: "Local (default)", Enabled: true, Priority: 100})
}
DB().Exec(`
WITH t AS (
select u.id as user_id, i.encrypted_password as encrypted_password
from users as u
join identities as i on u.id = i.user_id
where u.encrypted_password is NULL
)
UPDATE users
SET encrypted_password = t.encrypted_password
from t
where users.id = t.user_id
`)
log.Println("migrated DB.")
}
func prepareCanonicalPlansTable() error {
if !DB().Migrator().HasTable("plans") || DB().Migrator().HasColumn("plans", "code") {
return nil
}
return DB().Transaction(func(tx *gorm.DB) error {
return tx.Exec(`ALTER TABLE plans RENAME TO plans_legacy`).Error
})
}
func planForeignKeyReferences(tx *gorm.DB, table, target string) (bool, error) {
var references bool
err := tx.Raw(`SELECT EXISTS (
SELECT 1 FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.contype = 'f' AND c.conrelid = to_regclass(?)
AND c.confrelid = to_regclass(?) AND a.attname = 'plan_id'
)`, table, target).Scan(&references).Error
return references, err
}
func seedCanonicalPlansAndBackfill(remapAccounts bool) error {
plans := []Plan{
{Code: "free", NameRU: "Бесплатный", NameEN: "Free", Currency: "RUB", MonitorCap: 50, IntervalMinSeconds: 300, StatusPagesCap: 1, MaintenanceCap: -1, LoginSeatsIncluded: 3, NotifySeatsIncluded: 0, Integrations: []string{"email", "telegram"}, CheckKinds: []string{"http", "ssl", "dns", "whois", "ping"}, DataRetentionMonths: 3, IsDefault: true},
{Code: "solo", NameRU: "Соло", NameEN: "Solo", PriceMonthlyMinor: 74900, PriceAnnualMinor: 64900, Currency: "RUB", MonitorCap: 10, IntervalMinSeconds: 60, StatusPagesCap: 3, MaintenanceCap: 5, LoginSeatsIncluded: 5, NotifySeatsIncluded: 3, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping"}, DataRetentionMonths: 12, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 60},
{Code: "team", NameRU: "Команда", NameEN: "Team", PriceMonthlyMinor: 299000, PriceAnnualMinor: 254900, Currency: "RUB", MonitorCap: 100, IntervalMinSeconds: 60, StatusPagesCap: 100, MaintenanceCap: 50, LoginSeatsIncluded: 5, NotifySeatsIncluded: 5, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 24, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 45},
{Code: "enterprise", NameRU: "Предприятие", NameEN: "Enterprise", PriceMonthlyMinor: 549000, PriceAnnualMinor: 464900, Currency: "RUB", MonitorCap: 200, IntervalMinSeconds: 30, StatusPagesCap: 0, MaintenanceCap: 0, LoginSeatsIncluded: 0, NotifySeatsIncluded: 0, UnlimitedSeats: true, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook", "sso_saml"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 36, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 30, SOC2: true, GDPRDPA: true},
}
return DB().Transaction(func(tx *gorm.DB) error {
for i := range plans {
var existing Plan
err := tx.Where("code = ?", plans[i].Code).First(&existing).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
if err := tx.Create(&plans[i]).Error; err != nil {
return err
}
} else if err != nil {
return err
}
}
// Catalog rows already existed before maintenance_cap was introduced.
// Update this new entitlement only; do not overwrite customer-adjusted
// prices or other existing plan attributes during a normal migration.
if err := tx.Exec(`UPDATE plans SET maintenance_cap = CASE code
WHEN 'free' THEN -1 WHEN 'solo' THEN 5 WHEN 'team' THEN 50 WHEN 'enterprise' THEN 0 ELSE maintenance_cap END
WHERE code IN ('free', 'solo', 'team', 'enterprise')`).Error; err != nil {
return err
}
hasLegacy := tx.Migrator().HasTable("plans_legacy")
hasMigratedLegacy := tx.Migrator().HasTable("plans_legacy_migrated")
if !hasLegacy && !hasMigratedLegacy {
return nil
}
hasAccounts := tx.Migrator().HasTable("accounts")
accountsNeedRemap := false
if remapAccounts && hasLegacy && hasAccounts {
referencesCanonical, err := planForeignKeyReferences(tx, "accounts", "plans")
if err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
return err
}
accountsNeedRemap = !referencesCanonical
if accountsNeedRemap {
if err := tx.Exec(`UPDATE accounts SET plan_id = CASE
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id)`).Error; err != nil {
return err
}
}
}
hasSubscriptions := tx.Migrator().HasTable("subscriptions")
subscriptionsNeedRemap := false
if remapAccounts && hasLegacy && hasSubscriptions {
referencesCanonical, err := planForeignKeyReferences(tx, "subscriptions", "plans")
if err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
return err
}
subscriptionsNeedRemap = !referencesCanonical
if subscriptionsNeedRemap {
if err := tx.Exec(`UPDATE subscriptions SET plan_id = CASE
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id)`).Error; err != nil {
return err
}
}
}
if remapAccounts && hasLegacy && (accountsNeedRemap || subscriptionsNeedRemap) && tx.Migrator().HasTable("subscription_events") {
if err := tx.Exec(`UPDATE subscription_events e SET from_plan_id = CASE
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
FROM plans_legacy l WHERE e.from_plan_id = l.id`).Error; err != nil {
return err
}
if err := tx.Exec(`UPDATE subscription_events e SET to_plan_id = CASE
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
FROM plans_legacy l WHERE e.to_plan_id = l.id`).Error; err != nil {
return err
}
}
if !remapAccounts && hasAccounts && hasSubscriptions {
now := time.Now().UTC()
if err := tx.Exec(`INSERT INTO subscriptions (account_id, plan_id, provider, status, billing_cycle, current_period_start, current_period_end, currency, amount_minor, metadata_json, created_at, updated_at)
SELECT a.id, a.plan_id, 'manual', 'active', 'monthly', ?, ?, p.currency, p.price_monthly_minor, '{}'::jsonb, ?, ?
FROM accounts a JOIN plans p ON p.id = a.plan_id
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.account_id = a.id)`, now, now.AddDate(0, 0, 30), now, now).Error; err != nil {
return err
}
}
if !remapAccounts || !hasLegacy {
return nil
}
if hasAccounts {
// AutoMigrate names this association fk_accounts_plan, while an older
// migration used accounts_plan_id_fkey. Either name may still point at
// plans_legacy after the table rename, so replace both deterministically.
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts ADD CONSTRAINT fk_accounts_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
return err
}
}
if hasSubscriptions {
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions ADD CONSTRAINT fk_subscriptions_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
return err
}
}
return tx.Exec("ALTER TABLE plans_legacy RENAME TO plans_legacy_migrated").Error
})
}

162
app/models/migrate_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,162 @@
package models
import (
"context"
"fmt"
"os"
"strings"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
func TestMigrateUpgradesLegacyPlansWithoutClosingLockConnection(t *testing.T) {
var databaseName string
require.NoError(t, DB().Raw("SELECT current_database()").Scan(&databaseName).Error)
require.Contains(t, databaseName, "test")
original := db
schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
require.NoError(t, original.Exec("CREATE SCHEMA "+schema).Error)
var isolatedSQLDB interface{ Close() error }
t.Cleanup(func() {
SetDB(original)
if isolatedSQLDB != nil {
_ = isolatedSQLDB.Close()
}
original.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE")
})
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable search_path=%s",
testDatabaseEnv("DATABASE_HOST", "POSTGRES_HOST", "localhost"),
testDatabaseEnv("DATABASE_PORT", "POSTGRES_PORT", "5432"),
testDatabaseEnv("DATABASE_USER", "POSTGRES_USER", "rsmon"),
testDatabaseEnv("DATABASE_PASSWORD", "POSTGRES_PASSWORD", "rsmon"),
databaseName,
schema,
)
isolated, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
require.NoError(t, err)
sqlDB, err := isolated.DB()
require.NoError(t, err)
isolatedSQLDB = sqlDB
RegisterCallbacks(isolated)
SetDB(isolated.Set("gorm:association_autoupdate", false))
require.NoError(t, DB().Exec(`CREATE TABLE plans (
id BIGSERIAL PRIMARY KEY,
name TEXT NOT NULL,
price BIGINT NOT NULL DEFAULT 0,
total_monitors BIGINT NOT NULL DEFAULT 0,
"default" BOOLEAN NOT NULL DEFAULT FALSE
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE accounts (
id BIGSERIAL PRIMARY KEY,
name TEXT NOT NULL,
plan_id BIGINT REFERENCES plans(id)
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE subscriptions (
id BIGSERIAL PRIMARY KEY,
account_id BIGINT NOT NULL REFERENCES accounts(id),
plan_id BIGINT NOT NULL REFERENCES plans(id),
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
status VARCHAR(24) NOT NULL DEFAULT 'active',
billing_cycle VARCHAR(8) NOT NULL DEFAULT 'monthly',
currency VARCHAR(3) NOT NULL DEFAULT 'RUB',
amount_minor BIGINT NOT NULL DEFAULT 0,
metadata_json JSONB NOT NULL DEFAULT '{}'
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE subscription_events (
id BIGSERIAL PRIMARY KEY,
subscription_id BIGINT NOT NULL,
account_id BIGINT NOT NULL,
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
kind VARCHAR(32) NOT NULL,
from_plan_id BIGINT,
to_plan_id BIGINT,
payload_json JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)`).Error)
require.NoError(t, DB().Exec(`INSERT INTO plans (id, name, price, total_monitors, "default")
VALUES (42, 'Legacy free', 0, 10, TRUE)`).Error)
require.NoError(t, DB().Exec("INSERT INTO accounts (id, name, plan_id) VALUES (7, 'Legacy account', 42)").Error)
require.NoError(t, DB().Exec("INSERT INTO subscriptions (id, account_id, plan_id) VALUES (9, 7, 42)").Error)
require.NoError(t, DB().Exec(`INSERT INTO subscription_events
(id, subscription_id, account_id, kind, from_plan_id, to_plan_id)
VALUES (11, 9, 7, 'legacy_change', 42, 42)`).Error)
Migrate()
require.True(t, DB().Migrator().HasColumn("plans", "code"))
require.False(t, DB().Migrator().HasTable("plans_legacy"))
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
var codes []string
require.NoError(t, DB().Table("plans").Order("code").Pluck("code", &codes).Error)
require.ElementsMatch(t, CanonicalPlanCodes(), codes)
var accountPlanCode string
require.NoError(t, DB().Table("accounts").Select("plans.code").
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
Scan(&accountPlanCode).Error)
require.Equal(t, "free", accountPlanCode)
var subscriptionPlanCode string
require.NoError(t, DB().Table("subscriptions").Select("plans.code").
Joins("JOIN plans ON plans.id = subscriptions.plan_id").Where("subscriptions.id = 9").
Scan(&subscriptionPlanCode).Error)
require.Equal(t, "free", subscriptionPlanCode)
var eventPlanCodes struct {
FromCode string
ToCode string
}
require.NoError(t, DB().Table("subscription_events e").
Select("fp.code AS from_code, tp.code AS to_code").
Joins("JOIN plans fp ON fp.id = e.from_plan_id").
Joins("JOIN plans tp ON tp.id = e.to_plan_id").
Where("e.id = 11").Scan(&eventPlanCodes).Error)
require.Equal(t, "free", eventPlanCodes.FromCode)
require.Equal(t, "free", eventPlanCodes.ToCode)
require.NoError(t, DB().Exec("SELECT 1").Error)
// The previous implementation retained this name after remapping. Its
// canonical FKs must prevent a retry from remapping the same rows again.
require.NoError(t, DB().Exec("ALTER TABLE plans_legacy_migrated RENAME TO plans_legacy").Error)
require.NotPanics(t, Migrate)
accountPlanCode = ""
require.NoError(t, DB().Table("accounts").Select("plans.code").
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
Scan(&accountPlanCode).Error)
require.Equal(t, "free", accountPlanCode)
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
}
func TestMigrationAdvisoryLockIsReleasedAfterPanic(t *testing.T) {
sqlDB, err := DB().DB()
require.NoError(t, err)
otherConn, err := sqlDB.Conn(context.Background())
require.NoError(t, err)
t.Cleanup(func() { _ = otherConn.Close() })
require.PanicsWithValue(t, "migration failed", func() {
withMigrationAdvisoryLock(func() { panic("migration failed") })
})
const migrateAdvisoryLock = int64(1234567890)
var acquired bool
require.NoError(t, otherConn.QueryRowContext(context.Background(),
"SELECT pg_try_advisory_lock($1)", migrateAdvisoryLock).Scan(&acquired))
require.True(t, acquired)
_, err = otherConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock)
require.NoError(t, err)
}
func testDatabaseEnv(primary, fallback, defaultValue string) string {
if value := os.Getenv(primary); value != "" {
return value
}
if value := os.Getenv(fallback); value != "" {
return value
}
return defaultValue
}

534
app/models/monitor.go Обычный файл
Просмотреть файл

@@ -0,0 +1,534 @@
package models
import (
"log"
"sync"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/lib/pq"
"github.com/pkg/errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
stateOK = "OK"
stateERR = "ERR"
stateWARN = "WARN"
stateFail = "FAIL"
stateDegraded = "DEGRADED"
// Event states.
stateEnded = "ended"
// Check column names used in map[string]interface{} GORM updates. Defining
// them as constants keeps GORM column references in sync with model fields.
colLastStart = "last_start"
colLastEnd = "last_end"
colState = "state"
colWarnings = "warnings"
colInfos = "infos"
// Check kinds. Used to avoid sprinkling magic strings across the codebase.
kindHTTP = "http"
kindSSL = "ssl"
kindSSH = "ssh"
kindFTP = "ftp"
kindDNS = "dns"
kindWhois = "whois"
kindRKN = "rkn"
kindBSSL = "bssl"
kindLLM = "llm"
kindLLMHTTP = "llm-http"
kindPing = "ping"
kindTCP = "tcp"
kindUDP = "udp"
)
// Monitor monitor
type Monitor struct {
concerns.Model
// activity status
Enabled bool `gorm:"not null;default:true" json:"enabled"`
// check state, OK - all green, ERR - some checks have failed, UNK - new or not run, FAIL - unable to check
State string `gorm:"not null;default:'UNK'" json:"state"`
ConfirmState string `gorm:"size:32;not null;default:'none';index" json:"confirm_state"`
ConfirmAt *time.Time `json:"confirm_at,omitempty"`
ConfirmedByWorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"confirmed_by_worker_id,omitempty"`
// Group ID
GroupID int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty" validate:"required"`
Group *Group `json:"group,omitempty"`
// Optional inventory Site join (see docs/plans/inventory-management.md §6.3).
// Lets the operator navigate monitor → site → deployments → server in one query.
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
// Tags for monitor grouping/searching
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
// PreferredRegions is the optional whitelist of region codes a distributed
// worker must be in to receive this monitor's checks. Empty/NULL means
// "no preference" — any worker can pick it up. Used by Phase 2 of
// docs/plans/worker-notifier-mvp.md (regional job routing); see
// app/models/check_jobs.go applyRegionRouting.
PreferredRegions pq.StringArray `gorm:"type:varchar(64)[]" json:"preferred_regions,omitempty"`
// RegionMode controls how PreferredRegions is interpreted by the
// distributed-worker job router. Defaults to "any" so monitors without
// explicit routing still match every worker — backwards-compatible with
// Phase 1 deployments.
// "any" — no region filter; legacy behavior (default)
// "specific" — only workers whose region_code is in PreferredRegions
// "all" — Phase 3 placeholder; today behaves like "any". The
// multi-region quorum aggregation is not implemented yet,
// see docs/todo.md Phase 3.
RegionMode string `gorm:"size:16;not null;default:'any'" json:"region_mode" validate:"omitempty,oneof=any specific all"`
// Monitor name
Name *string `json:"name,omitempty"`
// Host to monitor
Host string `json:"host" validate:"required"`
// UserID specify user for this monitor (info field)
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
User *User `json:"user,omitempty"`
// Comment (info field)
Comment *string `json:"comment"`
Checks []Check `json:"checks,omitempty"`
DNSRecords []DNSRecord `json:"-"`
StatsData `gorm:"-:all" sql:"-" json:"stats"`
concerns.Timestamped
Audited
}
// KINDS Check kinds
var KINDS = []string{kindHTTP, kindSSL, kindSSH, kindFTP, kindDNS, kindWhois, kindRKN, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
// ValidCheckKind is the single canonical allow-list for user supplied check
// kinds. Keep it beside the kind constants so every transport validates the
// same set before a check reaches a worker.
func ValidCheckKind(kind string) bool {
for _, candidate := range KINDS {
if kind == candidate {
return true
}
}
return false
}
// Region routing mode constants used by Monitor.RegionMode and
// app/models/check_jobs.go applyRegionRouting. Centralized so the literal
// values are not sprinkled through the codebase.
const (
// RegionModeAny keeps the legacy behavior: every worker is eligible,
// PreferredRegions is ignored. Default for newly-created monitors.
RegionModeAny = "any"
// RegionModeSpecific limits eligible workers to those whose RegionCode
// is contained in PreferredRegions. Empty PreferredRegions falls back to
// RegionModeAny so the field is safe to leave blank in the UI.
RegionModeSpecific = "specific"
// RegionModeAll is the Phase 3 placeholder: a monitor pinned to all of
// its preferred regions for quorum aggregation. Phase 2 treats it as
// RegionModeAny and logs a TODO marker so it is easy to grep for.
RegionModeAll = "all"
)
// RegionCodesFromSlice is a convenience wrapper so callers (mainly tests
// and HTTP handlers) can pass a plain []string and get the pq.StringArray
// type the model expects. nil/empty input is preserved as a nil slice so
// the GORM column writes a SQL NULL instead of an empty array, matching
// the column default.
func RegionCodesFromSlice(in []string) pq.StringArray {
if len(in) == 0 {
return nil
}
out := make(pq.StringArray, len(in))
copy(out, in)
return out
}
// Int64ArrayFromSlice mirrors RegionCodesFromSlice for bigint[] columns
// such as status_pages.monitor_ids and status_page_maintenance.monitor_ids.
// The GORM pq.Int64Array driver expects a non-nil slice for ordered
// inserts; callers that always have a non-empty list (the dashboard list
// filter, the maintenance form) can rely on this to write a stable shape.
func Int64ArrayFromSlice(in []int64) pq.Int64Array {
if len(in) == 0 {
return pq.Int64Array{}
}
out := make(pq.Int64Array, len(in))
copy(out, in)
return out
}
// ValidateRegionMode returns an error when RegionMode is not one of the
// documented values ("any", "specific", "all"). Empty strings are treated as
// "any" for backwards compatibility with monitors persisted before the field
// existed; the DB column also defaults to "any".
func (m *Monitor) ValidateRegionMode() error {
switch m.RegionMode {
case "", RegionModeAny, RegionModeSpecific, RegionModeAll:
return nil
default:
return errors.Errorf("invalid region_mode %q (expected any|specific|all)", m.RegionMode)
}
}
// WantsRegion returns true when the monitor should be routed to a worker
// operating in the given region code. Callers use this in
// app/models/check_jobs.go to filter the eligible worker pool per check.
//
// - RegionModeAny: always true (no preference).
// - RegionModeAll (Phase 3 placeholder): behaves like Any today; returns
// true unconditionally so every region sees the check.
// - RegionModeSpecific: true when code is contained in PreferredRegions,
// or when PreferredRegions is empty (fall-back to Any).
func (m *Monitor) WantsRegion(code string) bool {
switch m.RegionMode {
case RegionModeSpecific:
if len(m.PreferredRegions) == 0 {
return true
}
for _, r := range m.PreferredRegions {
if r == code {
return true
}
}
return false
case RegionModeAll:
// TODO(phase3): enumerate PreferredRegions and emit one assignment
// per region so the result aggregator can do quorum. Today we
// behave like Any so existing workers keep getting checks.
return true
default:
return true
}
}
// GetLabel provides functionality.
func (m *Monitor) GetLabel() string {
if m.Name != nil {
return *m.Name
}
return m.Host
}
// ProcessChecks provides functionality.
func (m *Monitor) ProcessChecks(tx *gorm.DB) error {
log.Println("process checks")
checks := make([]Check, 0)
for _, c := range m.Checks { //nolint:gocritic // range copy is acceptable here
log.Println("maybe delete check", c.ID, c.Deleted, c.IsNew)
if c.Deleted {
if !c.IsNew {
log.Println("delete check", c.ID)
err := tx.Exec("delete from event_checks where check_id = ?", c.ID).Error
if err != nil {
return err
}
// First, find all message IDs for this check
var messageIDs []int64
err = tx.Model(&Message{}).Where("check_id = ?", c.ID).Pluck("id", &messageIDs).Error
if err != nil {
return err
}
// Delete event_messages (join table) first to avoid FK constraint violation
if len(messageIDs) > 0 {
err = tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error
if err != nil {
return err
}
}
// Now delete the messages
err = tx.Where("check_id = ?", c.ID).Delete(Message{}).Error
if err != nil {
return err
}
err = tx.Where("id = ? AND monitor_id = ?", c.ID, m.ID).Delete(Check{}).Error
if err != nil {
return err
}
}
continue
}
if c.IsNew {
c.ID = 0
}
err := c.ValidateSettings()
if err != nil {
return errors.Wrap(err, "check validation error")
}
checks = append(checks, c)
}
m.Checks = checks
return nil
}
// ActiveEvent provides functionality.
func (m *Monitor) ActiveEvent() Event {
evt := Event{}
DB().Where("monitor_id = ? AND state != 'old'", m.ID).First(&evt)
if evt.ID != 0 {
evt.MonitorID = m.ID
t := time.Now()
evt.StartTime = &t
}
return evt
}
var mutex sync.Mutex
// checkSeverityRank assigns an ordinal to each check state so the monitor
// aggregator can pick the highest-severity child deterministically.
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
// real check state takes precedence over them.
func checkSeverityRank(state string) int {
switch state {
case stateFail:
return 5
case stateERR:
return 4
case stateDegraded:
return 3
case stateWARN:
return 2
case stateOK:
return 1
default:
return 0
}
}
// UpdateStatusFromChecks updates the monitor status based on its checks.
func (m *Monitor) UpdateStatusFromChecks() {
mutex.Lock()
checks := make([]Check, 0)
tx := DB().Begin()
var locked Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&locked, m.ID).Error; err != nil {
tx.Rollback()
mutex.Unlock()
log.Println("UpdateStatusFromChecks lock monitor", err)
return
}
m.State = locked.State
_ = tx.Model(m).Association("Checks").Find(&checks)
prevState := m.State
m.State = stateOK
// Pick the highest-severity enabled check. The previous implementation
// inlined three if-statements with non-obvious precedence (a WARN that
// appeared AFTER an ERR in the iteration would never downgrade back,
// but a FAIL after ERR would silently get clobbered). Using a single
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
// independent of slice ordering — the same rule Phase 3 introduces
// for DEGRADED, applied uniformly to the existing states too.
bestRank := checkSeverityRank(stateOK)
bestState := stateOK
hasChecks := false
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
if check.Enabled == nil || !*check.Enabled {
continue
}
hasChecks = true
if r := checkSeverityRank(check.State); r > bestRank {
bestRank = r
bestState = check.State
}
}
m.State = bestState
if !hasChecks {
m.State = stateWARN
}
if m.State != prevState {
err := tx.Model(&m).UpdateColumn("state", m.State).Error
if err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks fail update state", err)
mutex.Unlock()
return
}
}
evt := Event{}
if err := tx.Where("monitor_id = ? AND state = ?", m.ID, "current").First(&evt).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
tx.Rollback()
mutex.Unlock()
log.Println("UpdateStatusFromChecks active event", err)
return
}
// log.Println("state", m.State, "active event:", evt.ID)
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
if check.State == stateERR || check.State == stateFail {
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
evt.Checks = append(evt.Checks, check)
if check.Error != nil {
evt.Reason = *check.Error
} else {
evt.Reason = "unknown error"
}
}
}
switch m.State {
case stateOK, stateWARN, stateDegraded:
// DEGRADED is treated like OK/WARN for the event lifecycle: we do
// NOT open a new "current" outage event for a partial regional
// failure. Operators see DEGRADED on the check detail page and
// the monitor list, but the existing notifier pipeline (down /
// restore events) only fires for full ERR/FAIL. A future
// improvement can add a separate "degraded" message kind.
if evt.ID != 0 {
upd := map[string]interface{}{
"duration": time.Since(*evt.StartTime).Seconds(),
"state": stateEnded,
"oks": evt.Oks + 1,
}
if evt.EndTime == nil {
upd["end_time"] = time.Now()
}
err := tx.Model(&evt).UpdateColumns(upd).Error
if err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks fail update to ended", err)
mutex.Unlock()
return
}
}
case stateERR, stateFail:
if evt.ID == 0 {
tn := time.Now()
evt.StartTime = &tn
evt.Duration = 0
evt.State = "current"
evt.MonitorID = m.ID
err := tx.Save(&evt).Error
if err != nil {
tx.Rollback()
spew.Dump(evt)
log.Println("UpdateStatusFromChecks fail create", err)
mutex.Unlock()
return
}
} else {
upd := map[string]interface{}{
"end_time": nil,
"state": "current",
"errors": evt.Errors + 1,
}
if evt.StartTime == nil {
upd["start_time"] = time.Now()
upd["duration"] = 0
} else {
upd["duration"] = time.Since(*evt.StartTime).Seconds()
}
err := tx.Model(&evt).UpdateColumns(upd).Error
if err != nil {
tx.Rollback()
spew.Dump(evt)
spew.Dump(upd)
log.Println("UpdateStatusFromChecks fail update to current", err)
mutex.Unlock()
return
}
}
}
if err := m.syncStatusPageIncidentsTx(tx, &evt); err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks status page incident", err)
mutex.Unlock()
return
}
err := tx.Commit().Error
mutex.Unlock()
if err != nil {
log.Println("UpdateStatusFromChecks commit fail", err)
return
}
if m.State != prevState {
m.invalidateStatusPages()
}
}
func (m *Monitor) invalidateStatusPages() {
var ids []int64
if err := DB().Model(&StatusPage{}).Where("? = ANY(monitor_ids)", m.ID).Pluck("id", &ids).Error; err != nil {
return
}
for _, id := range ids {
InvalidateStatusPageCache(id)
}
}
func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
if event == nil || event.ID == 0 {
return nil
}
var pages []StatusPage
if err := tx.Where("auto_open_incidents = TRUE AND ? = ANY(monitor_ids)", m.ID).Find(&pages).Error; err != nil {
return err
}
for i := range pages {
page := &pages[i]
var incident StatusPageIncident
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
if m.State == stateERR || m.State == "FAIL" {
if err != nil {
// The database uniqueness constraint makes concurrent state updates idempotent.
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
if err := tx.Create(&incident).Error; err != nil {
return err
}
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "opened"); err != nil {
return err
}
} else if incident.BodyMD != event.Reason {
if err := tx.Model(&incident).Update("body_md", event.Reason).Error; err != nil {
return err
}
incident.BodyMD, incident.UpdatedAt = event.Reason, time.Now()
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "updated"); err != nil {
return err
}
}
} else if err == nil && incident.ResolvedAt == nil {
now := time.Now()
if err := tx.Model(&incident).Update("resolved_at", now).Error; err != nil {
return err
}
incident.ResolvedAt, incident.UpdatedAt = &now, now
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "resolved"); err != nil {
return err
}
}
}
return nil
}

215
app/models/monitor_state_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,215 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// monitorStateTestWorld bundles the account/group/monitor fixture the
// monitor-state tests need. It deliberately re-creates the row in each
// test rather than sharing, because Monitor.UpdateStatusFromChecks
// mutates the row in place and the per-test assertions need a clean
// baseline.
type monitorStateTestWorld struct {
plan models.Plan
account models.Account
group models.Group
monitor models.Monitor
}
// seedMonitorStateWorld provisions one plan/account/group/monitor with
// the requested initial state. The monitor is enabled so
// UpdateStatusFromChecks treats its checks as live.
func seedMonitorStateWorld(t *testing.T, initialState string) monitorStateTestWorld {
t.Helper()
models.Drop()
models.Migrate()
plan := models.Plan{Name: "ms-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
account := models.Account{Name: "ms-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&account).Error)
group := models.Group{AccountID: account.ID, Name: "ms"}
require.NoError(t, models.DB().Create(&group).Error)
mon := models.Monitor{
Name: stringPtrAgg("mon.test"),
Host: "mon.test",
GroupID: group.ID,
Enabled: true,
State: initialState,
}
require.NoError(t, models.DB().Create(&mon).Error)
return monitorStateTestWorld{
plan: plan,
account: account,
group: group,
monitor: mon,
}
}
// attachCheck creates an enabled check on the monitor with the given
// state. Returns the persisted check so the test can re-load it.
func attachCheck(t *testing.T, monitorID int64, kind string, state string) models.Check {
t.Helper()
enTrue := true
c := models.Check{
MonitorID: monitorID,
Kind: kind,
Interval: 60,
Enabled: &enTrue,
State: state,
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&c).Error)
return c
}
// reloadMonitor pulls the latest monitor state from the DB so the test
// can compare against the post-UpdateStatusFromChecks row.
func reloadMonitor(t *testing.T, id int64) models.Monitor {
t.Helper()
var m models.Monitor
require.NoError(t, models.DB().First(&m, id).Error)
return m
}
// ---------------------------------------------------------------------------
// Phase 3 severity rules: FAIL > ERR > DEGRADED > WARN > OK.
// ---------------------------------------------------------------------------
// TestMonitorStateFromChecks_Degraded: monitor with 1 OK + 1 DEGRADED
// child → monitor.State = DEGRADED. The new severity rank must
// correctly promote DEGRADED above OK.
func TestMonitorStateFromChecks_Degraded(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "DEGRADED", got.State,
"DEGRADED child must promote monitor above OK")
}
// TestMonitorStateFromChecks_DegradedWithError: monitor with 1 ERR +
// 1 DEGRADED → monitor.State = ERR. ERR beats DEGRADED in the
// severity order.
func TestMonitorStateFromChecks_DegradedWithError(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "ERR")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "ERR", got.State,
"ERR must beat DEGRADED — the order is FAIL > ERR > DEGRADED > WARN > OK")
}
// TestMonitorStateFromChecks_DegradedOnlyOK covers the single-DEGRADED
// case explicitly so a regression that treats DEGRADED as "WARN-ish"
// would flip this assertion.
func TestMonitorStateFromChecks_DegradedOnlyOK(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "DEGRADED", got.State,
"mixed OK+DEGRADED monitor must be DEGRADED, not OK")
}
// TestMonitorStateFromChecks_SeverityOrderingTable is a table-driven
// sweep of the FAIL > ERR > DEGRADED > WARN > OK ladder. The case set
// is intentionally small — every pair that could reveal a wrong
// winner under the new severity rank. Keeping it table-driven makes
// it trivial to add more cases if a future state is introduced.
func TestMonitorStateFromChecks_SeverityOrderingTable(t *testing.T) {
cases := []struct {
name string
checks []string
wantMon string
}{
{"all_ok", []string{"OK", "OK"}, "OK"},
{"all_warn", []string{"WARN", "WARN"}, "WARN"},
{"ok_with_warn", []string{"OK", "WARN"}, "WARN"},
{"warn_with_ok", []string{"WARN", "OK"}, "WARN"}, // order independence
{"ok_with_degraded", []string{"OK", "DEGRADED"}, "DEGRADED"},
{"warn_with_degraded", []string{"WARN", "DEGRADED"}, "DEGRADED"},
{"degraded_with_warn", []string{"DEGRADED", "WARN"}, "DEGRADED"},
{"err_with_degraded", []string{"ERR", "DEGRADED"}, "ERR"},
{"degraded_with_err", []string{"DEGRADED", "ERR"}, "ERR"},
{"fail_with_err", []string{"FAIL", "ERR"}, "FAIL"},
{"err_with_fail", []string{"ERR", "FAIL"}, "FAIL"}, // order independence
{"fail_alone", []string{"FAIL"}, "FAIL"},
{"all_degraded", []string{"DEGRADED", "DEGRADED"}, "DEGRADED"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
for i, st := range c.checks {
attachCheck(t, world.monitor.ID,
"http-"+string(rune('a'+i)), st)
}
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, c.wantMon, got.State,
"severity winner for child states %v must be %s", c.checks, c.wantMon)
})
}
}
// TestMonitorStateFromChecks_NoChecksIsWarn pins the existing "no
// enabled checks → WARN" behavior — the new severity rank must not
// accidentally produce OK for an empty monitor.
func TestMonitorStateFromChecks_NoChecksIsWarn(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "WARN", got.State,
"an enabled monitor with zero checks must remain WARN")
}
// TestMonitorStateFromChecks_DisabledCheckIgnored verifies that a
// disabled check is not folded into the severity decision. Otherwise
// a stuck-in-ERR check that has been disabled would keep tripping the
// monitor forever.
func TestMonitorStateFromChecks_DisabledCheckIgnored(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
enFalse := false
disabled := models.Check{
MonitorID: world.monitor.ID,
Kind: "http-disabled",
Interval: 60,
Enabled: &enFalse,
State: "ERR",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&disabled).Error)
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "OK", got.State,
"disabled check must be ignored — only the enabled OK check counts")
}
// guard against time import being pruned by an editor when individual
// test bodies stop referencing it directly.
var _ = time.Second

94
app/models/monitor_transfer_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,94 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestMonitorTransferGroupSwap verifies that swapping a monitor's
// group_id between two accounts correctly re-homes the monitor without
// touching any other monitor data. This is the database primitive that
// POST /api/v1/monitors/:id/transfer relies on.
func TestMonitorTransferGroupSwap(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
// Two accounts, each with their own default group.
accA := models.Account{Name: "A", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&accA).Error)
accB := models.Account{Name: "B", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&accB).Error)
groupA := models.Group{AccountID: accA.ID, Name: "A-default"}
require.NoError(t, models.DB().Create(&groupA).Error)
groupB := models.Group{AccountID: accB.ID, Name: "B-default"}
require.NoError(t, models.DB().Create(&groupB).Error)
// Monitor lives in account A with one HTTP check.
monitor := models.Monitor{
GroupID: groupA.ID,
Host: "example-a.test",
}
require.NoError(t, models.DB().Create(&monitor).Error)
check := models.Check{
MonitorID: monitor.ID,
Kind: "http",
URL: ptrString("https://example-a.test/"),
Interval: 300,
Settings: datatypes.JSON([]byte("{}")),
}
require.NoError(t, models.DB().Create(&check).Error)
// Simulate the controller-side update.
require.NoError(t, models.DB().
Model(&models.Monitor{}).
Where("id = ?", monitor.ID).
Update("group_id", groupB.ID).Error)
// Monitor now lives in B; check follows by FK on monitor_id.
var reloaded models.Monitor
require.NoError(t, models.DB().Preload("Group").First(&reloaded, monitor.ID).Error)
assert.Equal(t, groupB.ID, reloaded.GroupID, "monitor group_id should now point at account B's group")
assert.Equal(t, accB.ID, reloaded.Group.AccountID, "preloaded group should belong to account B")
var checkCount int64
require.NoError(t, models.DB().Model(&models.Check{}).
Where("monitor_id = ?", monitor.ID).Count(&checkCount).Error)
assert.Equal(t, int64(1), checkCount, "check rows must follow the monitor across the move")
}
// TestMonitorTransferSameAccountGuard documents the early-return path: the
// controller must not silently no-op when the caller picks the current
// account, and the test exercises that the DB stays untouched.
func TestMonitorTransferSameAccountGuard(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "only", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "only"}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: "x.test"}
require.NoError(t, models.DB().Create(&monitor).Error)
// No update is issued because the controller rejects same-account moves
// before the SQL UPDATE. Verify the row is unchanged.
var reloaded models.Monitor
require.NoError(t, models.DB().First(&reloaded, monitor.ID).Error)
assert.Equal(t, group.ID, reloaded.GroupID)
}
func ptrString(s string) *string { return &s }

528
app/models/network_diagnostics.go Обычный файл
Просмотреть файл

@@ -0,0 +1,528 @@
package models
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/influx"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
const (
ConfirmStateNone = "none"
ConfirmStatePending = "pending"
ConfirmStateConfirmed = "confirmed" // A different worker reproduced the failure.
ConfirmStateTimeout = "confirmed_by_timeout"
AttemptKindRegular = "regular"
AttemptKindConfirm = "confirmation"
AttemptStateQueued = "queued"
AttemptStateLeased = "leased"
AttemptStateFinished = "finished"
diagnosticSoft = "soft"
diagnosticHard = "hard"
diagnosticRecovery = "recovery"
)
type diagnosticSettings struct {
confirmTimeout time.Duration
healthWindow time.Duration
healthRate float64
healthMin int64
}
func settingsForAccount(account *Account) diagnosticSettings {
settings := diagnosticSettings{confirmTimeout: 90 * time.Second, healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
if account == nil || account.Plan == nil {
return settings
}
plan := account.Plan
if plan.ConfirmTimeoutSec > 0 {
settings.confirmTimeout = time.Duration(plan.ConfirmTimeoutSec) * time.Second
}
if plan.HealthWindowSec > 0 {
settings.healthWindow = time.Duration(plan.HealthWindowSec) * time.Second
}
if plan.HealthRateThreshold > 0 && plan.HealthRateThreshold <= 1 {
settings.healthRate = plan.HealthRateThreshold
}
if plan.HealthMinAttempts > 0 {
settings.healthMin = int64(plan.HealthMinAttempts)
}
if !plan.Confirmations {
return settings
}
if account.ConfirmTimeoutSec != nil && *account.ConfirmTimeoutSec >= 15 {
settings.confirmTimeout = time.Duration(*account.ConfirmTimeoutSec) * time.Second
}
if account.HealthWindowSec != nil && *account.HealthWindowSec >= 60 {
settings.healthWindow = time.Duration(*account.HealthWindowSec) * time.Second
}
if account.HealthRateThreshold != nil && *account.HealthRateThreshold > 0 && *account.HealthRateThreshold <= 1 {
settings.healthRate = *account.HealthRateThreshold
}
if account.HealthMinAttempts != nil && *account.HealthMinAttempts > 0 {
settings.healthMin = int64(*account.HealthMinAttempts)
}
return settings
}
// CheckAttempt is the durable worker-attribution record. Unlike check state,
// it is append-only and therefore remains useful after a worker is deweighted.
type CheckAttempt struct {
concerns.Model
JobID string `gorm:"uniqueIndex;size:64;not null" json:"job_id"`
CheckID int64 `gorm:"index;not null" json:"check_id"`
MonitorID int64 `gorm:"index;not null" json:"monitor_id"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
WorkerNode *WorkerNode `json:"worker_node,omitempty"`
SourceWorkerNodeID *int64 `gorm:"index" json:"source_worker_node_id,omitempty"`
Kind string `gorm:"size:32;not null" json:"kind"`
State string `gorm:"size:32;not null" json:"state"`
ResultState string `gorm:"size:16" json:"result_state"`
Result datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"result"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
LeaseToken string `gorm:"size:64" json:"-"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
Deweighted bool `gorm:"not null;default:false" json:"deweighted"`
concerns.Timestamped
}
// DiagnosticAuditEvent is a compact, queryable control-plane audit record.
type DiagnosticAuditEvent struct {
concerns.Model
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
Kind string `gorm:"size:64;index;not null" json:"kind"`
Metadata datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"metadata"`
concerns.Timestamped
}
func auditDiagnostic(tx *gorm.DB, kind string, monitorID, workerID *int64, metadata map[string]interface{}) {
b, _ := json.Marshal(metadata)
_ = tx.Create(&DiagnosticAuditEvent{MonitorID: monitorID, WorkerNodeID: workerID, Kind: kind, Metadata: b}).Error
}
// AuditNetworkRecovery is used by the admin incident-response endpoint.
func AuditNetworkRecovery(tx *gorm.DB, workerID int64) {
auditDiagnostic(tx, "worker.network_problem_force_recover", nil, &workerID, nil)
}
// enqueueDiagnosticDelivery creates the message and its durable delivery task in
// the transition transaction. A transaction advisory lock prevents concurrent
// result frames from leaving duplicate messages when the task dedupe wins.
func enqueueDiagnosticDelivery(tx *gorm.DB, monitor *Monitor, account *Account, tier string, now time.Time) error {
if account == nil || account.Plan == nil {
return nil
}
if tier == diagnosticHard && !account.Plan.AllowHardAlerts {
return nil
}
var notifications []Notification
if err := tx.Joins("JOIN notification_groups ON notification_groups.notification_id = notifications.id").
Where("notifications.account_id = ? AND notifications.enabled AND notification_groups.group_id = ?", account.ID, monitor.GroupID).
Preload("Contacts", "enabled = ?", true).Find(&notifications).Error; err != nil {
return err
}
for i := range notifications {
for j := range notifications[i].Contacts {
contact := notifications[i].Contacts[j]
method := diagnosticContactMethod(contact.Kind)
if method == "" || (tier == diagnosticSoft && method != "email" && method != "telegram") {
continue
}
key := fmt.Sprintf("diagnostic:%d:%s:%d:%d", monitor.ID, tier, notifications[i].ID, contact.ID)
if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", key).Error; err != nil {
return err
}
var existing Task
if err := tx.Where("idempotency_key = ?", key).First(&existing).Error; err == nil {
continue
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
message := Message{NotificationID: notifications[i].ID, ContactID: contact.ID, Kind: "diagnostic_" + tier, State: TaskStateQueued}
if err := tx.Create(&message).Error; err != nil {
return err
}
monitorID, messageID := monitor.ID, message.ID
payload, err := json.Marshal(wire.NotificationTask{AccountID: account.ID, MessageID: messageID, NotificationID: notifications[i].ID, MonitorID: &monitorID, Method: method, Contact: wire.NotificationContact{ID: contact.ID, Kind: contact.Kind, Value: contact.Value, Name: contact.Name}, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind})
if err != nil {
return err
}
if _, err = EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notifications[i].ID, ContactID: contact.ID, MessageID: &messageID, MonitorID: &monitorID, Method: method, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind, NotBefore: now, Payload: payload, IdempotencyKey: key}); err != nil {
return err
}
}
}
return nil
}
func diagnosticContactMethod(kind string) string {
switch kind {
case "email":
return "email"
case "telegram_private", "telegram_group":
return "telegram"
case "webhook", "mattermost", "sms", "voice":
return kind
}
return ""
}
func diagnosticSubject(monitor *Monitor, tier string) string {
return fmt.Sprintf("Monitor %s: %s", monitor.Host, tier)
}
func diagnosticBody(monitor *Monitor, tier string) string {
return fmt.Sprintf("Network diagnostic %s for monitor %s.", tier, monitor.Host)
}
// ConfirmationJobsForWorker atomically leases confirmation jobs assigned to this
// worker or left unassigned by an expired lease. Unassigned attempts still retain
// SourceWorkerNodeID, so the original failing worker can never claim them.
func ConfirmationJobsForWorker(worker *WorkerNode, kinds []string, limit int) ([]wire.CheckJob, error) {
if worker == nil || worker.AccountID != nil || !worker.SupportsTaskEnvelope() || worker.NetworkProblemActive(time.Now()) || limit < 1 {
return nil, nil
}
var jobs []wire.CheckJob
err := DB().Transaction(func(tx *gorm.DB) error {
var attempts []CheckAttempt
if err := tx.Clauses(SkipLockedClause).Where("(worker_node_id = ? OR worker_node_id IS NULL) AND kind = ? AND state = ?", worker.ID, AttemptKindConfirm, AttemptStateQueued).Order("id").Limit(limit).Find(&attempts).Error; err != nil {
return err
}
for i := range attempts {
var check Check
if err := tx.Preload("Monitor").First(&check, attempts[i].CheckID).Error; err != nil {
continue
}
if check.Monitor == nil || !check.Monitor.Enabled {
continue
}
if attempts[i].SourceWorkerNodeID != nil && *attempts[i].SourceWorkerNodeID == worker.ID {
continue
}
if !containsString(kinds, check.Kind) || !containsString(worker.CheckTypes(), check.Kind) {
continue
}
now := time.Now()
leaseToken := uuid.NewString()
leaseUntil := now.Add(DefaultTaskLeaseTTL)
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateLeased, "started_at": now, "lease_token": leaseToken, "lease_expires_at": leaseUntil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
return err
}
jobs = append(jobs, wire.CheckJob{JobID: attempts[i].JobID, LeaseToken: leaseToken, CheckID: check.ID, MonitorID: check.MonitorID, Kind: check.Kind, Host: check.Monitor.Host, URL: check.URL, Interval: check.Interval, Settings: json.RawMessage(check.Settings)})
}
return nil
})
return jobs, err
}
// StartConfirmation creates exactly one targeted confirmation for a new outage.
func StartConfirmation(checkID, sourceWorkerID int64, now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
return StartConfirmationTx(tx, checkID, sourceWorkerID, now)
})
}
// StartConfirmationTx is StartConfirmation's transaction-aware form.
func StartConfirmationTx(tx *gorm.DB, checkID, sourceWorkerID int64, now time.Time) error {
if tx == nil {
return errors.New("start confirmation: nil transaction")
}
{
var check Check
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
return err
}
// Confirmations are a paid distributed-check entitlement. Free accounts
// retain the legacy direct soft alert path and do not consume worker budget.
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil || check.Monitor.Group.Account.Plan == nil || !check.Monitor.Group.Account.Plan.Confirmations {
return nil
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
return err
}
if monitor.ConfirmState == ConfirmStatePending || monitor.ConfirmState == ConfirmStateConfirmed {
return nil
}
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, 0, now)
if err != nil {
monitor.ConfirmState, monitor.ConfirmAt = ConfirmStateTimeout, &now
auditDiagnostic(tx, "check.confirm_unavailable", &monitor.ID, &sourceWorkerID, nil)
if err := tx.Save(&monitor).Error; err != nil {
return err
}
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticHard, now)
}
monitor.ConfirmState, monitor.ConfirmAt, monitor.ConfirmedByWorkerID = ConfirmStatePending, &now, &worker.ID
if err := tx.Save(&monitor).Error; err != nil {
return err
}
if err := enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticSoft, now); err != nil {
return err
}
attempt := CheckAttempt{JobID: uuid.NewString(), CheckID: check.ID, MonitorID: monitor.ID, WorkerNodeID: &worker.ID, SourceWorkerNodeID: &sourceWorkerID, Kind: AttemptKindConfirm, State: AttemptStateQueued}
if err := tx.Create(&attempt).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_assign", &monitor.ID, &worker.ID, map[string]interface{}{"exclude_worker_id": sourceWorkerID, "job_id": attempt.JobID})
return nil
}
}
// confirmationWorkerTx selects an independent active platform worker that can
// execute this exact kind. Capability filtering is deliberately performed in
// Go because the JSON capability format also supports legacy rows safely.
func confirmationWorkerTx(tx *gorm.DB, checkKind string, sourceWorkerID, excludeWorkerID int64, now time.Time) (*WorkerNode, error) {
var workers []WorkerNode
if err := tx.Where("id <> ? AND id <> ? AND account_id IS NULL AND status = 'active' AND (network_problems = FALSE OR network_problems_until <= ? OR network_problems_until IS NULL)", sourceWorkerID, excludeWorkerID, now).Order("id").Find(&workers).Error; err != nil {
return nil, err
}
for i := range workers {
if workers[i].SupportsTaskEnvelope() && containsString(workers[i].CheckTypes(), checkKind) {
return &workers[i], nil
}
}
return nil, gorm.ErrRecordNotFound
}
// ApplyDiagnosticResult resolves a targeted attempt once. Duplicate reports are ignored.
func ApplyDiagnosticResult(report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
if report.JobID == "" || worker == nil {
return false, nil
}
handled := true
err := DB().Transaction(func(tx *gorm.DB) error {
return ApplyDiagnosticResultTx(tx, report, worker, now, &handled)
})
return handled, err
}
// ApplyDiagnosticResultTx resolves a diagnostic attempt within the caller's
// transaction. handled distinguishes a normal check result from a diagnostic.
func ApplyDiagnosticResultTx(tx *gorm.DB, report wire.CheckResultReport, worker *WorkerNode, now time.Time, handled *bool) error {
if tx == nil {
return errors.New("apply diagnostic: nil transaction")
}
if handled == nil {
return errors.New("apply diagnostic: nil handled result")
}
*handled = true
{
var attempt CheckAttempt
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("job_id = ?", report.JobID).First(&attempt).Error; err != nil {
if err == gorm.ErrRecordNotFound {
*handled = false
return nil
}
return err
}
if attempt.Kind != AttemptKindConfirm {
*handled = false
return nil
}
if attempt.State == AttemptStateFinished {
*handled = false
return nil
}
if attempt.State != AttemptStateLeased || attempt.WorkerNodeID == nil || *attempt.WorkerNodeID != worker.ID {
return gorm.ErrRecordNotFound
}
if attempt.LeaseToken == "" || report.LeaseToken == "" || report.LeaseToken != attempt.LeaseToken || attempt.LeaseExpiresAt == nil || !attempt.LeaseExpiresAt.After(now) {
return errors.New("apply diagnostic: lease token is invalid or expired")
}
payload, _ := json.Marshal(report)
deweighted := worker.NetworkProblemActive(now)
if err := tx.Model(&attempt).Where("state = ? AND lease_token = ? AND lease_expires_at > ?", AttemptStateLeased, report.LeaseToken, now).Updates(map[string]interface{}{"state": AttemptStateFinished, "result_state": report.State, "result": payload, "finished_at": now, "lease_token": "", "lease_expires_at": nil, "deweighted": deweighted}).Error; err != nil {
return err
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, attempt.MonitorID).Error; err != nil {
return err
}
if attempt.Kind == AttemptKindConfirm && monitor.ConfirmState == ConfirmStatePending {
if report.State == stateERR || report.State == stateFail {
monitor.ConfirmState = ConfirmStateConfirmed
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticHard, now); err != nil {
return err
}
} else {
monitor.ConfirmState = ConfirmStateNone
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticRecovery, now); err != nil {
return err
}
}
if err := tx.Save(&monitor).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_result", &monitor.ID, &worker.ID, map[string]interface{}{"state": report.State, "deweighted": deweighted})
}
return nil
}
}
func monitorAccount(tx *gorm.DB, monitor *Monitor) *Account {
var group Group
if err := tx.Preload("Account.Plan").First(&group, monitor.GroupID).Error; err != nil {
return nil
}
return group.Account
}
// RecoverDiagnostic clears a completed hard escalation only once and queues the
// corresponding recovery tasks in the same transaction.
func RecoverDiagnostic(checkID int64, now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
return RecoverDiagnosticTx(tx, checkID, now)
})
}
// RecoverDiagnosticTx is RecoverDiagnostic's transaction-aware form.
func RecoverDiagnosticTx(tx *gorm.DB, checkID int64, now time.Time) error {
if tx == nil {
return errors.New("recover diagnostic: nil transaction")
}
{
var check Check
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
return err
}
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil {
return nil
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
return err
}
if monitor.ConfirmState != ConfirmStateConfirmed && monitor.ConfirmState != ConfirmStateTimeout {
return nil
}
monitor.ConfirmState = ConfirmStateNone
if err := tx.Save(&monitor).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_recovery", &monitor.ID, nil, nil)
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticRecovery, now)
}
}
// NetworkDiagnosticsTick expires confirmations and derives worker health from durable attempts.
func NetworkDiagnosticsTick(now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
if err := reapExpiredConfirmationAttemptsTx(tx, now); err != nil {
return err
}
var monitors []Monitor
if err := tx.Clauses(SkipLockedClause).Preload("Group.Account.Plan").Where("confirm_state = ?", ConfirmStatePending).Find(&monitors).Error; err != nil {
return err
}
for i := range monitors {
settings := settingsForAccount(monitors[i].Group.Account)
if monitors[i].ConfirmAt == nil || monitors[i].ConfirmAt.After(now.Add(-settings.confirmTimeout)) {
continue
}
if err := tx.Model(&monitors[i]).Update("confirm_state", ConfirmStateTimeout).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_timeout", &monitors[i].ID, nil, nil)
if err := enqueueDiagnosticDelivery(tx, &monitors[i], monitors[i].Group.Account, diagnosticHard, now); err != nil {
return err
}
}
var workers []WorkerNode
if err := tx.Find(&workers).Error; err != nil {
return err
}
for i := range workers {
var total, failed int64
// Operated workers serve accounts on different plans. Use the most
// sensitive entitled setting among their recent attempts.
settings := diagnosticSettings{healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
q := tx.Model(&CheckAttempt{}).Where("worker_node_id = ? AND finished_at >= ?", workers[i].ID, now.Add(-settings.healthWindow))
q.Count(&total)
q.Where("result_state IN ?", []string{stateERR, stateFail}).Count(&failed)
flagged := total >= settings.healthMin && float64(failed)/float64(total) >= settings.healthRate
updates := map[string]interface{}{"last_total_count": total, "last_failure_count": failed}
if flagged {
updates["network_problems"] = true
updates["network_problems_until"] = now.Add(10 * time.Minute)
}
if workers[i].NetworkProblems && workers[i].NetworkProblemsUntil != nil && workers[i].NetworkProblemsUntil.Before(now) && !flagged {
updates["network_problems"] = false
updates["network_problems_until"] = nil
auditDiagnostic(tx, "worker.network_problem_unflag", nil, &workers[i].ID, nil)
}
if flagged && !workers[i].NetworkProblems {
auditDiagnostic(tx, "worker.network_problem_flag", nil, &workers[i].ID, map[string]interface{}{"failures": failed, "total": total})
}
if err := tx.Model(&workers[i]).Updates(updates).Error; err != nil {
return err
}
_ = influx.WriteOne("worker_health", map[string]string{"worker_id": workers[i].WorkerID}, map[string]interface{}{"failures": failed, "total": total, "failure_rate": float64(failed) / float64(maxInt64(total, 1))})
}
return nil
})
}
func reapExpiredConfirmationAttemptsTx(tx *gorm.DB, now time.Time) error {
var attempts []CheckAttempt
if err := tx.Clauses(SkipLockedClause).Where("kind = ? AND state = ? AND lease_expires_at <= ?", AttemptKindConfirm, AttemptStateLeased, now).Find(&attempts).Error; err != nil {
return err
}
for i := range attempts {
var check Check
if err := tx.First(&check, attempts[i].CheckID).Error; err != nil {
return err
}
oldWorkerID := int64(0)
if attempts[i].WorkerNodeID != nil {
oldWorkerID = *attempts[i].WorkerNodeID
}
sourceWorkerID := int64(0)
if attempts[i].SourceWorkerNodeID != nil {
sourceWorkerID = *attempts[i].SourceWorkerNodeID
}
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, oldWorkerID, now)
if errors.Is(err, gorm.ErrRecordNotFound) {
// No replacement is available now. Remove the stale assignment so a
// later capable independent worker can claim this queued attempt.
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": nil, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", nil).Error; err != nil {
return err
}
continue
}
if err != nil {
return err
}
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
return err
}
}
return nil
}
func maxInt64(a, b int64) int64 {
if a > b {
return a
}
return b
}

193
app/models/notification.go Обычный файл
Просмотреть файл

@@ -0,0 +1,193 @@
package models
import (
"log"
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/workdays"
)
// Notification provides functionality.
type Notification struct {
concerns.Model
Name string `json:"name" gorm:"not null"`
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
ContactIDs []int64 `gorm:"-:all" json:"contact_ids"`
Contacts []Contact `gorm:"many2many:notification_contacts;" json:"contacts,omitempty"`
GroupIDs []int64 `gorm:"-:all" json:"group_ids"`
Groups []Group `gorm:"many2many:notification_groups;" json:"-"`
AlertDelay *int64 `json:"alert_delay,omitempty"`
// RepeatAlert *int64 `json:"repeat_alert,omitempty"`
BeforeExpiration *int64 `json:"before_expiration,omitempty"`
NotifyDown bool `gorm:"default:true" json:"notify_down"`
NotifyRestore bool `gorm:"default:true" json:"notify_restore"`
NotifyWHOIS bool `gorm:"default:true" json:"notify_whois"`
NotifySSL bool `gorm:"default:true" json:"notify_ssl"`
NotifyDays *int `json:"notify_days"`
NotifyDayStart *int `json:"notify_day_start"`
NotifyDayEnd *int `json:"notify_day_end"`
NotifyHolidays bool `gorm:"default:true" json:"notify_holidays"`
Messages []Message `json:"-"`
concerns.Timestamped
Audited
}
const notificationDebug = false
// EnabledNow checks if the notification is enabled at the given time.
func (n *Notification) EnabledNow(tn *time.Time) bool {
weekday := int(tn.Weekday())
// делаем из 0-воскр 1-пн 6-сб вариант 0-пн 6-воскр
if weekday == 0 {
weekday = 7
}
weekday--
if notificationDebug {
log.Println("notification", n.ID, "check enabled now at", tn, "for day", weekday)
}
if !n.NotifyHolidays {
c := workdays.GetCalendar()
if !c.IsWorkday(*tn) {
if notificationDebug {
log.Println("notification", n.ID, "is not enabled on holiday", tn)
}
return false
}
}
minusOneDay := false
notifyFromDay := true
if n.NotifyDayStart != nil && n.NotifyDayEnd != nil {
bod := BeginningOfDay(*tn)
secondsToday := int(tn.Sub(bod) / time.Second)
ds := *n.NotifyDayStart
de := *n.NotifyDayEnd
// вариант 9 утра - 2 часа ночи
if ds == de { //nolint:gocritic // complex condition chain
notifyFromDay = true
} else if ds > de {
// с 0 до DayEnd
if secondsToday < de { //nolint:gocritic // complex condition chain
minusOneDay = true
notifyFromDay = true
} else if secondsToday < ds {
// с DayEnd до DayStart
notifyFromDay = false
} else {
notifyFromDay = true
}
} else {
if secondsToday < ds { //nolint:gocritic // complex condition chain
// с 0 до DayStart
notifyFromDay = false
} else if secondsToday > de {
notifyFromDay = false
} else {
notifyFromDay = true
}
}
// log.Println(secondsToday)
}
if !notifyFromDay {
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled as notifyFromDay", tn)
}
return false
}
// Если время 0-dayStart считаем что это прошлый день
if minusOneDay {
weekday--
if weekday < 0 {
weekday = 6
}
}
if n.NotifyDays != nil {
if !HasBit(*n.NotifyDays, uint(weekday)) {
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled on weekday", tn.Weekday(), weekday, tn)
}
return false
}
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is enabled on weekday", tn.Weekday(), weekday, tn)
}
}
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is enabled", tn)
}
return true
}
// NotificationLoadIDs provides functionality.
func NotificationLoadIDs(notifications *[]Notification) {
for i, n := range *notifications { //nolint:gocritic // range copy is acceptable here
cids := make([]int64, len(n.Contacts))
for i, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
cids[i] = c.ID
}
(*notifications)[i].ContactIDs = cids
gids := make([]int64, len(n.Groups))
for j, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
gids[j] = g.ID
}
(*notifications)[i].GroupIDs = gids
}
}
// PersistRelations provides functionality.
func (n *Notification) PersistRelations() error {
cts := make([]Contact, len(n.ContactIDs))
for i, c := range n.ContactIDs {
ct := Contact{}
ct.ID = c
cts[i] = ct
}
err := DB().Model(&n).Association("Contacts").Replace(cts)
if err != nil {
return err
}
grp := make([]Group, len(n.GroupIDs))
for i, g := range n.GroupIDs {
gr := Group{}
gr.ID = g
grp[i] = gr
}
err = DB().Model(&n).Association("Groups").Replace(grp)
if err != nil {
return err
}
return nil
}
// GetContacts returns the contacts associated with this notification.
// Errors are logged and an empty slice is returned instead of panicking so
// that a single misconfigured notification cannot kill the scheduler goroutine
// that processes expiry alerts (see internal/notifier.RunExp).
func (n *Notification) GetContacts() []Contact {
contacts := make([]Contact, 0)
err := DB().Model(*n).Where("enabled = ?", true).Association("Contacts").Find(&contacts)
if err != nil {
log.Printf("notification %d: GetContacts failed: %v", n.ID, err)
return contacts
}
return contacts
}

167
app/models/notification_credential.go Обычный файл
Просмотреть файл

@@ -0,0 +1,167 @@
package models
import (
"encoding/base64"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// NotificationCredential kinds stored in the database.
const (
CredentialKindSMTP = "smtp"
CredentialKindTelegram = "telegram"
)
// NotificationCredential stores per-method delivery credentials (SMTP login,
// Telegram bot token, etc.) encrypted at rest. Credentials are pushed to
// workers through the init/config websocket refresh (see docs/worker-protocol.md
// "Credentials Push").
type NotificationCredential struct {
concerns.Model
// AccountID is nil for platform-managed credentials and set for credentials
// owned by one customer account.
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;index" json:"account_id,omitempty"`
Kind string `gorm:"not null" json:"kind"`
Name string `gorm:"not null" json:"name"`
// SMTP-specific
Server *string `json:"server,omitempty"`
Port *int `json:"port,omitempty"`
Login *string `json:"login,omitempty"`
FromName *string `json:"from_name,omitempty"`
FromAddr *string `json:"from_address,omitempty"`
InsecureSkipVerify bool `gorm:"default:false" json:"insecure_skip_verify"`
// Telegram-specific
BotName *string `json:"bot_name,omitempty"`
APIURL *string `json:"api_url,omitempty"`
WebhookToken string `gorm:"size:128" json:"webhook_token,omitempty"`
// SecretEnc holds the encrypted (or "plain:"-prefixed fallback) secret
// value — SMTP password or Telegram bot token. Decrypt via GetSecret.
SecretEnc string `gorm:"column:secret;type:text" json:"-"`
SecretMasked string `gorm:"-" json:"secret_masked,omitempty"`
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
Meta datatypes.JSON `gorm:"type:jsonb;default:'{}'" json:"meta,omitempty"`
concerns.Timestamped
Audited
}
// SetSecret encrypts and stores the plaintext secret value.
func (c *NotificationCredential) SetSecret(plaintext string) error {
enc, err := encryptSecret(plaintext)
if err != nil {
return err
}
c.SecretEnc = enc
return nil
}
// GetSecret decrypts and returns the secret value.
func (c *NotificationCredential) GetSecret() (string, error) {
return decryptSecret(c.SecretEnc)
}
// FillSecretMasked populates SecretMasked with a display-safe version of the credential secret.
func (c *NotificationCredential) FillSecretMasked() {
secret, err := c.GetSecret()
if err != nil {
return
}
c.SecretMasked = maskSecret(secret)
}
// EnsureWebhookToken creates the per-bot webhook URL token if it is missing.
func (c *NotificationCredential) EnsureWebhookToken() {
if c.WebhookToken != "" {
return
}
c.WebhookToken = base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
}
func maskSecret(secret string) string {
if secret == "" {
return ""
}
if len(secret) == 1 {
return secret
}
return secret[:1] + "***" + secret[len(secret)-1:]
}
// TableName overrides the default table name.
func (NotificationCredential) TableName() string {
return "notification_credentials"
}
// AllNotificationCredentials returns all credentials ordered by kind and name.
func AllNotificationCredentials() ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id IS NULL").Order("kind ASC, name ASC").Find(&creds).Error
for i := range creds {
creds[i].FillSecretMasked()
}
return creds, err
}
// AccountNotificationCredentials returns credentials owned by accountID.
func AccountNotificationCredentials(accountID int64) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id = ?", accountID).Order("kind ASC, name ASC").Find(&creds).Error
for i := range creds {
creds[i].FillSecretMasked()
}
return creds, err
}
// EnabledCredentialsByKind returns enabled credentials of the given kind.
func EnabledCredentialsByKind(kind string) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id IS NULL AND kind = ? AND enabled = ?", kind, true).Order("name ASC").Find(&creds).Error
return creds, err
}
// EnabledCredentialsByAccountAndKind returns only enabled credentials owned by
// accountID. It never falls back to platform credentials.
func EnabledCredentialsByAccountAndKind(accountID int64, kind string) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id = ? AND kind = ? AND enabled = ?", accountID, kind, true).Order("name ASC").Find(&creds).Error
return creds, err
}
// FindCredential returns a credential by id.
func FindCredential(id int64) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().First(&c, id).Error; err != nil {
return nil, err
}
c.FillSecretMasked()
return &c, nil
}
// FindCredentialByName returns a credential by kind and name.
func FindCredentialByName(kind, name string) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().Where("kind = ? AND name = ?", kind, name).First(&c).Error; err != nil {
return nil, err
}
return &c, nil
}
// FindTelegramCredentialByWebhookToken returns an enabled Telegram credential by webhook token.
func FindTelegramCredentialByWebhookToken(token string) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().Where("kind = ? AND webhook_token = ? AND enabled = ?", CredentialKindTelegram, token, true).First(&c).Error; err != nil {
return nil, err
}
return &c, nil
}
// DeleteCredential removes a credential by id.
func DeleteCredential(id int64) error {
return DB().Delete(&NotificationCredential{}, id).Error
}

186
app/models/notification_credential_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,186 @@
package models_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// TestNotificationCredential_EncryptDecryptRoundTrip verifies that SetSecret
// followed by GetSecret returns the original plaintext regardless of which
// encryption mode (AES-GCM or plain: fallback) is active.
func TestNotificationCredential_EncryptDecryptRoundTrip(t *testing.T) {
c := &models.NotificationCredential{}
plaintext := "super-secret-smtp-password"
require.NoError(t, c.SetSecret(plaintext))
got, err := c.GetSecret()
require.NoError(t, err)
assert.Equal(t, plaintext, got)
}
// TestNotificationCredential_EncryptDecryptWithKey verifies that when
// RSMON_CRED_KEY is configured the stored value is prefixed with "enc:" and
// can still be decrypted back to the original plaintext.
func TestNotificationCredential_EncryptDecryptWithKey(t *testing.T) {
t.Setenv("RSMON_CRED_KEY", "test-key-rotate-me-1234567890")
c := &models.NotificationCredential{}
plaintext := "bot-token-9876543210:ABCDEFG"
require.NoError(t, c.SetSecret(plaintext))
assert.True(t, strings.HasPrefix(c.SecretEnc, "enc:"),
"expected SecretEnc to start with 'enc:' prefix, got %q", c.SecretEnc)
assert.NotEqual(t, plaintext, c.SecretEnc, "encrypted value must not equal plaintext")
got, err := c.GetSecret()
require.NoError(t, err)
assert.Equal(t, plaintext, got)
}
// TestNotificationCredential_CRUD exercises create / find-by-id / find-by-name
// / delete against the test database.
func TestNotificationCredential_CRUD(t *testing.T) {
models.Drop()
models.Migrate()
server := "smtp.example.com"
port := 587
login := "alerts@example.com"
enabled := true
c := &models.NotificationCredential{
Kind: models.CredentialKindSMTP,
Name: "primary",
Server: &server,
Port: &port,
Login: &login,
Enabled: &enabled,
}
require.NoError(t, c.SetSecret("smtp-password-xyz"))
require.NoError(t, models.DB().Create(c).Error)
require.NotZero(t, c.ID, "expected ID to be assigned after Create")
found, err := models.FindCredential(c.ID)
require.NoError(t, err)
assert.Equal(t, "primary", found.Name)
assert.Equal(t, models.CredentialKindSMTP, found.Kind)
require.NotNil(t, found.Server)
assert.Equal(t, "smtp.example.com", *found.Server)
gotSecret, err := found.GetSecret()
require.NoError(t, err)
assert.Equal(t, "smtp-password-xyz", gotSecret)
assert.Equal(t, "s***z", found.SecretMasked)
byName, err := models.FindCredentialByName(models.CredentialKindSMTP, "primary")
require.NoError(t, err)
assert.Equal(t, c.ID, byName.ID)
require.NoError(t, models.DeleteCredential(c.ID))
_, err = models.FindCredential(c.ID)
assert.Error(t, err, "FindCredential should fail after delete")
}
// TestNotificationCredential_UniqueKindName verifies that two credentials with
// the same (kind, name) pair violate the unique index.
func TestNotificationCredential_UniqueKindName(t *testing.T) {
models.Drop()
models.Migrate()
enTrue := true
first := &models.NotificationCredential{
Kind: models.CredentialKindTelegram,
Name: "main-bot",
Enabled: &enTrue,
}
require.NoError(t, first.SetSecret("token-a"))
require.NoError(t, models.DB().Create(first).Error)
second := &models.NotificationCredential{
Kind: models.CredentialKindTelegram,
Name: "main-bot",
Enabled: &enTrue,
}
require.NoError(t, second.SetSecret("token-b"))
err := models.DB().Create(second).Error
require.Error(t, err, "expected unique constraint violation for duplicate (kind, name)")
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"expected error mentioning unique/duplicate, got: %v", err)
}
// TestEnabledCredentialsByKind verifies the kind+enabled filter.
func TestEnabledCredentialsByKind(t *testing.T) {
models.Drop()
models.Migrate()
enTrue := true
enFalse := false
enabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "enabled-1", Enabled: &enTrue}
disabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "disabled-1", Enabled: &enFalse}
require.NoError(t, models.DB().Create(enabled).Error)
require.NoError(t, models.DB().Create(disabled).Error)
got, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
require.NoError(t, err)
var names []string
for _, c := range got {
names = append(names, c.Name)
}
assert.Contains(t, names, "enabled-1")
assert.NotContains(t, names, "disabled-1")
}
func TestNotificationCredentialsAreScopedToSystemOrAccount(t *testing.T) {
models.Drop()
models.Migrate()
accountA := &models.Account{Name: "credential-a"}
accountB := &models.Account{Name: "credential-b"}
require.NoError(t, models.DB().Create(accountA).Error)
require.NoError(t, models.DB().Create(accountB).Error)
enabled := true
system := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "system", Enabled: &enabled}
ownedA := &models.NotificationCredential{AccountID: &accountA.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
ownedB := &models.NotificationCredential{AccountID: &accountB.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
require.NoError(t, models.DB().Create(system).Error)
require.NoError(t, models.DB().Create(ownedA).Error)
require.NoError(t, models.DB().Create(ownedB).Error)
systemCreds, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
require.NoError(t, err)
require.Len(t, systemCreds, 1)
assert.Equal(t, system.ID, systemCreds[0].ID)
accountCreds, err := models.EnabledCredentialsByAccountAndKind(accountA.ID, models.CredentialKindSMTP)
require.NoError(t, err)
require.Len(t, accountCreds, 1)
assert.Equal(t, ownedA.ID, accountCreds[0].ID)
}
func TestNotificationCredentialSecretMasked(t *testing.T) {
models.Drop()
models.Migrate()
cred := models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "smtp"}
require.NoError(t, cred.SetSecret("password"))
require.NoError(t, models.DB().Create(&cred).Error)
loaded, err := models.FindCredential(cred.ID)
require.NoError(t, err)
assert.Equal(t, "p***d", loaded.SecretMasked)
}

146
app/models/notification_get_contacts_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,146 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/spec/factories"
)
// TestNotificationGetContactsIncludesSystemContact exercises the regression
// reported when the production dump was restored into dev: the
// contacts.is_system column was missing and the GORM preload panicked on
// GetContacts. The fix has two layers:
//
// 1. AutoMigrate must add is_system (and deletion_requested_at) before the
// notifier scheduler starts running.
// 2. GetContacts itself must not panic on a query error so a single bad row
// cannot tear down the scheduler goroutine.
//
// This test verifies both layers by:
// - asserting that the schema post-Migrate includes is_system, so the
// production-like scenario no longer panics; and
// - building a notification that contains a contact flagged is_system=true
// and checking GetContacts returns it.
func TestNotificationGetContactsIncludesSystemContact(t *testing.T) {
models.Drop()
models.Migrate()
// Column must exist after Migrate; otherwise GetContacts would fail
// with the same panic we saw in production.
assertColumnExists(t, "contacts", "is_system")
account := &models.Account{Name: "acct-get-contacts"}
require.NoError(t, models.DB().Create(account).Error)
accountID := account.ID
trueVal := true
contact := &models.Contact{
AccountID: &accountID,
Name: "system-admin",
Kind: "email",
Value: "ops@example.com",
IsSystem: &trueVal,
}
require.NoError(t, models.DB().Create(contact).Error)
notification := factories.PersistedNotification(
account, []int64{contact.ID}, nil, 300, false,
)
// Reload so the model has its persisted ID; the factory's PersistRelations
// may have left ContactIDs empty on the returned value depending on the
// GORM version, so fetch fresh.
require.NoError(t, models.DB().
Preload("Contacts").
First(&notification, notification.ID).Error)
got := notification.GetContacts()
ids := make([]int64, 0, len(got))
for _, c := range got {
ids = append(ids, c.ID)
}
assert.Contains(t, ids, contact.ID, "GetContacts must include the is_system contact")
}
// TestUserDeletionRequestedAtColumnAndRoundTrip verifies the second missing
// column reported by the panic: users.deletion_requested_at. It asserts that
// AutoMigrate creates the column and that the field round-trips through the
// DB correctly. ProcessPendingDeletions (the consumer of this column) relies
// on it being present and queryable.
func TestUserDeletionRequestedAtColumnAndRoundTrip(t *testing.T) {
models.Drop()
models.Migrate()
assertColumnExists(t, "users", "deletion_requested_at")
user := factories.PersistedUser("deletion-roundtrip@test.ru", "secret")
now := time.Now().UTC().Truncate(time.Microsecond)
user.DeletionRequestedAt = &now
require.NoError(t, models.DB().Save(&user).Error)
reloaded := models.User{}
require.NoError(t, models.DB().First(&reloaded, user.ID).Error)
require.NotNil(t, reloaded.DeletionRequestedAt, "deletion_requested_at must round-trip via Save/First")
assert.True(t, reloaded.DeletionRequestedAt.Equal(now),
"deletion_requested_at must preserve the timestamp value (got %v, want %v)",
reloaded.DeletionRequestedAt, now)
// ProcessPendingDeletions should not panic on the populated schema and
// must respect the cutoff: a recently-set deletion_requested_at is
// still inside the 7-day grace period, so no hard-delete must occur.
deleted, err := models.ProcessPendingDeletions()
require.NoError(t, err)
assert.Equal(t, 0, deleted, "users within the 7-day grace period must not be hard-deleted")
}
// TestProcessPendingDeletionsQueriesMissingColumnGracefully asserts that even
// if the deletion_requested_at column were missing, ProcessPendingDeletions
// would not panic (the panic-on-error pattern was historically present in
// other notifier helpers). We force the failure by renaming the column back,
// calling ProcessPendingDeletions, then restoring the column.
func TestProcessPendingDeletionsQueriesMissingColumnGracefully(t *testing.T) {
models.Drop()
models.Migrate()
assertColumnExists(t, "users", "deletion_requested_at")
// Simulate the production-missing-column scenario in a contained way:
// rename the column so the SELECT against deletion_requested_at fails.
require.NoError(t, models.DB().
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at TO deletion_requested_at_hidden").Error)
t.Cleanup(func() {
// Restore so subsequent tests in this package keep working.
_ = models.DB().
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at_hidden TO deletion_requested_at").Error
})
// Must not panic; must return an error.
assert.NotPanics(t, func() {
_, err := models.ProcessPendingDeletions()
assert.Error(t, err, "missing column must surface as an error, not a panic")
})
}
// assertColumnExists checks that the given table has the given column by
// querying information_schema. It is the canary for the AutoMigrate step
// ordering bug: if the column is missing, every test that touches it will
// panic with SQLSTATE 42703.
func assertColumnExists(t *testing.T, table, column string) {
t.Helper()
var n int
err := models.DB().Raw(
`SELECT COUNT(*) FROM information_schema.columns
WHERE table_schema = current_schema()
AND table_name = ? AND column_name = ?`,
table, column,
).Scan(&n).Error
require.NoError(t, err, "information_schema query must succeed")
assert.Equal(t, 1, n, "table %q must have column %q after Migrate()", table, column)
}

128
app/models/notification_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,128 @@
package models
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var (
mondayFriday int
allDays int
)
var time1, time3, time8, time9, time10, time23 time.Time
func init() {
// database.ConfigFile = "." + database.ConfigFile
// database.Init()
// Migrate()
// Drop()
mondayFriday = 0
mondayFriday = SetBit(mondayFriday, 0)
mondayFriday = SetBit(mondayFriday, 1)
mondayFriday = SetBit(mondayFriday, 2)
mondayFriday = SetBit(mondayFriday, 3)
mondayFriday = SetBit(mondayFriday, 4)
allDays = 0
allDays = SetBit(allDays, 0)
allDays = SetBit(allDays, 1)
allDays = SetBit(allDays, 2)
allDays = SetBit(allDays, 3)
allDays = SetBit(allDays, 4)
allDays = SetBit(allDays, 5)
allDays = SetBit(allDays, 6)
time1 = time.Date(2019, time.May, 26, 1, 0, 0, 0, time.Local)
time3 = time.Date(2019, time.May, 26, 3, 0, 0, 0, time.Local)
time8 = time.Date(2019, time.May, 26, 8, 0, 0, 0, time.Local)
time9 = time.Date(2019, time.May, 26, 9, 0, 0, 0, time.Local)
time10 = time.Date(2019, time.May, 26, 10, 0, 0, 0, time.Local)
time23 = time.Date(2019, time.May, 26, 23, 0, 0, 0, time.Local)
}
func TestNotificationEnabledNowHolidays(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.ID = 1
n.NotifyHolidays = true
holiday := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.Local)
assert.Equal(t, true, n.EnabledNow(&holiday), "notification by default should be enabled on holidays")
n.ID = 2
n.NotifyHolidays = false
assert.Equal(t, false, n.EnabledNow(&holiday), "notification with NotifyHolidays=fasle should not be enabled on holiday")
}
func TestNotificationEnabledNowMondayFriday(t *testing.T) {
n := &Notification{}
n.NotifyHolidays = true
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
n.ID = 3
n.NotifyDays = &mondayFriday
assert.Equal(t, false, n.EnabledNow(&weekend), "notification mon-fri should not be enabled on sunday")
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-fri should be enabled on monday")
}
func TestNotificationEnabledNowAllDays(t *testing.T) {
n := &Notification{}
n.NotifyHolidays = true
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
n.ID = 4
n.NotifyDays = &allDays
assert.Equal(t, true, n.EnabledNow(&weekend), "notification mon-sat should be enabled on sunday")
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-sat should be enabled on monday")
}
func TestNotificationEnabledNowNormal(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.NotifyHolidays = true
// 9am - 18pm
n.ID = 5
start := 9 * 3600
end := 18 * 3600
n.NotifyDayStart = &start
n.NotifyDayEnd = &end
assert.Equal(t, false, n.EnabledNow(&time1), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
assert.Equal(t, false, n.EnabledNow(&time23), "notification should not be enabled outside day")
}
// TestNotificationEnabledNowRollover tests time range that crosses midnight
//
//nolint:dupl // Test structure similar to TestNotificationEnabledNowNormal but tests different behavior (rollover vs normal time range)
func TestNotificationEnabledNowRollover(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.NotifyHolidays = true
// 9am - 2am
n.ID = 6
start := 9 * 3600
end := 2 * 3600
n.NotifyDayStart = &start
n.NotifyDayEnd = &end
assert.Equal(t, true, n.EnabledNow(&time1), "notification should be enabled inside day")
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time23), "notification should be enabled inside day")
}

18
app/models/payment.go Обычный файл
Просмотреть файл

@@ -0,0 +1,18 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// Payment provides functionality.
type Payment struct {
concerns.Model
AccountID int64 `json:"account_id"`
Account User `json:"-"`
Kind string
ExtID string
Amount int
concerns.Timestamped
Audited
}

108
app/models/plan.go Обычный файл
Просмотреть файл

@@ -0,0 +1,108 @@
package models
import (
"log"
"github.com/google/uuid"
"github.com/lib/pq"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
var canonicalPlanCodes = []string{"free", "solo", "team", "enterprise"}
// CanonicalPlanCodes returns the supported public catalog codes.
func CanonicalPlanCodes() []string {
return append([]string(nil), canonicalPlanCodes...)
}
// Plan is a versioned-by-code catalog entry. A zero cap means unlimited.
type Plan struct {
concerns.Model
Code string `gorm:"size:32;not null" json:"code"`
NameRU string `gorm:"size:64;not null" json:"name_ru"`
NameEN string `gorm:"size:64;not null" json:"name_en"`
PriceMonthlyMinor int64 `gorm:"not null;default:0" json:"price_monthly_minor"`
PriceAnnualMinor int64 `gorm:"not null;default:0" json:"price_annual_minor"`
Currency string `gorm:"size:3;not null;default:'RUB'" json:"currency"`
MonitorCap int64 `gorm:"not null;default:0" json:"monitor_cap"`
IntervalMinSeconds int `gorm:"not null;default:30" json:"interval_min_seconds"`
StatusPagesCap int64 `gorm:"not null;default:0" json:"status_pages_cap"`
MaintenanceCap int64 `gorm:"not null;default:0" json:"maintenance_cap"`
LoginSeatsIncluded int64 `gorm:"not null;default:0" json:"login_seats_included"`
NotifySeatsIncluded int64 `gorm:"not null;default:0" json:"notify_seats_included"`
UnlimitedSeats bool `gorm:"not null;default:false" json:"unlimited_seats"`
Integrations pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"integrations"`
CheckKinds pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"check_kinds"`
DataRetentionMonths int `gorm:"not null;default:3" json:"data_retention_months"`
DistributedWorkers bool `gorm:"not null;default:false" json:"distributed_workers"`
Confirmations bool `gorm:"not null;default:false" json:"confirmations"`
AllowHardAlerts bool `gorm:"not null;default:false" json:"allow_hard_alerts"`
ConfirmTimeoutSec int `gorm:"not null;default:90" json:"confirm_timeout_sec"`
HealthWindowSec int `gorm:"not null;default:300" json:"health_window_sec"`
HealthRateThreshold float64 `gorm:"not null;default:0.5" json:"health_rate_threshold"`
HealthMinAttempts int `gorm:"not null;default:10" json:"health_min_attempts"`
SOC2 bool `gorm:"not null;default:false" json:"soc2"`
GDPRDPA bool `gorm:"not null;default:false" json:"gdpr_dpa"`
IsDefault bool `gorm:"not null;default:false" json:"is_default"`
Archived bool `gorm:"not null;default:false" json:"archived"`
// Deprecated source-compatibility fields. The legacy plans table is retained
// as plans_legacy; these values are never written to the canonical catalog.
Default bool `gorm:"-" json:"Default,omitempty"`
Name string `gorm:"-" json:"Name,omitempty"`
HTTPMonitors *int64 `gorm:"-" json:"HTTPMonitors,omitempty"`
DNSMonitors *int64 `gorm:"-" json:"DNSMonitors,omitempty"`
WHOISMonitors *int64 `gorm:"-" json:"WHOISMonitors,omitempty"`
TotalMonitors *int64 `gorm:"-" json:"TotalMonitors,omitempty"`
Price int `gorm:"-" json:"Price,omitempty"`
TrialPeriod int `gorm:"-" json:"TrialPeriod,omitempty"`
concerns.Timestamped
Audited
}
func (p *Plan) BeforeCreate(_ *gorm.DB) error {
if p.Code == "" {
p.Code = "legacy-" + uuid.NewString()[:24]
}
if p.NameRU == "" {
p.NameRU = p.Name
}
if p.NameEN == "" {
p.NameEN = p.NameRU
}
if p.Currency == "" {
p.Currency = "RUB"
}
if p.IntervalMinSeconds == 0 {
p.IntervalMinSeconds = 30
}
return nil
}
func (p *Plan) AfterFind(_ *gorm.DB) error {
p.Name = p.NameRU
p.Price = int(p.PriceMonthlyMinor / 100)
p.Default = p.IsDefault
p.TotalMonitors = &p.MonitorCap
return nil
}
// DefaultPlan returns the canonical free plan.
func DefaultPlan() Plan {
pl := Plan{}
if err := DB().Where("code = ? AND archived = FALSE", "free").First(&pl).Error; err != nil {
log.Println("unable to find default plan")
panic(err)
}
return pl
}
// AllowsDistributed keeps legacy callers working while using the canonical
// entitlement flag for catalog plans.
func (p *Plan) AllowsDistributed() bool {
return p != nil && (p.DistributedWorkers || p.Price > 0)
}

20
app/models/region.go Обычный файл
Просмотреть файл

@@ -0,0 +1,20 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DefaultRegionCode is the historical default region code seeded by
// Migrate() (see app/models/migrate.go). Centralized here so admin
// endpoints and check_jobs.go agree on the literal.
const DefaultRegionCode = "local"
// Region represents a geographic region where distributed workers can run
type Region struct {
concerns.Model
Code string `gorm:"uniqueIndex;size:20;not null" json:"code"` // e.g. "ru-msk", "us-east", "eu-west"
Name string `gorm:"not null" json:"name"` // "Moscow, Russia"
Enabled bool `gorm:"not null;default:true" json:"enabled"`
Priority int `gorm:"not null;default:0" json:"priority"`
concerns.Timestamped
}

38
app/models/repo.go Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
package models
import (
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Repo is a source repository shared by one or more account-scoped sites.
// Repositories themselves are global rstuff mirrors; SiteRepo supplies the
// account boundary through its Site.
type Repo struct {
concerns.Model
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
GitlabID *int64 `json:"gitlab_id,omitempty"`
Name string `gorm:"size:120;not null" json:"name"`
Namespace *string `gorm:"size:120" json:"namespace,omitempty"`
Path *string `gorm:"size:255" json:"path,omitempty"`
Description *string `gorm:"type:text" json:"description,omitempty"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
concerns.Timestamped
}
// TableName returns the repository table name.
func (Repo) TableName() string { return "repos" }
// SiteRepo is the explicit site/repository join. Role is intentionally data,
// rather than an enum, to preserve the rstuff contract as it evolves.
type SiteRepo struct {
SiteID int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE CASCADE;primaryKey" json:"site_id"`
RepoID int64 `gorm:"type:bigint REFERENCES repos(id) ON DELETE CASCADE;primaryKey" json:"repo_id"`
Role string `gorm:"size:32;not null;default:'primary'" json:"role"`
concerns.Timestamped
}
// TableName returns the repository assignment table name.
func (SiteRepo) TableName() string { return "site_repos" }

135
app/models/rkn_domain.go Обычный файл
Просмотреть файл

@@ -0,0 +1,135 @@
package models
import (
"strings"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// RknDomain is one row of the locally cached ru-blocked domains list. The
// `Domain` field carries a lowercased domain string and is the unique key.
type RknDomain struct {
concerns.Model
Domain string `gorm:"size:255;uniqueIndex;not null" json:"domain"`
}
// TableName pins the underlying table name so GORM migrations and raw SQL
// (used by IsRknDomainBlocked and the rkn updater) agree.
func (RknDomain) TableName() string { return "rkn_domains" }
// ReplaceRknDomains deletes every existing row and inserts the given domains
// in batches. Caller-supplied domains are lower-cased and de-duplicated, and
// blank entries are dropped. The whole operation runs in a single
// transaction so readers see either the old set or the new set — there is no
// in-between state where the table is half-flushed.
//
// Designed to be called once per parser-updater tick (default every 6h); the
// uniqueIndex on Domain guarantees idempotent re-runs even when the caller
// forgets to filter.
func ReplaceRknDomains(domains []string) error {
if domains == nil {
domains = []string{}
}
deduped := make([]string, 0, len(domains))
seen := make(map[string]struct{}, len(domains))
for _, d := range domains {
d = strings.ToLower(strings.TrimSpace(d))
if d == "" {
continue
}
if _, ok := seen[d]; ok {
continue
}
seen[d] = struct{}{}
deduped = append(deduped, d)
}
tx := DB().Begin()
if tx.Error != nil {
return tx.Error
}
// Step 1 — wipe the existing cache. Using a scoped Where("1 = 1") Delete
// instead of TRUNCATE so the advisory lock taken by Migrate() does not
// become a bottleneck and so any in-flight foreign-key checks against
// rkn_domains remain satisfied (the table has no FKs today, but this
// matches the convention used by Drop()).
if err := tx.Where("1 = 1").Delete(&RknDomain{}).Error; err != nil {
_ = tx.Rollback().Error
return err
}
// Step 2 — bulk-insert in chunks of 1000 rows. CreateInBatches runs N
// multi-row INSERT statements, which for the typical ~30k ru-blocked
// entries is ~3ms per batch — significantly cheaper than per-row
// Create() in tight loops (the previous AddRknDomain implementation).
const batchSize = 1000
for start := 0; start < len(deduped); start += batchSize {
end := start + batchSize
if end > len(deduped) {
end = len(deduped)
}
rows := make([]RknDomain, 0, end-start)
for _, d := range deduped[start:end] {
rows = append(rows, RknDomain{Domain: d})
}
if err := tx.CreateInBatches(rows, batchSize).Error; err != nil {
_ = tx.Rollback().Error
return err
}
}
return tx.Commit().Error
}
// IsRknDomainBlocked returns true iff `domain` (or its root label, or any
// parent suffix already recorded as `*.parent.tld`) is present in the
// rkn_domains table.
//
// Matching rules — see checks/crkn/rkn_init.go for the original logic we
// consolidate here:
// 1. Exact match against the stored domain string.
// 2. Root-domain match (last two labels of the input) — covers the case
// where the user passed a subdomain but the upstream only lists the
// apex.
// 3. Suffix match (`stored LIKE '%' || input || ?`) — covers the case
// where the user passed the apex (or a higher-level label) but the
// upstream lists a child subdomain.
//
// All three checks are combined into a single SQL statement via OR so the
// table is scanned at most once and the SQL planner can pick a single
// index access path.
func IsRknDomainBlocked(domain string) (bool, error) {
domain = strings.ToLower(strings.TrimSpace(domain))
if domain == "" {
return false, nil
}
rootDomain := rootDomainOf(domain)
// Build the suffix patterns once. Note: every ".X" entry in the table
// (i.e. a domain that begins with a dot) matches any subdomain whose
// suffix is domain.
suffixPattern := "%." + domain
var count int64
err := DB().Raw(
"SELECT COUNT(*) FROM rkn_domains WHERE domain = ? OR domain = ? OR domain LIKE ?",
domain, rootDomain, suffixPattern,
).Scan(&count).Error
if err != nil {
return false, err
}
return count > 0, nil
}
// rootDomainOf returns the last two labels of `domain` (e.g. "a.b.c" → "b.c").
// Returns `domain` unchanged when it has fewer than three labels, because a
// one- or two-label input IS already the apex/root domain.
func rootDomainOf(domain string) string {
i := strings.LastIndex(domain, ".")
if i < 0 {
return domain
}
j := strings.LastIndex(domain[:i], ".")
if j < 0 {
return domain
}
return domain[j+1:]
}

90
app/models/rkn_domain_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,90 @@
package models_test
import (
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestReplaceRknDomains_BulkAndDedup(t *testing.T) {
models.Drop()
models.Migrate()
input := []string{
"Foo.example",
"foo.example", // dup after lowercasing
" bar.example ",
"",
"baz.example",
"qux.example",
"qux.example", // dup within input
}
require.NoError(t, models.ReplaceRknDomains(input))
got := []string{}
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &got).Error)
want := []string{"bar.example", "baz.example", "foo.example", "qux.example"}
sort.Strings(want)
assert.Equal(t, want, got)
}
func TestReplaceRknDomains_ReplacesExisting(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknDomains([]string{"old1.example", "old2.example"}))
var n int64
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
assert.Equal(t, int64(2), n)
// Second call wipes and replaces — no overlap with old set.
require.NoError(t, models.ReplaceRknDomains([]string{"new1.example", "new2.example", "new3.example"}))
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
assert.Equal(t, int64(3), n)
var domains []string
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &domains).Error)
assert.Equal(t, []string{"new1.example", "new2.example", "new3.example"}, domains)
}
func TestIsRknDomainBlocked(t *testing.T) {
models.Drop()
models.Migrate()
input := []string{
"example.com",
"foo.bar",
"sub.test",
}
require.NoError(t, models.ReplaceRknDomains(input))
cases := []struct {
host string
want bool
}{
{"example.com", true}, // exact
{"EXAMPLE.com", true}, // case insensitive (caller lowercases)
{"www.example.com", true}, // stored has apex; query apex → root-domain match
{"deep.nested.example.com", true}, // suffix match via LIKE '%.X'
{"foo.bar", true},
{"sub.test", true},
{"a.sub.test", true},
{"unrelated.org", false},
{"two.labels", false}, // 2-label input not in list → must not collapse to "labels"
{"", false},
}
for _, c := range cases {
t.Run(c.host, func(t *testing.T) {
got, err := models.IsRknDomainBlocked(c.host)
require.NoError(t, err)
assert.Equal(t, c.want, got)
})
}
}

193
app/models/rkn_ip.go Обычный файл
Просмотреть файл

@@ -0,0 +1,193 @@
package models
import (
"fmt"
"log"
"net"
"strings"
"github.com/davecgh/go-spew/spew"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/netaddr"
)
// RknIP stores a single CIDR from the ru-blocked IP list. The cidr column
// uses Postgres's native cidr type so the (>>) containment operator works
// directly inside IsRknIPBlocked queries — GiST index recommended for any
// table large enough to amortize the planner cost; see EnsureRknIndexes.
type RknIP struct {
concerns.Model
Network *netaddr.Cidr `json:"network" gorm:"type:cidr;"`
}
// TableName pins the underlying table name.
func (RknIP) TableName() string { return "rkn_ips" }
// PanicOnErr panics with a spew-formatted error dump if err is non-nil.
// Kept for callers that still use the old add-by-row InsertInBatches path
// (none after this commit, but kept in case external scripts reference it).
func PanicOnErr(err error) {
if err != nil {
spew.Dump(err)
panic(err)
}
}
// FindRknIP returns true iff the IP inside `ip` (treated as a /32 or /128
// host) falls inside any CIDR already stored in rkn_ips. Kept for callers
// that already construct an internal/netaddr.Inet.
func FindRknIP(ip netaddr.Inet) (bool, error) {
ipstr := ip.Inet.String()
if !strings.Contains(ipstr, "/") {
if ip.Inet.To4() != nil {
ipstr += "/32"
} else {
ipstr += "/128"
}
}
_, network, err := net.ParseCIDR(ipstr)
if err != nil {
return false, err
}
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
var matched []RknIP
if err := DB().Raw("SELECT id FROM rkn_ips WHERE network >>= ?", &cidr).Scan(&matched).Error; err != nil {
return false, err
}
return len(matched) > 0, nil
}
// AddRknIP is the legacy per-row insert path. Deprecated: callers should
// invoke ReplaceRknIPs from the rkn updater. Kept around so existing cron
// scripts that import the symbol keep compiling.
func AddRknIP(data []string, count int) {
tx := DB().Begin()
for k, dataIP := range data {
if count > 0 && k > count-1 {
break
}
if !strings.Contains(dataIP, "/") {
dataIP += "/32"
}
_, network, err := net.ParseCIDR(dataIP)
PanicOnErr(err)
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
var rknIPs []RknIP
err = tx.Raw("SELECT id FROM rkn_ips where network >>= ?", &cidr).Scan(&rknIPs).Error
PanicOnErr(err)
if len(rknIPs) == 0 {
rknIPs = []RknIP{}
err = tx.Raw("SELECT id FROM rkn_ips where network << ?", &cidr).Scan(&rknIPs).Error
PanicOnErr(err)
if len(rknIPs) > 0 {
for _, r := range rknIPs {
PanicOnErr(tx.Delete(&r).Error)
}
}
PanicOnErr(tx.Create(&RknIP{Network: &cidr}).Error)
}
log.Println("created:", dataIP)
}
PanicOnErr(tx.Commit().Error)
}
// ReplaceRknIPs deletes every existing row and bulk-inserts the given CIDRs
// in batches of 1000. Whole operation runs in a single transaction so a
// partially-applied update can never leave the table in a hybrid state.
//
// CIDR.parseCIDR-loop uses net.ParseCIDR to canonicalise the address —
// upstream .dat files occasionally contain range/mask pairs that aren't
// already reduced (e.g. 192.168.0.0/16 spelled as 192.168.5.0/16); the
// Postgres cidr type normalises on insert.
func ReplaceRknIPs(nets []*net.IPNet) error {
tx := DB().Begin()
if tx.Error != nil {
return tx.Error
}
rows := make([]RknIP, 0, len(nets))
seen := make(map[string]struct{}, len(nets))
for _, n := range nets {
if n == nil || n.IP == nil {
continue
}
// Canonicalise by routing through net.ParseCIDR. This drops the
// host bits (a common bug in upstream dumps where a /24 range
// is written with the .5 host bit set) and stamps the right
// address family flag for Postgres.
canonical := n.String()
if _, parsed, err := net.ParseCIDR(canonical); err == nil {
n = parsed
canonical = parsed.String()
}
if _, ok := seen[canonical]; ok {
continue
}
seen[canonical] = struct{}{}
cidr := netaddr.Cidr{Cidr: *n, Valid: true}
rows = append(rows, RknIP{Network: &cidr})
}
// Wipe + re-insert in one transaction.
if err := tx.Where("1 = 1").Delete(&RknIP{}).Error; err != nil {
_ = tx.Rollback().Error
return err
}
const batchSize = 1000
for start := 0; start < len(rows); start += batchSize {
end := start + batchSize
if end > len(rows) {
end = len(rows)
}
if err := tx.CreateInBatches(rows[start:end], batchSize).Error; err != nil {
_ = tx.Rollback().Error
return err
}
}
return tx.Commit().Error
}
// IsRknIPBlocked returns true iff `ip` (any textual form ParseCIDR accepts)
// falls inside any CIDR stored in the rkn_ips table. The query uses the
// cidr >>= inet containment operator — see EnsureRknIndexes for the GiST
// index that makes this fast at scale.
func IsRknIPBlocked(ip string) (bool, error) {
ip = strings.TrimSpace(ip)
if ip == "" {
return false, nil
}
var count int64
if err := DB().Raw("SELECT COUNT(*) FROM rkn_ips WHERE network >>= ?::inet", ip).Scan(&count).Error; err != nil {
return false, err
}
return count > 0, nil
}
// EnsureRknIndexes creates the indexes that AutoMigrate cannot express —
// the GiST index on rkn_ips.network uses cidr >>= cidr containment (the
// expression index `network` already covers equality and prefix ranges,
// but the planner benefits from a GiST for `network >>= <other cidr>`
// queries against ~30k rows). The unique index on rkn_domains.domain is
// also declared in the GORM tag, this function only adds what GORM can't
// (GiST) and is idempotent so it's safe to call repeatedly during boot
// or migration.
func EnsureRknIndexes() error {
// GiST on cidr requires the btree_gist contrib — its `cidr_ops`
// opclass exposes cidr to GiST. CREATE EXTENSION IF NOT EXISTS is
// idempotent.
if err := DB().Exec("CREATE EXTENSION IF NOT EXISTS btree_gist").Error; err != nil {
return fmt.Errorf("ensure btree_gist: %w", err)
}
// rkn_ips GiST index on the cidr column supports the >>= containment
// operator that IsRknIPBlocked uses. Without it a 30k-row table makes
// every IP check a sequential scan; with it each check is an index
// probe.
if err := DB().Exec(
"CREATE INDEX IF NOT EXISTS idx_rkn_ips_network ON rkn_ips USING gist (network)",
).Error; err != nil {
return fmt.Errorf("ensure idx_rkn_ips_network: %w", err)
}
return nil
}

114
app/models/rkn_ip_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,114 @@
package models_test
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func mustIPNet(t *testing.T, cidr string) *net.IPNet {
t.Helper()
_, n, err := net.ParseCIDR(cidr)
require.NoError(t, err)
return n
}
func TestReplaceRknIPs_BulkAndDedup(t *testing.T) {
models.Drop()
models.Migrate()
input := []*net.IPNet{
mustIPNet(t, "10.0.0.0/8"),
mustIPNet(t, "10.5.0.0/8"), // same canonical /8 — dedup
mustIPNet(t, "192.168.1.0/24"),
mustIPNet(t, "2001:db8::/32"),
}
require.NoError(t, models.ReplaceRknIPs(input))
// The uniqueIndex on rkn_ips.network ensures the dedup actually drops
// duplicates; ReplaceRknIPs does an in-memory dedup, but the DB-level
// constraint is the guarantee.
var count int64
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&count).Error)
assert.Equal(t, int64(3), count, "expected dedup to 3 unique CIDRs")
}
func TestReplaceRknIPs_ReplacesExisting(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "8.8.8.0/24"),
}))
var n int64
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
assert.Equal(t, int64(1), n)
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "1.0.0.0/8"),
mustIPNet(t, "2.0.0.0/8"),
}))
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
assert.Equal(t, int64(2), n)
}
func TestIsRknIPBlocked(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "10.0.0.0/8"),
mustIPNet(t, "192.168.1.0/24"),
mustIPNet(t, "2001:db8::/32"),
}))
cases := []struct {
ip string
want bool
}{
{"10.5.7.9", true},
{"10.255.255.255", true},
{"192.168.1.42", true},
{"11.0.0.1", false},
{"8.8.8.8", false},
{"2001:db8::1", true},
{"2001:db9::1", false},
{"", false},
}
for _, c := range cases {
t.Run(c.ip, func(t *testing.T) {
got, err := models.IsRknIPBlocked(c.ip)
require.NoError(t, err)
assert.Equal(t, c.want, got, "ip=%s", c.ip)
})
}
}
func TestEnsureRknIndexes_Idempotent(t *testing.T) {
// Calling EnsureRknIndexes twice must not error — it's used both by
// Migrate() and could be called from boot scripts.
models.Drop()
models.Migrate()
require.NoError(t, models.EnsureRknIndexes())
require.NoError(t, models.EnsureRknIndexes())
// GiST index must exist on rkn_ips.network.
var exists bool
err := models.DB().Raw(`
SELECT EXISTS (
SELECT 1 FROM pg_indexes
WHERE schemaname='public' AND tablename='rkn_ips'
AND indexname='idx_rkn_ips_network'
)
`).Scan(&exists).Error
require.NoError(t, err)
assert.True(t, exists, "idx_rkn_ips_network must exist")
}

55
app/models/selfcheck.go Обычный файл
Просмотреть файл

@@ -0,0 +1,55 @@
package models
import (
"time"
"github.com/pkg/errors"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// SelfCheck provides functionality.
type SelfCheck struct {
concerns.Model
Kind string `gorm:"not null;uniqueIndex:selfchecks" json:"kind"`
Server *string `gorm:"uniqueIndex:selfchecks" json:"server"`
Info string `json:"info"`
LastCheck time.Time `json:"created_at"`
}
// LogCheck provides functionality.
func LogCheck(kind string) error {
m := SelfCheck{
Kind: kind,
Server: nil,
}
DB().FirstOrInit(&m, m)
m.LastCheck = time.Now()
return DB().Save(&m).Error
}
// IsOk checks if the selfcheck for the given kind ran recently.
func IsOk(kind string) (bool, string, error) {
m := SelfCheck{
Kind: kind,
Server: nil,
}
DB().First(&m, m)
if m.ID == 0 {
return false, "", errors.New("not run")
}
var ago time.Time
if kind == "exp" {
ago = time.Now().Add(-3 * time.Hour)
} else {
ago = time.Now().Add(-15 * time.Minute)
}
isOk := m.LastCheck.After(ago)
return isOk, m.LastCheck.Format(time.RFC3339), nil
}

388
app/models/server.go Обычный файл
Просмотреть файл

@@ -0,0 +1,388 @@
package models
import (
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"database/sql/driver"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// AccountMCPToken stores a one-way verifier. MCP tokens are bearer credentials
// and must remain valid even when the deployment has no encryption key.
type AccountMCPToken struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
Name string `gorm:"size:120;not null" json:"name"`
TokenEnc string `gorm:"column:token;type:char(64);not null;index" json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
concerns.Timestamped
}
func (AccountMCPToken) TableName() string { return "account_mcp_tokens" }
func (t *AccountMCPToken) SetToken(token string) error {
sum := sha256.Sum256([]byte(token))
t.TokenEnc = hex.EncodeToString(sum[:])
return nil
}
func (t *AccountMCPToken) TokenMatches(token string) bool {
sum := sha256.Sum256([]byte(token))
encoded := hex.EncodeToString(sum[:])
return subtle.ConstantTimeCompare([]byte(t.TokenEnc), []byte(encoded)) == 1
}
func GenerateMCPToken() string {
return "mcp_" + base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
}
// Server health states — worst-of-monitor-states rollup; see
// docs/plans/servers-and-hardware-metrics.md §5.1 for the data model.
const (
ServerHealthDown = "down"
ServerHealthWarn = "warn"
ServerHealthUp = "up"
ServerHealthPaused = "paused"
ServerHealthUnknown = "unknown"
)
// ServerEnvironments is the allow-list for Server.Environment.
// New environments require an explicit edit so they are visible in
// tests.
var ServerEnvironments = []string{"production", "staging", "dev", "test"}
// ServerKind is the rstuff-mirrored lifecycle label for a Server
// (production / staging / old). See
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable mapping.
type ServerKind string
// ServerKind values match rstuff's Server.kind enum exactly.
// New values require adding a Postgres enum value via
// app/models/migrate.go.
const (
ServerKindProduction ServerKind = "production"
ServerKindStaging ServerKind = "staging"
ServerKindOld ServerKind = "old"
)
// Scan implements sql.Scanner for ServerKind.
func (k *ServerKind) Scan(src any) error {
if src == nil {
*k = ""
return nil
}
switch v := src.(type) {
case string:
*k = ServerKind(v)
case []byte:
*k = ServerKind(string(v))
default:
return fmt.Errorf("server_kind: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for ServerKind.
func (k ServerKind) Value() (driver.Value, error) {
if k == "" {
return nil, nil
}
return string(k), nil
}
// Server represents a customer-facing logical host (e.g. "prod-web-01").
//
// One Server can host many WorkerNodes (HA after a VM migration). Each
// WorkerNode carries a nullable ServerID so legacy "no server assigned"
// rows keep working. The 1:N relation is stored as a nullable FK on
// worker_nodes.server_id. The N:M relation to Monitor is the
// monitor_servers join table. See
// docs/plans/servers-and-hardware-metrics.md §3 for the layer model
// and §5.1 for the schema.
//
// The inventory fields (ExtID, Kind, Token, PriceCents, Comment,
// Meta) are added per docs/plans/inventory-management.md §6.2 so
// rstuff can push the same row in via Valkey Streams and
// deploymentd can authenticate via Token.
type Server struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
Name string `gorm:"size:120;not null" json:"name"`
Slug string `gorm:"size:120;not null;index" json:"slug"`
Description *string `gorm:"type:text" json:"description"`
Region string `gorm:"size:64;not null;index" json:"region"`
Environment string `gorm:"size:32;not null;default:'production'" json:"environment"`
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
Icon *string `gorm:"size:16" json:"icon"`
Color *string `gorm:"size:16" json:"color"`
Paused bool `gorm:"not null;default:false" json:"paused"`
// Inventory fields (rstuff mirror; see inventory-management.md §6.2).
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
Kind ServerKind `gorm:"type:server_kind;not null;default:'production'" json:"kind"`
// Token is omitted from JSON because it is a bearer credential.
// Read it back only via /api/v1/servers/:id/token (operator-only)
// and never echoed in list/show responses.
Token *string `gorm:"size:64" json:"-"`
PriceCents int `gorm:"not null;default:0" json:"price_cents"`
Comment *string `gorm:"type:text" json:"comment,omitempty"`
// Meta is rstuff-style free-form jsonb; serialized via JSON
// encoding (gin renders it as a nested object).
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
HealthState string `gorm:"size:16;not null;default:'unknown';index" json:"health_state"`
LastCheckAt *time.Time `json:"last_check_at"`
Uptime24h float64 `gorm:"not null;default:1.0" json:"uptime_24h"`
Uptime30d float64 `gorm:"not null;default:1.0" json:"uptime_30d"`
Monitors []Monitor `gorm:"many2many:monitor_servers;joinForeignKey:server_id;joinReferences:monitor_id;" json:"monitors,omitempty"`
Workers []WorkerNode `gorm:"foreignKey:ServerID" json:"workers,omitempty"`
concerns.Timestamped
Audited
}
// TableName provides functionality.
func (Server) TableName() string { return "servers" }
// MonitorServer is the join row for the N:M relation between Monitor and
// Server. One Monitor can be hosted on many Servers (multi-region
// failover); one Server can host many Monitors.
type MonitorServer struct {
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id) ON DELETE CASCADE;primaryKey" json:"monitor_id"`
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;primaryKey" json:"server_id"`
Position int `gorm:"not null;default:0" json:"position"`
concerns.Timestamped
}
// TableName provides functionality.
func (MonitorServer) TableName() string { return "monitor_servers" }
// ServerMetric is the last-N-point cache written alongside VictoriaMetrics.
// The full time-series lives in TSDB; Postgres only keeps the most recent
// row per (server, source) for fast health badges and "last seen" cells.
// See docs/plans/servers-and-hardware-metrics.md §5.3.
type ServerMetric struct {
concerns.Model
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
WorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"worker_id"`
Source string `gorm:"size:32;not null;default:'worker'" json:"source"`
CPUPercent *float64 `json:"cpu_percent"`
MemUsed *int64 `json:"mem_used"`
MemTotal *int64 `json:"mem_total"`
DiskUsed *int64 `json:"disk_used"`
DiskTotal *int64 `json:"disk_total"`
NetRx *int64 `json:"net_rx"`
NetTx *int64 `json:"net_tx"`
HostUptimeSec *int64 `json:"host_uptime_sec"`
Load1 *float64 `json:"load1"`
Load5 *float64 `json:"load5"`
Load15 *float64 `json:"load15"`
ProcessCount *int `json:"process_count"`
Processes datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"processes"`
Networks datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"networks"`
concerns.Timestamped
}
// ServerAlertRule defines one account-owned threshold for a server metric.
// ClearThreshold implements hysteresis: a firing rule only recovers after the
// value drops below it, avoiding alert flapping around Threshold.
type ServerAlertRule struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
Metric string `gorm:"size:32;not null" json:"metric"`
Threshold float64 `gorm:"not null" json:"threshold"`
ClearThreshold float64 `gorm:"not null" json:"clear_threshold"`
DurationSec int `gorm:"not null;default:300" json:"duration_sec"`
NotificationID int64 `gorm:"type:bigint REFERENCES notifications(id) ON DELETE CASCADE;not null" json:"notification_id"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
State string `gorm:"size:16;not null;default:'ok'" json:"state"`
BreachSince *time.Time `json:"breach_since"`
LastValue *float64 `json:"last_value"`
LastMetricID *int64 `json:"last_metric_id"`
LastFiredAt *time.Time `json:"last_fired_at"`
concerns.Timestamped
}
func (ServerAlertRule) TableName() string { return "server_alert_rules" }
// ServerAlertEvent is the durable dedupe/audit record for threshold changes.
type ServerAlertEvent struct {
concerns.Model
RuleID int64 `gorm:"type:bigint REFERENCES server_alert_rules(id) ON DELETE CASCADE;not null;index" json:"rule_id"`
State string `gorm:"size:16;not null" json:"state"`
Value float64 `gorm:"not null" json:"value"`
concerns.Timestamped
}
func (ServerAlertEvent) TableName() string { return "server_alert_events" }
// TableName provides functionality.
func (ServerMetric) TableName() string { return "server_metrics" }
// ValidateEnvironment returns nil iff env is in the allow list.
func ValidateEnvironment(env string) error {
for _, e := range ServerEnvironments {
if env == e {
return nil
}
}
return errors.New("invalid environment")
}
// Slugify turns a server name into a URL-safe slug.
func Slugify(name string) string {
slug := strings.ToLower(strings.TrimSpace(name))
var b strings.Builder
for _, r := range slug {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == ' ', r == '_', r == '-', r == '.':
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
out = "server"
}
return out
}
// AssignMonitors replaces the full set of monitors for a server. Saves
// the join table explicitly because gorm:association_autoupdate is
// disabled globally (CLAUDE.md GORM Conventions).
func (s *Server) AssignMonitors(tx *gorm.DB, monitorIDs []uint) error {
if tx == nil {
tx = DB()
}
if err := tx.Exec("DELETE FROM monitor_servers WHERE server_id = ?", s.ID).Error; err != nil {
return err
}
if len(monitorIDs) == 0 {
return nil
}
seen := make(map[uint]struct{}, len(monitorIDs))
for i, mid := range monitorIDs {
if _, ok := seen[mid]; ok {
continue
}
seen[mid] = struct{}{}
row := MonitorServer{ServerID: s.ID, MonitorID: int64(mid), Position: i}
if err := tx.Create(&row).Error; err != nil {
return err
}
}
return nil
}
// RollupHealthState computes the worst-of-monitor-states. Returns one of
// ServerHealth{Down,Warn,Up,Paused,Unknown}.
func (s *Server) RollupHealthState(monitors []Monitor) string {
if s.Paused {
return ServerHealthPaused
}
if len(monitors) == 0 {
return ServerHealthUnknown
}
allPaused := true
worst := ServerHealthUp
for i := range monitors {
m := &monitors[i]
if m.Enabled {
allPaused = false
}
switch m.State {
case stateERR, stateFail:
return ServerHealthDown
case stateWARN:
worst = ServerHealthWarn
}
}
if allPaused {
return ServerHealthPaused
}
return worst
}
// SortedTagList returns tags sorted ascending; helper for stable JSON.
func (s *Server) SortedTagList() []string {
out := make([]string, len(s.Tags))
copy(out, s.Tags)
sort.Strings(out)
return out
}
// HealthForServer recomputes and persists health_state + last_check_at
// for one server. Called by the distworker health ticker.
func HealthForServer(serverID int64) error {
server := Server{}
if err := DB().First(&server, serverID).Error; err != nil {
return err
}
var monitors []Monitor
if err := DB().Joins("JOIN monitor_servers ms ON ms.monitor_id = monitors.id").
Where("ms.server_id = ?", serverID).Find(&monitors).Error; err != nil {
return err
}
state := server.RollupHealthState(monitors)
now := time.Now()
updates := map[string]interface{}{"health_state": state}
if len(monitors) > 0 {
updates["last_check_at"] = &now
}
return DB().Model(&server).Updates(updates).Error
}
// LatestServerMetric returns the newest accepted worker snapshot for a server.
func LatestServerMetric(serverID int64) (*ServerMetric, error) {
metric := ServerMetric{}
err := DB().Where("server_id = ?", serverID).Order("id DESC").First(&metric).Error
if err != nil {
return nil, err
}
return &metric, nil
}
// FindServerByToken returns the Server whose token column equals
// the given hex value, or nil with gorm.ErrRecordNotFound when no
// row matches. Used by the deploymentd receiver middleware.
func FindServerByToken(token string) (*Server, error) {
var s Server
err := DB().Where("token = ?", token).First(&s).Error
if err != nil {
return nil, err
}
return &s, nil
}
// GenerateServerToken returns a 32-byte random hex string. Caller
// stores the plaintext exactly once (via /servers/:id/rotate-token)
// and updates Server.Token; the old value is no longer recoverable.
func GenerateServerToken() string {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
// crypto/rand should not fail on Linux; panic keeps the
// contract simple for callers in the rare fatal case.
panic(err)
}
return hex.EncodeToString(b)
}

43
app/models/server_health_ticker.go Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package models
import (
"context"
"log"
"sync"
"time"
)
// HealthTickInterval is exported so focused tests can exercise the same
// lifecycle with a short interval.
var HealthTickInterval = time.Minute
var serverHealthTickerOnce sync.Once
// StartServerHealthTicker periodically refreshes derived server health. The
// first production tick is delayed to keep CLI migration paths side-effect
// free; assignment/pause paths recompute synchronously.
func StartServerHealthTicker(parent context.Context) {
serverHealthTickerOnce.Do(func() {
go func() {
ticker := time.NewTicker(HealthTickInterval)
defer ticker.Stop()
for {
select {
case <-parent.Done():
return
case <-ticker.C:
var ids []int64
if err := DB().Model(&Server{}).Pluck("id", &ids).Error; err != nil {
log.Printf("server health: list: %v", err)
continue
}
for _, id := range ids {
if err := HealthForServer(id); err != nil {
log.Printf("server health %d: %v", id, err)
}
}
}
}
}()
})
}

36
app/models/server_ip.go Обычный файл
Просмотреть файл

@@ -0,0 +1,36 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// ServerIp is a single IP (v4 or v6) bound to a Server. The
// canonical source for these rows is the deploymentd server-inventory
// receiver (M1) and the network-diagnostics partial plan; today the
// only writer is operator-entered via /api/v1/servers/:id/ips.
//
// `address` is Postgres `inet` so range queries (`<<` / `>>`) work
// without parsing text. One row per (server_id, address). The
// `is_primary` flag is set when more than one IP exists and the
// deploymentd payload signals a primary; otherwise the first row wins.
//
//nolint:revive // ServerIP rename deferred to M3; rstuff schema uses ServerIp verbatim and parity test depends on it.
type ServerIp struct {
concerns.Model
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
Server *Server `json:"-"`
// Address is mapped to Postgres inet via raw SQL in Migrate(); the
// GORM `type:` tag is not enough because gorm.io/driver/postgres
// does not register `inet` in its type map. Field is stored as a
// string and validated by the API layer (see
// app/controllers/api/server.go ServerIPsAdd).
Address string `gorm:"type:inet;not null" json:"address"`
IsPrimary bool `gorm:"not null;default:false" json:"is_primary"`
RelatedSitesCount int `gorm:"not null;default:0" json:"related_sites_count"`
concerns.Timestamped
}
// TableName provides functionality.
func (ServerIp) TableName() string { return "server_ips" }

98
app/models/site.go Обычный файл
Просмотреть файл

@@ -0,0 +1,98 @@
package models
import (
"strings"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Site represents a customer-facing website or app. One Site can be
// hosted on zero or one Server (server_id nullable) and exposes one
// or more Deployments (compose services or nginx vhosts) plus zero
// or more Repos (via site_repos). The RSMon slice mirrors rstuff's
// `sites` table verbatim — see docs/parity/rstuff-inventory.md §2
// and docs/plans/inventory-management.md §4.
type Site struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
Name string `gorm:"size:120;not null" json:"name"`
Slug string `gorm:"size:120;not null;index" json:"slug"`
URL *string `gorm:"type:text" json:"url,omitempty"`
Description *string `gorm:"type:text" json:"description,omitempty"`
// Kind is a free-text label (not the PG enum) so we can absorb
// rstuff additions without a migration. Default "production".
Kind string `gorm:"size:32;not null;default:'production'" json:"kind"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
Deployments []Deployment `gorm:"foreignKey:SiteID" json:"deployments,omitempty"`
Repos []Repo `gorm:"many2many:site_repos;" json:"repos,omitempty"`
concerns.Timestamped
Audited
}
// TableName returns the table name used for Site. Matches rstuff's
// `sites` plural exactly so the parity test stays trivial.
func (Site) TableName() string { return "sites" }
// SiteSlugify turns a name into a URL-safe slug. Mirrors the rules
// in app/models/server.go:Slugify so /sites/:slug looks the same as
// /servers/:slug. Consecutive separators collapse to a single dash;
// non-ASCII letters are stripped (the same rule as Slugify — we
// don't transliterate in v1, see docs/plans/inventory-management.md
// §11 for transliteration as a future hardening item).
func SiteSlugify(name string) string {
slug := strings.ToLower(strings.TrimSpace(name))
var b strings.Builder
prevDash := false
for _, r := range slug {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
prevDash = false
case r == ' ', r == '_', r == '-', r == '.':
if !prevDash && b.Len() > 0 {
b.WriteByte('-')
prevDash = true
}
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
out = "site"
}
return out
}
// FindOrCreateSiteBySlug returns the site with the given slug for the
// given account, creating an empty row (Name=slug, Kind=production,
// IsActive=true) when no match exists. The caller's tx wraps the
// operation so docker payload ingestion stays atomic.
// See docs/plans/inventory-management.md §7.2 — Docker receiver.
func FindOrCreateSiteBySlug(tx *gorm.DB, accountID int64, slug string) (*Site, error) {
if tx == nil {
tx = DB()
}
var site Site
err := tx.Where("account_id = ? AND slug = ?", accountID, slug).First(&site).Error
if err == nil {
return &site, nil
}
if err != gorm.ErrRecordNotFound {
return nil, err
}
site = Site{AccountID: accountID, Slug: slug, Name: slug, Kind: "production", IsActive: true}
if err := tx.Create(&site).Error; err != nil {
return nil, err
}
return &site, nil
}

120
app/models/stats_data.go Обычный файл
Просмотреть файл

@@ -0,0 +1,120 @@
package models
// StatsData provides functionality.
type StatsData struct {
MonitorID *int64 `json:"monitor_id,omitempty"`
Up30d *float64 `json:"up_30d"`
Up7d *float64 `json:"up_7d"`
Up1d *float64 `json:"up_1d"`
}
// Process provides functionality.
func (data *StatsData) Process() {
if data.Up1d == nil {
dv := 100.0
data.Up1d = &dv
}
if data.Up7d == nil {
dv := 100.0
data.Up7d = &dv
}
if data.Up30d == nil {
dv := 100.0
data.Up30d = &dv
}
}
// 86400 seconds / 1d
// 604800 seconds / 7d
// 2592000 seconds / 30d
// UptimeSQL provides functionality.
const UptimeSQL = `
SELECT
monitors.id AS monitor_id,
round(up1.up::numeric, 3) AS up1d,
round(up7.up::numeric, 3) AS up7d,
round(up30.up::numeric, 3) AS up30d
FROM monitors
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '1' day
group by monitor_id
) up1 ON monitors.id = up1.monitor_id
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '7' day
group by monitor_id
) up7 ON monitors.id = up7.monitor_id
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '30' day
group by monitor_id
) up30 ON monitors.id = up30.monitor_id
`
// UptimeAllSQL provides functionality.
const UptimeAllSQL = `
SELECT
round(up1.up::numeric, 3) AS up1d,
round(up7.up::numeric, 3) AS up7d,
round(up30.up::numeric, 3) AS up30d
FROM (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '1' day AND events.monitor_id IN (?)
) up1, (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '7' day AND events.monitor_id IN (?)
) up7, (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '30' day AND events.monitor_id IN (?)
) up30
`
// MonitorStats provides functionality.
func MonitorStats(monitors *[]Monitor) error {
ids := make([]int64, len(*monitors))
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
ids[i] = m.ID
}
rows, err := DB().Raw(UptimeSQL+"WHERE monitors.id IN (?)", ids).Rows()
if err != nil {
return err
}
stats := make(map[int64]StatsData, 0)
defer rows.Close() //nolint:errcheck
for rows.Next() {
data := StatsData{}
_ = DB().ScanRows(rows, &data)
data.Process()
stats[*data.MonitorID] = data
}
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
(*monitors)[i].StatsData = stats[m.ID]
}
return nil
}

461
app/models/status_page.go Обычный файл
Просмотреть файл

@@ -0,0 +1,461 @@
// Package models — status page subsystem (docs/plans/status-pages.md).
//
// M0 ships the schema for status_pages and its five related tables
// (subscribers, incidents, maintenance, domains). The M0 milestone is
// read-only at the dashboard level — no editor and no public render yet —
// but landing the schema now lets downstream milestones wire public
// routes, editor flows, and the notifier→subscriber bridge without
// further ALTER TABLE churn. M5 (custom domain) only fills in
// status_page_domains rows; the table itself is reserved here so the
// M5 migration is just data, not DDL.
//
// All tables follow the existing RSMon conventions: concerns.Model +
// concerns.Timestamped + Audited mixins, gorm.DeletedAt for soft delete
// on the top-level status_pages row, pq.Int64Array for the monitor_ids
// bigint[] join columns (same shape as sites and the Check.Warnings
// slice). Partial-unique indexes (slug, subscriber email) are added via
// raw SQL in app/models/migrate.go because the GORM tag language cannot
// express a WHERE deleted_at IS NULL predicate.
package models
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"net"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/lib/pq"
"golang.org/x/net/idna"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/config/credis"
)
// Status page color defaults. Matches the existing landing-page primary
// green and the accent blue used in the /settings UI, so a freshly
// created page already blends in with the rest of the app.
const (
statusPageDefaultPrimaryColor = "#62c600"
statusPageDefaultAccentColor = "#1a73e8"
statusPageDefaultHistoryDays = 90
statusPageMaxSlugLen = 64
statusPageMaxNameLen = 120
)
// Hex color regex — accepts #RGB and #RRGGBB. Centralized so the
// controller/model validation agree on the same shape.
var hexColorRegex = regexp.MustCompile(`^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$`)
// StatusPageSlugRegex mirrors the slug format enforced in
// StatusPage.NormalizeSlug / ValidateSlug. Lowercase alphanumerics and
// dashes, must start and end with an alphanumeric. Length is checked
// separately so the regex stays readable.
var statusPageSlugRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`)
// StatusPage represents a public status page owned by an account. One
// account may own many pages (gated by plan in M2+); slugs are globally
// unique because public URLs do not contain the account ID. Each page exposes
// a curated subset of the account's monitors and a recent-incidents
// feed. Soft-deleted rows remain in the table so the partial-unique
// index on (account_id, slug) WHERE deleted_at IS NULL still rejects
// duplicate slugs against historical records — see the comment on
// StatusPagesAccountSlugUnique in migrate.go.
type StatusPage struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
Slug string `gorm:"size:64;not null;index" json:"slug"`
Name string `gorm:"size:120;not null" json:"name"`
Description *string `gorm:"type:text" json:"description,omitempty"`
LogoURL *string `gorm:"size:255" json:"logo_url,omitempty"`
PrimaryColor string `gorm:"size:7;not null;default:'#62c600'" json:"primary_color"`
AccentColor string `gorm:"size:7;not null;default:'#1a73e8'" json:"accent_color"`
// MonitorIDs is the curated subset of account monitors the page
// exposes. Order is preserved so the dashboard list and the public
// render show the same ordering. Stored as bigint[] to keep
// monitor-to-page mapping lookup-free on the read path; M2 will
// add a UI to maintain this set.
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
ShowUptimeBars bool `gorm:"not null;default:true" json:"show_uptime_bars"`
ShowResponseTimes bool `gorm:"not null;default:true" json:"show_response_times"`
ShowHistoryDays int `gorm:"not null;default:90" json:"show_history_days"`
// PasswordHash is populated in M4 only. Stored at length 255 so
// a future bcrypt cost bump does not need a column resize.
PasswordHash *string `gorm:"size:255" json:"-"`
// GATrackingID — Google Analytics 4 measurement ID; emitted by
// the public renderer in M4.
GATrackingID *string `gorm:"size:32" json:"ga_tracking_id,omitempty"`
// NoIndex emits <meta name="robots" content="noindex"> so the
// page can be staged without polluting search indexes.
NoIndex bool `gorm:"not null;default:false" json:"no_index"`
// IsPublished gates the public /status/:slug render. Until M2
// ships the editor the default value keeps M0 pages invisible.
IsPublished bool `gorm:"not null;default:false" json:"is_published"`
// AutoOpenIncidents is opt-in so publishing a page does not change
// existing alert behavior until an owner explicitly enables it.
AutoOpenIncidents bool `gorm:"not null;default:false" json:"auto_open_incidents"`
concerns.Timestamped
Audited
// DeletedAt is the GORM soft-delete marker. Using gorm.DeletedAt
// rather than concerns.SoftDelete because the latter adds a
// DeleterID users(id) FK that we do not yet need on status_pages.
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
}
// TableName returns the explicit table name so GORM does not try to
// pluralize to "status_pages" via inflection. The plural is already
// correct; we declare it anyway for clarity.
func (StatusPage) TableName() string { return "status_pages" }
// StatusPageSubscriberKind — values stored in status_page_subscribers.kind.
// "alert" subscribes to incident-driven notifications; "digest_daily"
// receives the morning summary (M3). New kinds should be appended so
// the JSON serializations stay stable.
const (
StatusPageSubscriberKindAlert = "alert"
StatusPageSubscriberKindDigestDaily = "digest_daily"
)
// StatusPageSubscriber is a row in status_page_subscribers. Email is
// stored verbatim (no citext) because the codebase already persists
// contact emails as-is; case-insensitive uniqueness is enforced via
// the partial unique index in migrate.go using lower(email).
type StatusPageSubscriber struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
// ContactID is an internal delivery endpoint. It is never returned by public
// subscription APIs; tasks use it to preserve the normal worker email path.
ContactID *int64 `gorm:"type:bigint REFERENCES contacts(id) ON DELETE SET NULL;index" json:"-"`
Email string `gorm:"size:255;not null" json:"email"`
Kind string `gorm:"size:16;not null;default:'alert'" json:"kind"`
ConfirmTokenHash string `gorm:"size:64" json:"-"`
// LegacyConfirmToken is retained only for confirmation links issued before
// token hashing shipped. It is cleared on first use or resend.
LegacyConfirmToken *string `gorm:"column:confirm_token;size:255" json:"-"`
TokenExpiresAt time.Time `json:"-"`
UnsubscribeTokenHash string `gorm:"size:64;default:''" json:"-"`
ConfirmedAt *time.Time `json:"confirmed_at,omitempty"`
UnsubscribedAt *time.Time `json:"unsubscribed_at,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_subscribers table name.
func (StatusPageSubscriber) TableName() string { return "status_page_subscribers" }
// StatusPageIncident severity values. info = heads-up notices, warn =
// degradation, crit = full outage. Used for color-coding in the public
// render (M1) and for filtering in the dashboard list (M0).
const (
StatusPageIncidentSeverityInfo = "info"
StatusPageIncidentSeverityWarn = "warn"
StatusPageIncidentSeverityCrit = "crit"
)
// StatusPageIncident represents a single incident entry on a status
// page. event_id is a soft link back to the existing Event model so
// "auto-open on monitor error" can be wired later without a second
// migration. posted_by_user_id is nullable so external integrations
// can write incidents anonymously.
type StatusPageIncident struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
EventID *int64 `gorm:"type:bigint REFERENCES events(id) ON DELETE SET NULL;index" json:"event_id,omitempty"`
Title string `gorm:"size:200;not null" json:"title"`
BodyMD string `gorm:"type:text" json:"body_md,omitempty"`
Severity string `gorm:"size:16;not null;default:'info'" json:"severity"`
StartedAt time.Time `gorm:"not null;index" json:"started_at"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
PostedByUserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"posted_by_user_id,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_incidents table name.
func (StatusPageIncident) TableName() string { return "status_page_incidents" }
// StatusPageMaintenance is a scheduled maintenance window. The
// monitor_ids column is the set of monitors the window covers; empty
// means "all monitors on the page".
type StatusPageMaintenance struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
Title string `gorm:"size:200;not null" json:"title"`
Description string `gorm:"type:text" json:"description,omitempty"`
StartsAt time.Time `gorm:"not null;index" json:"starts_at"`
EndsAt time.Time `gorm:"not null" json:"ends_at"`
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
NotifySubscribers bool `gorm:"not null;default:true" json:"notify_subscribers"`
concerns.Timestamped
}
// TableName returns the explicit status_page_maintenance table name.
func (StatusPageMaintenance) TableName() string { return "status_page_maintenance" }
// StatusPageDomain is the M5 custom-domain mapping. Reserved in M0 so
// the table does not need to be created at M5 — only rows are written
// then. domain is unique globally (CNAMEs are hostnames, they cannot
// be reused across pages), txt_token is the value the user adds as a
// DNS TXT record to prove ownership.
type StatusPageDomain struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
Domain string `gorm:"size:255;not null;uniqueIndex" json:"domain"`
VerifiedAt *time.Time `json:"verified_at,omitempty"`
TXTToken string `gorm:"size:64;not null" json:"txt_token,omitempty"`
VerifyError string `gorm:"type:text" json:"verify_error,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_domains table name.
func (StatusPageDomain) TableName() string { return "status_page_domains" }
// NormalizeStatusPageDomain accepts a hostname only. URLs, ports, IP literals,
// wildcard names, and invalid IDNA are deliberately rejected before DNS work.
func NormalizeStatusPageDomain(in string) (string, error) {
domain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(in)), ".")
if domain == "" || len(domain) > 253 || strings.ContainsAny(domain, "/:@") || net.ParseIP(domain) != nil {
return "", errStatusPage("domain must be a hostname")
}
ascii, err := idna.Lookup.ToASCII(domain)
if err != nil || ascii == "" || len(ascii) > 253 || !strings.Contains(ascii, ".") {
return "", errStatusPage("domain must be a valid hostname")
}
for _, label := range strings.Split(ascii, ".") {
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
return "", errStatusPage("domain must be a valid hostname")
}
for _, r := range label {
if r != '-' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
return "", errStatusPage("domain must be a valid hostname")
}
}
}
return ascii, nil
}
// HashStatusPageToken keeps bearer-style subscription URLs out of the database.
func HashStatusPageToken(token string) string {
sum := sha256.Sum256([]byte(token))
return fmt.Sprintf("%x", sum[:])
}
func (s *StatusPageSubscriber) TokenMatches(token string) bool {
return s != nil && s.TokenExpiresAt.After(time.Now()) && s.ConfirmTokenHash == HashStatusPageToken(token)
}
func (s *StatusPageSubscriber) UnsubscribeTokenMatches(token string) bool {
return s != nil && s.UnsubscribeTokenHash != "" && s.UnsubscribeTokenHash == HashStatusPageToken(token)
}
// StatusPagePlatformDomain and StatusPagePublicIPs are deployment-owned DNS
// targets. Customer domains must point here before they can become routable.
func StatusPagePlatformDomain() string {
return strings.TrimSuffix(strings.ToLower(os.Getenv("STATUS_PAGE_PLATFORM_DOMAIN")), ".")
}
func StatusPagePublicIPs() []string {
return strings.FieldsFunc(os.Getenv("STATUS_PAGE_PUBLIC_IPS"), func(r rune) bool { return r == ',' || r == ' ' })
}
var (
statusPageLookupCNAME = net.LookupCNAME
statusPageLookupHost = net.LookupHost
statusPageLookupTXT = net.LookupTXT
)
// VerifyStatusPageDomain performs the BYO-DNS preflight. A customer must keep
// the ownership TXT record and point either a CNAME at the platform hostname or
// an A/AAAA record at one of the explicitly configured public addresses.
func VerifyStatusPageDomain(domain *StatusPageDomain) error {
if domain == nil {
return errStatusPage("domain is required")
}
want, err := NormalizeStatusPageDomain(domain.Domain)
if err != nil {
return err
}
platform := StatusPagePlatformDomain()
publicIPs := StatusPagePublicIPs()
if platform == "" && len(publicIPs) == 0 {
return errStatusPage("custom domain verification is not configured")
}
matchedTarget := false
if platform != "" {
if cname, lookupErr := statusPageLookupCNAME(want); lookupErr == nil {
matchedTarget = strings.TrimSuffix(strings.ToLower(cname), ".") == platform
}
}
if !matchedTarget && len(publicIPs) > 0 {
if hosts, lookupErr := statusPageLookupHost(want); lookupErr == nil {
for _, host := range hosts {
for _, allowed := range publicIPs {
if host == allowed {
matchedTarget = true
}
}
}
}
}
if !matchedTarget {
return errStatusPage("DNS must contain the configured CNAME or public A/AAAA address")
}
txt, lookupErr := statusPageLookupTXT(want)
if lookupErr != nil {
return errStatusPage("ownership TXT record was not found")
}
for _, value := range txt {
if value == "rsmon-verify="+domain.TXTToken {
return nil
}
}
return errStatusPage("ownership TXT record does not match")
}
// NormalizeStatusPageSlug lowercases and trims a candidate slug so the
// global partial unique index on slug is satisfied
// regardless of how the caller capitalizes the input. Returns the
// empty string when the result would be unusable as a URL path;
// callers should fall back to a name-derived slug in that case.
func NormalizeStatusPageSlug(in string) string {
slug := strings.ToLower(strings.TrimSpace(in))
return slug
}
// ValidateStatusPageSlug enforces the slug format we expose to users:
// lowercase alphanumeric plus dash, must start and end with an
// alphanumeric, max length 64. Used by the editor before save (M2).
// Returns nil when the slug is acceptable.
func ValidateStatusPageSlug(slug string) error {
if slug == "" {
return errStatusPage("slug is required")
}
if len(slug) > statusPageMaxSlugLen {
return errStatusPage("slug is too long")
}
if !statusPageSlugRegex.MatchString(slug) {
return errStatusPage("slug must be lowercase alphanumeric with dashes")
}
return nil
}
// ValidateStatusPageColors returns an error if either color is set but
// not a valid CSS hex string. Empty strings fall back to the model
// defaults when written via BeforeSave hooks.
func ValidateStatusPageColors(primary, accent string) error {
if primary != "" && !hexColorRegex.MatchString(primary) {
return errStatusPage("primary_color must be #RGB or #RRGGBB")
}
if accent != "" && !hexColorRegex.MatchString(accent) {
return errStatusPage("accent_color must be #RGB or #RRGGBB")
}
return nil
}
// BeforeSave is the GORM hook that fills in the canonical defaults
// (colors, history days) so callers can pass an empty struct and still
// get a usable page. Hook is also the single place where slugs are
// normalized, so the unique index never has to chase trailing spaces.
// The gorm.DB parameter is required by the hook signature but unused —
// the validation here is purely local to the model.
func (p *StatusPage) BeforeSave(_ *gorm.DB) error {
if p == nil {
return nil
}
p.Slug = NormalizeStatusPageSlug(p.Slug)
if err := ValidateStatusPageSlug(p.Slug); err != nil {
return err
}
if p.PrimaryColor == "" {
p.PrimaryColor = statusPageDefaultPrimaryColor
}
if p.AccentColor == "" {
p.AccentColor = statusPageDefaultAccentColor
}
if p.ShowHistoryDays == 0 {
p.ShowHistoryDays = statusPageDefaultHistoryDays
}
return ValidateStatusPageColors(p.PrimaryColor, p.AccentColor)
}
// IsPublishedNow reports whether the page is publicly visible. M0
// always returns false because the editor (M2) is the only thing that
// flips IsPublished to true; this helper centralizes that contract.
func (p *StatusPage) IsPublishedNow() bool {
return p != nil && p.IsPublished && !p.DeletedAt.Valid
}
// errStatusPage builds a validation error carrying the message. The
// returned error is a plain error; controllers translate it into a
// 422 response.
func errStatusPage(msg string) error {
if msg == "" {
return errors.New("status_page: invalid")
}
return fmt.Errorf("status_page: %s", msg)
}
// IsActiveSubscriber returns true when the subscriber has confirmed and
// has not unsubscribed. Used by the M3 incident-notification loop.
func (s *StatusPageSubscriber) IsActiveSubscriber() bool {
if s == nil {
return false
}
return s.ConfirmedAt != nil && s.UnsubscribedAt == nil
}
func (s *StatusPageSubscriber) BeforeCreate(_ *gorm.DB) error {
if s.TokenExpiresAt.IsZero() {
s.TokenExpiresAt = time.Now().Add(24 * time.Hour)
}
if s.ConfirmTokenHash == "" {
s.ConfirmTokenHash = HashStatusPageToken(fmt.Sprintf("legacy-%d-%s", time.Now().UnixNano(), s.Email))
}
return nil
}
// InvalidateStatusPageCache removes the public HTML cache without making
// Redis availability part of the monitor or management write path.
func InvalidateStatusPageCache(pageID int64) {
if credis.Redis != nil {
_ = credis.Redis.Del(context.Background(), "statuspage:html:"+strconv.FormatInt(pageID, 10)).Err()
}
}

408
app/models/status_page_delivery.go Обычный файл
Просмотреть файл

@@ -0,0 +1,408 @@
package models
import (
"context"
"encoding/json"
"errors"
"fmt"
"html"
"log"
"strings"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
const (
statusPageDigestHour = 9
statusPageOutboxPending = "pending"
statusPageOutboxDispatched = "dispatched"
statusPageOutboxCanceled = "canceled"
)
// StatusPageDelivery is an outbox row. It does not retain email addresses;
// confirmation links are the only transient body data and are redacted on
// cancellation. The subscriber/contact data is read while dispatching.
type StatusPageDelivery struct {
ID int64 `gorm:"primarykey"`
StatusPageID int64 `gorm:"not null;index"`
SubscriberID int64 `gorm:"not null;index"`
IncidentID *int64 `gorm:"index"`
Kind string `gorm:"size:32;not null"`
Version string `gorm:"size:64;not null"`
LocalDate string `gorm:"size:10"`
State string `gorm:"size:16;not null;index"`
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
MessageID *int64 `gorm:"index"`
TaskID *int64 `gorm:"index"`
LastError string `gorm:"type:text"`
Subject string `gorm:"type:text"`
BodyText string `gorm:"type:text"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
type StatusPageDigestSchedule struct {
ID int64 `gorm:"primarykey"`
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
DueAt time.Time `gorm:"not null;index"`
CreatedAt time.Time
}
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
if subscriber.ContactID != nil {
return nil
}
accountID := page.AccountID
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
if err := tx.Create(contact).Error; err != nil {
return err
}
subscriber.ContactID = &contact.ID
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
}
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
}
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
}
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
}
func valueOrZero(v *int64) int64 {
if v == nil {
return 0
}
return *v
}
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
// transaction, so an incident can never become visible without its delivery
// intent being recoverable by the dispatcher.
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
return err
}
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
for _, subscriber := range subscribers {
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
return err
}
}
return nil
}
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
var notification Notification
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(&notification).Error
if err == nil {
return &notification, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
return &notification, tx.Create(&notification).Error
}
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
// Message and Task rows in the same transaction, and preserves a pending row on
// transient worker/capability failure for the next tick.
func DispatchStatusPageDeliveries(now time.Time) error {
var rows []StatusPageDelivery
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
return err
}
var errs []error
for _, row := range rows {
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func dispatchStatusPageDelivery(id int64, now time.Time) error {
err := DB().Transaction(func(tx *gorm.DB) error {
var outbox StatusPageDelivery
if err := tx.First(&outbox, id).Error; err != nil {
return err
}
var subscriber StatusPageSubscriber
var page StatusPage
// Subscriber then outbox is the global lock order shared with
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
}
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
return err
}
if outbox.State != statusPageOutboxPending {
return nil
}
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
}
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
return err
}
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
return err
}
notification, err := statusPageNotificationTx(tx, page.AccountID)
if err != nil {
return err
}
subject, text := outbox.Subject, outbox.BodyText
if subject == "" {
subject, text = "["+page.Name+"] status update", "Status update"
}
if outbox.IncidentID != nil {
var incident StatusPageIncident
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
return err
}
subject = "[" + page.Name + "] " + incident.Title
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
} else if outbox.BodyText == "" {
text = "Open incidents daily digest"
}
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
if err := tx.Create(message).Error; err != nil {
return err
}
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
if err != nil {
return err
}
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
if err != nil {
return err
}
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
})
if err != nil {
// This update intentionally runs after rollback: diagnostic state must not
// disappear with the failed message/task transaction.
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
}
return err
}
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
return err
}
if outbox.TaskID != nil {
// Redact every task, including terminal audit rows. State is preserved for
// terminal rows, while the payload can no longer disclose the address.
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
return err
}
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
}
if outbox.MessageID != nil {
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
}
return nil
}
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
var subscriber StatusPageSubscriber
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
var rows []StatusPageDelivery
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
return err
}
}
return nil
}
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
// retained status-page lineage without touching unrelated account messages.
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
return err
}
for i := range rows {
row := &rows[i]
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if row.TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if row.MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if rows[i].TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if rows[i].MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
// endpoint inert and irreversibly removing subscriber PII.
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
return err
}
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
return err
}
now := time.Now()
for i := range subscribers {
s := &subscribers[i]
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
return err
}
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
return err
}
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
return err
}
if s.ContactID != nil {
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
return err
}
}
}
return tx.Delete(page).Error
}
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
func EnqueueStatusPageDailyDigests(now time.Time) error {
var subscribers []StatusPageSubscriber
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
return err
}
var errs []error
for i := range subscribers {
s := &subscribers[i]
if s.StatusPage == nil || s.StatusPage.Account == nil {
continue
}
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
if err != nil {
loc = time.UTC
}
localNow := now.In(loc)
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
// Only persist today's actual due date. Older pending outbox rows are
// retried by DispatchStatusPageDeliveries; never invent history from a
// current incident snapshot after downtime.
if now.Before(due) || s.ConfirmedAt.After(due) {
continue
}
var incidents []StatusPageIncident
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
errs = append(errs, err)
continue
}
if len(incidents) == 0 {
continue
} // No digest is better than a misleading outage summary.
lines := make([]string, 0, len(incidents))
for _, incident := range incidents {
lines = append(lines, "- "+incident.Title)
}
err = DB().Transaction(func(tx *gorm.DB) error {
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
return err
}
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
})
if err != nil {
errs = append(errs, err)
}
}
if err := DispatchStatusPageDeliveries(now); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func StartStatusPageDigestScheduler(ctx context.Context) {
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
log.Printf("status-page delivery dispatch: %v", err)
}
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
log.Printf("status-page digest: %v", err)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
}

401
app/models/status_page_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,401 @@
package models_test
import (
"strings"
"testing"
"time"
"gorm.io/gorm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// statusPageSeedAcc returns a fresh account for status-page tests.
func statusPageSeedAcc(t *testing.T) *models.Account {
t.Helper()
acc := &models.Account{Name: "status-page-" + strings.ReplaceAll(t.Name(), "/", "_")}
require.NoError(t, models.DB().Create(acc).Error)
return acc
}
// TestStatusPage_Defaults asserts that creating a StatusPage with the
// minimum required fields fills in the documented defaults (colors,
// history days, soft-delete marker) and leaves IsPublished false.
func TestStatusPage_Defaults(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{
AccountID: acc.ID,
Slug: "acme-status",
Name: "Acme Status",
}
require.NoError(t, models.DB().Create(page).Error)
got := models.StatusPage{}
require.NoError(t, models.DB().First(&got, page.ID).Error)
assert.Equal(t, "#62c600", got.PrimaryColor, "default primary color")
assert.Equal(t, "#1a73e8", got.AccentColor, "default accent color")
assert.Equal(t, 90, got.ShowHistoryDays, "default history days")
assert.True(t, got.ShowUptimeBars, "show_uptime_bars defaults true")
assert.True(t, got.ShowResponseTimes, "show_response_times defaults true")
assert.False(t, got.IsPublished, "is_published defaults false")
assert.False(t, got.IsPublishedNow(), "IsPublishedNow() returns false until publish")
assert.True(t, got.DeletedAt.Valid == false, "no soft-delete timestamp on fresh row")
}
// TestStatusPage_SlugUniquenessSoftDelete verifies the partial unique
// index allows recreating a slug after soft-deleting the previous row.
func TestStatusPage_SlugUniquenessSoftDelete(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
first := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout",
}
require.NoError(t, models.DB().Create(first).Error)
// Second live row with the same slug must fail.
dup := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout copy",
}
err := models.DB().Create(dup).Error
require.Error(t, err, "duplicate slug on live rows must fail")
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"unexpected error: %v", err,
)
// Soft-delete the first row. After that the slug is reusable.
require.NoError(t, models.DB().Delete(first).Error)
again := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout v2",
}
require.NoError(t, models.DB().Create(again).Error,
"recreating a slug after a soft-delete must succeed (partial index)")
assert.NotEqual(t, first.ID, again.ID)
}
// TestStatusPage_SlugIsGlobal verifies public URLs are unambiguous: the
// account is not part of /status/:slug, so another account cannot claim it.
func TestStatusPage_SlugIsGlobal(t *testing.T) {
models.Drop()
models.Migrate()
accA := statusPageSeedAcc(t)
accB := &models.Account{Name: "status-page-other"}
require.NoError(t, models.DB().Create(accB).Error)
a := &models.StatusPage{AccountID: accA.ID, Slug: "shared", Name: "A"}
require.NoError(t, models.DB().Create(a).Error)
b := &models.StatusPage{AccountID: accB.ID, Slug: "shared", Name: "B"}
require.Error(t, models.DB().Create(b).Error,
"different accounts must not be able to reuse a public slug")
}
func TestStatusPageIncidentDeliveryIsIdempotentAndAccountScoped(t *testing.T) {
models.Drop()
models.Migrate()
account := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: account.ID, Slug: "delivery-page", Name: "Delivery"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "subscriber@example.test", Kind: models.StatusPageSubscriberKindAlert, ConfirmedAt: &now}
require.NoError(t, models.DB().Create(subscriber).Error)
incident := &models.StatusPageIncident{StatusPageID: page.ID, Title: "API unavailable", Severity: models.StatusPageIncidentSeverityCrit, StartedAt: now}
require.NoError(t, models.DB().Create(incident).Error)
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
}))
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
}))
var deliveries []models.StatusPageDelivery
require.NoError(t, models.DB().Where("idempotency_key LIKE ?", "status-page:opened:%").Find(&deliveries).Error)
require.Len(t, deliveries, 1)
assert.Equal(t, page.ID, deliveries[0].StatusPageID)
}
func TestStatusPageDailyDigestIsTimezoneDateIdempotent(t *testing.T) {
models.Drop()
models.Migrate()
account := statusPageSeedAcc(t)
account.Timezone = "UTC"
require.NoError(t, models.DB().Save(account).Error)
page := &models.StatusPage{AccountID: account.ID, Slug: "digest-page", Name: "Digest"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Date(2026, time.July, 13, 9, 15, 0, 0, time.UTC)
confirmedAt := now.Add(-time.Hour)
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "digest@example.test", Kind: models.StatusPageSubscriberKindDigestDaily, ConfirmedAt: &confirmedAt}
require.NoError(t, models.DB().Create(subscriber).Error)
require.NoError(t, models.DB().Create(&models.StatusPageIncident{StatusPageID: page.ID, Title: "Still open", Severity: models.StatusPageIncidentSeverityWarn, StartedAt: now}).Error)
_ = models.EnqueueStatusPageDailyDigests(now)
_ = models.EnqueueStatusPageDailyDigests(now.Add(30 * time.Second))
var count int64
require.NoError(t, models.DB().Model(&models.StatusPageDelivery{}).Where("idempotency_key LIKE ?", "status-page:digest:%").Count(&count).Error)
assert.GreaterOrEqual(t, count, int64(1), "catch-up may include earlier local due dates")
}
// TestStatusPage_NormalizeSlugAndValidate exercises the slug normalizer
// and validator — uppercase input becomes lowercase, and an invalid
// slug (leading dash) is rejected on save via BeforeSave.
func TestStatusPage_NormalizeSlugAndValidate(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
assert.Equal(t, "lower-case", models.NormalizeStatusPageSlug(" Lower-Case "))
assert.NoError(t, models.ValidateStatusPageSlug("acme-status"))
bad := []string{"", "-leading-dash", "trailing-dash-", "UPPER", "with spaces", "with_underscore"}
for _, slug := range bad {
err := models.ValidateStatusPageSlug(slug)
assert.Error(t, err, "expected error for slug %q", slug)
}
// Save enforces the same rules — uppercase gets normalized, invalid
// characters reject.
good := &models.StatusPage{AccountID: acc.ID, Slug: "ACME-Status", Name: "Acme"}
require.NoError(t, models.DB().Create(good).Error)
assert.Equal(t, "acme-status", good.Slug)
bad2 := &models.StatusPage{AccountID: acc.ID, Slug: "-bad", Name: "Bad"}
err := models.DB().Create(bad2).Error
require.Error(t, err, "invalid slug should be rejected by BeforeSave")
}
// TestStatusPage_ColorValidation rejects non-hex colors at save time.
func TestStatusPage_ColorValidation(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
bad := &models.StatusPage{
AccountID: acc.ID,
Slug: "badcolor",
Name: "BadColor",
PrimaryColor: "not-a-color",
}
err := models.DB().Create(bad).Error
require.Error(t, err, "non-hex color must be rejected")
good := &models.StatusPage{
AccountID: acc.ID,
Slug: "goodcolor",
Name: "GoodColor",
PrimaryColor: "#0a1b2c",
AccentColor: "#abc",
}
require.NoError(t, models.DB().Create(good).Error)
reloaded := models.StatusPage{}
require.NoError(t, models.DB().First(&reloaded, good.ID).Error)
assert.Equal(t, "#0a1b2c", reloaded.PrimaryColor)
assert.Equal(t, "#abc", reloaded.AccentColor)
}
// TestStatusPageSubscriber_UniqueActive verifies the partial unique
// index on (status_page_id, lower(email)) only fires while a
// subscriber is not unsubscribed.
func TestStatusPageSubscriber_UniqueActive(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{
AccountID: acc.ID,
Slug: "sub",
Name: "Sub",
}
require.NoError(t, models.DB().Create(page).Error)
mk := func(email string, unsub *time.Time) *models.StatusPageSubscriber {
s := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: email,
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-" + email),
}
s.UnsubscribedAt = unsub
return s
}
first := mk("Alice@Example.com", nil)
require.NoError(t, models.DB().Create(first).Error)
// Same email with mixed case must conflict (lower(email) is the
// unique key), so a true case-insensitive uniqueness is in place.
dup := mk("alice@example.com", nil)
err := models.DB().Create(dup).Error
require.Error(t, err, "active subscriber with same lowercased email must conflict")
// Soft-unsubscribe the original then re-add a fresh row with the
// same email — this must succeed because the partial index
// excludes unsubscribed rows.
now := time.Now()
first.UnsubscribedAt = &now
require.NoError(t, models.DB().Save(first).Error)
resurrected := mk("alice@example.com", nil)
require.NoError(t, models.DB().Create(resurrected).Error,
"creating a fresh subscriber after the previous one unsubscribed must succeed")
assert.NotEqual(t, first.ID, resurrected.ID)
// ActiveSubscriber reflects both confirmed and unsubscribed flags.
confirmed := time.Now()
active := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: "bob@example.com",
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-bob"),
ConfirmedAt: &confirmed,
}
require.NoError(t, models.DB().Create(active).Error)
assert.True(t, active.IsActiveSubscriber(), "confirmed and not unsubscribed → active")
pending := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: "carol@example.com",
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-carol"),
}
require.NoError(t, models.DB().Create(pending).Error)
assert.False(t, pending.IsActiveSubscriber(), "unconfirmed → not active")
}
// TestStatusPageIncident_SeverityAndEventFK checks that the FK to
// events is wired correctly and that severity defaults to "info".
func TestStatusPageIncident_SeverityAndEventFK(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "inc", Name: "Inc"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
inc := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "API slowdown",
BodyMD: "Investigating.",
Severity: models.StatusPageIncidentSeverityWarn,
StartedAt: now,
}
require.NoError(t, models.DB().Create(inc).Error)
got := models.StatusPageIncident{}
require.NoError(t, models.DB().First(&got, inc.ID).Error)
assert.Equal(t, "warn", got.Severity)
// Default severity is "info" when omitted.
auto := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "heads up",
StartedAt: now,
}
require.NoError(t, models.DB().Create(auto).Error)
gotAuto := models.StatusPageIncident{}
require.NoError(t, models.DB().First(&gotAuto, auto.ID).Error)
assert.Equal(t, models.StatusPageIncidentSeverityInfo, gotAuto.Severity)
// EventID stays nullable and the FK tolerates a NULL event.
var nilEvt *int64
noEvt := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "no event",
Severity: models.StatusPageIncidentSeverityCrit,
StartedAt: now,
EventID: nilEvt,
}
require.NoError(t, models.DB().Create(noEvt).Error)
}
// TestStatusPageMaintenance_BigintArray ensures the monitor_ids bigint[]
// column round-trips through the GORM pq.Int64Array driver correctly.
func TestStatusPageMaintenance_BigintArray(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "mt", Name: "Maintenance"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
maint := &models.StatusPageMaintenance{
StatusPageID: page.ID,
Title: "DB upgrade",
Description: "Rolling upgrade.",
StartsAt: now,
EndsAt: now.Add(2 * time.Hour),
MonitorIDs: models.Int64ArrayFromSlice([]int64{1, 7, 42}),
NotifySubscribers: true,
}
require.NoError(t, models.DB().Create(maint).Error)
got := models.StatusPageMaintenance{}
require.NoError(t, models.DB().First(&got, maint.ID).Error)
assert.Equal(t, []int64{1, 7, 42}, []int64(got.MonitorIDs))
assert.True(t, got.NotifySubscribers)
}
// TestStatusPageDomain_TableReservation verifies the M5 table is
// created with the right unique constraints even though M0 does not
// yet populate it.
func TestStatusPageDomain_TableReservation(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "dom", Name: "Domain"}
require.NoError(t, models.DB().Create(page).Error)
d := &models.StatusPageDomain{
StatusPageID: page.ID,
Domain: "status.example.com",
TXTToken: "rsmon-verify=abc123",
}
require.NoError(t, models.DB().Create(d).Error)
require.NoError(t, models.DB().Create(&models.StatusPageDomain{
StatusPageID: page.ID,
Domain: "www.status.example.com",
TXTToken: "rsmon-verify=second",
}).Error, "one status page may own multiple domains")
// Second page claiming the same domain must fail because domain
// is globally unique.
other := &models.StatusPage{AccountID: acc.ID, Slug: "dom2", Name: "Domain2"}
require.NoError(t, models.DB().Create(other).Error)
dup := &models.StatusPageDomain{
StatusPageID: other.ID,
Domain: "status.example.com",
TXTToken: "rsmon-verify=xyz789",
}
err := models.DB().Create(dup).Error
require.Error(t, err)
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"expected unique-constraint error, got %v", err)
}
func TestNormalizeStatusPageDomain(t *testing.T) {
got, err := models.NormalizeStatusPageDomain(" Status.Example.COM. ")
require.NoError(t, err)
assert.Equal(t, "status.example.com", got)
for _, input := range []string{"https://example.com", "example.com:443", "127.0.0.1", "*.example.com", "-bad.example.com"} {
_, err := models.NormalizeStatusPageDomain(input)
assert.Error(t, err, input)
}
}

61
app/models/subscription.go Обычный файл
Просмотреть файл

@@ -0,0 +1,61 @@
package models
import (
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
SubscriptionStatusTrialing = "trialing"
SubscriptionStatusActive = "active"
SubscriptionStatusPastDue = "past_due"
SubscriptionStatusPaused = "paused"
SubscriptionStatusCanceled = "canceled"
SubscriptionStatusExpired = "expired"
)
// Subscription is the current billable entitlement for one account. M0 only
// writes manual subscriptions; provider flows are deliberately deferred.
type Subscription struct {
concerns.Model
AccountID int64 `gorm:"not null;index" json:"account_id"`
Account *Account `json:"-"`
PlanID int64 `gorm:"not null;index" json:"plan_id"`
Plan *Plan `json:"plan,omitempty"`
Provider string `gorm:"size:16;not null;default:'manual'" json:"provider"`
ProviderSubscriptionID *string `gorm:"size:128" json:"provider_subscription_id,omitempty"`
ProviderCustomerID *string `gorm:"size:128" json:"provider_customer_id,omitempty"`
Status string `gorm:"size:24;not null;default:'active';index" json:"status"`
BillingCycle string `gorm:"size:8;not null;default:'monthly'" json:"billing_cycle"`
CurrentPeriodStart *time.Time `json:"current_period_start,omitempty"`
CurrentPeriodEnd *time.Time `gorm:"index" json:"current_period_end,omitempty"`
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
CancelAtPeriodEnd bool `gorm:"not null;default:false" json:"cancel_at_period_end"`
CanceledAt *time.Time `json:"canceled_at,omitempty"`
Currency string `gorm:"size:3;not null" json:"currency"`
AmountMinor int64 `gorm:"not null;default:0" json:"amount_minor"`
MetadataJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"metadata_json"`
concerns.Timestamped
}
// SubscriptionEvent is an append-only audit record for entitlement changes
// and provider deliveries. ProviderEventID makes future webhook processing
// idempotent without coupling the ledger to a specific PSP.
type SubscriptionEvent struct {
concerns.Model
SubscriptionID int64 `gorm:"not null;index" json:"subscription_id"`
AccountID int64 `gorm:"not null;index" json:"account_id"`
Provider string `gorm:"size:16;not null;default:'manual';uniqueIndex:idx_subscription_events_provider_event,priority:1" json:"provider"`
Kind string `gorm:"size:32;not null" json:"kind"`
FromPlanID *int64 `json:"from_plan_id,omitempty"`
ToPlanID *int64 `json:"to_plan_id,omitempty"`
AmountMinor *int64 `json:"amount_minor,omitempty"`
Currency string `gorm:"size:3" json:"currency,omitempty"`
ActorUserID *int64 `json:"actor_user_id,omitempty"`
ProviderEventID *string `gorm:"size:128;uniqueIndex:idx_subscription_events_provider_event,priority:2" json:"provider_event_id,omitempty"`
PayloadJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"payload_json"`
CreatedAt time.Time `json:"created_at"`
}

47
app/models/tag.go Обычный файл
Просмотреть файл

@@ -0,0 +1,47 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Tag stores metadata for a single tag name within an account: the
// user-chosen color (hex string like "#FF5733") and icon (FontAwesome
// icon name like "faTag"). The (account_id, name) pair is unique.
//
// Tags are still attached to monitors via the monitors.tags text[]
// column (see Monitor.Tags). This table only holds the per-tag display
// metadata so the same tag renders consistently across the SPA — it
// does not affect monitor scoping or filtering.
//
// Rows are created lazily: the SPA POSTs a Tag the first time a user
// customizes its color/icon, and GET /tags?with_counts=1 LEFT JOINs
// the unnested monitors.tags array against this table to enrich the
// (name, count) pairs with display metadata. A Tag row may exist with
// zero monitors using it (count=0) — that happens when a user creates
// a tag from /settings/tags/new but has not yet applied it.
type Tag struct {
concerns.Model
AccountID int64 `gorm:"index;not null" json:"account_id"`
Account *Account `gorm:"foreignKey:AccountID" json:"account,omitempty"`
Name string `gorm:"type:varchar(255);not null" json:"name"`
Color string `gorm:"type:varchar(32);not null;default:'#6c757d'" json:"color"`
Icon string `gorm:"type:varchar(64);not null;default:'faTag'" json:"icon"`
Audited
concerns.Timestamped
}
// TagInfo is a tag with the number of monitors using it plus optional
// display metadata (color, icon) coming from the Tag table.
type TagInfo struct {
Name string `json:"name"`
Count int64 `json:"count"`
Color string `json:"color"`
Icon string `json:"icon"`
}
// DefaultTagColor is the hex color used when a Tag has no metadata row.
const DefaultTagColor = "#6c757d"
// DefaultTagIcon is the FontAwesome icon name used when a Tag has no
// metadata row. Matches the icon the SPA renders by default.
const DefaultTagIcon = "faTag"

135
app/models/tags_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,135 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// seedTagsMonitors creates one group with four monitors:
//
// a.test [prod, web]
// b.test [prod, api]
// c.test [staging]
// d.test [] (NULL tags)
//
// and returns the group id. It mirrors the exact SQL the /tags rename, delete
// and with_counts endpoints run, pinning the contract.
func seedTagsMonitors(t *testing.T) int64 {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "A", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "A-default"}
require.NoError(t, models.DB().Create(&group).Error)
monitors := []models.Monitor{
{GroupID: group.ID, Host: "a.test", Tags: []string{"prod", "web"}},
{GroupID: group.ID, Host: "b.test", Tags: []string{"prod", "api"}},
{GroupID: group.ID, Host: "c.test", Tags: []string{"staging"}},
{GroupID: group.ID, Host: "d.test"},
}
for i := range monitors {
require.NoError(t, models.DB().Create(&monitors[i]).Error)
}
return group.ID
}
func TestTagsRenameSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
res := models.DB().Exec(
`UPDATE monitors SET tags = (
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", "production", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected, "only a.test and b.test contain prod")
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"production", "web"}, []string(a.Tags))
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.ElementsMatch(t, []string{"production", "api"}, []string(b.Tags))
var c models.Monitor
require.NoError(t, models.DB().Where("host = ?", "c.test").First(&c).Error)
assert.Equal(t, []string{"staging"}, []string(c.Tags), "staging-only monitor untouched")
}
func TestTagsRenameSQL_Dedup(t *testing.T) {
groupID := seedTagsMonitors(t)
// Rename prod -> web. a.test has both prod and web: must collapse to a
// single "web" entry (array_agg DISTINCT), not [web, web].
res := models.DB().Exec(
`UPDATE monitors SET tags = (
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", "web", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected)
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"web"}, []string(a.Tags), "duplicate must be deduped")
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.ElementsMatch(t, []string{"web", "api"}, []string(b.Tags))
}
func TestTagsDeleteSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
res := models.DB().Exec(
`UPDATE monitors SET tags = array_remove(tags, ?)
WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected)
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"web"}, []string(a.Tags))
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.Equal(t, []string{"api"}, []string(b.Tags))
}
func TestTagsCountSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
type tagInfo struct {
Name string
Count int64
}
var tags []tagInfo
err := models.DB().Raw(
`SELECT tag AS name, COUNT(*) AS count
FROM (SELECT unnest(tags) AS tag FROM monitors
WHERE tags IS NOT NULL AND group_id IN (?)) sub
GROUP BY tag
ORDER BY tag`,
[]int64{groupID},
).Scan(&tags).Error
require.NoError(t, err)
got := map[string]int64{}
for _, tg := range tags {
got[tg.Name] = tg.Count
}
assert.Equal(t, map[string]int64{"prod": 2, "web": 1, "api": 1, "staging": 1}, got)
}

145
app/models/task.go Обычный файл
Просмотреть файл

@@ -0,0 +1,145 @@
package models
import (
"time"
"gorm.io/datatypes"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Task kinds stored in the tasks.kind column. The plan (docs/plans/worker-notifier-mvp.md
// section 4.1) reserves the same enum for checks and notifications; this phase only
// emits notification rows but the enum is shared so the selector can stay a single
// function.
const (
TaskKindNotification = "notification"
TaskKindCheck = "check"
TaskKindServerMetric = "server_metric"
)
// Task states for the durable task envelope.
const (
TaskStateQueued = "queued"
TaskStateLeased = "leased"
TaskStateSucceeded = "succeeded"
TaskStateFailedRetry = "failed_retry"
TaskStateFailedPerm = "failed_perm"
TaskStateDead = "dead"
)
// Notification result statuses reported by the worker (mirrors the wire enum so the
// result handler can decode without re-typing the constants).
const (
NotificationResultDelivered = "delivered"
NotificationResultRetryable = "retryable"
NotificationResultPermanent = "permanent"
NotificationResultPartial = "partial"
)
// SkipLockedClause is the SELECT ... FOR UPDATE SKIP LOCKED clause used by
// every worker-pool selector (checks in check_jobs.ChecksForWorker and
// tasks in task_selector.TasksForWorker / TasksForWorkerNotification).
// Sharing the value keeps the SQL identical across selectors so goconst
// does not flag the literal, and a future change (e.g. NOWAIT) only has
// to touch one place.
var SkipLockedClause = clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}
// Task is the durable envelope for both check and notification work executed by the
// distributed worker pool. Selection uses FOR UPDATE SKIP LOCKED per worker poll so
// a single primary key or sequence never becomes the bottleneck.
//
// Phase 1 (this commit) only emits notification tasks. The `kind` discriminator and
// capability filters are designed to accept checks in phase 2 without a schema change.
type Task struct {
concerns.Model
JobID string `gorm:"uniqueIndex;size:64" json:"job_id"`
Kind string `gorm:"size:32;index" json:"kind"`
State string `gorm:"size:32;index" json:"state"`
LastError string `gorm:"type:text" json:"last_error"`
// Tenancy + audit anchor. AccountID is required for the capability match in
// TasksForWorker; monitor_id / message_id / contact_id are denormalized for
// fast admin queries.
AccountID int64 `gorm:"index" json:"account_id"`
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
CheckID *int64 `json:"check_id,omitempty"`
MessageID *int64 `json:"message_id,omitempty"`
ContactID *int64 `json:"contact_id,omitempty"`
// Payload is the kind-specific blob the worker needs to execute. For
// notifications the producer pre-renders subject/body so the worker does not
// need templating context (see RenderNotificationContent in internal/notifier).
Payload datatypes.JSON `gorm:"type:jsonb" json:"payload"`
// Scheduling + retry envelope. NotBefore is set to NOW() by the producer and
// bumped by the result handler on retryable failures. Deadline is a soft cap
// the selector can use to skip stale tasks.
NotBefore time.Time `json:"not_before"`
Deadline *time.Time `json:"deadline,omitempty"`
// LeaseOwner + LeaseExpiresAt are owned by the selector while the task is
// in state=leased. The reaper clears them when the lease expires.
LeaseOwner string `gorm:"size:128" json:"lease_owner"`
LeaseToken string `gorm:"size:64" json:"-"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
Attempts int `json:"attempts"`
MaxAttempts int `json:"max_attempts"`
// IdempotencyKey is unique per logical event so a retry of the producer's
// enqueue never produces a second Task row. See EnqueueNotificationTask.
IdempotencyKey string `gorm:"uniqueIndex;size:255" json:"idempotency_key"`
// Result holds the most recent worker result body (NotificationResultReport or
// CheckResultReport shape, depending on Kind). It is JSONB so the admin UI can
// pretty-print without a separate result table for transient lookups.
Result datatypes.JSON `gorm:"type:jsonb" json:"result"`
SucceededAt *time.Time `json:"succeeded_at,omitempty"`
concerns.Timestamped
}
// TaskReplay records the single operator-initiated replay of a dead task.
// OriginalTaskID is unique, making repeated clicks/API retries idempotent.
type TaskReplay struct {
concerns.Model
OriginalTaskID int64 `gorm:"uniqueIndex" json:"original_task_id"`
RequeuedTaskID int64 `gorm:"uniqueIndex" json:"requeued_task_id"`
OperatorUserID int64 `gorm:"index" json:"operator_user_id"`
concerns.Timestamped
}
// TableName overrides the default table name so pluralization stays consistent
// with the rest of the schema (tasks, not "task" or "taskses").
func (Task) TableName() string {
return "tasks"
}
// NotificationDelivery is the per-attempt audit row required by section 7.5 of the
// plan ("Audit rows: each successful or failed delivery writes a row in a new
// notification_deliveries table"). The result handler appends one row per result
// frame, which lets support answer "did the customer ever get this alert" without
// scanning application logs.
type NotificationDelivery struct {
concerns.Model
MessageID int64 `gorm:"index" json:"message_id"`
WorkerID string `gorm:"size:128;index" json:"worker_id"`
TaskID int64 `gorm:"index" json:"task_id"`
Status string `gorm:"size:32" json:"status"`
Error string `gorm:"type:text" json:"error"`
DurationMs int `json:"duration_ms"`
ProviderResponse string `gorm:"type:text" json:"provider_response"`
concerns.Timestamped
}
// TableName mirrors the plan's preferred lowercase plural.
func (NotificationDelivery) TableName() string {
return "notification_deliveries"
}

136
app/models/task_reaper.go Обычный файл
Просмотреть файл

@@ -0,0 +1,136 @@
package models
import (
"context"
"log"
"time"
"gorm.io/gorm"
)
// ReapExpiredTasks is the periodic cleanup function described in
// docs/plans/worker-notifier-mvp.md section 8.5:
//
// - tasks in state='leased' whose lease_expires_at is past are returned to
// state='queued' and have their lease_owner cleared, so the next selector
// poll can pick them up.
// - tasks in state='failed_retry' whose not_before is past AND
// attempts >= max_attempts are moved to state='dead' so they show up on
// the admin dead-letter page and stop consuming selector bandwidth.
//
// It returns the number of rows it touched so the caller can log a metric.
// Cheap enough to run from the web process every 30s.
func ReapExpiredTasks() (reaped int, deaded int, err error) {
now := time.Now()
err = DB().Transaction(func(tx *gorm.DB) error {
if err := expireQueuedNotificationTasksTx(tx, now); err != nil {
return err
}
var expired []Task
if err := tx.Where("state = ? AND lease_expires_at IS NOT NULL AND lease_expires_at < ?", TaskStateLeased, now).Find(&expired).Error; err != nil {
return err
}
for i := range expired {
if expired[i].Attempts >= expired[i].MaxAttempts {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateDead, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "last_error": "lease expired after max attempts", "updated_at": now})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 1 {
deaded++
if err := FinalizeNotificationTaskTx(tx, &expired[i], "dead", "lease expired after max attempts"); err != nil {
return err
}
}
continue
}
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateQueued, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "updated_at": now})
if result.Error != nil {
return result.Error
}
reaped += int(result.RowsAffected)
}
var exhausted []Task
if err := tx.Where("state = ? AND not_before <= ? AND attempts >= max_attempts", TaskStateFailedRetry, now).Find(&exhausted).Error; err != nil {
return err
}
for i := range exhausted {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", exhausted[i].ID, TaskStateFailedRetry).Updates(map[string]interface{}{"state": TaskStateDead, "updated_at": now})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 1 {
deaded++
if err := FinalizeNotificationTaskTx(tx, &exhausted[i], "dead", exhausted[i].LastError); err != nil {
return err
}
}
}
return nil
})
return reaped, deaded, err
}
// FinalizeNotificationTaskTx makes a terminal notification task customer-visible
// and auditable. The caller owns the task state transition in this transaction.
func FinalizeNotificationTaskTx(tx *gorm.DB, task *Task, status, reason string) error {
if task == nil || task.Kind != TaskKindNotification || task.MessageID == nil {
return nil
}
if err := tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *task.MessageID, []string{"sent", "error"}).Updates(map[string]interface{}{"state": "error", "error": reason}).Error; err != nil {
return err
}
return tx.Create(&NotificationDelivery{MessageID: *task.MessageID, TaskID: task.ID, Status: status, Error: reason}).Error
}
func expireQueuedNotificationTasksTx(tx *gorm.DB, now time.Time) error {
var tasks []Task
if err := tx.Where("state = ? AND kind = ? AND deadline IS NOT NULL AND deadline <= ?", TaskStateQueued, TaskKindNotification, now).Find(&tasks).Error; err != nil {
return err
}
for i := range tasks {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", tasks[i].ID, TaskStateQueued).Updates(map[string]interface{}{
"state": TaskStateDead, "last_error": "notification deadline expired", "updated_at": now,
})
if result.Error != nil || result.RowsAffected == 0 {
if result.Error != nil {
return result.Error
}
continue
}
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "expired", "notification deadline expired"); err != nil {
return err
}
}
return nil
}
// StartTaskReaper launches a goroutine that runs ReapExpiredTasks on the given
// interval. It honors ctx.Done() so the caller can wind it down without
// leaking. The function is safe to call once per process; the control plane
// runs the reaper from main.init() so only one ticker ever exists in a single
// web process.
func StartTaskReaper(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 30 * time.Second
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
reaped, deaded, err := ReapExpiredTasks()
if err != nil {
log.Printf("task_reaper: error: %v", err)
continue
}
if reaped > 0 || deaded > 0 {
log.Printf("task_reaper: reaped=%d dead=%d", reaped, deaded)
}
}
}
}()
}

391
app/models/task_selector.go Обычный файл
Просмотреть файл

@@ -0,0 +1,391 @@
package models
import (
"errors"
"fmt"
"log"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// DefaultCheckTaskLeaseTTL is how long a leased check remains owned before the reaper
// returns it to the queue. It is deliberately larger than the worker's heartbeat
// (10s) so a healthy worker can finish a 30s check without the reaper stealing
// the lease, and deliberately smaller than the heartbeat timeout (2m) so a
// crashed worker sees its tasks reaped within one watchdog cycle.
const DefaultCheckTaskLeaseTTL = 60 * time.Second
// DefaultTaskLeaseTTL remains the check-task default for existing confirmation
// callers. Generic task selection must use TaskLeaseTTL so notification work is
// not reclaimed during its longer execution window.
const DefaultTaskLeaseTTL = DefaultCheckTaskLeaseTTL
// Notification execution is bounded by the worker runner at five minutes. The
// additional minute covers result serialization, websocket/HTTP transport, and
// a control-plane scheduling delay before the persisted lease may be reaped.
const (
DefaultNotificationExecutionTimeout = 5 * time.Minute
NotificationTaskReportMargin = time.Minute
DefaultNotificationTaskLeaseTTL = DefaultNotificationExecutionTimeout + NotificationTaskReportMargin
)
// TaskLeaseTTL returns the persisted lease lifetime for a task kind.
func TaskLeaseTTL(kind string) time.Duration {
if kind == TaskKindNotification {
return DefaultNotificationTaskLeaseTTL
}
return DefaultCheckTaskLeaseTTL
}
// DefaultTaskMaxAttempts is the retry budget for a task before it moves to dead.
const DefaultTaskMaxAttempts = 5
// DefaultNotificationTaskDeadline is assigned to manually replayed notification
// dead letters. Normal producer tasks may be deadline-free, but a replay must
// never inherit an already-expired deadline.
const DefaultNotificationTaskDeadline = 15 * time.Minute
// ErrNotificationMethodNotAuthorized is returned by EnqueueNotificationTask
// when the producer can prove no worker in the pool is authorized for the
// (method, account) pair. The caller may skip the enqueue or log + continue.
var ErrNotificationMethodNotAuthorized = errors.New("no worker authorized for method/account")
// EnqueueNotificationTaskInput is the pre-rendered envelope produced by the
// notifier producer. All slices are required; the selector never reads them.
type EnqueueNotificationTaskInput struct {
AccountID int64
NotificationID int64
ContactID int64
MessageID *int64
MonitorID *int64
CheckID *int64
EventIDs []int64
Method string // "email", "telegram", "webhook", "mattermost", "sms", "voice"
Subject string
BodyText string
BodyHTML string
BodyMarkdown string
Language string
MessageKind string // "down", "up", "exp", "test"
NotBefore time.Time
Deadline *time.Time
MaxAttempts int
Payload []byte // marshaled task-specific data
IdempotencyKey string // optional; manual test tasks use a unique key and do not have event IDs
}
// EnqueueNotificationTask writes one Task row keyed by a stable idempotency key.
// A second call with the same key (same notification/contact/event triple) is a
// no-op so the producer is safe to call more than once per pass.
//
// The capability precheck uses the same NotificationMethods + NotificationAccounts
// rule the selector does, so the producer can skip enqueueing work that no
// operated worker could ever pick up (sms/voice until phase 4).
func EnqueueNotificationTask(input *EnqueueNotificationTaskInput) (*Task, error) {
return EnqueueNotificationTaskTx(DB(), input)
}
// EnqueueNotificationTaskTx is the transactional form used by state machines
// that must commit their transition, audit event, message, and task together.
func EnqueueNotificationTaskTx(tx *gorm.DB, input *EnqueueNotificationTaskInput) (*Task, error) {
if tx == nil {
return nil, errors.New("enqueue: nil transaction")
}
if input.AccountID == 0 || input.ContactID == 0 {
return nil, errors.New("enqueue: account_id and contact_id are required")
}
idempotencyKey := input.IdempotencyKey
if idempotencyKey == "" {
if input.NotificationID == 0 || len(input.EventIDs) == 0 {
return nil, errors.New("enqueue: notification_id and event_ids are required without explicit idempotency_key")
}
idempotencyKey = notificationIdempotencyKey(input.NotificationID, input.ContactID, input.EventIDs[0])
}
// Fast path: row already exists from a previous producer tick. Returning
// the existing row is the idempotency guarantee — second calls return the
// same id, no second INSERT.
var existing Task
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
return &existing, nil
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
maxAttempts := input.MaxAttempts
if maxAttempts <= 0 {
maxAttempts = DefaultTaskMaxAttempts
}
if !anyWorkerCanDeliver(input.Method, input.AccountID) {
return nil, fmt.Errorf("%w: method=%s account=%d", ErrNotificationMethodNotAuthorized, input.Method, input.AccountID)
}
notBefore := input.NotBefore
if notBefore.IsZero() {
notBefore = time.Now()
}
now := time.Now()
task := &Task{
JobID: uuid.New().String(),
Kind: TaskKindNotification,
State: TaskStateQueued,
AccountID: input.AccountID,
MessageID: input.MessageID,
ContactID: &input.ContactID,
MonitorID: input.MonitorID,
CheckID: input.CheckID,
NotBefore: notBefore,
Deadline: input.Deadline,
Attempts: 0,
MaxAttempts: maxAttempts,
IdempotencyKey: idempotencyKey,
}
if len(input.Payload) > 0 {
task.Payload = input.Payload
}
task.CreatedAt = now
task.UpdatedAt = now
// ON CONFLICT DO NOTHING so a concurrent producer tick racing with us on
// the same idempotency_key loses the race but does not duplicate the row.
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(task).Error; err != nil {
return nil, err
}
if task.ID == 0 {
// Lost the race. Re-read and return the winner.
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err != nil {
return nil, err
}
return &existing, nil
}
return task, nil
}
// NotificationIdempotencyKey exposes the producer's idempotency key so the
// result handler and admin tooling can match a Task row back to the logical
// (notification, contact, event) tuple without re-deriving the format.
func NotificationIdempotencyKey(notificationID, contactID, eventID int64) string {
return notificationIdempotencyKey(notificationID, contactID, eventID)
}
func notificationIdempotencyKey(notificationID, contactID, eventID int64) string {
return fmt.Sprintf("notif:%d:contact:%d:event:%d", notificationID, contactID, eventID)
}
// anyWorkerCanDeliver returns true if at least one active worker in the pool is
// authorized to deliver the given (method, account) pair. Used by the producer
// to skip enqueues no worker could ever pick up.
func anyWorkerCanDeliver(method string, accountID int64) bool {
var nodes []WorkerNode
if err := DB().Where("status = ? AND last_seen > ?", "active", time.Now().Add(-WorkerHeartbeatFreshness)).Find(&nodes).Error; err != nil {
log.Printf("task_selector: cannot list workers: %v", err)
// Be permissive on lookup failure: the selector's own filter would still
// hold the lease back, so the worst case is a queued task nobody picks
// up — which the reaper eventually dead-letters.
return true
}
for i := range nodes {
if nodes[i].SupportsTaskEnvelope() && nodes[i].CanDeliverNotification(method, accountID) {
return true
}
}
return false
}
// TasksForWorker leases up to `limit` due tasks for the worker. The selection
// is one transaction so the FOR UPDATE SKIP LOCKED + UPDATE that flips state
// from queued to leased is atomic. Notification tasks are filtered by the worker's
// notification_methods + notification_accounts capability set; check tasks are
// filtered by check_types in their payload.
//
// The function is safe to call from multiple goroutines for different workers.
// Two workers that hit the DB at the same time will see disjoint task sets.
func TasksForWorker(worker *WorkerNode, limit int) ([]Task, error) {
if worker == nil {
return nil, errors.New("TasksForWorker: worker is nil")
}
if !worker.SupportsTaskEnvelope() {
return nil, nil
}
if limit <= 0 {
limit = 1
}
notifMethods := worker.NotificationMethods()
notifAccounts := worker.AccessibleAccountIDs()
hasNotif := len(notifMethods) > 0
tx := DB().Begin()
if tx.Error != nil {
return nil, tx.Error
}
defer func() {
if r := recover(); r != nil {
_ = tx.Rollback().Error
panic(r)
}
}()
now := time.Now()
var out []Task
// First pass: notification tasks the worker is authorized to deliver. We
// also bump attempts and flip state to leased in the same row so the
// outer selector+lease is atomic. The method filter is a JSONB extract on
// payload->>'method' so a single worker query can target one method list.
if hasNotif {
notifQuery := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'method' IN (?)", notifMethods).
Where("payload->>'method' <> ''")
if len(notifAccounts) > 0 {
notifQuery = notifQuery.Where("account_id IN (?)", notifAccounts)
}
var picked []Task
if err := notifQuery.Limit(limit).Find(&picked).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
for i := range picked {
row := picked[i]
newAttempts := row.Attempts + 1
leaseToken := uuid.NewString()
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
if err := tx.Model(&row).Updates(map[string]interface{}{
colState: TaskStateLeased,
"lease_owner": worker.WorkerID,
"lease_expires_at": leaseUntil,
"attempts": newAttempts,
"lease_token": leaseToken,
"updated_at": now,
}).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
row.State = TaskStateLeased
row.LeaseOwner = worker.WorkerID
row.LeaseExpiresAt = &leaseUntil
row.Attempts = newAttempts
row.LeaseToken = leaseToken
out = append(out, row)
}
}
remaining := limit - len(out)
if checkTypes := worker.CheckTypes(); remaining > 0 && len(checkTypes) > 0 {
checkQuery := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindCheck).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'kind' IN (?)", checkTypes)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
checkQuery = checkQuery.Where("account_id IN (?)", accounts)
}
var picked []Task
if err := checkQuery.Limit(remaining).Find(&picked).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
for i := range picked {
row := picked[i]
newAttempts := row.Attempts + 1
leaseToken := uuid.NewString()
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
if err := tx.Model(&row).Updates(map[string]interface{}{
colState: TaskStateLeased, "lease_owner": worker.WorkerID,
"lease_expires_at": leaseUntil, "attempts": newAttempts, "lease_token": leaseToken, "updated_at": now,
}).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
row.State, row.LeaseOwner, row.LeaseExpiresAt, row.Attempts, row.LeaseToken = TaskStateLeased, worker.WorkerID, &leaseUntil, newAttempts, leaseToken
out = append(out, row)
}
}
if err := tx.Commit().Error; err != nil {
return nil, err
}
return out, nil
}
// AvailableWorkerTaskCapacity returns unoccupied local worker slots. Durable
// unexpired leases and the worker's heartbeat-reported active/queued workload
// describe the same work from different sides, so the larger value is used to
// avoid both over-dispatching and double-counting a healthy worker.
func AvailableWorkerTaskCapacity(worker *WorkerNode) (int, error) {
if worker == nil {
return 0, errors.New("worker capacity: worker is nil")
}
concurrency := worker.Concurrency
if concurrency < 1 {
concurrency = 1
}
var leased int64
if err := DB().Model(&Task{}).Where("state = ? AND lease_owner = ? AND lease_expires_at > ?", TaskStateLeased, worker.WorkerID, time.Now()).Count(&leased).Error; err != nil {
return 0, err
}
var confirmationLeases int64
if err := DB().Model(&CheckAttempt{}).Where("worker_node_id = ? AND kind = ? AND state = ? AND lease_expires_at > ?", worker.ID, AttemptKindConfirm, AttemptStateLeased, time.Now()).Count(&confirmationLeases).Error; err != nil {
return 0, err
}
used := int(leased + confirmationLeases)
if reported := worker.ReportedWorkload(); reported > used {
used = reported
}
if used >= concurrency {
return 0, nil
}
return concurrency - used, nil
}
// GetTaskByJobID returns one task row keyed by its unique job_id. The result
// handler uses this to validate that the incoming JobID exists and matches the
// calling worker before it mutates state.
func GetTaskByJobID(jobID string) (*Task, error) {
if jobID == "" {
return nil, errors.New("GetTaskByJobID: empty job_id")
}
var task Task
if err := DB().Where("job_id = ?", jobID).First(&task).Error; err != nil {
return nil, err
}
return &task, nil
}
// TasksForWorkerTx is the variant exposed for tests so a single SELECT inside a
// caller-provided transaction can be inspected without the auto-commit wrapper.
// Production code should use TasksForWorker.
func TasksForWorkerTx(tx *gorm.DB, worker *WorkerNode, limit int) ([]Task, error) {
if tx == nil {
return nil, errors.New("TasksForWorkerTx: nil tx")
}
notifMethods := worker.NotificationMethods()
if len(notifMethods) == 0 {
return nil, nil
}
now := time.Now()
var out []Task
// Use SKIP LOCKED to avoid contention between workers (mirror of ChecksForWorker).
q := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'method' IN (?)", notifMethods)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Where("account_id IN (?)", accounts)
}
if err := q.Limit(limit).Find(&out).Error; err != nil {
return nil, err
}
return out, nil
}

681
app/models/task_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,681 @@
package models_test
import (
"encoding/json"
"errors"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// seedRegion creates a Region row before a worker fixture inserts, so the FK
// from worker_nodes -> regions holds. Idempotent: Drop() cleans up.
func seedRegion(t *testing.T, code string) models.Region {
t.Helper()
r := models.Region{}
err := models.DB().Where("code = ?", code).First(&r).Error
if err == nil {
return r
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("seed region lookup: %v", err)
}
r = models.Region{Code: code, Name: code, Enabled: true}
require.NoError(t, models.DB().Create(&r).Error)
return r
}
// seedAccountUserPlan returns an account with a default plan and the first user
// for the FK chain that contacts/notifications walk. Cleanup is the caller's
// responsibility (Drop() at end of test).
func seedAccountUserPlan(t *testing.T) (models.Account, models.User) {
t.Helper()
plan := models.Plan{Name: "test-plan", Default: false}
if err := models.DB().Create(&plan).Error; err != nil {
t.Fatalf("seed plan: %v", err)
}
user := models.User{Name: "test-user", Email: taskStringPtr("test-" + uuid.NewString() + "@example.com"), Timezone: "UTC"}
if err := models.DB().Create(&user).Error; err != nil {
t.Fatalf("seed user: %v", err)
}
account := models.Account{Name: "test-account", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
if err := models.DB().Create(&account).Error; err != nil {
t.Fatalf("seed account: %v", err)
}
return account, user
}
func seedNotification(t *testing.T, accountID int64) models.Notification {
t.Helper()
n := models.Notification{Name: "default", AccountID: accountID, Enabled: true, NotifyDown: true, NotifyRestore: true}
require.NoError(t, models.DB().Create(&n).Error)
return n
}
func seedEmailContact(t *testing.T, accountID int64) models.Contact {
t.Helper()
c := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &accountID}
require.NoError(t, models.DB().Create(&c).Error)
return c
}
func taskStringPtr(s string) *string { return &s }
func TestTaskSchemaMigration(t *testing.T) {
models.Drop()
models.Migrate()
require.True(t, models.DB().Migrator().HasTable(&models.Task{}), "tasks table must exist after Migrate()")
require.True(t, models.DB().Migrator().HasTable(&models.NotificationDelivery{}), "notification_deliveries table must exist after Migrate()")
}
func TestEnqueueNotificationTask_Idempotency(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
// Producer precheck requires at least one worker authorized for the
// (method, account) pair. Add an operated-style email worker.
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-idempotency-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
LastSeen: &now,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
payload := []byte(`{"method":"email","subject":"[rsmon] x is down","body_text":"down","body_html":"<p>down</p>","body_markdown":"**down**","language":"en","message_kind":"down"}`)
input := models.EnqueueNotificationTaskInput{
AccountID: account.ID,
NotificationID: notification.ID,
ContactID: contact.ID,
Method: "email",
Subject: "[rsmon] x is down",
BodyText: "down",
BodyHTML: "<p>down</p>",
Language: "en",
MessageKind: "down",
EventIDs: []int64{42},
Payload: payload,
}
first, err := models.EnqueueNotificationTask(&input)
require.NoError(t, err)
require.NotZero(t, first.ID)
// Second call with the same logical event must not create a duplicate row.
second, err := models.EnqueueNotificationTask(&input)
require.NoError(t, err)
assert.Equal(t, first.ID, second.ID, "idempotency: second enqueue should return the same row")
var count int64
require.NoError(t, models.DB().Model(&models.Task{}).Where("idempotency_key = ?", first.IdempotencyKey).Count(&count).Error)
assert.EqualValues(t, 1, count, "exactly one Task row per idempotency key")
}
// TestEnqueueNotificationTask_AuthorizationSkip makes sure the producer can
// observe ErrNotificationMethodNotAuthorized when no worker is eligible for
// the (method, account) pair. The producer uses this to avoid enqueueing work
// no operated worker could ever pick up.
func TestEnqueueNotificationTask_AuthorizationSkip(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
// Register a worker that only knows telegram. An email task must fail the
// precheck.
w := &models.WorkerNode{
WorkerID: "worker-tg-only-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"telegram"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
_, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
AccountID: account.ID,
NotificationID: notification.ID,
ContactID: contact.ID,
Method: "email",
EventIDs: []int64{1},
Payload: []byte(`{"method":"email"}`),
})
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
}
func TestEnqueueNotificationTask_RequiresTaskEnvelopeWorker(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
worker := &models.WorkerNode{WorkerID: "legacy-notify-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"}, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
}))}
require.NoError(t, models.DB().Create(worker).Error)
require.NoError(t, models.DB().Model(&models.WorkerNode{}).Where("id <> ?", worker.ID).Update("status", "dead").Error)
input := &models.EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notification.ID, ContactID: contact.ID, Method: "email", EventIDs: []int64{77}, Payload: []byte(`{"method":"email"}`)}
_, err := models.EnqueueNotificationTask(input)
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
var count int64
require.NoError(t, models.DB().Model(&models.Task{}).Count(&count).Error)
assert.Zero(t, count)
worker.Capabilities = datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"}, "task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
}))
require.NoError(t, models.DB().Save(worker).Error)
task, err := models.EnqueueNotificationTask(input)
require.NoError(t, err)
assert.Equal(t, models.TaskStateQueued, task.State)
}
func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-email-only-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
LastSeen: &now,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
// Enqueue 3 tasks of different methods; only the email ones should be
// picked up by the worker. We use the raw helper because the producer's
// precheck would refuse the telegram row when no worker handles telegram —
// the selector test must exercise the SELECT-side filter, not the
// producer-side authorization.
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{1})
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "telegram", []int64{2})
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{3})
picked, err := models.TasksForWorker(w, 10)
require.NoError(t, err)
require.Len(t, picked, 2, "only email tasks should be leased")
for _, p := range picked {
assert.Equal(t, models.TaskStateLeased, p.State)
assert.Equal(t, w.WorkerID, p.LeaseOwner)
assert.NotNil(t, p.LeaseExpiresAt)
assert.Equal(t, 1, p.Attempts)
}
// A second call must not return the same rows.
picked2, err := models.TasksForWorker(w, 10)
require.NoError(t, err)
assert.Empty(t, picked2, "second selector poll should see an empty queue while leased")
}
func TestEnqueueDueCheckTasks_UsesGenericEnvelope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
account, _ := seedAccountUserPlan(t)
group := models.Group{Name: "checks", AccountID: account.ID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: "example.com", Enabled: true}
require.NoError(t, models.DB().Create(&monitor).Error)
enabled := true
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
require.NoError(t, models.DB().Create(&check).Error)
worker := &models.WorkerNode{
WorkerID: "generic-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
require.NoError(t, models.DB().Create(worker).Error)
require.NoError(t, models.EnqueueDueCheckTasks(worker, []string{"http"}, 1))
picked, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
require.Len(t, picked, 1)
assert.Equal(t, models.TaskKindCheck, picked[0].Kind)
assert.Equal(t, models.TaskStateLeased, picked[0].State)
assert.Equal(t, check.ID, *picked[0].CheckID)
}
func TestTasksForWorker_ChecksRespectPrivateAccountScope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
accountA, _ := seedAccountUserPlan(t)
accountB, _ := seedAccountUserPlan(t)
accountID := accountA.ID
private := &models.WorkerNode{
WorkerID: "private-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
platform := &models.WorkerNode{
WorkerID: "platform-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(),
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
require.NoError(t, models.DB().Create(private).Error)
require.NoError(t, models.DB().Create(platform).Error)
task := models.Task{JobID: uuid.NewString(), Kind: models.TaskKindCheck, State: models.TaskStateQueued, AccountID: accountB.ID, Payload: datatypes.JSON([]byte(`{"kind":"http"}`)), NotBefore: time.Now().Add(-time.Second), MaxAttempts: 5, IdempotencyKey: "cross-account-" + uuid.NewString()}
require.NoError(t, models.DB().Create(&task).Error)
picked, err := models.TasksForWorker(private, 1)
require.NoError(t, err)
assert.Empty(t, picked)
picked, err = models.TasksForWorker(platform, 1)
require.NoError(t, err)
require.Len(t, picked, 1)
assert.Equal(t, task.ID, picked[0].ID)
}
func TestChecksForWorker_RespectsPrivateAccountScope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
accountA, _ := seedAccountUserPlan(t)
accountB, _ := seedAccountUserPlan(t)
makeCheck := func(accountID int64, host string) models.Check {
group := models.Group{Name: host, AccountID: accountID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: host, Enabled: true}
require.NoError(t, models.DB().Create(&monitor).Error)
enabled := true
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
require.NoError(t, models.DB().Create(&check).Error)
return check
}
owned := makeCheck(accountA.ID, "owned.example")
_ = makeCheck(accountB.ID, "other.example")
accountID := accountA.ID
private := &models.WorkerNode{
WorkerID: "private-legacy-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
}
require.NoError(t, models.DB().Create(private).Error)
checks := models.ChecksForWorker(private, []string{"http"}, 10)
require.Len(t, checks, 1)
assert.Equal(t, owned.ID, checks[0].ID)
}
func TestTasksForWorker_SkipsExpiredDeadline(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
worker := &models.WorkerNode{
WorkerID: "worker-deadline-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
}
require.NoError(t, models.DB().Create(worker).Error)
expired := time.Now().Add(-time.Second)
task := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{91})
require.NoError(t, models.DB().Model(&task).Update("deadline", expired).Error)
picked, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
assert.Empty(t, picked)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateQueued, stored.State)
assert.Equal(t, 0, stored.Attempts)
}
func TestReapExpiredTasksTerminatesExpiredQueuedNotification(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
require.NoError(t, models.DB().Create(&message).Error)
deadline := time.Now().Add(-time.Minute)
// This reaper test intentionally has no eligible worker; insert directly
// so it tests deadline handling rather than producer capability validation.
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{92})
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{"message_id": message.ID, "deadline": deadline}).Error)
_, _, err := models.ReapExpiredTasks()
require.NoError(t, err)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateDead, stored.State)
assert.Equal(t, "notification deadline expired", stored.LastError)
var storedMessage models.Message
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
assert.Equal(t, "error", storedMessage.State)
var auditCount int64
require.NoError(t, models.DB().Model(&models.NotificationDelivery{}).Where("task_id = ? AND status = ?", task.ID, "expired").Count(&auditCount).Error)
assert.EqualValues(t, 1, auditCount)
}
func TestReapExpiredTasks_RecyclesLeasesAndDeadsExhaustedRetries(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-reap-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
LastSeen: &now,
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
// 1) A leased task whose lease expired — should go back to queued.
expiredLease := time.Now().Add(-time.Minute)
leased := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{10})
require.NoError(t, models.DB().Model(&leased).Updates(map[string]interface{}{
"state": models.TaskStateLeased,
"lease_owner": w.WorkerID,
"lease_expires_at": expiredLease,
"attempts": 1,
}).Error)
// 2) A failed_retry task past its not_before and at max_attempts — should move to dead.
failedRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{11})
require.NoError(t, models.DB().Model(&failedRetry).Updates(map[string]interface{}{
"state": models.TaskStateFailedRetry,
"attempts": 5,
"max_attempts": 5,
"not_before": time.Now().Add(-time.Minute),
}).Error)
// 3) A failed_retry task past not_before but attempts < max_attempts — must stay failed_retry.
pendingRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{12})
require.NoError(t, models.DB().Model(&pendingRetry).Updates(map[string]interface{}{
"state": models.TaskStateFailedRetry,
"attempts": 2,
"max_attempts": 5,
"not_before": time.Now().Add(-time.Minute),
}).Error)
reaped, deaded, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, reaped, "one expired lease should be returned to queue")
assert.Equal(t, 1, deaded, "one exhausted retry should move to dead")
var leasedRow models.Task
require.NoError(t, models.DB().First(&leasedRow, leased.ID).Error)
assert.Equal(t, models.TaskStateQueued, leasedRow.State)
assert.Empty(t, leasedRow.LeaseOwner)
assert.Nil(t, leasedRow.LeaseExpiresAt)
var deadRow models.Task
require.NoError(t, models.DB().First(&deadRow, failedRetry.ID).Error)
assert.Equal(t, models.TaskStateDead, deadRow.State)
var pendingRow models.Task
require.NoError(t, models.DB().First(&pendingRow, pendingRetry.ID).Error)
assert.Equal(t, models.TaskStateFailedRetry, pendingRow.State)
}
func TestReapExpiredTasks_ExpiredNotificationLeaseExhaustionFinalizesMessage(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
require.NoError(t, models.DB().Create(&message).Error)
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{99})
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{
"message_id": message.ID, "state": models.TaskStateLeased, "attempts": 5, "max_attempts": 5,
"lease_expires_at": time.Now().Add(-time.Minute),
}).Error)
_, deaded, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, deaded)
var storedTask models.Task
require.NoError(t, models.DB().First(&storedTask, task.ID).Error)
assert.Equal(t, models.TaskStateDead, storedTask.State)
assert.Equal(t, "lease expired after max attempts", storedTask.LastError)
var storedMessage models.Message
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
assert.Equal(t, "error", storedMessage.State)
require.NotNil(t, storedMessage.Error)
assert.Equal(t, "lease expired after max attempts", *storedMessage.Error)
var delivery models.NotificationDelivery
require.NoError(t, models.DB().Where("task_id = ?", task.ID).First(&delivery).Error)
assert.Equal(t, "dead", delivery.Status)
assert.Equal(t, "lease expired after max attempts", delivery.Error)
}
func TestNotificationTaskLeaseOutlivesExecutionTimeoutAndReapsAfterExpiry(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
worker := &models.WorkerNode{
WorkerID: "notification-lease-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
}
require.NoError(t, models.DB().Create(worker).Error)
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{101})
leased, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
require.Len(t, leased, 1)
require.NotNil(t, leased[0].LeaseExpiresAt)
assert.GreaterOrEqual(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationTaskLeaseTTL-time.Second)
assert.Greater(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationExecutionTimeout)
withinExecution := time.Now().Add(models.DefaultNotificationExecutionTimeout)
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", withinExecution).Error)
reaped, _, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Zero(t, reaped)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateLeased, stored.State)
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error)
reaped, _, err = models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, reaped)
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateQueued, stored.State)
}
func mustEnqueue(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
t.Helper()
payload := []byte(`{"method":"` + method + `"}`)
task, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
AccountID: accountID,
NotificationID: notificationID,
ContactID: contactID,
Method: method,
EventIDs: eventIDs,
Payload: payload,
NotBefore: time.Now().Add(-time.Second),
})
require.NoError(t, err)
require.NotZero(t, task.ID)
return *task
}
// mustEnqueueRaw inserts a Task row directly without going through the producer
// precheck. The selector test deliberately mixes methods (email + telegram) on a
// worker that only handles email; the producer would refuse the telegram row,
// which is the wrong thing to assert about in a selector test.
func mustEnqueueRaw(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
t.Helper()
payload := datatypes.JSON([]byte(`{"method":"` + method + `"}`))
contact := contactID
task := models.Task{
JobID: uuid.New().String(),
Kind: models.TaskKindNotification,
State: models.TaskStateQueued,
AccountID: accountID,
ContactID: &contact,
Payload: payload,
NotBefore: time.Now().Add(-time.Second),
Attempts: 0,
MaxAttempts: 5,
IdempotencyKey: models.NotificationIdempotencyKey(notificationID, contactID, eventIDs[0]),
}
require.NoError(t, models.DB().Create(&task).Error)
require.NotZero(t, task.ID)
return task
}
func mustJSON(t *testing.T, v interface{}) []byte {
t.Helper()
b, err := json.Marshal(v)
require.NoError(t, err)
return b
}
// TestWorkerNodeCapabilities_NotificationFlags confirms that the JSON-backed
// capabilities blob correctly exposes the notification_methods and
// notification_accounts arrays that the selector and credential push depend on.
func TestWorkerNodeCapabilities_NotificationFlags(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "caps-worker-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"email", "telegram"},
"notification_accounts": []int64{7, 8},
})),
}
require.NoError(t, models.DB().Create(w).Error)
got := models.WorkerNode{}
require.NoError(t, models.DB().First(&got, w.ID).Error)
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
assert.Equal(t, []int64{7, 8}, got.NotificationAccounts())
assert.True(t, got.CanDeliverNotification("email", 7))
assert.False(t, got.CanDeliverNotification("email", 9), "account 9 is not in the allowed list")
assert.False(t, got.CanDeliverNotification("mattermost", 7), "method not authorized")
// Operated-style worker: empty accounts list means "all accounts".
w2 := &models.WorkerNode{
WorkerID: "ops-worker-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w2).Error)
got2 := models.WorkerNode{}
require.NoError(t, models.DB().First(&got2, w2.ID).Error)
assert.True(t, got2.CanDeliverNotification("email", 9999), "empty accounts list means all accounts")
}
func TestWorkerNodeReportedWorkloadUsesDisjointHeartbeatFields(t *testing.T) {
w := &models.WorkerNode{Capabilities: datatypes.JSON([]byte(`{"active_checks":2,"queue_depth":3,"active_notifications":5,"notification_queue_depth":7}`))}
assert.Equal(t, 17, w.ReportedWorkload())
}
// TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults verifies that
// EnsureConfiguredWorkerNode (the in-cluster worker provisioner) populates
// notification_methods / notification_accounts on the JSON blob so the new
// selector and the new credential push work out of the box for the bundled
// worker.
func TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults(t *testing.T) {
models.Drop()
models.Migrate()
t.Setenv("WORKER_AUTH_TOKEN", "secret-token-for-test-xyz")
t.Setenv("DEPLOY_ENV", "test-env")
t.Setenv("RSMON_WORKER_ID", "worker-test")
t.Setenv("NOTIFICATION_METHODS", "email,telegram")
t.Setenv("NOTIFICATION_ACCOUNTS", "11,22")
models.EnsureConfiguredWorkerNode()
var got models.WorkerNode
require.NoError(t, models.DB().Where("worker_id = ?", "worker-test").First(&got).Error)
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
assert.Equal(t, []int64{11, 22}, got.NotificationAccounts())
}
// TestNotificationIdempotencyKeyFormat pins the producer-side key shape so the
// result handler can re-derive it for matching without depending on internal
// package state.
func TestNotificationIdempotencyKeyFormat(t *testing.T) {
key := models.NotificationIdempotencyKey(7, 9, 13)
assert.Equal(t, "notif:7:contact:9:event:13", key)
}
// guard against uuid being accidentally dropped from the imports.
var _ = uuid.New

77
app/models/telegram_bot.go Обычный файл
Просмотреть файл

@@ -0,0 +1,77 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
// TelegramBotMessageReceived marks inbound bot messages.
TelegramBotMessageReceived = "received"
// TelegramBotMessageSent marks outbound bot replies.
TelegramBotMessageSent = "sent"
// TelegramBotStatusMain is the singleton status row name for the bot.
TelegramBotStatusMain = "main"
)
// TelegramBotMessage stores a Telegram bot chat message for admin history.
type TelegramBotMessage struct {
concerns.Model
Direction string `gorm:"size:16;index" json:"direction"`
ChatID int64 `gorm:"index" json:"chat_id"`
ChatType string `gorm:"size:32" json:"chat_type"`
Username string `gorm:"size:255" json:"username"`
Text string `gorm:"type:text" json:"text"`
Command string `gorm:"size:64" json:"command"`
ContactID *int64 `gorm:"index" json:"contact_id,omitempty"`
Error string `gorm:"type:text" json:"error,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
// TableName overrides the default table name.
func (TelegramBotMessage) TableName() string {
return "telegram_bot_messages"
}
// TelegramBotStatus stores the current Telegram bot heartbeat/status.
type TelegramBotStatus struct {
concerns.Model
Name string `gorm:"uniqueIndex;size:64;not null" json:"name"`
Username string `gorm:"size:255" json:"username"`
Online bool `gorm:"not null;default:false" json:"online"`
LastSeen *time.Time `json:"last_seen,omitempty"`
LastError string `gorm:"type:text" json:"last_error,omitempty"`
concerns.Timestamped
}
// TableName overrides the default table name.
func (TelegramBotStatus) TableName() string {
return "telegram_bot_statuses"
}
// RecentTelegramBotMessages returns the latest Telegram bot messages, capped to a safe limit.
func RecentTelegramBotMessages(limit int) ([]TelegramBotMessage, error) {
if limit <= 0 || limit > 500 {
limit = 100
}
messages := []TelegramBotMessage{}
err := DB().Order("id DESC").Limit(limit).Find(&messages).Error
return messages, err
}
// TelegramBotCurrentStatus returns the singleton Telegram bot status row.
func TelegramBotCurrentStatus() (*TelegramBotStatus, error) {
status := TelegramBotStatus{}
if err := DB().Where("name = ?", TelegramBotStatusMain).First(&status).Error; err != nil {
return nil, err
}
if status.LastSeen == nil || time.Since(*status.LastSeen) > 2*time.Minute {
status.Online = false
}
return &status, nil
}

24
app/models/telegram_bot_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,24 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestTelegramBotCurrentStatusMarksStaleOffline(t *testing.T) {
models.Drop()
models.Migrate()
seen := time.Now().Add(-3 * time.Minute)
status := models.TelegramBotStatus{Name: models.TelegramBotStatusMain, Online: true, LastSeen: &seen}
require.NoError(t, models.DB().Create(&status).Error)
got, err := models.TelegramBotCurrentStatus()
require.NoError(t, err)
assert.False(t, got.Online)
}

233
app/models/user.go Обычный файл
Просмотреть файл

@@ -0,0 +1,233 @@
package models
import (
"crypto/md5"
"fmt"
"log"
"time"
"github.com/lib/pq"
"github.com/pkg/errors"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// User represents a platform user.
//
// A User is a global identity that may belong to many tenants via the
// Access join table (see Access). Contacts created by or assigned to a
// User carry that UserID; admins reach them via the cross-tenant
// /admin/users page, while per-account management is via
// /settings/users (which shows Accesses preloaded with User + Invite).
//
// Authentication state (password, confirmation, lock, recover) lives
// here, not on Access, because those attributes are account-independent.
type User struct {
// concerns.Model
ID int64 `gorm:"primarykey" json:"id"`
Email *string `gorm:"uniqueIndex;size:255" json:"email" validate:"required"`
Name string `json:"name"`
Enabled bool `gorm:"not null;default:true" json:"-"`
// Operator grants platform-wide operational access. Account ownership alone
// must never grant cross-tenant task inspection or replay.
Operator bool `gorm:"not null;default:false" json:"operator"`
Timezone string `json:"timezone"`
Language string `gorm:"default:ru" json:"language"`
// Settings holds small per-user UI preferences. It intentionally stays
// separate from account settings because the sidebar is a personal view.
Settings datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"settings"`
Phone string `gorm:"index,size:255" json:"phone"`
TelegramID *int64 `gorm:"uniqueIndex" json:"telegram_id"`
TelegramUsername string `gorm:"size:255" json:"telegram_username"`
Accesses []Access `json:"-"`
Contacts []Contact `json:"-"`
LastActiveAt *time.Time `json:"last_active_at"`
LastActiveIP *string `json:"last_active_ip"`
EncryptedPassword *string `json:"-"`
PasswordSetAt *time.Time `json:"-"`
// Confirm
ConfirmationToken *string `json:"-"`
Confirmed bool `json:"confirmed"`
// Lock
AttemptCount int `json:"-"`
LastAttempt *time.Time `json:"-"`
LockedAt *time.Time `json:"-"`
// Recover
RecoverToken *string `json:"-"`
RecoverTokenAt *time.Time `json:"-"`
RememberTokens pq.StringArray `gorm:"index;type:varchar(100)[]" json:"-"`
// DeletionRequestedAt is set when the user requests account deletion.
// During the 7-day grace period the user can cancel the deletion
// (clearing this field). After 7 days the user and all related data
// are hard-deleted by a scheduled job.
DeletionRequestedAt *time.Time `json:"deletion_requested_at"`
concerns.Timestamped `json:"-"`
}
// DeletionPending returns true if the user has requested deletion and is
// still within the 7-day grace period.
func (u *User) DeletionPending() bool {
return u.DeletionRequestedAt != nil
}
// GetLabel returns info label for user.
func (u *User) GetLabel() string {
return u.DisplayName()
}
// DisplayName implements qor.CurrentUser for admin.
func (u *User) DisplayName() string {
if u.Email != nil {
return u.Name + " " + *u.Email
}
return u.Name
}
// AfterSocialLogin is a callback after social login.
func (u *User) AfterSocialLogin(inviteID int64) (*User, error) {
oldUser := User{}
DB().Where("email = ?", u.Email).Where("id != ?", u.ID).First(&oldUser)
if oldUser.ID > 0 {
log.Println("new user", u.ID, "has same email", u.Email, "as old user", oldUser.ID, "so replacing")
err := DB().Model(&authidentity.AuthIdentity{}).Where("user_id = ?", u.ID).Updates(
authidentity.Basic{
UserID: &oldUser.ID,
},
).Error
return &oldUser, err
}
err := u.AfterRegister(inviteID)
return u, err
}
// AfterInvite is a callback after invite acceptance.
func (u *User) AfterInvite(invite *Invite) error {
invite.InviteeID = &u.ID
err := DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
if err != nil {
return errors.Wrap(err, "invite: failed to add accesses to invited user")
}
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{
"confirmed_at": time.Now(),
}).Error
if err != nil {
return errors.Wrap(err, "invite: failed set user as confirmed")
}
eml := invite.Email
u.Email = &eml
if invite.Name != "" {
u.Name = invite.Name
}
err = DB().Save(&u).Error
if err != nil {
return errors.Wrap(err, "invite: failed to save user")
}
if invite.Name == "" {
log.Println("set name", u.Name)
invite.Name = u.Name
}
invite.State = stateOK
err = DB().Save(&invite).Error
if err != nil {
return errors.Wrap(err, "invite: failed to save invite")
}
invite.Invitee = u
return nil
}
// AfterRegister is a callback after registration.
func (u *User) AfterRegister(inviteID int64) error {
var err error
if inviteID > 0 {
invite := Invite{}
err := DB().First(&invite, inviteID).Error
if err != nil {
return errors.Wrap(err, "invite: failed to find invite")
}
err = u.AfterInvite(&invite)
if err != nil {
return err
}
}
_, err = CreateAccountForUser("", u)
if err != nil {
return err
}
return nil
}
// AfterLogin is a callback after login.
func (u *User) AfterLogin(inviteID int64) error {
var err error
if inviteID > 0 {
invite := Invite{}
err = DB().First(&invite, inviteID).Error
if err == nil {
invite.InviteeID = &u.ID
invite.State = stateOK
err = DB().Save(&invite).Error
if err != nil {
return errors.Wrap(err, "failed to save invite")
}
err = DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
if err != nil {
return errors.Wrap(err, "failed to add accesses to invited user")
}
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{ //nolint:lll
"confirmed_at": time.Now(),
}).Error
if err != nil {
return errors.Wrap(err, "failed set user as confirmed")
}
}
}
return nil
}
// Gravatar returns the Gravatar URL for the user.
func (u *User) Gravatar(size int) string {
if u.Email == nil {
return ""
}
hash := md5.Sum([]byte(*u.Email))
return fmt.Sprintf("https://www.gravatar.com/avatar/%x?s=%d&d=blank", hash, size)
}
// AsJSON returns a JSON representation of user.
func (u User) AsJSON() map[string]interface{} { //nolint:gocritic // hugeParam: accepted for interface compatibility
r := map[string]interface{}{
"id": u.ID,
"email": u.Email,
"avatar": u.Gravatar(32),
"deletion_requested_at": u.DeletionRequestedAt,
}
return r
}

23
app/models/whois.go Обычный файл
Просмотреть файл

@@ -0,0 +1,23 @@
package models
import (
"time"
"github.com/lib/pq"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Whois provides functionality.
type Whois struct {
concerns.Model
MonitorID *int64
Monitor *Monitor
Tld string
Nameservers pq.StringArray `gorm:"type:varchar(255)[]"`
Expires *time.Time
UpdatedAt *time.Time
Data string
}

31
app/models/worker_log_event.go Обычный файл
Просмотреть файл

@@ -0,0 +1,31 @@
package models
import (
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// WorkerLogEvent stores critical log events sent by worker nodes. The same
// payload may also be forwarded to VictoriaLogs; Postgres keeps a compact audit
// copy so the control plane can show recent critical events even if external log
// storage is temporarily unavailable.
type WorkerLogEvent struct {
concerns.Model
WorkerID int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL;index" json:"worker_id"`
WorkerNodeID string `gorm:"size:100;not null;index" json:"worker_node_id"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Level string `gorm:"size:16;not null;index" json:"level"`
Message string `gorm:"type:text;not null" json:"message"`
Source string `gorm:"size:64" json:"source"`
Payload datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"payload"`
OccurredAt time.Time `gorm:"not null;index" json:"occurred_at"`
concerns.Timestamped
}
// TableName provides functionality.
func (WorkerLogEvent) TableName() string { return "worker_log_events" }

452
app/models/worker_node.go Обычный файл
Просмотреть файл

@@ -0,0 +1,452 @@
package models
import (
"encoding/json"
"log"
"os"
"strconv"
"strings"
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// WorkerNode represents a distributed monitoring worker
type WorkerNode struct {
concerns.Model
WorkerID string `gorm:"uniqueIndex;size:100;not null" json:"worker_id"` // UUID or configured ID
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
Status string `gorm:"not null;default:'registered'" json:"status"` // registered, active, inactive, dead
AuthToken string `gorm:"uniqueIndex;size:64;not null" json:"-"`
LastSeen *time.Time `json:"last_seen"`
Version string `gorm:"size:50" json:"version"`
URL string `gorm:"size:500" json:"url"` //nolint:lll // publicly-advertised URL; may differ from bind host:port when behind Traefik
Capabilities datatypes.JSON `gorm:"not null;default:'{}'" json:"capabilities"` // {"check_types": ["http","ssl","dns",...]}
Concurrency int `gorm:"not null;default:20" json:"concurrency"`
NetworkProblems bool `gorm:"not null;default:false;index" json:"network_problems"`
NetworkProblemsUntil *time.Time `json:"network_problems_until,omitempty"`
LastFailureCount int `gorm:"not null;default:0" json:"last_failure_count"`
LastTotalCount int `gorm:"not null;default:0" json:"last_total_count"`
// Capability flags (see docs/plans/inventory-management.md §3).
// All default to true so an existing worker row that pre-dates
// this migration keeps running checks. Toggle from the admin UI
// or POST a boolean to /api/v1/workers to disable any one of
// them; the distworker client re-reads these flags on every
// task poll.
//
// Pointer types so GORM can distinguish "client didn't supply
// the key, fall back to the DB default" from "client explicitly
// set false". A plain `bool` would be silently re-overwritten
// by the column default on Save (default:true kicks in when
// GORM sees the zero value, regardless of whether the handler
// asked for false). See the controller tests for the
// partial-update case.
RunChecks *bool `gorm:"not null;default:true" json:"run_checks"`
CollectMetrics *bool `gorm:"not null;default:true" json:"collect_metrics"`
DetectProjects *bool `gorm:"not null;default:true" json:"detect_projects"`
// ServerID is the optional inventory Server this worker daemon
// is running on (see docs/plans/servers-and-hardware-metrics.md
// §3 and docs/plans/inventory-management.md §1). Nullable so
// legacy "no server assigned" rows keep working. Indexed because
// the distworker health ticker joins servers→workers frequently.
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
// AccountID scopes a worker to a single customer account (private
// worker per docs/distributed/private-workers.md). NULL means a
// platform-operated worker eligible to serve any account; non-NULL
// is a customer-operated worker pinned to one account. ON DELETE
// SET NULL keeps an operated worker valid if its account row is
// ever removed without an explicit private-worker cleanup.
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE SET NULL;index" json:"account_id,omitempty"`
LLMs []LLM `json:"llms,omitempty" gorm:"many2many:worker_llms;"`
concerns.Timestamped
}
const WorkerHeartbeatFreshness = 2 * time.Minute
func (w *WorkerNode) NetworkProblemActive(now time.Time) bool {
return w != nil && w.NetworkProblems && (w.NetworkProblemsUntil == nil || w.NetworkProblemsUntil.After(now))
}
// WorkerStatuses provides functionality.
// WorkerStatus represents the possible worker statuses
var WorkerStatuses = []string{"registered", "active", "inactive", "dead"}
// AllWorkerCheckKinds is the complete distributed-worker capability set.
func AllWorkerCheckKinds() []string {
return []string{kindHTTP, kindSSL, kindDNS, kindSSH, kindFTP, kindWhois, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
}
// NormalizeWorkerCapabilities expands aliases and removes unknown or duplicate capabilities.
func NormalizeWorkerCapabilities(capabilities []string) []string {
allowed := make(map[string]bool)
for _, kind := range AllWorkerCheckKinds() {
allowed[kind] = true
}
seen := make(map[string]bool)
normalized := make([]string, 0, len(capabilities))
for _, capability := range capabilities {
capability = strings.TrimSpace(strings.ToLower(capability))
if capability == "all" || capability == "*" {
return AllWorkerCheckKinds()
}
if !allowed[capability] || seen[capability] {
continue
}
seen[capability] = true
normalized = append(normalized, capability)
}
if len(normalized) == 0 {
return AllWorkerCheckKinds()
}
return normalized
}
// IsAlive returns true if the worker is considered alive based on last_seen
func (w *WorkerNode) IsAlive() bool {
if w.LastSeen == nil {
return false
}
// Worker is considered dead if no heartbeat for 2 minutes
return w.LastSeen.After(time.Now().Add(-WorkerHeartbeatFreshness))
}
// NotificationMethods returns the notification methods the worker is authorized
// to deliver (e.g. ["email", "telegram"]). Empty slice means "no notification
// delivery authorized". See docs/plans/worker-notifier-mvp.md section 4.3.
func (w *WorkerNode) NotificationMethods() []string {
caps := w.capabilitiesMap()
if caps == nil {
return nil
}
raw, ok := caps["notification_methods"]
if !ok {
return nil
}
return parseStringList(raw)
}
// NotificationAccounts returns the account IDs the worker is authorized to
// serve for notifications. Empty slice means "owned by RSMon, all accounts".
// Customer-hosted workers (phase 4) ship a non-empty slice to scope credentials.
func (w *WorkerNode) NotificationAccounts() []int64 {
caps := w.capabilitiesMap()
if caps == nil {
return nil
}
raw, ok := caps["notification_accounts"]
if !ok {
return nil
}
return parseInt64List(raw)
}
// AccessibleAccountIDs returns the accounts this worker may access. Empty means
// RSMon-operated/global worker. It combines the legacy single AccountID field
// with the newer notification_accounts capability list.
func (w *WorkerNode) AccessibleAccountIDs() []int64 {
if w == nil {
return nil
}
if w.AccountID != nil && *w.AccountID > 0 {
// A private worker cannot widen its account scope through a mutable
// capability JSON blob.
return []int64{*w.AccountID}
}
seen := map[int64]bool{}
out := []int64{}
for _, id := range w.NotificationAccounts() {
if id > 0 && !seen[id] {
seen[id] = true
out = append(out, id)
}
}
return out
}
// CanDeliverNotification returns true when the worker is allowed to deliver
// the given method for the given account. An empty NotificationAccounts slice
// means the worker is RSMon-operated and may serve any account.
func (w *WorkerNode) CanDeliverNotification(method string, accountID int64) bool {
methods := w.NotificationMethods()
if len(methods) == 0 {
return false
}
if !containsString(methods, method) {
return false
}
accounts := w.AccessibleAccountIDs()
if len(accounts) == 0 {
return true
}
return containsInt64(accounts, accountID)
}
// CheckTypes returns the check kinds this worker may execute.
func (w *WorkerNode) CheckTypes() []string {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return nil
}
return parseStringList(capabilities["check_types"])
}
// SupportsTaskEnvelope is an explicit protocol capability. Version labels are
// build metadata (and may be "latest" or a commit SHA), not a wire contract.
// Rows created before this capability existed intentionally remain v1.
func (w *WorkerNode) SupportsTaskEnvelope() bool {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return false
}
supported, _ := capabilities["task_envelope"].(bool)
return supported
}
// ReportedWorkload is the worker's local active and queued work across both
// checks and notifications. It is advisory; durable leases remain authoritative.
func (w *WorkerNode) ReportedWorkload() int {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return 0
}
keys := []string{"active_checks", "queue_depth", "active_notifications", "notification_queue_depth"}
total := 0
for _, key := range keys {
switch value := capabilities[key].(type) {
case float64:
if value > 0 {
total += int(value)
}
case int:
if value > 0 {
total += value
}
}
}
return total
}
func (w *WorkerNode) capabilitiesMap() map[string]interface{} {
if w == nil || len(w.Capabilities) == 0 {
return nil
}
var out map[string]interface{}
if err := json.Unmarshal(w.Capabilities, &out); err != nil {
return nil
}
return out
}
func parseStringList(raw interface{}) []string {
switch v := raw.(type) {
case []interface{}:
out := make([]string, 0, len(v))
for _, item := range v {
if s, ok := item.(string); ok && s != "" {
out = append(out, s)
}
}
return out
case []string:
out := make([]string, 0, len(v))
for _, s := range v {
if s != "" {
out = append(out, s)
}
}
return out
}
return nil
}
func parseInt64List(raw interface{}) []int64 {
switch v := raw.(type) {
case []interface{}:
out := make([]int64, 0, len(v))
for _, item := range v {
switch n := item.(type) {
case float64:
out = append(out, int64(n))
case int64:
out = append(out, n)
}
}
return out
case []int64:
return v
}
return nil
}
func containsString(haystack []string, needle string) bool {
for _, s := range haystack {
if s == needle {
return true
}
}
return false
}
func containsInt64(haystack []int64, needle int64) bool {
for _, n := range haystack {
if n == needle {
return true
}
}
return false
}
// EnsureConfiguredWorkerNode creates or updates the bundled Docker Compose worker from environment variables.
func EnsureConfiguredWorkerNode() {
token := envFirst("WORKER_AUTH_TOKEN", "RSMON_AUTH_TOKEN")
if token == "" {
return
}
deployEnv := envDefault("DEPLOY_ENV", "production")
workerID := envDefault("RSMON_WORKER_ID", "worker-"+deployEnv+"-01")
regionCode := envDefault("RSMON_REGION_CODE", deployEnv)
version := envDefault("RSMON_WORKER_VERSION", envDefault("IMAGE_TAG", "latest"))
concurrency := envInt("WORKER_CONCURRENCY", 20)
workerURL := strings.TrimSpace(os.Getenv("WORKER_URL"))
capabilities := NormalizeWorkerCapabilities(splitEnvList(envDefault(
"RSMON_CAPABILITIES",
"http,ssl,dns,ssh,ftp,whois,bssl,llm,llm-http,ping,tcp,udp",
)))
region := Region{}
if err := DB().Where("code = ?", regionCode).First(&region).Error; err != nil {
region = Region{Code: regionCode, Name: regionCode, Enabled: true}
if err := DB().Create(&region).Error; err != nil {
log.Printf("worker: failed to create configured worker region %s: %v", regionCode, err)
return
}
}
capJSON, err := json.Marshal(map[string]interface{}{
"check_types": capabilities,
"task_envelope": true,
"notification_methods": parseStringList(notificationMethodsFromEnv()),
"notification_accounts": parseInt64List(notificationAccountsFromEnv()),
})
if err != nil {
log.Printf("worker: failed to marshal configured worker capabilities: %v", err)
return
}
worker := WorkerNode{}
DB().Where("worker_id = ? OR auth_token = ?", workerID, token).First(&worker)
created := worker.ID == 0
worker.WorkerID = workerID
worker.RegionCode = regionCode
worker.Status = "registered"
worker.AuthToken = token
worker.Version = version
worker.URL = workerURL
worker.Capabilities = datatypes.JSON(capJSON)
worker.Concurrency = concurrency
if err := DB().Save(&worker).Error; err != nil {
log.Printf("worker: failed to provision configured worker %s: %v", workerID, err)
return
}
if created {
log.Printf("worker: provisioned configured worker %s in region %s", workerID, regionCode)
} else {
log.Printf("worker: updated configured worker %s in region %s", workerID, regionCode)
}
}
func splitEnvList(value string) []string {
parts := strings.Split(value, ",")
items := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
items = append(items, part)
}
}
return items
}
func envFirst(keys ...string) string {
for _, key := range keys {
if value := os.Getenv(key); value != "" {
return value
}
}
return ""
}
func envDefault(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
func envInt(key string, defaultValue int) int {
value, err := strconv.Atoi(os.Getenv(key))
if err != nil || value <= 0 {
return defaultValue
}
return value
}
// Notification method constants used across the package. Centralized here so
// the literal does not appear three or more times (goconst).
const (
methodEmail = "email"
methodTelegram = "telegram"
methodWebhook = "webhook"
methodMattermost = "mattermost"
)
// defaultNotificationMethods is the operated-worker default notification method
// list. Lives at package scope so goconst does not flag the literal across
// the package (account.go and user.go already reference "email").
var defaultNotificationMethods = []string{methodEmail, methodTelegram, methodWebhook, methodMattermost}
// notificationMethodsFromEnv reads the optional NOTIFICATION_METHODS env var.
// Empty result yields the default "all four" list so the operated worker can
// deliver email / telegram / webhook / mattermost out of the box.
func notificationMethodsFromEnv() []string {
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_METHODS"))
if raw == "" {
return append([]string{}, defaultNotificationMethods...)
}
out := make([]string, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
// notificationAccountsFromEnv reads the optional NOTIFICATION_ACCOUNTS env var.
// Empty result means "all accounts allowed" (the RSMon-operated default).
func notificationAccountsFromEnv() []int64 {
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_ACCOUNTS"))
if raw == "" {
return nil
}
out := make([]int64, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
n, err := strconv.ParseInt(part, 10, 64)
if err != nil || n <= 0 {
continue
}
out = append(out, n)
}
return out
}