feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
60
app/models/access.go
Обычный файл
60
app/models/access.go
Обычный файл
@@ -0,0 +1,60 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Access represents membership of a User (or ApiKey) within a tenant
|
||||
// Account, optionally scoped to a Group or Monitor.
|
||||
//
|
||||
// A User can hold many Access rows across many Accounts — the Access
|
||||
// table is the source of truth for "who can see what". Each row answers:
|
||||
//
|
||||
// "Does user U have permission P on scope S of account A?"
|
||||
//
|
||||
// Where:
|
||||
//
|
||||
// - A = AccountID (tenant)
|
||||
// - U = UserID (or ApiKeyID for service tokens)
|
||||
// - P = Role ("owner" | "admin" | "manager" | "view" |
|
||||
// "notify")
|
||||
// - S = Kind + (GroupID | MonitorID) — defaults to account-wide when
|
||||
// Kind = "account" and both ids are
|
||||
// nil.
|
||||
//
|
||||
// One Access row may also reference the Invite that produced it via
|
||||
// InviteID. The Invite is preserved after registration so the access
|
||||
// history stays auditable — system-registered users and admin-added
|
||||
// users have nil InviteID.
|
||||
//
|
||||
// See docs/plans/users-and-rbac.md for the full RBAC matrix.
|
||||
type Access struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
|
||||
// Kind access kind, account \ group \ monitor
|
||||
Kind string `gorm:"not null;default:'account'" json:"kind"`
|
||||
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
User *User `json:"-"`
|
||||
ApiKeyID *int64 `gorm:"type:bigint REFERENCES api_keys(id)" json:"-"` //nolint:revive // accepted lint exception
|
||||
ApiKey *ApiKey `json:"-"` //nolint:revive // accepted lint exception
|
||||
InviteID *int64 `gorm:"type:bigint REFERENCES invites(id)" json:"-"`
|
||||
Invite *Invite `json:"-"`
|
||||
|
||||
GroupID *int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty"`
|
||||
MonitorID *int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
|
||||
Role string `json:"role"`
|
||||
|
||||
// SeatType is additive to Role: role remains the authorization decision,
|
||||
// while seat type is the billing entitlement.
|
||||
SeatType string `gorm:"size:16;not null;default:'login'" json:"seat_type"`
|
||||
Status string `gorm:"size:16;not null;default:'active'" json:"status"`
|
||||
NotifyOnly bool `gorm:"not null;default:false" json:"notify_only"`
|
||||
SeatAddonID *int64 `json:"seat_addon_id,omitempty"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
Audited
|
||||
}
|
||||
117
app/models/account.go
Обычный файл
117
app/models/account.go
Обычный файл
@@ -0,0 +1,117 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Account represents a tenant — an isolated billing/permissions boundary
|
||||
// that groups users, contacts, groups, monitors and notifications.
|
||||
//
|
||||
// Access to an Account is granted via the Access join table (see Access).
|
||||
// `Role` on Account is a gorm:"-":all" virtual column populated by the
|
||||
// controllers for the current session — it is the role the calling user
|
||||
// holds on THIS account, not a property of the account itself.
|
||||
type Account struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name"`
|
||||
Accesses []Access `json:"-"`
|
||||
Contacts []Contact `json:"-"`
|
||||
Groups []Group `json:"-"`
|
||||
Notifications []Notification `json:"-"`
|
||||
PlanID *int64 `gorm:"type:bigint REFERENCES plans(id)" json:"-"`
|
||||
Plan *Plan `json:"plan"`
|
||||
// Diagnostic overrides are available only to plans that include confirmations.
|
||||
// Nil keeps the catalog value; bounds are enforced by DiagnosticSettings.
|
||||
ConfirmTimeoutSec *int `json:"confirm_timeout_sec,omitempty"`
|
||||
HealthWindowSec *int `json:"health_window_sec,omitempty"`
|
||||
HealthRateThreshold *float64 `json:"health_rate_threshold,omitempty"`
|
||||
HealthMinAttempts *int `json:"health_min_attempts,omitempty"`
|
||||
Role string `gorm:"-:all" json:"role"`
|
||||
Timezone string `json:"timezone"`
|
||||
Language string `gorm:"default:'ru'" json:"language"`
|
||||
Disabled bool `gorm:"not null;default:false" json:"disabled"`
|
||||
Blocked bool `gorm:"not null;default:false" json:"blocked"`
|
||||
PaidUntil *time.Time `json:"paid_until"`
|
||||
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
|
||||
Deleted bool `gorm:"not null;default:false"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// Users provides functionality.
|
||||
func (a Account) Users() []User { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
users := make([]User, 0)
|
||||
err := DB().Where("id IN (SELECT user_id FROM accesses WHERE account_id = ?)", a.ID).Find(&users).Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
// CreateAccountForUser provides functionality.
|
||||
func CreateAccountForUser(name string, u *User) (*Account, error) {
|
||||
trialPlan := Plan{}
|
||||
if err := DB().Where("code = ? AND archived = FALSE", "team").First(&trialPlan).Error; err != nil {
|
||||
return nil, errors.Wrap(err, "failed to find trial plan")
|
||||
}
|
||||
trialEndsAt := time.Now().UTC().AddDate(0, 0, 14)
|
||||
account := Account{PlanID: &trialPlan.ID, TrialEndsAt: &trialEndsAt}
|
||||
if name != "" {
|
||||
account.Name = name
|
||||
}
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Create(&account).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create account")
|
||||
}
|
||||
if err := tx.Create(&Subscription{
|
||||
AccountID: account.ID, PlanID: trialPlan.ID, Provider: "manual", Status: SubscriptionStatusTrialing,
|
||||
BillingCycle: "monthly", Currency: trialPlan.Currency, AmountMinor: trialPlan.PriceMonthlyMinor, CurrentPeriodEnd: &trialEndsAt, TrialEndsAt: &trialEndsAt,
|
||||
}).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create subscription")
|
||||
}
|
||||
var subscription Subscription
|
||||
if err := tx.Where("account_id = ?", account.ID).First(&subscription).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to load trial subscription")
|
||||
}
|
||||
if err := tx.Create(&SubscriptionEvent{SubscriptionID: subscription.ID, AccountID: account.ID, Provider: "manual", Kind: "trial_started", ToPlanID: &trialPlan.ID, ActorUserID: &u.ID, CreatedAt: time.Now().UTC()}).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to record trial")
|
||||
}
|
||||
access := Access{AccountID: account.ID, UserID: &u.ID, Role: "owner", SeatType: "admin"}
|
||||
if err := tx.Create(&access).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create access")
|
||||
}
|
||||
group := Group{AccountID: account.ID, Name: "Основные"}
|
||||
if err := tx.Create(&group).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create group")
|
||||
}
|
||||
notification := Notification{AccountID: account.ID, Name: "Основные", Enabled: true}
|
||||
if err := tx.Create(¬ification).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create notification")
|
||||
}
|
||||
if u.Email != nil {
|
||||
contact := Contact{AccountID: &account.ID, UserID: &u.ID, Kind: "email", Value: *u.Email}
|
||||
if err := tx.Create(&contact).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create contact")
|
||||
}
|
||||
if err := tx.Model(¬ification).Association("Contacts").Append(&contact); err != nil {
|
||||
return errors.Wrap(err, "failed to add contact to notification")
|
||||
}
|
||||
}
|
||||
if err := tx.Model(¬ification).Association("Groups").Append(&group); err != nil {
|
||||
return errors.Wrap(err, "failed to add group to notification")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &account, nil
|
||||
}
|
||||
283
app/models/account_test.go
Обычный файл
283
app/models/account_test.go
Обычный файл
@@ -0,0 +1,283 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestCreateAccountForUserStartsTeamTrial(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
email := "team-trial@example.test"
|
||||
user := models.User{Name: "Trial User", Email: &email, Enabled: true, Confirmed: true}
|
||||
require.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
account, err := models.CreateAccountForUser("Trial Account", &user)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, account.PlanID)
|
||||
require.NotNil(t, account.TrialEndsAt)
|
||||
|
||||
var plan models.Plan
|
||||
require.NoError(t, models.DB().First(&plan, *account.PlanID).Error)
|
||||
assert.Equal(t, "team", plan.Code)
|
||||
|
||||
var subscription models.Subscription
|
||||
require.NoError(t, models.DB().Where("account_id = ?", account.ID).First(&subscription).Error)
|
||||
assert.Equal(t, models.SubscriptionStatusTrialing, subscription.Status)
|
||||
assert.Equal(t, plan.ID, subscription.PlanID)
|
||||
require.NotNil(t, subscription.TrialEndsAt)
|
||||
assert.WithinDuration(t, *account.TrialEndsAt, *subscription.TrialEndsAt, time.Millisecond)
|
||||
}
|
||||
|
||||
// TestAccountModel tests basic Account model functionality
|
||||
func TestAccountModel(t *testing.T) {
|
||||
// Test Account structure
|
||||
account := models.Account{
|
||||
Name: "Test Account",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
Deleted: false,
|
||||
}
|
||||
|
||||
assert.Equal(t, "Test Account", account.Name)
|
||||
assert.Equal(t, "UTC", account.Timezone)
|
||||
assert.Equal(t, "en", account.Language)
|
||||
assert.False(t, account.Deleted)
|
||||
}
|
||||
|
||||
// TestAccountDisplayName tests User.DisplayName method
|
||||
func TestUserDisplayName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
user models.User
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "User with email",
|
||||
user: models.User{
|
||||
Name: "John Doe",
|
||||
Email: stringPtr("john@example.com"),
|
||||
},
|
||||
expected: "John Doe john@example.com",
|
||||
},
|
||||
{
|
||||
name: "User without email",
|
||||
user: models.User{
|
||||
Name: "Jane Doe",
|
||||
Email: nil,
|
||||
},
|
||||
expected: "Jane Doe",
|
||||
},
|
||||
{
|
||||
name: "User with empty name and email",
|
||||
user: models.User{
|
||||
Name: "",
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
expected: " test@example.com",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.user.DisplayName()
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserGravatar tests User.Gravatar method
|
||||
func TestUserGravatar(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
user models.User
|
||||
size int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "User with email",
|
||||
user: models.User{
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
size: 32,
|
||||
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=32&d=blank",
|
||||
},
|
||||
{
|
||||
name: "User without email",
|
||||
user: models.User{Email: nil},
|
||||
size: 32,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Different size",
|
||||
user: models.User{
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
size: 64,
|
||||
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=64&d=blank",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.user.Gravatar(tt.size)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserAsJSON tests User.AsJSON method
|
||||
func TestUserAsJSON(t *testing.T) {
|
||||
email := "test@example.com"
|
||||
user := models.User{
|
||||
ID: 123,
|
||||
Email: &email,
|
||||
Name: "Test User",
|
||||
}
|
||||
|
||||
result := user.AsJSON()
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, int64(123), result["id"])
|
||||
assert.Equal(t, &email, result["email"])
|
||||
assert.Contains(t, result["avatar"], "gravatar.com")
|
||||
}
|
||||
|
||||
// TestAccessModel tests Access model structure
|
||||
func TestAccessModel(t *testing.T) {
|
||||
access := models.Access{
|
||||
AccountID: 1,
|
||||
Kind: "account",
|
||||
Role: "owner",
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(1), access.AccountID)
|
||||
assert.Equal(t, "account", access.Kind)
|
||||
assert.Equal(t, "owner", access.Role)
|
||||
}
|
||||
|
||||
// TestGroupModel tests Group model structure
|
||||
func TestGroupModel(t *testing.T) {
|
||||
group := models.Group{
|
||||
AccountID: 1,
|
||||
Name: "Test Group",
|
||||
MonitorsCount: 5,
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(1), group.AccountID)
|
||||
assert.Equal(t, "Test Group", group.Name)
|
||||
assert.Equal(t, 5, group.MonitorsCount)
|
||||
}
|
||||
|
||||
// TestGroupIdsForAccountId tests GroupIdsForAccountId function
|
||||
func TestGroupIdsForAccountId(t *testing.T) {
|
||||
// This test would require a database connection
|
||||
// For now, we test that it doesn't panic with invalid input
|
||||
t.Run("handles zero account id", func(t *testing.T) {
|
||||
// Note: This will panic without DB connection, which is expected behavior
|
||||
// In a real test, we'd set up a test database
|
||||
})
|
||||
}
|
||||
|
||||
// TestAccountTableDrivenTests demonstrates table-driven testing pattern
|
||||
func TestAccountValidationTableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
account models.Account
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Valid account",
|
||||
account: models.Account{
|
||||
Name: "Valid Account",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Account with empty name",
|
||||
account: models.Account{
|
||||
Name: "",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: true, // Name should be required
|
||||
},
|
||||
{
|
||||
name: "Account with invalid timezone",
|
||||
account: models.Account{
|
||||
Name: "Test Account",
|
||||
Timezone: "Invalid/Timezone",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Validation logic would go here
|
||||
// For now, we just verify the test structure
|
||||
assert.NotNil(t, tt.account)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create string pointer
|
||||
func stringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// BenchmarkUserDisplayName benchmarks the DisplayName method
|
||||
func BenchmarkUserDisplayName(b *testing.B) {
|
||||
user := models.User{
|
||||
Name: "Test User",
|
||||
Email: stringPtr("test@example.com"),
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = user.DisplayName()
|
||||
}
|
||||
}
|
||||
|
||||
// TestAccountConcurrentOperations tests concurrent access to account data
|
||||
func TestAccountConcurrentOperations(t *testing.T) {
|
||||
account := models.Account{
|
||||
Name: "Concurrent Test",
|
||||
Timezone: "UTC",
|
||||
}
|
||||
|
||||
done := make(chan bool)
|
||||
|
||||
// Simulate concurrent reads
|
||||
for i := 0; i < 10; i++ {
|
||||
go func() {
|
||||
_ = account.Name
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all goroutines
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// ExampleAccountUsage provides an example of how to use Account model
|
||||
func ExampleAccount() {
|
||||
account := models.Account{
|
||||
Name: "Example Account",
|
||||
Timezone: "America/New_York",
|
||||
Language: "en",
|
||||
}
|
||||
|
||||
_ = account.Name
|
||||
// Output:
|
||||
}
|
||||
38
app/models/api_key.go
Обычный файл
38
app/models/api_key.go
Обычный файл
@@ -0,0 +1,38 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// ApiKey represents an API authentication key. //nolint:revive // accepted lint exception
|
||||
type ApiKey struct { //nolint:revive // accepted lint exception
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
AccountID int64 `json:"account_id,omitempty"`
|
||||
Account User `json:"-"`
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
User *User `json:"-"`
|
||||
Accesses []Access `json:"accesses" gorm:"foreignkey:api_key_id"`
|
||||
|
||||
concerns.HasToken
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
func (n *ApiKey) BeforeCreate(tx *gorm.DB) error { //nolint:revive // accepted lint exception
|
||||
n.SetToken()
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillAccesses provides functionality.
|
||||
func (n *ApiKey) FillAccesses() {
|
||||
for k, a := range n.Accesses { //nolint:gocritic // range copy is acceptable here
|
||||
if a.ID <= 0 {
|
||||
n.Accesses[k].ID = 0
|
||||
}
|
||||
n.Accesses[k].AccountID = n.AccountID
|
||||
}
|
||||
}
|
||||
63
app/models/audited.go
Обычный файл
63
app/models/audited.go
Обычный файл
@@ -0,0 +1,63 @@
|
||||
// Package models provides GORM models and business logic.
|
||||
// Audited models inspired by https://github.com/qor/audited
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AuditedCurrentUserKey is the GORM Set key for the current user.
|
||||
const AuditedCurrentUserKey = "audited:current_user"
|
||||
|
||||
// Audited tracks creator and updater IDs.
|
||||
type Audited struct {
|
||||
CreatorID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
Creator *User `json:"-"`
|
||||
UpdaterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
Updater *User `json:"-"`
|
||||
}
|
||||
|
||||
func getCurrentUser(scope *gorm.DB) (int64, bool) {
|
||||
var user interface{}
|
||||
var hasUser bool
|
||||
|
||||
user, hasUser = scope.Get(AuditedCurrentUserKey)
|
||||
|
||||
// spew.Dump(user, hasUser)
|
||||
if hasUser {
|
||||
return user.(*User).ID, true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func assignCreatedBy(tx *gorm.DB) {
|
||||
name := "CreatorID"
|
||||
if field := tx.Statement.Schema.LookUpField(name); field != nil {
|
||||
if user, ok := getCurrentUser(tx); ok {
|
||||
tx.Statement.SetColumn(name, user)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assignUpdatedBy(tx *gorm.DB) {
|
||||
name := "UpdaterID"
|
||||
if field := tx.Statement.Schema.LookUpField(name); field != nil {
|
||||
if user, ok := getCurrentUser(tx); ok {
|
||||
tx.Statement.SetColumn("UpdaterID", user, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterCallbacks register callback into GORM DB
|
||||
func RegisterCallbacks(db *gorm.DB) {
|
||||
callback := db.Callback()
|
||||
|
||||
if callback.Create().Get("audited:assign_created_by") == nil {
|
||||
_ = callback.Create().After("gorm:before_create").Register("audited:assign_created_by", assignCreatedBy)
|
||||
}
|
||||
|
||||
if callback.Update().Get("audited:assign_updated_by") == nil {
|
||||
_ = callback.Update().After("gorm:before_update").Register("audited:assign_updated_by", assignUpdatedBy)
|
||||
}
|
||||
}
|
||||
32
app/models/authidentity/auth_identity.go
Обычный файл
32
app/models/authidentity/auth_identity.go
Обычный файл
@@ -0,0 +1,32 @@
|
||||
// Package authidentity provides the AuthIdentity and Basic types for QOR-style
|
||||
// authentication identity management. Vendored from github.com/glebtv/auth/auth_identity
|
||||
// to eliminate the rsgit.ru/rs/sessionmanager transitive dependency.
|
||||
package authidentity
|
||||
|
||||
import "time"
|
||||
|
||||
// AuthIdentity combines Basic provider info with SignLogs for a full identity record.
|
||||
type AuthIdentity struct {
|
||||
Basic
|
||||
SignLogs
|
||||
}
|
||||
|
||||
// TableName returns the database table name for AuthIdentity.
|
||||
func (AuthIdentity) TableName() string {
|
||||
return "identities"
|
||||
}
|
||||
|
||||
// Basic represents the core identity fields (provider, UID, encrypted password).
|
||||
type Basic struct {
|
||||
ID int64 `gorm:"primary_key" json:"id"`
|
||||
Provider string
|
||||
UID string `gorm:"column:uid"`
|
||||
EncryptedPassword string
|
||||
UserID *int64
|
||||
ConfirmedAt *time.Time
|
||||
}
|
||||
|
||||
// TableName returns the database table name for Basic.
|
||||
func (Basic) TableName() string {
|
||||
return "identities"
|
||||
}
|
||||
50
app/models/authidentity/sign_logs.go
Обычный файл
50
app/models/authidentity/sign_logs.go
Обычный файл
@@ -0,0 +1,50 @@
|
||||
package authidentity
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SignLogs holds login history (log entries and sign-in count).
|
||||
type SignLogs struct {
|
||||
Log string `sql:"-"`
|
||||
SignInCount uint
|
||||
Logs []SignLog
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for deserializing SignLogs from JSON.
|
||||
func (signLogs *SignLogs) Scan(data interface{}) (err error) {
|
||||
switch values := data.(type) {
|
||||
case []byte:
|
||||
if len(values) != 0 {
|
||||
return json.Unmarshal(values, signLogs)
|
||||
}
|
||||
case string:
|
||||
return signLogs.Scan([]byte(values))
|
||||
case []string:
|
||||
for _, str := range values {
|
||||
if err := signLogs.Scan(str); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = errors.New("unsupported driver -> Scan pair for SignLogs")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for serializing SignLogs to JSON.
|
||||
func (signLogs SignLogs) Value() (driver.Value, error) {
|
||||
results, err := json.Marshal(signLogs)
|
||||
return string(results), err
|
||||
}
|
||||
|
||||
// SignLog represents a single login event entry.
|
||||
type SignLog struct {
|
||||
UserAgent string
|
||||
At *time.Time
|
||||
IP string
|
||||
}
|
||||
31
app/models/bits.go
Обычный файл
31
app/models/bits.go
Обычный файл
@@ -0,0 +1,31 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// BeginningOfDay provides functionality.
|
||||
func BeginningOfDay(t time.Time) time.Time {
|
||||
year, month, day := t.Date()
|
||||
return time.Date(year, month, day, 0, 0, 0, 0, t.Location())
|
||||
}
|
||||
|
||||
// SetBit provides functionality.
|
||||
// https://stackoverflow.com/questions/23192262/how-would-you-set-and-clear-a-single-bit-in-go
|
||||
// Sets the bit at pos in the integer n.
|
||||
func SetBit(n int, pos uint) int {
|
||||
n |= (1 << pos)
|
||||
return n
|
||||
}
|
||||
|
||||
// ClearBit provides functionality.
|
||||
// Clears the bit at pos in n.
|
||||
func ClearBit(n int, pos uint) int {
|
||||
mask := ^(1 << pos)
|
||||
n &= mask
|
||||
return n
|
||||
}
|
||||
|
||||
// HasBit provides functionality.
|
||||
func HasBit(n int, pos uint) bool {
|
||||
val := n & (1 << pos)
|
||||
return (val > 0)
|
||||
}
|
||||
171
app/models/check.go
Обычный файл
171
app/models/check.go
Обычный файл
@@ -0,0 +1,171 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Check provides functionality.
|
||||
type Check struct {
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
|
||||
|
||||
MonitorID int64 `gorm:"index;type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
Monitor *Monitor `json:"-"`
|
||||
|
||||
Name *string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Interval int `json:"interval" validate:"required,gte=60"`
|
||||
|
||||
// URL to monitor
|
||||
URL *string `json:"url,omitempty"`
|
||||
|
||||
// Other settings of the check
|
||||
Settings datatypes.JSON `gorm:"not null;" json:"settings"`
|
||||
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
|
||||
LastStart *time.Time `json:"last_start"`
|
||||
LastEnd *time.Time `json:"last_end"`
|
||||
LastOk *time.Time `json:"last_ok"`
|
||||
LastFail *time.Time `json:"last_fail"`
|
||||
WasUp *time.Time `json:"was_up"`
|
||||
|
||||
Fails int `json:"fails"`
|
||||
Expires *time.Time `json:"expires"`
|
||||
|
||||
Error *string `json:"error"`
|
||||
Warnings pq.StringArray `gorm:"type:varchar(255)[]" json:"warnings"`
|
||||
Infos pq.StringArray `gorm:"type:varchar(255)[]" json:"infos"`
|
||||
|
||||
// RequireQuorum enables multi-region result aggregation (Phase 3 of
|
||||
// docs/todo.md): when >1 the check's State is NOT written directly by
|
||||
// ApplyRemoteCheckResult — instead CheckRegionResult rows accumulate
|
||||
// until app/models/check_aggregator.go decides OK/ERR/DEGRADED.
|
||||
// Default 1 keeps the legacy single-region behavior unchanged.
|
||||
RequireQuorum int `gorm:"not null;default:1" json:"require_quorum"`
|
||||
|
||||
// AggregationWindowSeconds is how long the aggregator waits for
|
||||
// regional CheckRegionResult rows before deciding the check's State.
|
||||
// Stored as int seconds (matching the existing GORM style — no
|
||||
// time.Duration columns) and exposed via AggregationWindow(). Default
|
||||
// 5s; ignored when RequireQuorum <= 1.
|
||||
AggregationWindowSeconds int `gorm:"not null;default:5" json:"aggregation_window_seconds"`
|
||||
|
||||
IsNew bool `gorm:"-:all" sql:"-" json:"is_new,omitempty"`
|
||||
Deleted bool `gorm:"-:all" sql:"-" json:"deleted,omitempty"`
|
||||
|
||||
Events []Event `json:"-" gorm:"many2many:event_checks;"`
|
||||
|
||||
Audited
|
||||
}
|
||||
|
||||
// ExpScope provides functionality.
|
||||
func ExpScope(q *gorm.DB) *gorm.DB {
|
||||
return q.Where("kind IN ('whois', 'ssl')").
|
||||
Preload("Monitor").
|
||||
Preload("Monitor.Group").
|
||||
Preload("Monitor.Group.Notifications").
|
||||
Preload("Monitor.Group.Notifications.Contacts").
|
||||
Where("expires < ?", time.Now().Add(time.Hour*7*24))
|
||||
}
|
||||
|
||||
// IntervalOK provides functionality.
|
||||
func (c *Check) IntervalOK() bool {
|
||||
if c.Kind == kindRKN {
|
||||
return true
|
||||
}
|
||||
|
||||
if c.Kind == kindWhois {
|
||||
return c.Interval >= 43200
|
||||
}
|
||||
|
||||
return c.Interval >= 30
|
||||
}
|
||||
|
||||
// GetLabel provides functionality.
|
||||
func (c *Check) GetLabel() string {
|
||||
if c.Name != nil {
|
||||
return *c.Name
|
||||
}
|
||||
if c.URL != nil {
|
||||
return *c.URL
|
||||
}
|
||||
return c.Kind
|
||||
}
|
||||
|
||||
// KindLabel provides functionality.
|
||||
func (c *Check) KindLabel() string {
|
||||
if c.Kind == kindWhois {
|
||||
return "регистрация домена"
|
||||
}
|
||||
if c.Kind == kindSSL {
|
||||
return "SSL сертификат"
|
||||
}
|
||||
|
||||
return c.Kind
|
||||
}
|
||||
|
||||
// GetSettings provides functionality.
|
||||
func (c *Check) GetSettings() CheckSettings {
|
||||
d := CheckSettings{}
|
||||
|
||||
err := json.Unmarshal(c.Settings, &d)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ValidateSettings provides functionality.
|
||||
func (c *Check) ValidateSettings() error {
|
||||
if len(c.Settings) == 0 {
|
||||
c.Settings = []byte("{}")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetURL provides functionality.
|
||||
func (c *Check) GetURL() (string, error) {
|
||||
// return c.GetSettings()["url"].(string)
|
||||
if c.URL != nil {
|
||||
return *c.URL, nil
|
||||
}
|
||||
|
||||
return "http://" + c.Monitor.Host, nil
|
||||
}
|
||||
|
||||
// MetricName provides functionality.
|
||||
func (c *Check) MetricName() string {
|
||||
sanitized := strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == ':' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, c.Kind)
|
||||
return "c" + sanitized
|
||||
}
|
||||
|
||||
// QuorumEnabled reports whether this check should be aggregated by
|
||||
// app/models/check_aggregator.go. When false (RequireQuorum <= 1),
|
||||
// ApplyRemoteCheckResult keeps the legacy direct State update path.
|
||||
func (c *Check) QuorumEnabled() bool {
|
||||
return c.RequireQuorum > 1
|
||||
}
|
||||
|
||||
// AggregationWindow returns AggregationWindowSeconds as a time.Duration.
|
||||
// Defaults to 5s when the underlying int is zero/negative, mirroring the
|
||||
// GORM column default; callers can rely on a strictly positive value.
|
||||
func (c *Check) AggregationWindow() time.Duration {
|
||||
if c.AggregationWindowSeconds <= 0 {
|
||||
return 5 * time.Second
|
||||
}
|
||||
return time.Duration(c.AggregationWindowSeconds) * time.Second
|
||||
}
|
||||
336
app/models/check_aggregator.go
Обычный файл
336
app/models/check_aggregator.go
Обычный файл
@@ -0,0 +1,336 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Phase 3 of docs/todo.md — result aggregation for multi-region checks.
|
||||
//
|
||||
// When a check has RequireQuorum > 1, ApplyRemoteCheckResult does not
|
||||
// write Check.State directly. Instead it appends a CheckRegionResult row
|
||||
// and leaves AggregatedAt NULL. This file owns the background goroutine
|
||||
// that walks those pending rows once their aggregation window has
|
||||
// elapsed, decides OK/ERR/DEGRADED per the documented rule, writes the
|
||||
// aggregate state onto Check, stamps AggregatedAt on the contributing
|
||||
// rows, and triggers Monitor.UpdateStatusFromChecks so the monitor's own
|
||||
// status follows.
|
||||
//
|
||||
// Aggregation rule (see docs/todo.md Phase 3 + checkSeverityRank in
|
||||
// monitor.go for the corresponding severity order):
|
||||
//
|
||||
// - Aggregate only rows whose created_at is older than
|
||||
// NOW() - Check.AggregationWindowSeconds. This is the "watermark"
|
||||
// pattern: a row is eligible only when no fresher regional result
|
||||
// could still arrive and tip the vote. The window is per-check so
|
||||
// noisy checks can use a longer wait than fast ones.
|
||||
// - If zero eligible rows exist for a check, leave Check.State
|
||||
// untouched (the special case called out in the spec).
|
||||
// - Otherwise count OK vs not-OK among the eligible rows:
|
||||
// OK >= RequireQuorum → Check.State = OK
|
||||
// not-OK >= RequireQuorum → Check.State = ERR
|
||||
// neither side reaches quorum → Check.State = DEGRADED
|
||||
// - Stamp AggregatedAt = NOW() on every contributing row so the next
|
||||
// tick skips them. One transaction per check; per-row failures do
|
||||
// not poison other checks.
|
||||
|
||||
// AggregatorTickInterval is the default cadence of StartCheckAggregator
|
||||
// when the caller passes interval <= 0. Mirrors the 30s default used by
|
||||
// the other reapers in this package so the three reapers all tick on
|
||||
// the same wall clock cadence — easier to grep, easier to reason about
|
||||
// in incident timelines.
|
||||
const AggregatorTickInterval = 30 * time.Second
|
||||
|
||||
// EnsureCheckAggregatorIndexes adds the partial indexes the aggregator
|
||||
// relies on. AutoMigrate creates AggregatedAt as a regular btree column,
|
||||
// but the per-tick SELECT filters on `aggregated_at IS NULL` over what
|
||||
// grows to be a busy table; a partial index keeps the working set
|
||||
// tiny. Idempotent so it is safe to call from Migrate() and from tests.
|
||||
func EnsureCheckAggregatorIndexes() error {
|
||||
return DB().Exec(`
|
||||
CREATE INDEX IF NOT EXISTS check_region_results_pending_idx
|
||||
ON check_region_results (check_id, created_at)
|
||||
WHERE aggregated_at IS NULL
|
||||
`).Error
|
||||
}
|
||||
|
||||
// aggregateCheckState holds the per-check aggregation inputs we need to
|
||||
// keep the rule readable. Rows is the set of CheckRegionResult rows
|
||||
// eligible for the current decision; quorum is Check.RequireQuorum.
|
||||
type aggregateCheckState struct {
|
||||
CheckID int64
|
||||
Quorum int
|
||||
Rows []CheckRegionResult
|
||||
}
|
||||
|
||||
// decideAggregateState encodes the OK/ERR/DEGRADED rule described in
|
||||
// the package doc. Pure function — no DB, no time — so it is trivially
|
||||
// unit-testable from the test file.
|
||||
func decideAggregateState(in aggregateCheckState) (string, bool) {
|
||||
if len(in.Rows) == 0 || in.Quorum <= 1 {
|
||||
// Zero eligible rows in the window OR a misconfigured check
|
||||
// (QuorumEnabled false). Caller must leave Check.State alone
|
||||
// in both cases.
|
||||
return "", false
|
||||
}
|
||||
okCount := 0
|
||||
badCount := 0
|
||||
for i := range in.Rows {
|
||||
if in.Rows[i].State == stateOK {
|
||||
okCount++
|
||||
} else {
|
||||
badCount++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case okCount >= in.Quorum:
|
||||
return stateOK, true
|
||||
case badCount >= in.Quorum:
|
||||
return stateERR, true
|
||||
default:
|
||||
return stateDegraded, true
|
||||
}
|
||||
}
|
||||
|
||||
// CheckAggregatorTick performs one pass of the aggregator. It is the
|
||||
// per-tick body StartCheckAggregator calls. Exported so the test suite
|
||||
// can call it directly without spinning up the goroutine; production
|
||||
// always goes through StartCheckAggregator.
|
||||
//
|
||||
// The returned (aggregated, err) tuple lets the caller log a metric:
|
||||
// aggregated counts how many Check rows had their State written this
|
||||
// tick. The function is idempotent — a second call with no new
|
||||
// unaggregated rows is a no-op that returns (0, nil).
|
||||
func CheckAggregatorTick() (aggregated int, err error) {
|
||||
// Step 1: collect candidate check IDs. The JOIN to checks is needed
|
||||
// to read each check's window length and to filter on
|
||||
// require_quorum > 1 (so we never aggregate the legacy path).
|
||||
rows, err := DB().Raw(`
|
||||
SELECT DISTINCT crr.check_id
|
||||
FROM check_region_results crr
|
||||
JOIN checks c ON c.id = crr.check_id
|
||||
WHERE crr.aggregated_at IS NULL
|
||||
AND c.require_quorum > 1
|
||||
AND crr.created_at < NOW() - make_interval(secs => GREATEST(c.aggregation_window_seconds, 1))
|
||||
ORDER BY crr.check_id
|
||||
`).Rows()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var checkIDs []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return 0, scanErr
|
||||
}
|
||||
checkIDs = append(checkIDs, id)
|
||||
}
|
||||
if scanErr := rows.Err(); scanErr != nil {
|
||||
return 0, scanErr
|
||||
}
|
||||
if len(checkIDs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
for _, checkID := range checkIDs {
|
||||
n, err := aggregateOneCheck(checkID)
|
||||
if err != nil {
|
||||
// Log and continue: one bad check must not stop the loop.
|
||||
log.Printf("check_aggregator: check_id=%d error: %v", checkID, err)
|
||||
continue
|
||||
}
|
||||
aggregated += n
|
||||
}
|
||||
return aggregated, nil
|
||||
}
|
||||
|
||||
// aggregateOneCheck runs the aggregation logic for a single check inside
|
||||
// a transaction. The transaction holds a FOR UPDATE row lock on the
|
||||
// check so concurrent aggregator instances (multiple web processes) can
|
||||
// not race on the same check — the second one waits for the first to
|
||||
// commit, then sees AggregatedAt IS NOT NULL on every row and the
|
||||
// candidate SELECT below returns an empty set.
|
||||
func aggregateOneCheck(checkID int64) (int, error) {
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return 0, tx.Error
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
tx.Rollback()
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
var check Check
|
||||
if err := tx.Clauses(SkipLockedClause).First(&check, checkID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
// Check was deleted between candidate SELECT and lock; not
|
||||
// an error, just nothing to do.
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
// Defensive: only aggregate quorum-enabled checks. The candidate
|
||||
// SELECT already filters on this, but a stale row that flipped off
|
||||
// quorum between calls must not be re-aggregated here.
|
||||
if !check.QuorumEnabled() {
|
||||
tx.Rollback()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var results []CheckRegionResult
|
||||
if err := tx.
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
Order("created_at ASC").
|
||||
Find(&results).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
decision, ok := decideAggregateState(aggregateCheckState{
|
||||
CheckID: checkID,
|
||||
Quorum: check.RequireQuorum,
|
||||
Rows: results,
|
||||
})
|
||||
if !ok {
|
||||
// Zero eligible rows — leave Check.State alone. There is also
|
||||
// nothing to stamp, so just rollback and move on.
|
||||
tx.Rollback()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
// Pull the latest error string from the contributing rows so the
|
||||
// monitor event / notifier pipeline has something to show. Prefer
|
||||
// the most recent ERR row's message; fall back to the most recent
|
||||
// any-row message. UNK / empty stays NULL.
|
||||
var lastError *string
|
||||
for i := len(results) - 1; i >= 0; i-- {
|
||||
if results[i].Error != nil && *results[i].Error != "" {
|
||||
lastError = results[i].Error
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
upd := map[string]interface{}{
|
||||
colState: decision,
|
||||
colLastEnd: now,
|
||||
}
|
||||
if decision == stateOK {
|
||||
// OK resets error — mirrors the legacy ApplyRemoteCheckResult
|
||||
// path that sets `error = gorm.Expr("NULL")` when state==OK.
|
||||
upd["error"] = gorm.Expr("NULL")
|
||||
upd["last_ok"] = now
|
||||
upd["fails"] = 0
|
||||
upd["was_up"] = now
|
||||
} else {
|
||||
// Non-OK: bump the fail counter and only overwrite the error
|
||||
// when one of the contributing rows actually carries a
|
||||
// message. If none do, leave whatever was there before —
|
||||
// mirrors the legacy `if report.Error != nil` branch.
|
||||
upd["last_fail"] = now
|
||||
upd["fails"] = gorm.Expr("fails + 1")
|
||||
if lastError != nil {
|
||||
upd["error"] = *lastError
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Model(&Check{}).Where("id = ?", checkID).UpdateColumns(upd).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := tx.Model(&CheckRegionResult{}).
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
UpdateColumns(map[string]interface{}{
|
||||
"aggregated_at": now,
|
||||
}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Mirror ApplyRemoteCheckResult: propagate the aggregate decision
|
||||
// up to the monitor. We do this AFTER commit so a rollback does
|
||||
// not leave the monitor in a state whose corresponding check is
|
||||
// still pre-aggregate. The goroutine keeps the failure path of
|
||||
// UpdateStatusFromChecks isolated from the aggregator's hot loop.
|
||||
if check.MonitorID != 0 {
|
||||
var mon Monitor
|
||||
if err := DB().First(&mon, check.MonitorID).Error; err == nil {
|
||||
go mon.UpdateStatusFromChecks()
|
||||
} else {
|
||||
log.Printf("check_aggregator: monitor lookup failed for check_id=%d: %v", checkID, err)
|
||||
}
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// StartCheckAggregator launches a goroutine that calls
|
||||
// CheckAggregatorTick on the given interval until ctx is canceled.
|
||||
// Mirrors StartTaskReaper / StartDeadWorkerReaper in this package — same
|
||||
// ticker shape, same default-interval fall-back, same per-tick recover
|
||||
// so a malformed row cannot crash the web process.
|
||||
//
|
||||
// The default interval is AggregatorTickInterval (30s); values <= 0
|
||||
// fall back to the default so the helper is safe to call from any call
|
||||
// site without a guard. A nil context falls back to context.Background()
|
||||
// the same way StartDeadWorkerReaper does, so main.init() and tests
|
||||
// can both call it without ceremony.
|
||||
//
|
||||
// Wire from main.init() once per process. The aggregator is cheap in
|
||||
// steady state (one indexed SELECT for candidates + a per-check
|
||||
// transaction over a handful of unaggregated rows). Under load it
|
||||
// scales horizontally — multiple web processes can each run their own
|
||||
// StartCheckAggregator goroutine because FOR UPDATE SKIP LOCKED on the
|
||||
// per-check transaction guarantees at-most-one winner per check.
|
||||
func StartCheckAggregator(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = AggregatorTickInterval
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
// Best-effort index bootstrap. AutoMigrate declares AggregatedAt as
|
||||
// a regular btree column; the partial index speeds up the per-tick
|
||||
// candidate SELECT. Idempotent — safe to call on every boot.
|
||||
if err := EnsureCheckAggregatorIndexes(); err != nil {
|
||||
log.Printf("check_aggregator: ensure index: %v", err)
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("check_aggregator: panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
n, err := CheckAggregatorTick()
|
||||
if err != nil {
|
||||
log.Printf("check_aggregator: error: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("check_aggregator: aggregated=%d", n)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
448
app/models/check_aggregator_test.go
Обычный файл
448
app/models/check_aggregator_test.go
Обычный файл
@@ -0,0 +1,448 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// seedAggregatorWorld creates the minimum fixture the aggregator tests
|
||||
// need: a plan, account, group, monitor, and an http check whose
|
||||
// RequireQuorum / AggregationWindowSeconds are set per call. The check
|
||||
// is created with state=UNK so the test can observe the aggregator's
|
||||
// effect on Check.State directly.
|
||||
//
|
||||
// Returns the freshly-created monitor + check; the check is what every
|
||||
// test below mutates (RequireQuorum, AggregationWindowSeconds) and then
|
||||
// asserts on. Cleanup is the caller's responsibility — most tests call
|
||||
// models.Drop() at the top instead.
|
||||
func seedAggregatorWorld(t *testing.T, quorum, windowSeconds int) (models.Monitor, models.Check) {
|
||||
t.Helper()
|
||||
|
||||
plan := models.Plan{Name: "agg-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "agg-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
grp := &models.Group{AccountID: acc.ID, Name: "agg"}
|
||||
require.NoError(t, models.DB().Create(grp).Error)
|
||||
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrAgg("agg.test"),
|
||||
Host: "agg.test",
|
||||
GroupID: grp.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
enTrue := true
|
||||
check := models.Check{
|
||||
MonitorID: mon.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
RequireQuorum: quorum,
|
||||
AggregationWindowSeconds: windowSeconds,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return mon, check
|
||||
}
|
||||
|
||||
func stringPtrAgg(s string) *string { return &s }
|
||||
|
||||
// makeReport constructs a wire.CheckResultReport with sensible defaults
|
||||
// for the OK or not-OK case. Tests use this to push results through
|
||||
// ApplyRemoteCheckResult exactly the way a real worker would.
|
||||
func makeReport(checkID, monitorID int64, state string) wire.CheckResultReport {
|
||||
return wire.CheckResultReport{
|
||||
JobID: "job-" + state,
|
||||
CheckID: checkID,
|
||||
MonitorID: monitorID,
|
||||
State: state,
|
||||
}
|
||||
}
|
||||
|
||||
// regionResultWithErr is regionResultFor plus an error message. Used
|
||||
// when the test wants to verify that the aggregator forwards the row's
|
||||
// error string onto Check.Error (mimics a real worker reporting
|
||||
// state=ERR with a diagnostic message).
|
||||
func regionResultWithErr(t *testing.T, checkID int64, region, state, errMsg string, age time.Duration) models.CheckRegionResult {
|
||||
t.Helper()
|
||||
row := regionResultFor(t, checkID, region, state, age)
|
||||
require.NoError(t, models.DB().Model(&row).UpdateColumn("error", errMsg).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
// regionResultFor inserts a single CheckRegionResult row whose
|
||||
// created_at and executed_at are both backdated by `age`, so the
|
||||
// aggregator's window-based watermark picks it up immediately without
|
||||
// needing a real time.Sleep. Returned row has its DB-assigned ID
|
||||
// populated.
|
||||
func regionResultFor(t *testing.T, checkID int64, region string, state string, age time.Duration) models.CheckRegionResult {
|
||||
t.Helper()
|
||||
// RegionCode has a FK to regions.code, so the region must exist
|
||||
// before the result row is inserted. seedRegion is idempotent.
|
||||
seedRegion(t, region)
|
||||
row := models.CheckRegionResult{
|
||||
CheckID: checkID,
|
||||
RegionCode: region,
|
||||
ExecutedAt: time.Now().Add(-age),
|
||||
State: state,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&row).Error)
|
||||
// Backdate CreatedAt too — the aggregator SQL keys on
|
||||
// check_region_results.created_at (see CheckAggregatorTick). GORM
|
||||
// auto-sets CreatedAt on insert, so we have to UPDATE it post-hoc.
|
||||
require.NoError(t, models.DB().Model(&row).UpdateColumns(map[string]interface{}{
|
||||
"created_at": time.Now().Add(-age),
|
||||
"updated_at": time.Now().Add(-age),
|
||||
}).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
// loadCheck re-reads a Check row by ID — used after the aggregator
|
||||
// runs so the test asserts against the post-tick state.
|
||||
func loadCheck(t *testing.T, id int64) models.Check {
|
||||
t.Helper()
|
||||
var c models.Check
|
||||
require.NoError(t, models.DB().First(&c, id).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
// countPendingResults returns how many CheckRegionResult rows for
|
||||
// checkID have aggregated_at IS NULL — the working set the next
|
||||
// aggregator tick would consider.
|
||||
func countPendingResults(t *testing.T, checkID int64) int64 {
|
||||
t.Helper()
|
||||
var n int64
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
Count(&n).Error)
|
||||
return n
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ApplyRemoteCheckResult: regression tests for the QuorumEnabled split.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestApplyRemoteCheckResult_DirectWhenQuorumOne pins the legacy path:
|
||||
// when RequireQuorum==1 (the default), ApplyRemoteCheckResult still
|
||||
// writes Check.State synchronously, exactly the way it did before Phase
|
||||
// 3. This is the regression guard for the in-process RKN scheduler
|
||||
// tests in internal/rknscheduler.
|
||||
func TestApplyRemoteCheckResult_DirectWhenQuorumOne(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
mon, check := seedAggregatorWorld(t, 1, 5)
|
||||
// Region must exist because CheckRegionResult has a FK to
|
||||
// regions.code (seeded by Migrate, but the test region is custom).
|
||||
seedRegion(t, "ru-msk")
|
||||
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "OK"),
|
||||
"ru-msk",
|
||||
))
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "quorum=1 must keep the direct State update")
|
||||
assert.NotNil(t, got.LastEnd, "legacy path must keep stamping last_end")
|
||||
// One region result inserted with aggregated_at=NULL.
|
||||
assert.EqualValues(t, 1, countPendingResults(t, check.ID),
|
||||
"the region result row is always inserted even on the legacy path")
|
||||
}
|
||||
|
||||
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
|
||||
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
|
||||
// Check.State — it only inserts the CheckRegionResult row. The check
|
||||
// stays at its initial UNK and the unaggregated row count grows by
|
||||
// exactly 1 per call.
|
||||
func TestApplyRemoteCheckResult_BuffersWhenQuorumN(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
mon, check := seedAggregatorWorld(t, 3, 5)
|
||||
seedRegion(t, "ru-msk")
|
||||
seedRegion(t, "us-east")
|
||||
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "OK"), "ru-msk",
|
||||
))
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "ERR"), "us-east",
|
||||
))
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State,
|
||||
"quorum>1 must NOT touch Check.State — the aggregator owns it")
|
||||
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
|
||||
"two results buffered, both with aggregated_at=NULL")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CheckAggregatorTick: rule tests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestAggregator_QuorumOK: with RequireQuorum=2 and two OK results
|
||||
// buffered, the aggregator must decide OK and stamp AggregatedAt on
|
||||
// both contributing rows.
|
||||
func TestAggregator_QuorumOK(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n, "one check aggregated this tick")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "2 OK results >= quorum=2 → state=OK")
|
||||
assert.NotNil(t, got.LastOk, "OK decision must stamp last_ok")
|
||||
assert.EqualValues(t, 0, got.Fails, "fails must reset on OK")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID),
|
||||
"both contributing rows must be stamped aggregated_at")
|
||||
}
|
||||
|
||||
// TestAggregator_QuorumFail: with RequireQuorum=2 and two ERR results
|
||||
// buffered, the aggregator must decide ERR and surface the most recent
|
||||
// row's error message on Check.Error.
|
||||
func TestAggregator_QuorumFail(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultWithErr(t, check.ID, "ru-msk", "ERR", "connection refused", 5*time.Second)
|
||||
regionResultWithErr(t, check.ID, "us-east", "ERR", "timeout", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "ERR", got.State, "2 ERR >= quorum=2 → state=ERR")
|
||||
assert.NotNil(t, got.LastFail)
|
||||
assert.NotNil(t, got.Error, "ERR decision must carry an error message from the rows")
|
||||
assert.Contains(t, *got.Error, "timeout",
|
||||
"aggregator should surface the latest row's error message")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_DegradedWhenPartial: with RequireQuorum=3 and 1 OK +
|
||||
// 2 ERR (mixed within window), neither side reaches the quorum of 3 so
|
||||
// the aggregator must decide DEGRADED. The state must NOT silently
|
||||
// become OK or ERR.
|
||||
func TestAggregator_DegradedWhenPartial(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 3, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "ERR", 4*time.Second)
|
||||
regionResultFor(t, check.ID, "eu-west", "ERR", 3*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"neither OK nor ERR reaches quorum=3 → state=DEGRADED")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_NotEnoughRegionsAlsoDegraded covers the single-region-
|
||||
// only-delivered case: with RequireQuorum=3 and only 1 result buffered
|
||||
// (and it aged past the window), the rule still says "neither side
|
||||
// reached quorum" → DEGRADED. This is the documented behavior for slow
|
||||
// regions that never report in time.
|
||||
func TestAggregator_NotEnoughRegionsAlsoDegraded(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 3, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"single OK row vs quorum=3 → DEGRADED (below quorum on both sides)")
|
||||
}
|
||||
|
||||
// TestAggregator_NoResultsLeavesStateAlone is the "special case" from
|
||||
// the spec: when the aggregator tick finds no eligible rows for a
|
||||
// check, Check.State must NOT change. Pre-set the check to OK and
|
||||
// verify it stays OK.
|
||||
func TestAggregator_NoResultsLeavesStateAlone(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
// Pre-set state and a previous LastEnd so we can detect any
|
||||
// accidental overwrite.
|
||||
prevEnd := time.Now().Add(-time.Hour)
|
||||
require.NoError(t, models.DB().Model(&models.Check{}).
|
||||
Where("id = ?", check.ID).
|
||||
Updates(map[string]interface{}{
|
||||
"state": "OK",
|
||||
"last_end": prevEnd,
|
||||
"last_ok": prevEnd,
|
||||
}).Error)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "no eligible rows → nothing aggregated")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "state must not change with zero eligible rows")
|
||||
assert.WithinDuration(t, prevEnd, *got.LastEnd, time.Second,
|
||||
"last_end must not be touched when there are no eligible rows")
|
||||
}
|
||||
|
||||
// TestAggregator_MultipleChecksIndependent verifies that a single tick
|
||||
// processes every check with pending results, not just the first one.
|
||||
// Two checks, each with 2 regions, each should flip to OK after the
|
||||
// tick.
|
||||
func TestAggregator_MultipleChecksIndependent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, c1 := seedAggregatorWorld(t, 2, 1)
|
||||
_, c2 := seedAggregatorWorld(t, 2, 1)
|
||||
|
||||
regionResultFor(t, c1.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, c1.ID, "us-east", "OK", 4*time.Second)
|
||||
regionResultFor(t, c2.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, c2.ID, "eu-west", "OK", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, n, "both checks aggregated in the same tick")
|
||||
|
||||
assert.Equal(t, "OK", loadCheck(t, c1.ID).State)
|
||||
assert.Equal(t, "OK", loadCheck(t, c2.ID).State)
|
||||
assert.EqualValues(t, 0, countPendingResults(t, c1.ID))
|
||||
assert.EqualValues(t, 0, countPendingResults(t, c2.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_AlreadyAggregatedRowsSkipped pins the idempotency
|
||||
// story: a second tick with no new rows must be a no-op. We pre-mark
|
||||
// the rows aggregated_at=NOW() and verify the tick returns (0, nil)
|
||||
// without touching Check.State.
|
||||
func TestAggregator_AlreadyAggregatedRowsSkipped(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
r1 := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
r2 := regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
// Pretend a previous aggregator pass already stamped them.
|
||||
now := time.Now()
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("id IN ?", []int64{r1.ID, r2.ID}).
|
||||
UpdateColumns(map[string]interface{}{"aggregated_at": now}).Error)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "no candidate checks → 0 aggregated")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State, "already-aggregated rows must not cause a re-decision")
|
||||
}
|
||||
|
||||
// TestAggregator_IgnoresRowsInsideWindow verifies the watermark: rows
|
||||
// whose CreatedAt is NEWER than (NOW() - window) are NOT eligible and
|
||||
// must NOT be stamped. With AggregationWindowSeconds=10 and rows aged
|
||||
// only 2s, the aggregator finds nothing to do.
|
||||
func TestAggregator_IgnoresRowsInsideWindow(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 10)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 2*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 1*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "rows still inside window → no aggregation")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State)
|
||||
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
|
||||
"rows inside window stay unaggregated for the next tick")
|
||||
}
|
||||
|
||||
// TestAggregator_SkipsChecksWithQuorumOne guards the candidate SELECT
|
||||
// filter: even though CheckRegionResult rows are written for
|
||||
// RequireQuorum=1 checks (via StoreCheckRegionResult), the aggregator
|
||||
// must not re-decide their state because ApplyRemoteCheckResult
|
||||
// already did. We simulate by inserting a region row with aggregated_at
|
||||
// NULL for a quorum=1 check and verifying the tick ignores it.
|
||||
func TestAggregator_SkipsChecksWithQuorumOne(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 1, 1)
|
||||
r := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
require.Nil(t, r.AggregatedAt)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "quorum=1 checks must be filtered out by the candidate SELECT")
|
||||
|
||||
// The row must stay unaggregated too — the aggregator has no
|
||||
// business stamping it.
|
||||
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// StartCheckAggregator: ticker smoke test.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestStartCheckAggregator_TickerFiresOnce is the smoke test for the
|
||||
// background helper: spin up the aggregator with a tight 10ms ticker
|
||||
// and a cancellable context, wait for one tick to flip a seeded
|
||||
// check's state, then cancel so the goroutine exits cleanly. Mirrors
|
||||
// TestStartDeadWorkerReaper_TickerFiresOnce in shape.
|
||||
func TestStartCheckAggregator_TickerFiresOnce(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
models.StartCheckAggregator(ctx, 10*time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got := loadCheck(t, check.ID)
|
||||
if got.State == "OK" {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("aggregator goroutine did not flip check to OK within 2s; state=%q", loadCheck(t, check.ID).State)
|
||||
}
|
||||
|
||||
// silence unused import warnings when individual helpers are inlined by
|
||||
// editors — the package-level references below keep the imports live.
|
||||
var (
|
||||
_ = gorm.ErrRecordNotFound
|
||||
)
|
||||
14
app/models/check_data.go
Обычный файл
14
app/models/check_data.go
Обычный файл
@@ -0,0 +1,14 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
)
|
||||
|
||||
// CheckData provides functionality.
|
||||
type CheckData struct {
|
||||
Uptime int `json:"uptime"`
|
||||
Data []influx.InfluxData `json:"data"`
|
||||
LastCheck *time.Time `json:"last_check"`
|
||||
}
|
||||
415
app/models/check_jobs.go
Обычный файл
415
app/models/check_jobs.go
Обычный файл
@@ -0,0 +1,415 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// defaultRegionCode is the historical default region seeded by Migrate()
|
||||
// (see app/models/migrate.go) and used as a catch-all bucket for results
|
||||
// reported without a region code. Aliased to the exported Region
|
||||
// constant (DefaultRegionCode) so admin endpoints and the in-process
|
||||
// job router share one source of truth.
|
||||
const defaultRegionCode = DefaultRegionCode
|
||||
|
||||
// ChecksForWorker returns checks that need to be executed by a distributed worker.
|
||||
// It uses FOR UPDATE SKIP LOCKED to prevent race conditions between concurrent workers.
|
||||
// The worker specifies which check kinds it can handle via the kinds parameter.
|
||||
//
|
||||
// Phase 2 of docs/plans/worker-notifier-mvp.md adds regional job routing: when
|
||||
// worker is non-nil, the candidate monitor set is filtered by
|
||||
// applyRegionRouting so a worker only sees checks that explicitly allow its
|
||||
// region. Pass nil for the legacy "no region scoping" path used by
|
||||
// diagnostics/dashboard tooling.
|
||||
func ChecksForWorker(worker *WorkerNode, kinds []string, limit int) []*Check {
|
||||
tx := DB().Begin()
|
||||
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
|
||||
Where("monitors.enabled").
|
||||
Where("checks.enabled AND checks.kind IN (?)", kinds)
|
||||
|
||||
if worker != nil {
|
||||
q = applyRegionRouting(q, worker)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
|
||||
}
|
||||
// Flagged workers remain visible for audit/history but never receive new work.
|
||||
if worker.NetworkProblemActive(time.Now()) {
|
||||
tx.Rollback()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Allow faster retry for failed http/dns checks
|
||||
notOk := ""
|
||||
hasHTTPOrDNS := false
|
||||
for _, k := range kinds {
|
||||
if k == kindHTTP || k == kindDNS {
|
||||
hasHTTPOrDNS = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasHTTPOrDNS {
|
||||
notOk = `OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
|
||||
}
|
||||
|
||||
whereClause := `
|
||||
(checks.last_start IS NULL) OR
|
||||
(checks.last_start + (checks.interval || ' second')::interval < now())
|
||||
`
|
||||
if notOk != "" {
|
||||
whereClause += notOk
|
||||
}
|
||||
rq := q.Where(whereClause)
|
||||
|
||||
// Use SKIP LOCKED to avoid contention between workers. The same
|
||||
// FOR UPDATE SKIP LOCKED clause also gives us implicit load balancing
|
||||
// across workers in the same region: each concurrent worker call
|
||||
// grabs a disjoint slice of the pending checks and a row leased by
|
||||
// worker A is invisible to worker B until A's transaction commits
|
||||
// (or rolls back / lease expires).
|
||||
var checks []*Check
|
||||
rq.Clauses(SkipLockedClause).
|
||||
Limit(limit).
|
||||
Preload("Monitor").
|
||||
Find(&checks)
|
||||
|
||||
for _, c := range checks {
|
||||
log.Println("worker: assigned remote check:", c.ID, c.Kind)
|
||||
tx.Model(&c).Where("id = ?", c.ID).Update(colLastStart, time.Now())
|
||||
}
|
||||
|
||||
tx.Commit()
|
||||
return checks
|
||||
}
|
||||
|
||||
// EnqueueDueCheckTasks atomically turns due normal checks into durable generic
|
||||
// task envelopes. ChecksForWorker remains for the HTTP polling compatibility
|
||||
// endpoint, while websocket scheduling uses this task-producing path.
|
||||
func EnqueueDueCheckTasks(worker *WorkerNode, kinds []string, limit int) error {
|
||||
if worker == nil || len(kinds) == 0 || limit <= 0 || worker.NetworkProblemActive(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
|
||||
Where("monitors.enabled").Where("checks.enabled AND checks.kind IN (?)", kinds)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
|
||||
}
|
||||
q = applyRegionRouting(q, worker)
|
||||
where := `(checks.last_start IS NULL) OR (checks.last_start + (checks.interval || ' second')::interval < now())`
|
||||
for _, kind := range kinds {
|
||||
if kind == kindHTTP || kind == kindDNS {
|
||||
where += ` OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
|
||||
break
|
||||
}
|
||||
}
|
||||
var checks []*Check
|
||||
if err := q.Where(where).Clauses(SkipLockedClause).Limit(limit).Preload("Monitor.Group").Find(&checks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
for _, check := range checks {
|
||||
if check.Monitor == nil || check.Monitor.Group == nil {
|
||||
continue
|
||||
}
|
||||
job := JobForCheck(check)
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checkID, monitorID := check.ID, check.MonitorID
|
||||
bucket := now.UTC().Unix() / int64(check.Interval)
|
||||
task := Task{
|
||||
JobID: job.JobID, Kind: TaskKindCheck, State: TaskStateQueued,
|
||||
AccountID: check.Monitor.Group.AccountID, CheckID: &checkID, MonitorID: &monitorID,
|
||||
Payload: payload, NotBefore: now, MaxAttempts: DefaultTaskMaxAttempts,
|
||||
IdempotencyKey: fmt.Sprintf("check:%d:%d", check.ID, bucket),
|
||||
}
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&task).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(check).Update(colLastStart, now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// applyRegionRouting narrows the monitor JOIN in ChecksForWorker to the
|
||||
// subset whose routing rule matches the calling worker's region. The filter
|
||||
// is applied at the SQL layer so the SKIP LOCKED page only scans/leases
|
||||
// rows that this worker is allowed to run, instead of leasing and then
|
||||
// discarding forbidden checks.
|
||||
//
|
||||
// The function intentionally mirrors Monitor.WantsRegion so the helper can
|
||||
// be reused from non-SQL callers (UI preview, plan validation, etc.).
|
||||
//
|
||||
// SQL form:
|
||||
//
|
||||
// - monitors.region_mode IN ('any', 'all', ”)
|
||||
// → unconditional match; legacy / Phase 3 placeholder behavior.
|
||||
// - monitors.region_mode = 'specific' AND
|
||||
// (monitors.preferred_regions IS NULL OR
|
||||
// cardinality(monitors.preferred_regions) = 0 OR
|
||||
// ? = ANY(monitors.preferred_regions))
|
||||
// → empty array falls back to "any"; otherwise the worker code must
|
||||
// be in the whitelist.
|
||||
func applyRegionRouting(q *gorm.DB, worker *WorkerNode) *gorm.DB {
|
||||
if worker == nil || worker.RegionCode == "" {
|
||||
return q
|
||||
}
|
||||
if worker.RegionCode == defaultRegionCode {
|
||||
// The default "local" region is the historical catch-all; the
|
||||
// in-process scheduler (not ChecksForWorker) handles those
|
||||
// monitors. Skip regional filtering entirely so we don't leak
|
||||
// Phase 1 in-process workers through the new router.
|
||||
return q
|
||||
}
|
||||
// TODO(phase3): split RegionMode="all" into N assignments, one per
|
||||
// preferred region, so the result aggregator can build a quorum.
|
||||
// Today it is treated as "any" so existing checks keep flowing.
|
||||
return q.Where(
|
||||
`(monitors.region_mode IN ('any', 'all', '') OR `+
|
||||
`(monitors.region_mode = 'specific' AND `+
|
||||
`(monitors.preferred_regions IS NULL OR `+
|
||||
`coalesce(array_length(monitors.preferred_regions, 1), 0) = 0 OR `+
|
||||
`? = ANY(monitors.preferred_regions))))`,
|
||||
worker.RegionCode,
|
||||
)
|
||||
}
|
||||
|
||||
// JobForCheck creates a CheckJob from a Check model for sending to a worker
|
||||
func JobForCheck(c *Check) wire.CheckJob {
|
||||
jobID := uuid.New().String()
|
||||
var urlStr *string
|
||||
if c.URL != nil {
|
||||
urlStr = c.URL
|
||||
}
|
||||
return wire.CheckJob{
|
||||
JobID: jobID,
|
||||
CheckID: c.ID,
|
||||
MonitorID: c.MonitorID,
|
||||
Kind: c.Kind,
|
||||
Host: c.Monitor.Host,
|
||||
URL: urlStr,
|
||||
Interval: c.Interval,
|
||||
Settings: json.RawMessage(c.Settings),
|
||||
}
|
||||
}
|
||||
|
||||
// QueueMonitorChecks makes enabled checks for a monitor immediately eligible for remote assignment.
|
||||
func QueueMonitorChecks(monitorID int64) error {
|
||||
return DB().Model(&Check{}).
|
||||
Where("monitor_id = ? AND enabled", monitorID).
|
||||
Updates(map[string]interface{}{
|
||||
colLastStart: nil,
|
||||
colLastEnd: nil,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// QueueMonitorChecksKind makes enabled checks of one kind immediately eligible for remote assignment.
|
||||
func QueueMonitorChecksKind(monitorID int64, kind string) error {
|
||||
return DB().Model(&Check{}).
|
||||
Where("monitor_id = ? AND kind = ? AND enabled", monitorID, kind).
|
||||
Updates(map[string]interface{}{
|
||||
colLastStart: nil,
|
||||
colLastEnd: nil,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResult applies a check result reported by a distributed worker.
|
||||
// It updates the check state in the database and triggers monitor status aggregation.
|
||||
//
|
||||
// Phase 3 of docs/todo.md (multi-region quorum aggregation): when the
|
||||
// check has RequireQuorum > 1, the per-region result is recorded in
|
||||
// check_region_results but Check.State is NOT touched here — that is
|
||||
// the job of app/models/check_aggregator.go, which decides OK/ERR/
|
||||
// DEGRADED once enough regional results have arrived or the aggregation
|
||||
// window has elapsed. QuorumEnabled() == false preserves the legacy
|
||||
// direct-update path so single-region / non-aggregated monitors keep
|
||||
// the same behavior.
|
||||
func ApplyRemoteCheckResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
return ApplyRemoteCheckResultFromWorker(report, regionCode, nil)
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResultFromWorker persists worker attribution before changing
|
||||
// legacy check state. A confirmation result is consumed exactly once and never
|
||||
// overwrites the original check result.
|
||||
func ApplyRemoteCheckResultFromWorker(report wire.CheckResultReport, regionCode string, worker *WorkerNode) error { //nolint:gocritic,lll // hugeParam: wire compatibility
|
||||
var monitor *Monitor
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var err error
|
||||
monitor, err = ApplyRemoteCheckResultFromWorkerTx(tx, report, regionCode, worker)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor != nil {
|
||||
monitor.UpdateStatusFromChecks()
|
||||
}
|
||||
// VictoriaMetrics is outside PostgreSQL and is deliberately post-commit.
|
||||
// A caller that retries after this error will not duplicate relational state;
|
||||
// metric points are external at-least-once observations and need TSDB repair
|
||||
// if the write remains unavailable.
|
||||
return StoreRemoteCheckMetrics(report.Metrics)
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResultFromWorkerTx applies all relational result effects using
|
||||
// the caller's transaction. It intentionally does not write VictoriaMetrics or
|
||||
// aggregate monitor state: both must happen only after the transaction commits.
|
||||
// A nil monitor means the report was a consumed diagnostic attempt.
|
||||
func ApplyRemoteCheckResultFromWorkerTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, worker *WorkerNode) (*Monitor, error) { //nolint:gocritic,lll // hugeParam: wire compatibility
|
||||
if tx == nil {
|
||||
return nil, fmt.Errorf("apply check result: nil transaction")
|
||||
}
|
||||
now := time.Now()
|
||||
if worker != nil {
|
||||
handled := false
|
||||
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if handled {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
check := Check{}
|
||||
if err := tx.Preload("Monitor").First(&check, report.CheckID).Error; err != nil {
|
||||
log.Println("worker: check not found:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Always persist the per-region result first so the aggregator can
|
||||
// pick it up regardless of which path we take next. We rely on
|
||||
// StoreCheckRegionResult to default AggregatedAt=NULL (the column
|
||||
// type is *time.Time, so a zero value writes SQL NULL).
|
||||
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
|
||||
log.Println("worker: error storing region result:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Quorum-enabled checks: write nothing to Check.State here. The
|
||||
// aggregator will compute the aggregate state once the window has
|
||||
// elapsed (or enough regions have reported) and stamp AggregatedAt on
|
||||
// the contributing CheckRegionResult rows.
|
||||
if check.QuorumEnabled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
update := map[string]interface{}{
|
||||
colState: report.State,
|
||||
colLastEnd: now,
|
||||
colWarnings: pq.StringArray(report.Warnings),
|
||||
colInfos: pq.StringArray(report.Infos),
|
||||
}
|
||||
|
||||
if report.State == "OK" {
|
||||
update["was_up"] = now
|
||||
update["last_ok"] = now
|
||||
update["fails"] = 0
|
||||
update["error"] = gorm.Expr("NULL")
|
||||
} else {
|
||||
update["last_fail"] = now
|
||||
update["fails"] = gorm.Expr("fails + 1")
|
||||
if report.Error != nil {
|
||||
update["error"] = *report.Error
|
||||
}
|
||||
}
|
||||
|
||||
if report.ExpiresAt != nil {
|
||||
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
|
||||
if err == nil {
|
||||
update["expires"] = t
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Model(&check).UpdateColumns(update).Error; err != nil {
|
||||
log.Println("worker: error updating check:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
if worker != nil {
|
||||
payload, _ := json.Marshal(report)
|
||||
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
|
||||
if attempt.JobID == "" {
|
||||
attempt.JobID = uuid.New().String()
|
||||
}
|
||||
// A duplicate websocket/HTTP delivery must not create another attempt.
|
||||
if err := tx.Where("job_id = ?", attempt.JobID).FirstOrCreate(&attempt).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch report.State {
|
||||
case stateERR, stateFail:
|
||||
if err := StartConfirmationTx(tx, check.ID, worker.ID, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case stateOK:
|
||||
if err := RecoverDiagnosticTx(tx, check.ID, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return check.Monitor, nil
|
||||
}
|
||||
|
||||
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
|
||||
func StoreRemoteCheckMetrics(metrics []wire.MetricPoint) error {
|
||||
for _, metric := range metrics {
|
||||
if metric.Metric == "" || len(metric.Fields) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := influx.WriteOne(metric.Metric, metric.Tags, metric.Fields); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StoreCheckRegionResult stores a per-region check result for distributed monitoring analytics
|
||||
func StoreCheckRegionResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
return StoreCheckRegionResultTx(DB(), report, regionCode, time.Now())
|
||||
}
|
||||
|
||||
// StoreCheckRegionResultTx stores a regional result in the caller's transaction.
|
||||
func StoreCheckRegionResultTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, executedAt time.Time) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
if tx == nil {
|
||||
return fmt.Errorf("store region result: nil transaction")
|
||||
}
|
||||
if regionCode == "" {
|
||||
regionCode = defaultRegionCode
|
||||
}
|
||||
|
||||
result := CheckRegionResult{
|
||||
CheckID: report.CheckID,
|
||||
RegionCode: regionCode,
|
||||
ExecutedAt: executedAt,
|
||||
State: report.State,
|
||||
DurationMs: report.DurationMs,
|
||||
Error: report.Error,
|
||||
}
|
||||
|
||||
return tx.Create(&result).Error
|
||||
}
|
||||
|
||||
// StaleWorkers marks workers as inactive or dead based on last_seen time
|
||||
func StaleWorkers() {
|
||||
// Mark workers with no heartbeat for 2 minutes as inactive
|
||||
DB().Model(&WorkerNode{}).
|
||||
Where("status = ? AND last_seen < ?", "active", time.Now().Add(-2*time.Minute)).
|
||||
Update("status", "inactive")
|
||||
|
||||
// Mark workers with no heartbeat for 5 minutes as dead
|
||||
DB().Model(&WorkerNode{}).
|
||||
Where("status IN (?, ?) AND last_seen < ?", "active", "inactive", time.Now().Add(-5*time.Minute)).
|
||||
Update("status", "dead")
|
||||
}
|
||||
451
app/models/check_jobs_test.go
Обычный файл
451
app/models/check_jobs_test.go
Обычный файл
@@ -0,0 +1,451 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// seedRegionRoutingWorld builds two regions and three groups (one per
|
||||
// monitor) plus three workers. The monitors and their PreferredRegions /
|
||||
// RegionMode are configured by the caller via a callback so each test can
|
||||
// express the exact routing scenario it wants to assert on.
|
||||
//
|
||||
// Returns a routerTestWorld that knows how to look up each fixture by name
|
||||
// for readable assertions.
|
||||
type routerTestWorld struct {
|
||||
groupAny *models.Group
|
||||
groupSpec *models.Group
|
||||
groupAll *models.Group
|
||||
|
||||
workerMsk *models.WorkerNode
|
||||
workerUSEast *models.WorkerNode
|
||||
workerLocal *models.WorkerNode
|
||||
}
|
||||
|
||||
// seedWorker creates a worker bound to regionCode. AuthToken is unique per
|
||||
// worker so the FOR UPDATE SKIP LOCKED path can exercise two distinct
|
||||
// concurrent callers.
|
||||
func seedWorker(t *testing.T, id, regionCode string) *models.WorkerNode {
|
||||
t.Helper()
|
||||
seedRegion(t, regionCode)
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: id,
|
||||
RegionCode: regionCode,
|
||||
Status: "active",
|
||||
AuthToken: "tok-" + id,
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
return w
|
||||
}
|
||||
|
||||
// seedRouterMonitor creates a Monitor with the given region routing
|
||||
// attributes and one ready-to-run http Check. The check has Interval=60
|
||||
// (minimum allowed) and LastStart=nil so ChecksForWorker picks it up
|
||||
// immediately on the next poll.
|
||||
func seedRouterMonitor(t *testing.T, group *models.Group, host, regionMode string, preferred []string) (models.Monitor, models.Check) {
|
||||
t.Helper()
|
||||
enTrue := true
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
if regionMode != "" {
|
||||
mon.RegionMode = regionMode
|
||||
}
|
||||
if preferred != nil {
|
||||
mon.PreferredRegions = models.RegionCodesFromSlice(preferred)
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
check := models.Check{
|
||||
MonitorID: mon.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return mon, check
|
||||
}
|
||||
|
||||
func stringPtrRouter(s string) *string { return &s }
|
||||
|
||||
// seedRouterWorld is the common fixture for the TestRegionRouting_* table.
|
||||
// It provisions two regions (ru-msk, us-east) and three monitors pinned to
|
||||
// different routing modes; the workers are created lazily by the caller.
|
||||
func seedRouterWorld(t *testing.T) routerTestWorld {
|
||||
t.Helper()
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "ru-msk")
|
||||
seedRegion(t, "us-east")
|
||||
|
||||
plan := models.Plan{Name: "router", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "router-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
groupAny := &models.Group{AccountID: acc.ID, Name: "any"}
|
||||
groupSpec := &models.Group{AccountID: acc.ID, Name: "spec"}
|
||||
groupAll := &models.Group{AccountID: acc.ID, Name: "all"}
|
||||
require.NoError(t, models.DB().Create(groupAny).Error)
|
||||
require.NoError(t, models.DB().Create(groupSpec).Error)
|
||||
require.NoError(t, models.DB().Create(groupAll).Error)
|
||||
|
||||
seedRouterMonitor(t, groupAny, "any.test", models.RegionModeAny, nil)
|
||||
seedRouterMonitor(t, groupSpec, "spec-msk.test", models.RegionModeSpecific, []string{"ru-msk"})
|
||||
seedRouterMonitor(t, groupSpec, "spec-us.test", models.RegionModeSpecific, []string{"us-east"})
|
||||
seedRouterMonitor(t, groupAll, "all.test", models.RegionModeAll, []string{"ru-msk", "us-east"})
|
||||
|
||||
return routerTestWorld{
|
||||
groupAny: groupAny,
|
||||
groupSpec: groupSpec,
|
||||
groupAll: groupAll,
|
||||
workerMsk: seedWorker(t, "w-msk", "ru-msk"),
|
||||
workerUSEast: seedWorker(t, "w-us", "us-east"),
|
||||
workerLocal: seedWorker(t, "w-local", "local"),
|
||||
}
|
||||
}
|
||||
|
||||
// idsOf returns a sorted list of monitor IDs assigned to the worker for
|
||||
// easier assertions across checks.
|
||||
func idsOf(checks []*models.Check) []int64 {
|
||||
out := make([]int64, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestRegionRouting_AnyReturnsAll confirms the SQL filter preserves the
|
||||
// legacy behavior for region_mode='any' monitors and the Phase 3
|
||||
// placeholder 'all' monitors: a worker in a region nobody explicitly
|
||||
// whitelisted must still see them, while monitors pinned to specific
|
||||
// regions stay filtered out.
|
||||
//
|
||||
// seedRouterWorld configures four monitors:
|
||||
// - any.test → region_mode='any', no PreferredRegions
|
||||
// - spec-msk.test → region_mode='specific', preferred=[ru-msk]
|
||||
// - spec-us.test → region_mode='specific', preferred=[us-east]
|
||||
// - all.test → region_mode='all', preferred=[ru-msk, us-east]
|
||||
//
|
||||
// A worker in region "remote" (whitelisted by nobody) must see exactly
|
||||
// {any.test, all.test} — the two monitors whose region_mode bypasses the
|
||||
// whitelist — and nothing else.
|
||||
func TestRegionRouting_AnyReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
seedRegion(t, "remote")
|
||||
w := seedWorker(t, "w-remote", "remote")
|
||||
|
||||
checks := models.ChecksForWorker(w, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
|
||||
assert.ElementsMatch(t, []string{"any.test", "all.test"}, hosts,
|
||||
"region_mode='any' and the Phase 3 'all' placeholder must bypass the whitelist")
|
||||
assert.NotContains(t, hosts, "spec-msk.test",
|
||||
"specific-mode monitor with whitelisted ru-msk must NOT reach a remote worker")
|
||||
assert.NotContains(t, hosts, "spec-us.test",
|
||||
"specific-mode monitor with whitelisted us-east must NOT reach a remote worker")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificFiltersByRegion proves the core Phase 2 promise:
|
||||
// workers in different regions never receive a monitor whose PreferredRegions
|
||||
// does not include their region code. The check is run with parallel
|
||||
// goroutines because ChecksForWorker stamps `last_start` on every row it
|
||||
// leases — a sequential second poll would always see an already-claimed
|
||||
// queue and the filter would have nothing to test against.
|
||||
//
|
||||
// Under the FOR UPDATE SKIP LOCKED race, whichever SELECT fires first grabs
|
||||
// every matching row, so the *exact* per-worker host list is non-deterministic.
|
||||
// The deterministic invariant the test asserts is the no-leak property: a
|
||||
// worker in ru-msk must never see spec-us.test, and vice versa.
|
||||
func TestRegionRouting_SpecificFiltersByRegion(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mskChecks []*models.Check
|
||||
usChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
mskChecks = models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
usChecks = models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
mskHosts := hostsOf(mskChecks)
|
||||
usHosts := hostsOf(usChecks)
|
||||
|
||||
// Aggregate coverage: together the two workers must see every check
|
||||
// the routing layer would ever allow them — the four seeded monitors.
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
},
|
||||
append(append([]string{}, mskHosts...), usHosts...),
|
||||
"union of both workers' slices must cover every seeded monitor (any/specific/all × region)")
|
||||
|
||||
// Core Phase 2 invariant: regional filtering never leaks across
|
||||
// PreferredRegions boundaries. This is the only assertion a
|
||||
// concurrent SKIP LOCKED race lets us pin deterministically.
|
||||
assert.NotContains(t, mskHosts, "spec-us.test",
|
||||
"ru-msk worker must never see a monitor whitelisted for us-east only")
|
||||
assert.NotContains(t, usHosts, "spec-msk.test",
|
||||
"us-east worker must never see a monitor whitelisted for ru-msk only")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificEmptyPreferredFallsBackToAny confirms the
|
||||
// documented fall-back: a monitor in RegionModeSpecific with no
|
||||
// PreferredRegions behaves like RegionModeAny so the field is safe to
|
||||
// leave blank. We poll from the us-east worker — without the fall-back it
|
||||
// would only see any.test + all.test + spec-us.test.
|
||||
func TestRegionRouting_SpecificEmptyPreferredFallsBackToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
// Reset the spec-msk monitor to have an empty PreferredRegions list
|
||||
// (the seed above gave it one). The Monitor row's RegionMode stays
|
||||
// 'specific'.
|
||||
require.NoError(t, models.DB().Model(&models.Monitor{}).
|
||||
Where("host = ?", "spec-msk.test").
|
||||
Update("preferred_regions", models.RegionCodesFromSlice(nil)).Error)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "spec-msk.test",
|
||||
"empty PreferredRegions with region_mode=specific must fall back to 'any'")
|
||||
}
|
||||
|
||||
// TestRegionRouting_AllDeferredToAny pins the Phase 3 placeholder behavior:
|
||||
// region_mode='all' is logged and treated as 'any' today. The test asserts
|
||||
// the monitor flows to a worker in any region (the TODO log marker is
|
||||
// emitted from applyRegionRouting — pinned here as a code-grep contract).
|
||||
func TestRegionRouting_AllDeferredToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "all.test",
|
||||
"region_mode='all' must currently behave like 'any' so existing checks keep flowing")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LocalWorkerBypass ensures the historic "local" region
|
||||
// still routes everything: the in-process scheduler handles those monitors
|
||||
// and we don't want the Phase 2 filter to leak platform workers through it.
|
||||
func TestRegionRouting_LocalWorkerBypass(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerLocal, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "worker in region 'local' must receive every check (bypass)")
|
||||
}
|
||||
|
||||
// TestRegionRouting_NilWorkerReturnsAll asserts the diagnostic-friendly
|
||||
// escape hatch: passing nil for the worker skips the routing filter and
|
||||
// returns every check the kinds/limit envelope allows.
|
||||
func TestRegionRouting_NilWorkerReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(nil, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "nil worker must bypass the routing filter")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LoadBalanceImplicit confirms the SKIP LOCKED implicit
|
||||
// load-balancing story: when two workers in the same region race for a pool
|
||||
// of pending checks, each of them receives a non-empty disjoint slice. The
|
||||
// two polls run in parallel goroutines so the FOR UPDATE SKIP LOCKED race
|
||||
// window is actually exercised.
|
||||
//
|
||||
// IMPORTANT: SKIP LOCKED with a large LIMIT is unfair — whichever
|
||||
// transaction's SELECT fires first grabs everything. The test therefore
|
||||
// uses LIMIT=4 with 10 pending rows so each worker is forced to leave some
|
||||
// rows unlocked for the other worker to pick up. Together they must cover
|
||||
// at most 8 rows (LIMIT × workers) without overlap; the remaining rows are
|
||||
// intentionally left for a future poll cycle, which mirrors production
|
||||
// behavior where workers continually drain a backlog.
|
||||
func TestRegionRouting_LoadBalanceImplicit(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "shared")
|
||||
plan := models.Plan{Name: "lb-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "lb", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := &models.Group{AccountID: acc.ID, Name: "lb-g"}
|
||||
require.NoError(t, models.DB().Create(group).Error)
|
||||
|
||||
enTrue := true
|
||||
for i := 0; i < 10; i++ {
|
||||
host := "lb-" + string(rune('a'+i)) + ".test"
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
ck := models.Check{
|
||||
MonitorID: mon.ID, Kind: "http", Interval: 60,
|
||||
Enabled: &enTrue, State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&ck).Error)
|
||||
}
|
||||
|
||||
w1 := seedWorker(t, "lb-w1", "shared")
|
||||
w2 := seedWorker(t, "lb-w2", "shared")
|
||||
|
||||
const limitPerWorker = 4
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
aChecks, bChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
aChecks = models.ChecksForWorker(w1, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
bChecks = models.ChecksForWorker(w2, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
assert.Greater(t, len(aChecks), 0, "worker 1 must receive at least one check")
|
||||
assert.Greater(t, len(bChecks), 0, "worker 2 must receive at least one check")
|
||||
assert.LessOrEqual(t, len(aChecks)+len(bChecks), 2*limitPerWorker,
|
||||
"two concurrent workers with LIMIT each can lease at most LIMIT*2 rows per cycle")
|
||||
assert.Empty(t, intersectHosts(aChecks, bChecks),
|
||||
"the two slices must be disjoint (FOR UPDATE SKIP LOCKED must not double-lease)")
|
||||
}
|
||||
|
||||
// TestMonitorValidateRegionMode exercises the documented enum on the
|
||||
// Monitor type so the validator surface does not regress.
|
||||
func TestMonitorValidateRegionMode(t *testing.T) {
|
||||
cases := []struct {
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", false},
|
||||
{"any", false},
|
||||
{"specific", false},
|
||||
{"all", false},
|
||||
{"round-robin", true},
|
||||
{"RANDOM", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run("mode="+c.mode, func(t *testing.T) {
|
||||
m := models.Monitor{RegionMode: c.mode}
|
||||
err := m.ValidateRegionMode()
|
||||
if c.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMonitorWantsRegion pins the public WantsRegion helper that powers the
|
||||
// non-SQL callers (UI preview, plan validation). Phase 3 will swap the
|
||||
// 'all' branch's behavior, so the table covers all three modes today.
|
||||
func TestMonitorWantsRegion(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mode string
|
||||
regions []string
|
||||
workerRC string
|
||||
want bool
|
||||
}{
|
||||
{"any_always_true", "any", []string{"ru-msk"}, "us-east", true},
|
||||
{"any_empty_pref_still_true", "any", nil, "us-east", true},
|
||||
{"specific_match", "specific", []string{"ru-msk", "eu-west"}, "ru-msk", true},
|
||||
{"specific_no_match", "specific", []string{"ru-msk", "eu-west"}, "us-east", false},
|
||||
{"specific_empty_pref_fallback", "specific", nil, "us-east", true},
|
||||
{"all_placeholder_true", "all", []string{"ru-msk", "us-east"}, "ru-msk", true},
|
||||
{"all_placeholder_foreign_region", "all", []string{"ru-msk", "us-east"}, "eu-west", true},
|
||||
{"empty_mode_defaults_to_any", "", nil, "us-east", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
m := models.Monitor{
|
||||
RegionMode: c.mode,
|
||||
PreferredRegions: models.RegionCodesFromSlice(c.regions),
|
||||
}
|
||||
assert.Equal(t, c.want, m.WantsRegion(c.workerRC))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hostsOf extracts the hostnames from the assigned checks for readable
|
||||
// assertions in table-driven tests.
|
||||
func hostsOf(checks []*models.Check) []string {
|
||||
out := make([]string, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// intersectHosts returns the hostnames present in both slices — used to
|
||||
// prove two concurrent workers did not lease the same check twice.
|
||||
func intersectHosts(a, b []*models.Check) []string {
|
||||
set := make(map[string]struct{}, len(a))
|
||||
for _, c := range a {
|
||||
if c.Monitor != nil {
|
||||
set[c.Monitor.Host] = struct{}{}
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, c := range b {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[c.Monitor.Host]; ok {
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
42
app/models/check_metric_test.go
Обычный файл
42
app/models/check_metric_test.go
Обычный файл
@@ -0,0 +1,42 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckMetricName(t *testing.T) {
|
||||
tests := []struct {
|
||||
kind string
|
||||
want string
|
||||
}{
|
||||
{"http", "chttp"},
|
||||
{"ssl", "cssl"},
|
||||
{"bssl", "cbssl"},
|
||||
{"ssh", "cssh"},
|
||||
{"ftp", "cftp"},
|
||||
{"dns", "cdns"},
|
||||
{"whois", "cwhois"},
|
||||
{"rkn", "crkn"},
|
||||
{"llm", "cllm"},
|
||||
{"llm-http", "cllm_http"},
|
||||
{"weird-kind", "cweird_kind"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.kind, func(t *testing.T) {
|
||||
c := &Check{Kind: tt.kind, ID: 1}
|
||||
got := c.MetricName()
|
||||
assert.Equal(t, tt.want, got, "MetricName() for kind %q", tt.kind)
|
||||
|
||||
metric := got
|
||||
for _, ch := range metric {
|
||||
if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' && ch != ':' {
|
||||
t.Errorf("metric name %q contains invalid character %q", metric, ch)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
33
app/models/check_region_result.go
Обычный файл
33
app/models/check_region_result.go
Обычный файл
@@ -0,0 +1,33 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// CheckRegionResult stores check results from distributed workers by region
|
||||
type CheckRegionResult struct {
|
||||
concerns.Model
|
||||
CheckID int64 `gorm:"index;not null" json:"check_id"`
|
||||
Check *Check `gorm:"foreignKey:CheckID" json:"check,omitempty"`
|
||||
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
|
||||
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id"`
|
||||
WorkerNode *WorkerNode `gorm:"foreignKey:WorkerNodeID" json:"worker_node,omitempty"`
|
||||
ExecutedAt time.Time `gorm:"not null" json:"executed_at"`
|
||||
State string `gorm:"size:10;not null" json:"state"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Error *string `json:"error"`
|
||||
|
||||
// AggregatedAt is set by app/models/check_aggregator.go once a row has
|
||||
// been folded into a Check.State decision. NULL means "still waiting
|
||||
// for the aggregator"; non-NULL means "this row has already been
|
||||
// counted in a quorum decision and must not be re-counted". The
|
||||
// column is indexed (see check_aggregator.go index helper) so the
|
||||
// per-tick SELECT that finds unaggregated rows is O(matching rows)
|
||||
// rather than scanning the whole table.
|
||||
AggregatedAt *time.Time `gorm:"index" json:"aggregated_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
155
app/models/check_settings.go
Обычный файл
155
app/models/check_settings.go
Обычный файл
@@ -0,0 +1,155 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CheckHeader provides functionality.
|
||||
type CheckHeader struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// CheckSettings provides functionality.
|
||||
type CheckSettings struct {
|
||||
// HTTP Basic Auth Username
|
||||
HTTPUsername string `json:"http_username,omitempty"`
|
||||
// HTTP Basic Auth Password
|
||||
HTTPPassword string `json:"http_password,omitempty"`
|
||||
|
||||
// ExpectedAnswer (default, redirect, custom)
|
||||
ExpectedAnswer string `json:"expected_answer,omitempty"`
|
||||
// Expected redirect location
|
||||
ExpectedLocation string `json:"expected_location,omitempty"`
|
||||
// Expected HTTP status codes for custom
|
||||
ExpectedHTTPCode int `json:"expected_http_code,omitempty"`
|
||||
|
||||
// Keyword search type (off, present or absent)
|
||||
KeywordType string `json:"keyword_type,omitempty"`
|
||||
// Keyword to search for
|
||||
KeywordValue string `json:"keyword_value,omitempty"`
|
||||
|
||||
SlowTime int `json:"slow_time"`
|
||||
Timeout int `json:"timeout"`
|
||||
// Request Settings
|
||||
CheckIp bool `json:"checkip"` //nolint:revive // accepted lint exception
|
||||
CheckIPv6 bool `json:"checkipv6"`
|
||||
RequestHeader []CheckHeader `json:"request_headers"`
|
||||
RequestMethod string `json:"request_method"`
|
||||
RequestType string `json:"request_type"`
|
||||
RequestContent string `json:"request_content"`
|
||||
// SSH Settings
|
||||
Port string `json:"port"`
|
||||
|
||||
// Host optionally overrides the monitor host for checks that do
|
||||
// not naturally target the Monitor.Host (e.g. an ICMP/TCP/UDP
|
||||
// probe to a separate machine, or a different IP family). Used by
|
||||
// cping/ctcp/cudp.
|
||||
Host string `json:"host,omitempty"`
|
||||
|
||||
// Count is the per-check packet count for ping. Defaults to 1 in
|
||||
// cping when zero or negative.
|
||||
Count int `json:"count,omitempty"`
|
||||
|
||||
// PacketSize is the ICMP payload size for ping (bytes). Defaults
|
||||
// to 56 in cping when zero or negative.
|
||||
PacketSize int `json:"packet_size,omitempty"`
|
||||
|
||||
// Distributed marks this check as eligible for execution on the
|
||||
// distributed worker pool (multi-region, multi-worker) instead of
|
||||
// only the in-process scheduler. Reserved for paid plans; the
|
||||
// controller layer enforces the plan check.
|
||||
Distributed bool `json:"distributed"`
|
||||
}
|
||||
|
||||
// redirectCodes contains all HTTP redirect status codes.
|
||||
var redirectCodes = []int{300, 301, 302, 303, 307, 308}
|
||||
|
||||
// CheckKeyword provides functionality.
|
||||
func (s *CheckSettings) CheckKeyword(body []byte, warns []string) ([]string, error) {
|
||||
var err error
|
||||
|
||||
switch s.KeywordType {
|
||||
case "", "off":
|
||||
return warns, nil
|
||||
case "present":
|
||||
if !strings.Contains(string(body), s.KeywordValue) {
|
||||
err = errors.New("expected keyword " + s.KeywordValue + " not found")
|
||||
}
|
||||
case "absent":
|
||||
if strings.Contains(string(body), s.KeywordValue) {
|
||||
err = errors.New("unexpected keyword " + s.KeywordValue + " found")
|
||||
}
|
||||
}
|
||||
|
||||
return warns, err
|
||||
}
|
||||
|
||||
// CheckAnswer provides functionality.
|
||||
func (s *CheckSettings) CheckAnswer(resp *http.Response, body []byte) ([]string, error) {
|
||||
// log.Println("check answer, settings:")
|
||||
// spew.Dump(s)
|
||||
|
||||
isRedirect := false
|
||||
|
||||
for _, rc := range redirectCodes {
|
||||
if resp.StatusCode == rc {
|
||||
isRedirect = true
|
||||
}
|
||||
}
|
||||
|
||||
location := resp.Header.Get("location")
|
||||
|
||||
// log.Println("redirect?", isRedirect, location)
|
||||
|
||||
switch s.ExpectedAnswer {
|
||||
case "", "default":
|
||||
if isRedirect {
|
||||
return s.CheckKeyword(body, []string{"redirect: " + location})
|
||||
} else if resp.StatusCode != 200 {
|
||||
return []string{}, fmt.Errorf("bad status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
case "redirect":
|
||||
if !isRedirect {
|
||||
return []string{}, fmt.Errorf("bad status code: %d (expected redirect)", resp.StatusCode)
|
||||
}
|
||||
|
||||
if s.ExpectedLocation != "" {
|
||||
if location != s.ExpectedLocation {
|
||||
return []string{}, fmt.Errorf(
|
||||
"bad location: %s (expected %s)",
|
||||
location,
|
||||
s.ExpectedLocation,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case "custom":
|
||||
if s.ExpectedHTTPCode == 0 {
|
||||
s.ExpectedHTTPCode = 200
|
||||
}
|
||||
if resp.StatusCode != s.ExpectedHTTPCode {
|
||||
return []string{}, fmt.Errorf("bad status code: %d (expected %d)", resp.StatusCode, s.ExpectedHTTPCode)
|
||||
}
|
||||
|
||||
if s.ExpectedLocation != "" {
|
||||
if location != s.ExpectedLocation {
|
||||
return []string{}, fmt.Errorf(
|
||||
"bad location: %s (expected %s)",
|
||||
location,
|
||||
s.ExpectedLocation,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
panic("bad expectedAnswer")
|
||||
}
|
||||
|
||||
// return []string{}, nil
|
||||
return s.CheckKeyword(body, []string{})
|
||||
}
|
||||
60
app/models/check_settings_test.go
Обычный файл
60
app/models/check_settings_test.go
Обычный файл
@@ -0,0 +1,60 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckSettingsDistributed_Default(t *testing.T) {
|
||||
s := CheckSettings{}
|
||||
assert.False(t, s.Distributed, "new CheckSettings should default to Distributed=false")
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_JSON(t *testing.T) {
|
||||
original := CheckSettings{
|
||||
HTTPUsername: "user",
|
||||
HTTPPassword: "pass",
|
||||
ExpectedAnswer: "default",
|
||||
Timeout: 30,
|
||||
Distributed: true,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(original)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, string(data), `"distributed":true`)
|
||||
|
||||
var decoded CheckSettings
|
||||
err = json.Unmarshal(data, &decoded)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, original, decoded)
|
||||
assert.True(t, decoded.Distributed)
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_OmitFalse(t *testing.T) {
|
||||
s := CheckSettings{HTTPUsername: "user"}
|
||||
data, err := json.Marshal(s)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, string(data), `"distributed":false`)
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_CheckUnmarshal(t *testing.T) {
|
||||
c := &Check{
|
||||
Settings: []byte(`{"distributed": true, "timeout": 60}`),
|
||||
}
|
||||
got := c.GetSettings()
|
||||
assert.True(t, got.Distributed)
|
||||
assert.Equal(t, 60, got.Timeout)
|
||||
}
|
||||
|
||||
func TestPlanAllowsDistributed(t *testing.T) {
|
||||
var nilPlan *Plan
|
||||
assert.False(t, nilPlan.AllowsDistributed(), "nil plan must not allow distributed")
|
||||
|
||||
free := &Plan{Price: 0}
|
||||
assert.False(t, free.AllowsDistributed(), "free plan must not allow distributed")
|
||||
|
||||
paid := &Plan{Price: 100}
|
||||
assert.True(t, paid.AllowsDistributed(), "paid plan must allow distributed")
|
||||
}
|
||||
181
app/models/cleanup_stale_test.go
Обычный файл
181
app/models/cleanup_stale_test.go
Обычный файл
@@ -0,0 +1,181 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// makeStaleUser inserts a user whose LastActiveAt is older than the
|
||||
// 3-month cutoff used by the cleanup filter. The returned user is what
|
||||
// the candidates query should pick up.
|
||||
func makeStaleUser(t *testing.T, email string, lastActive *time.Time) *models.User {
|
||||
t.Helper()
|
||||
u := &models.User{
|
||||
Email: &email,
|
||||
Name: "stale " + email,
|
||||
Enabled: true,
|
||||
Confirmed: true,
|
||||
LastActiveAt: lastActive,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(u).Error)
|
||||
return u
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_EmptyWhenNoCandidates checks the obvious
|
||||
// negative case: a fresh account with an active owner is not eligible.
|
||||
func TestFindStaleAccounts_EmptyWhenNoCandidates(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "fresh", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
// Group + owner access + active user
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
recent := time.Now().Add(-1 * time.Hour)
|
||||
user := makeStaleUser(t, "active@example.com", &recent)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "an account with an active owner is not stale")
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_PicksStaleEmptyAccount checks the happy path:
|
||||
// account with no monitors + single user + last login > 3 months ago.
|
||||
func TestFindStaleAccounts_PicksStaleEmptyAccount(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "ghost", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
neverLoggedIn := (*time.Time)(nil)
|
||||
user := makeStaleUser(t, "ghost@example.com", neverLoggedIn)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1, "the empty stale account should be picked up")
|
||||
assert.Equal(t, acc.ID, got[0].AccountID)
|
||||
assert.Equal(t, user.ID, got[0].UserID)
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_SkipsAccountWithMonitors makes sure the
|
||||
// "zero monitors" gate is enforced.
|
||||
func TestFindStaleAccounts_SkipsAccountWithMonitors(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "active", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
// One monitor → account is NOT eligible even if the user is stale.
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "example.com"}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
stale := time.Now().Add(-365 * 24 * time.Hour)
|
||||
user := makeStaleUser(t, "owner@example.com", &stale)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_SkipsUserWithMultipleAccounts verifies that a
|
||||
// user holding two accounts disqualifies BOTH accounts.
|
||||
func TestFindStaleAccounts_SkipsUserWithMultipleAccounts(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc1 := models.Account{Name: "acc1", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc1).Error)
|
||||
acc2 := models.Account{Name: "acc2", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc2).Error)
|
||||
models.DB().Create(&models.Group{AccountID: acc1.ID, Name: "g1"})
|
||||
models.DB().Create(&models.Group{AccountID: acc2.ID, Name: "g2"})
|
||||
|
||||
stale := time.Now().Add(-365 * 24 * time.Hour)
|
||||
user := makeStaleUser(t, "shared@example.com", &stale)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc1.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc2.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "user with two accounts disqualifies both accounts")
|
||||
}
|
||||
|
||||
// TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans verifies the
|
||||
// end-to-end cleanup: matching account + user are removed, and
|
||||
// recently-active accounts survive.
|
||||
func TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
// Stale account with a stale user that has only this one account.
|
||||
staleAcc := models.Account{Name: "ghost", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&staleAcc).Error)
|
||||
staleGroup := models.Group{AccountID: staleAcc.ID, Name: "g"}
|
||||
require.NoError(t, models.DB().Create(&staleGroup).Error)
|
||||
staleUser := makeStaleUser(t, "ghost@example.com", nil)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: staleAcc.ID, UserID: &staleUser.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
// Active account with a recent user — must NOT be touched.
|
||||
freshAcc := models.Account{Name: "fresh", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&freshAcc).Error)
|
||||
require.NoError(t, models.DB().Create(&models.Group{AccountID: freshAcc.ID, Name: "g"}).Error)
|
||||
recent := time.Now().Add(-1 * time.Hour)
|
||||
freshUser := makeStaleUser(t, "fresh@example.com", &recent)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: freshAcc.ID, UserID: &freshUser.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
deleted, err := models.CleanupStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, deleted, "only the stale empty account should be deleted")
|
||||
|
||||
// Stale account is gone.
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", staleAcc.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(0), count)
|
||||
|
||||
// Stale user is orphaned → also hard-deleted by the cleanup pass.
|
||||
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", staleUser.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(0), count)
|
||||
|
||||
// Fresh account and user survive.
|
||||
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", freshAcc.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", freshUser.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
41
app/models/concerns/has_token.go
Обычный файл
41
app/models/concerns/has_token.go
Обычный файл
@@ -0,0 +1,41 @@
|
||||
// Package concerns provides functionality.
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
)
|
||||
|
||||
// HasToken provides functionality.
|
||||
type HasToken struct {
|
||||
Token string `json:"-" gorm:"unique_index"`
|
||||
}
|
||||
|
||||
// SetToken provides functionality.
|
||||
func (m *HasToken) SetToken() {
|
||||
tk := RandomToken(32)
|
||||
m.Token = base64.RawURLEncoding.EncodeToString(tk)
|
||||
if m.Token == "" {
|
||||
panic("RandomToken failed: token not set")
|
||||
}
|
||||
if len(m.Token) < 32 {
|
||||
panic("RandomToken failed: token too short")
|
||||
}
|
||||
}
|
||||
|
||||
// RandomToken provides functionality.
|
||||
func RandomToken(tokenLen int) []byte {
|
||||
b := make([]byte, tokenLen)
|
||||
n, err := rand.Read(b)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if n != tokenLen {
|
||||
panic("RandomToken failed: bad len")
|
||||
}
|
||||
if bytes.Equal(b, make([]byte, tokenLen)) {
|
||||
panic("RandomToken failed: generated empty token")
|
||||
}
|
||||
return b
|
||||
}
|
||||
14
app/models/concerns/model.go
Обычный файл
14
app/models/concerns/model.go
Обычный файл
@@ -0,0 +1,14 @@
|
||||
// Source: https://gorm.io/gorm/blob/master/model.go
|
||||
// Use 64 bit keys
|
||||
|
||||
package concerns
|
||||
|
||||
// Model base model definition, including fields `ID`, `CreatedAt`, `UpdatedAt`, `DeletedAt`, which could be embedded in your models
|
||||
// type User struct {
|
||||
// gorm.Model
|
||||
// }
|
||||
|
||||
// Model provides functionality.
|
||||
type Model struct {
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
}
|
||||
7
app/models/concerns/renderable.go
Обычный файл
7
app/models/concerns/renderable.go
Обычный файл
@@ -0,0 +1,7 @@
|
||||
package concerns
|
||||
|
||||
// Renderable provides functionality.
|
||||
type Renderable struct {
|
||||
Raw string
|
||||
Rendered string
|
||||
}
|
||||
12
app/models/concerns/soft_delete.go
Обычный файл
12
app/models/concerns/soft_delete.go
Обычный файл
@@ -0,0 +1,12 @@
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// SoftDelete provides functionality.
|
||||
type SoftDelete struct {
|
||||
DeletedAt *time.Time `gorm:"index" json:"-"`
|
||||
DeleterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
// Deleter *User `json:"-"`
|
||||
}
|
||||
11
app/models/concerns/timestamped.go
Обычный файл
11
app/models/concerns/timestamped.go
Обычный файл
@@ -0,0 +1,11 @@
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Timestamped provides functionality.
|
||||
type Timestamped struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
86
app/models/contact.go
Обычный файл
86
app/models/contact.go
Обычный файл
@@ -0,0 +1,86 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Contact represents a notification contact.
|
||||
//
|
||||
// Ownership / tenancy:
|
||||
//
|
||||
// - A Contact belongs to exactly one Account (tenant). AccountID may be
|
||||
// nil for system-level admin contacts (see SystemContacts).
|
||||
// - A Contact may belong to at most one User. UserID is set when the
|
||||
// contact was created on behalf of a specific user (the typical
|
||||
// case for self-service "My email" / "My Telegram" contacts) and is
|
||||
// nil for account-wide contacts.
|
||||
//
|
||||
// `User` is omitempty because most list payloads don't preload it; the
|
||||
// `/settings/users` page loads it server-side via the AccountUserRow
|
||||
// payload.
|
||||
type Contact struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id)"`
|
||||
Account *Account `json:"-"`
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
|
||||
User *User `json:"user,omitempty"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
Kind string `json:"kind" gorm:"not null;index:contact_value"`
|
||||
Value string `json:"value" gorm:"index:contact_value"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
Data datatypes.JSON `json:"data"`
|
||||
IsSystem *bool `json:"is_system" gorm:"default:false"`
|
||||
Notifications []Notification `json:"-" gorm:"many2many:notification_contacts;"`
|
||||
NotificationsCount int `gorm:"-:all" json:"notifications_count"`
|
||||
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
|
||||
Messages []Message `json:"-"`
|
||||
|
||||
concerns.Timestamped
|
||||
concerns.HasToken
|
||||
Audited
|
||||
}
|
||||
|
||||
// SystemContacts returns all contacts marked as system/admin (is_system=true).
|
||||
// These are the contacts distributed workers notify directly when the main
|
||||
// API is unreachable (see docs/worker-protocol.md "System Selfcheck").
|
||||
func SystemContacts() ([]Contact, error) {
|
||||
var contacts []Contact
|
||||
err := DB().Where("is_system = ? AND enabled = ?", true, true).Find(&contacts).Error
|
||||
return contacts, err
|
||||
}
|
||||
|
||||
// ContactsCounts fills NotificationsCount and MonitorsCount for each contact.
|
||||
func ContactsCounts(contacts *[]Contact) {
|
||||
groupIDs := make(map[int64]bool, 0)
|
||||
groupCount := make(map[int64]int, 0)
|
||||
|
||||
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
|
||||
(*contacts)[i].NotificationsCount = len(c.Notifications)
|
||||
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
|
||||
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
groupIDs[g.ID] = true
|
||||
groupCount[g.ID] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gids := make([]int64, 0, len(groupIDs))
|
||||
for g := range groupIDs {
|
||||
gids = append(gids, g)
|
||||
}
|
||||
|
||||
CountGroups(gids, &groupCount)
|
||||
|
||||
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
|
||||
cnt := 0
|
||||
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
|
||||
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
cnt += groupCount[g.ID]
|
||||
}
|
||||
}
|
||||
(*contacts)[i].MonitorsCount = cnt
|
||||
}
|
||||
}
|
||||
65
app/models/contact_test.go
Обычный файл
65
app/models/contact_test.go
Обычный файл
@@ -0,0 +1,65 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestSystemContacts verifies that SystemContacts returns only contacts with
|
||||
// is_system=true and ignores contacts with is_system=false or nil.
|
||||
func TestSystemContacts(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account := &models.Account{Name: "test-account"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
accountID := account.ID
|
||||
|
||||
trueVal, falseVal := true, false
|
||||
|
||||
systemContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "system-admin",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
regularContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "regular",
|
||||
Kind: "email",
|
||||
Value: "user@example.com",
|
||||
IsSystem: &falseVal,
|
||||
}
|
||||
nilSystemContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "nil-system",
|
||||
Kind: "email",
|
||||
Value: "nil@example.com",
|
||||
}
|
||||
|
||||
require.NoError(t, models.DB().Create(systemContact).Error)
|
||||
require.NoError(t, models.DB().Create(regularContact).Error)
|
||||
require.NoError(t, models.DB().Create(nilSystemContact).Error)
|
||||
|
||||
got, err := models.SystemContacts()
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []int64
|
||||
var names []string
|
||||
for _, c := range got {
|
||||
ids = append(ids, c.ID)
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
|
||||
assert.Contains(t, names, "system-admin")
|
||||
assert.NotContains(t, names, "regular")
|
||||
assert.NotContains(t, names, "nil-system")
|
||||
assert.Contains(t, ids, systemContact.ID)
|
||||
assert.NotContains(t, ids, regularContact.ID)
|
||||
assert.NotContains(t, ids, nilSystemContact.ID)
|
||||
}
|
||||
97
app/models/credential_crypto.go
Обычный файл
97
app/models/credential_crypto.go
Обычный файл
@@ -0,0 +1,97 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/config/secrets"
|
||||
)
|
||||
|
||||
const credentialKeyEnv = "RSMON_CRED_KEY"
|
||||
|
||||
// encryptSecret encrypts plaintext with AES-GCM. If no key is configured
|
||||
// (dev/test), stores the value with a "plain:" prefix. Production deployments
|
||||
// must set RSMON_CRED_KEY or config/secrets.yml crypto.pepper.
|
||||
func encryptSecret(plaintext string) (string, error) {
|
||||
key := credentialKey()
|
||||
if key == "" {
|
||||
return "plain:" + plaintext, nil
|
||||
}
|
||||
block, err := aes.NewCipher(deriveKey(key))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return "", err
|
||||
}
|
||||
sealed := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
|
||||
return "enc:" + base64.StdEncoding.EncodeToString(sealed), nil
|
||||
}
|
||||
|
||||
// decryptSecret decrypts a value produced by encryptSecret. Values without a
|
||||
// known prefix are returned as-is for backward compatibility with legacy
|
||||
// plaintext records.
|
||||
func decryptSecret(stored string) (string, error) {
|
||||
if stored == "" {
|
||||
return "", nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(stored, "plain:"):
|
||||
return strings.TrimPrefix(stored, "plain:"), nil
|
||||
case strings.HasPrefix(stored, "enc:"):
|
||||
key := credentialKey()
|
||||
if key == "" {
|
||||
return "", errors.New("credential key not configured but secret is encrypted")
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(stored, "enc:"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
block, err := aes.NewCipher(deriveKey(key))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(raw) < gcm.NonceSize() {
|
||||
return "", errors.New("ciphertext too short")
|
||||
}
|
||||
nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
|
||||
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(plaintext), nil
|
||||
default:
|
||||
return stored, nil
|
||||
}
|
||||
}
|
||||
|
||||
func deriveKey(s string) []byte {
|
||||
h := sha256.Sum256([]byte(s))
|
||||
return h[:32]
|
||||
}
|
||||
|
||||
func credentialKey() string {
|
||||
if k := os.Getenv(credentialKeyEnv); k != "" {
|
||||
return k
|
||||
}
|
||||
if secrets.Crypto != nil && secrets.Crypto.Pepper != "" {
|
||||
return secrets.Crypto.Pepper
|
||||
}
|
||||
return ""
|
||||
}
|
||||
120
app/models/dead_worker_reaper.go
Обычный файл
120
app/models/dead_worker_reaper.go
Обычный файл
@@ -0,0 +1,120 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeadWorkerHeartbeatTimeout is the threshold for ReapDeadWorkers — a worker
|
||||
// whose last_seen is older than this is considered dead and any leased tasks
|
||||
// it owns are reassigned to the pool. Five minutes mirrors the StaleWorkers()
|
||||
// check in check_jobs.go so the two reapers cannot disagree about who is
|
||||
// dead. See docs/todo.md Phase 4 §5.
|
||||
const DeadWorkerHeartbeatTimeout = 5 * time.Minute
|
||||
|
||||
// ReapDeadWorkers marks any non-dead worker whose last_seen is older than
|
||||
// DeadWorkerHeartbeatTimeout as "dead", then reassigns its leased tasks
|
||||
// back to the queued pool so other workers (or freshly registered ones)
|
||||
// can pick them up. It mirrors the structure of ReapExpiredTasks — two
|
||||
// short UPDATE statements, cheap enough to run from the web process
|
||||
// every 30s.
|
||||
//
|
||||
// The returned tuple is (reaped, reassigned, err): reaped counts the
|
||||
// workers that flipped to dead during this call; reassigned counts the
|
||||
// leased tasks that were given back to the pool. A zero count on either
|
||||
// is normal — the reaper is idempotent and the call is silent when
|
||||
// nothing is due.
|
||||
func ReapDeadWorkers() (reaped int, reassigned int, err error) {
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-DeadWorkerHeartbeatTimeout)
|
||||
|
||||
// First flip the workers to dead so the second UPDATE can match the
|
||||
// freshly-stamped ids without having to re-derive them in Go.
|
||||
r := DB().Exec(`
|
||||
UPDATE worker_nodes
|
||||
SET status = ?, updated_at = ?
|
||||
WHERE status <> ? AND last_seen IS NOT NULL AND last_seen < ?`,
|
||||
"dead", now, "dead", cutoff,
|
||||
)
|
||||
if r.Error != nil {
|
||||
return 0, 0, r.Error
|
||||
}
|
||||
reaped = int(r.RowsAffected)
|
||||
|
||||
// Nothing flipped → no tasks to return. Cheaper than running an
|
||||
// UPDATE that touches 0 rows on every tick when the fleet is healthy.
|
||||
if reaped == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
// Second: clear any leased tasks owned by the now-dead workers.
|
||||
// The selector stashes the worker's WorkerID string in lease_owner;
|
||||
// matching against the (now-stale) worker row's WorkerID is the
|
||||
// same identifier the selector uses, so we don't need an extra
|
||||
// join. Tasks in other states (queued, succeeded, dead, …) are
|
||||
// unaffected — only leased work the dead worker still owned has
|
||||
// to go back to the queue.
|
||||
r2 := DB().Exec(`
|
||||
UPDATE tasks
|
||||
SET state = ?, lease_owner = '', lease_expires_at = NULL, updated_at = ?
|
||||
WHERE state = ? AND lease_owner IN (
|
||||
SELECT worker_id FROM worker_nodes WHERE status = ?
|
||||
)`,
|
||||
TaskStateQueued, now, TaskStateLeased, "dead",
|
||||
)
|
||||
if r2.Error != nil {
|
||||
return reaped, 0, r2.Error
|
||||
}
|
||||
reassigned = int(r2.RowsAffected)
|
||||
return reaped, reassigned, nil
|
||||
}
|
||||
|
||||
// StartDeadWorkerReaper launches a goroutine that runs ReapDeadWorkers
|
||||
// on the given interval until ctx is canceled. Mirrors StartTaskReaper
|
||||
// in this package: same ticker pattern, same logging style, same
|
||||
// recover() safety net so a malformed row cannot crash the web process.
|
||||
//
|
||||
// The default interval is 30s; values <= 0 fall back to the default so
|
||||
// the helper is safe to call from any call site without a guard. Wire
|
||||
// from main.init() once per process — the reaper uses short row-level
|
||||
// locks and is cheap under load (two indexed UPDATEs of <= a few
|
||||
// hundred rows in steady state).
|
||||
//
|
||||
// Passing a nil context falls back to context.Background() so callers
|
||||
// can write `models.StartDeadWorkerReaper(nil, ...)` in one-liners
|
||||
// (main, tests) without having to import "context" first.
|
||||
func StartDeadWorkerReaper(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("dead_worker_reaper: panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
reaped, reassigned, err := ReapDeadWorkers()
|
||||
if err != nil {
|
||||
log.Printf("dead_worker_reaper: error: %v", err)
|
||||
return
|
||||
}
|
||||
if reaped > 0 || reassigned > 0 {
|
||||
log.Printf("dead_worker_reaper: reaped=%d reassigned=%d", reaped, reassigned)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
274
app/models/dead_worker_reaper_test.go
Обычный файл
274
app/models/dead_worker_reaper_test.go
Обычный файл
@@ -0,0 +1,274 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestReapDeadWorkers_SkipsHealthyAndKillsSilent mirrors the
|
||||
// reaper-shape test for ReapExpiredTasks in task_test.go: seed three
|
||||
// workers (heartbeat fresh / heartbeat stale / already dead) plus two
|
||||
// leased tasks owned by the stale worker and one leased task on a
|
||||
// healthy worker (which must NOT be touched). Then assert:
|
||||
//
|
||||
// - the fresh worker is left active
|
||||
// - the silent worker flips to "dead"
|
||||
// - the already-dead worker is left dead (idempotent)
|
||||
// - the silent worker's leased tasks are reset to queued
|
||||
// - the healthy worker's leased task is untouched
|
||||
func TestReapDeadWorkers_SkipsHealthyAndKillsSilent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "test")
|
||||
|
||||
healthy := &models.WorkerNode{
|
||||
WorkerID: "w-healthy",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "tok-healthy",
|
||||
LastSeen: timePtr(time.Now()),
|
||||
}
|
||||
stale := &models.WorkerNode{
|
||||
WorkerID: "w-stale",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "tok-stale",
|
||||
LastSeen: timePtr(time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)),
|
||||
}
|
||||
alreadyDead := &models.WorkerNode{
|
||||
WorkerID: "w-dead",
|
||||
RegionCode: "test",
|
||||
Status: "dead",
|
||||
AuthToken: "tok-dead",
|
||||
LastSeen: timePtr(time.Now().Add(-time.Hour)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(healthy).Error)
|
||||
require.NoError(t, models.DB().Create(stale).Error)
|
||||
require.NoError(t, models.DB().Create(alreadyDead).Error)
|
||||
|
||||
// Two leased tasks on the stale worker — both must come back.
|
||||
task1 := mustLeaseTask(t, stale.WorkerID, "test-acct")
|
||||
task2 := mustLeaseTask(t, stale.WorkerID, "test-acct")
|
||||
// One leased task on the healthy worker — must stay leased.
|
||||
healthyTask := mustLeaseTask(t, healthy.WorkerID, "test-acct")
|
||||
// Already-dead worker with a leased task — not part of THIS reaper's
|
||||
// reaping set (it would only be touched by a fresh reaper pass), so
|
||||
// leave it leased to demonstrate that we don't accidentally reassign
|
||||
// from prior-dead leases too.
|
||||
deadPriorTask := mustLeaseTask(t, alreadyDead.WorkerID, "test-acct")
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped, "only the stale worker should flip (already-dead is left untouched)")
|
||||
// 3 tasks come back: the 2 leased by the stale worker (just flipped
|
||||
// to dead) + the 1 leased by the prior-dead worker, which had never
|
||||
// been cleaned up because no previous reaper ran. The reaper matches
|
||||
// dead workers by status, so any leased task on a dead worker is a
|
||||
// stranded lease and must be returned to the pool regardless of when
|
||||
// the worker flipped.
|
||||
assert.Equal(t, 3, reassigned)
|
||||
|
||||
var healthyRow, staleRow, deadRow models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&healthyRow, healthy.ID).Error)
|
||||
assert.Equal(t, "active", healthyRow.Status, "healthy worker must stay active")
|
||||
|
||||
require.NoError(t, models.DB().First(&staleRow, stale.ID).Error)
|
||||
assert.Equal(t, "dead", staleRow.Status, "stale worker should flip to dead")
|
||||
|
||||
require.NoError(t, models.DB().First(&deadRow, alreadyDead.ID).Error)
|
||||
assert.Equal(t, "dead", deadRow.Status, "already-dead worker should not be touched")
|
||||
|
||||
got := func(id int64) models.Task {
|
||||
var row models.Task
|
||||
require.NoError(t, models.DB().First(&row, id).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
t1 := got(task1.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, t1.State, "stale worker task must come back to queued")
|
||||
assert.Empty(t, t1.LeaseOwner)
|
||||
assert.Nil(t, t1.LeaseExpiresAt)
|
||||
|
||||
t2 := got(task2.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, t2.State)
|
||||
assert.Empty(t, t2.LeaseOwner)
|
||||
assert.Nil(t, t2.LeaseExpiresAt)
|
||||
|
||||
ht := got(healthyTask.ID)
|
||||
assert.Equal(t, models.TaskStateLeased, ht.State, "healthy worker's lease must be untouched")
|
||||
assert.Equal(t, healthy.WorkerID, ht.LeaseOwner)
|
||||
|
||||
dt := got(deadPriorTask.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, dt.State,
|
||||
"prior-dead task must also be returned to the pool — any leased task on a dead worker is a stranded lease")
|
||||
assert.Empty(t, dt.LeaseOwner)
|
||||
}
|
||||
|
||||
// TestReapDeadWorkers_NoOpOnHealthyFleet verifies the cheap path: when
|
||||
// no workers are stale the reaper returns (0, 0, nil) without doing any
|
||||
// work. Mirrors the "reaped = int(r.RowsAffected)" branch in
|
||||
// ReapExpiredTasks.
|
||||
func TestReapDeadWorkers_NoOpOnHealthyFleet(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-only-healthy",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: timePtr(time.Now()),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, reaped)
|
||||
assert.Equal(t, 0, reassigned)
|
||||
|
||||
var row models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&row, w.ID).Error)
|
||||
assert.Equal(t, "active", row.Status)
|
||||
}
|
||||
|
||||
// TestReapDeadWorkers_OnlyReassignsLeasedNotOthers ensures that the
|
||||
// reaper does not touch queued/succeeded/failed_retry tasks on the dead
|
||||
// worker — only leased ones need to be returned to the queue. Tasks in
|
||||
// other states either belong to no one (queued) or are terminal/semi-
|
||||
// terminal and have their own audit trail.
|
||||
func TestReapDeadWorkers_OnlyReassignsLeasedNotOthers(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-mix",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: &now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
leased := mustLeaseTask(t, w.WorkerID, "acct-mix")
|
||||
succeeded := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateSucceeded)
|
||||
failedRetry := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedRetry)
|
||||
failedPerm := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedPerm)
|
||||
otherWorker := mustLeaseTask(t, "w-other", "acct-mix")
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
assert.Equal(t, 1, reassigned, "exactly the one leased task on the dead worker")
|
||||
|
||||
got := func(id int64) string {
|
||||
var row models.Task
|
||||
require.NoError(t, models.DB().First(&row, id).Error)
|
||||
return row.State
|
||||
}
|
||||
assert.Equal(t, models.TaskStateQueued, got(leased.ID))
|
||||
assert.Equal(t, models.TaskStateSucceeded, got(succeeded.ID), "succeeded must not move")
|
||||
assert.Equal(t, models.TaskStateFailedRetry, got(failedRetry.ID), "failed_retry must not move")
|
||||
assert.Equal(t, models.TaskStateFailedPerm, got(failedPerm.ID), "failed_perm must not move")
|
||||
assert.Equal(t, models.TaskStateLeased, got(otherWorker.ID),
|
||||
"tasks leased by another worker must not move")
|
||||
}
|
||||
|
||||
// TestStartDeadWorkerReaper_TickerFiresOnce is a smoke test for the
|
||||
// background helper: spin up the reaper with a tight 10ms ticker and
|
||||
// a cancellable context, wait for the first tick, then cancel the
|
||||
// context so the goroutine exits cleanly without leaking. Mirrors the
|
||||
// shape of how main.init() uses StartTaskReaper (it can't be torn
|
||||
// down, but for tests we always pass a cancellable context).
|
||||
func TestStartDeadWorkerReaper_TickerFiresOnce(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-ticker",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: timePtr(time.Now().Add(-2 * models.DeadWorkerHeartbeatTimeout)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
models.StartDeadWorkerReaper(ctx, 10*time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var got models.WorkerNode
|
||||
for time.Now().Before(deadline) {
|
||||
require.NoError(t, models.DB().First(&got, w.ID).Error)
|
||||
if got.Status == "dead" {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("reaper goroutine did not flip worker to dead within 2s; last status=%q", got.Status)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func timePtr(t time.Time) *time.Time { return &t }
|
||||
|
||||
// mustLeaseTask inserts a leased Task row owned by workerID. The state
|
||||
// is the only field that matters for reaper tests; payload/idempotency
|
||||
// are stubs.
|
||||
func mustLeaseTask(t *testing.T, workerID, accountLabel string) models.Task {
|
||||
t.Helper()
|
||||
return mustInsertTask(t, workerID, accountLabel, models.TaskStateLeased)
|
||||
}
|
||||
|
||||
func mustInsertTask(t *testing.T, workerID, accountLabel string, state string) models.Task {
|
||||
t.Helper()
|
||||
leaseUntil := time.Now().Add(time.Hour)
|
||||
stamp := time.Now().UnixNano()
|
||||
jobID := fmt.Sprintf("reap-%s-%s-%d", state, workerID, stamp)
|
||||
// Cap at 64 chars (tasks.job_id VARCHAR(64)). The components above
|
||||
// already stay under the limit because mustLeaseTask keeps workerID
|
||||
// short ("w-mix", "w-stale", …) and state is bounded.
|
||||
if len(jobID) > 64 {
|
||||
jobID = jobID[:64]
|
||||
}
|
||||
idemp := fmt.Sprintf("reap:%s:%d", workerID, stamp)
|
||||
if len(idemp) > 255 {
|
||||
idemp = idemp[:255]
|
||||
}
|
||||
task := models.Task{
|
||||
JobID: jobID,
|
||||
Kind: models.TaskKindNotification,
|
||||
State: state,
|
||||
AccountID: 1,
|
||||
Payload: datatypes.JSON([]byte(`{"method":"email"}`)),
|
||||
NotBefore: time.Now().Add(-time.Minute),
|
||||
LeaseOwner: workerID,
|
||||
LeaseExpiresAt: &leaseUntil,
|
||||
Attempts: 1,
|
||||
MaxAttempts: 5,
|
||||
IdempotencyKey: idemp,
|
||||
}
|
||||
if state != models.TaskStateLeased {
|
||||
task.LeaseOwner = ""
|
||||
task.LeaseExpiresAt = nil
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
require.NotZero(t, task.ID)
|
||||
return task
|
||||
}
|
||||
582
app/models/deletion.go
Обычный файл
582
app/models/deletion.go
Обычный файл
@@ -0,0 +1,582 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// OnUserCacheInvalidate is called when a user is mutated in a way that
|
||||
// invalidates the cached representation in auth/cache.go. It is wired up
|
||||
// from the auth package during init() to avoid an import cycle.
|
||||
var OnUserCacheInvalidate func(userID int64)
|
||||
|
||||
// DeletionGracePeriod is the time between a user requesting account deletion
|
||||
// and the scheduled hard-delete. During this period the user can cancel the
|
||||
// request and monitoring is paused for all of the user's accounts.
|
||||
const DeletionGracePeriod = 7 * 24 * time.Hour
|
||||
|
||||
// RequestUserDeletion sets the user's DeletionRequestedAt to now,
|
||||
// starting the 7-day grace period.
|
||||
func RequestUserDeletion(user *User) error {
|
||||
if user == nil {
|
||||
return errors.New("nil user")
|
||||
}
|
||||
if user.DeletionPending() {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
user.DeletionRequestedAt = &now
|
||||
if err := DB().Save(user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if OnUserCacheInvalidate != nil {
|
||||
OnUserCacheInvalidate(user.ID)
|
||||
}
|
||||
log.Printf("user %d requested account deletion (grace until %s)", user.ID, now.Add(DeletionGracePeriod))
|
||||
return nil
|
||||
}
|
||||
|
||||
// CancelUserDeletion clears the user's DeletionRequestedAt field,
|
||||
// canceling the pending account deletion.
|
||||
func CancelUserDeletion(user *User) error {
|
||||
if user == nil {
|
||||
return errors.New("nil user")
|
||||
}
|
||||
if !user.DeletionPending() {
|
||||
return nil
|
||||
}
|
||||
user.DeletionRequestedAt = nil
|
||||
if err := DB().Save(user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if OnUserCacheInvalidate != nil {
|
||||
OnUserCacheInvalidate(user.ID)
|
||||
}
|
||||
log.Printf("user %d canceled account deletion", user.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// HardDeleteAccount removes an account and all of its data (monitors, checks,
|
||||
// events, messages, contacts, groups, notifications, etc.) in a single
|
||||
// transaction. Accesses and invites pointing at the account are also cleaned
|
||||
// up. Users themselves are kept (they may belong to other accounts). This is
|
||||
// the admin "purge" path used to remove spam/error accounts immediately.
|
||||
func HardDeleteAccount(accountID int64) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
// Remove accesses pointing at this account first (FK constraint).
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Access{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove invites for this account.
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Invite{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Delete all account data and the account row itself.
|
||||
return deleteAccountData(tx, accountID)
|
||||
})
|
||||
}
|
||||
|
||||
// HardDeleteUser removes the user and all of their owned data: accesses,
|
||||
// contacts, accounts, monitors, checks, events and messages. The deletion
|
||||
// is wrapped in a transaction to make sure partial failures don't leave
|
||||
// the database in a broken state.
|
||||
func HardDeleteUser(userID int64) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
user := User{}
|
||||
if err := tx.First(&user, userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1) Find all accounts the user has any access to.
|
||||
accesses := make([]Access, 0)
|
||||
if err := tx.Where("user_id = ?", userID).Find(&accesses).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
accountIDs := make([]int64, 0, len(accesses))
|
||||
for i := range accesses {
|
||||
accountIDs = append(accountIDs, accesses[i].AccountID)
|
||||
}
|
||||
|
||||
// 2) Find all contacts that belong to those accounts or directly to the user.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&Contact{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 3) For each account: delete the related data, then the account itself.
|
||||
for _, accountID := range accountIDs {
|
||||
if err := deleteAccountData(tx, accountID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 4) Remove accesses.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&Access{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 5) Remove invites issued by or for this user.
|
||||
if err := tx.Exec("DELETE FROM invites WHERE invitee_id = ? OR account_id IN (?)",
|
||||
userID, accountIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 6) Remove auth identities (password, social).
|
||||
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 7) Remove api keys owned by the user.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 8) Remove sessions.
|
||||
if err := tx.Exec("DELETE FROM remember_tokens WHERE user_id = ?", userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 9) Finally remove the user row.
|
||||
if err := tx.Delete(&user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("user %d hard-deleted (cascade accounts=%v)", userID, accountIDs)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// deleteAccountData removes everything associated with a single account:
|
||||
// contacts, monitors, checks, events, messages, notifications, groups and
|
||||
// the account itself. The user accesses are removed separately.
|
||||
func deleteAccountData(tx *gorm.DB, accountID int64) error {
|
||||
// All contacts that reference this account (both account-scoped ones
|
||||
// with user_id IS NULL and per-user ones created in
|
||||
// CreateAccountForUser that set both account_id and user_id).
|
||||
// User-only contacts (account_id IS NULL) are independent of the
|
||||
// account and survive the deletion. contacts.account_id has a FK
|
||||
// to accounts(id) without ON DELETE CASCADE, so all referencing
|
||||
// rows must be removed before the account row goes away.
|
||||
contactIDs := make([]int64, 0)
|
||||
if err := tx.Model(&Contact{}).Where("account_id = ?", accountID).Pluck("id", &contactIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
notificationIDs := make([]int64, 0)
|
||||
if err := tx.Model(&Notification{}).Where("account_id = ?", accountID).Pluck("id", ¬ificationIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(contactIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM notification_contacts WHERE contact_id IN (?)", contactIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(notificationIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM notification_contacts WHERE notification_id IN (?)", notificationIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
messageQuery := tx.Model(&Message{})
|
||||
switch {
|
||||
case len(contactIDs) > 0 && len(notificationIDs) > 0:
|
||||
messageQuery = messageQuery.Where("contact_id IN (?) OR notification_id IN (?)", contactIDs, notificationIDs)
|
||||
case len(contactIDs) > 0:
|
||||
messageQuery = messageQuery.Where("contact_id IN (?)", contactIDs)
|
||||
case len(notificationIDs) > 0:
|
||||
messageQuery = messageQuery.Where("notification_id IN (?)", notificationIDs)
|
||||
default:
|
||||
messageQuery = nil
|
||||
}
|
||||
if messageQuery != nil {
|
||||
messageIDs := make([]int64, 0)
|
||||
if err := messageQuery.Pluck("id", &messageIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(messageIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("id IN (?)", messageIDs).Delete(&Message{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Contact{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Monitors (and their checks/events/messages via cascade below).
|
||||
monitors := make([]Monitor, 0)
|
||||
if err := tx.Joins("JOIN groups ON monitors.group_id = groups.id").
|
||||
Where("groups.account_id = ?", accountID).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
monitorIDs := make([]int64, 0, len(monitors))
|
||||
for i := range monitors {
|
||||
monitorIDs = append(monitorIDs, monitors[i].ID)
|
||||
}
|
||||
|
||||
if len(monitorIDs) > 0 {
|
||||
// Checks
|
||||
checks := make([]Check, 0)
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&checks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
checkIDs := make([]int64, 0, len(checks))
|
||||
for i := range checks {
|
||||
checkIDs = append(checkIDs, checks[i].ID)
|
||||
}
|
||||
|
||||
// Events
|
||||
events := make([]Event, 0)
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&events).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
eventIDs := make([]int64, 0, len(events))
|
||||
for i := range events {
|
||||
eventIDs = append(eventIDs, events[i].ID)
|
||||
}
|
||||
|
||||
// Join tables first to avoid FK violations
|
||||
if len(checkIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_checks WHERE check_id IN (?)", checkIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// check_region_results.check_id has a FK to checks(id) without
|
||||
// ON DELETE CASCADE, so it must be purged before checks go away.
|
||||
if err := tx.Exec("DELETE FROM check_region_results WHERE check_id IN (?)", checkIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(eventIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_messages WHERE event_id IN (?)", eventIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(checkIDs) > 0 {
|
||||
if err := tx.Where("id IN (?)", checkIDs).Delete(&Check{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(eventIDs) > 0 {
|
||||
if err := tx.Where("id IN (?)", eventIDs).Delete(&Event{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// DNS records
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Delete(&DNSRecord{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Monitors themselves
|
||||
if err := tx.Where("id IN (?)", monitorIDs).Delete(&Monitor{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Notification <-> group links
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM notification_groups WHERE group_id IN (SELECT id FROM groups WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Notifications
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Notification{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// LLMs scoped to this account. worker_llms.llm_id has a FK to llms(id)
|
||||
// without ON DELETE CASCADE, so the join rows must go first.
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM worker_llms WHERE llm_id IN (SELECT id FROM llms WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&LLM{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Groups
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Group{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Inventory entities (docs/plans/inventory-management.md §6). All
|
||||
// four tables hold account_id FKs to accounts(id) without ON
|
||||
// DELETE CASCADE, so we wipe them in dependency order: deployments
|
||||
// and domains first (both FK into sites and servers), then
|
||||
// server_ips, then sites, then servers. Anything the account does
|
||||
// not own is left alone (e.g. shared infra servers are filtered
|
||||
// by account_id and survive).
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Deployment{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Domain{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM server_ips WHERE server_id IN (SELECT id FROM servers WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Site{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Server{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Private workers scoped to this account (NULL account_id workers
|
||||
// are platform-operated and survive account deletion). The
|
||||
// worker_llms and check_region_results joins both FK to
|
||||
// worker_nodes(id) without ON DELETE CASCADE, so the join rows
|
||||
// must be cleared before the worker rows go away.
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM worker_llms WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM check_region_results WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&WorkerNode{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The account itself
|
||||
return tx.Delete(&Account{}, accountID).Error
|
||||
}
|
||||
|
||||
// ProcessPendingDeletions hard-deletes users whose 7-day grace period has
|
||||
// elapsed. Designed to be called from a periodic scheduler.
|
||||
func ProcessPendingDeletions() (int, error) {
|
||||
cutoff := time.Now().Add(-DeletionGracePeriod)
|
||||
users := make([]User, 0)
|
||||
if err := DB().Where("deletion_requested_at IS NOT NULL AND deletion_requested_at < ?", cutoff).
|
||||
Find(&users).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
deleted := 0
|
||||
for i := range users {
|
||||
if err := HardDeleteUser(users[i].ID); err != nil {
|
||||
log.Printf("ProcessPendingDeletions: failed to delete user %d: %v", users[i].ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("ProcessPendingDeletions: hard-deleted %d user(s)", deleted)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// StaleAccountInactivity is the minimum inactivity window before a stale
|
||||
// account becomes eligible for admin cleanup. Picked at 3 months per the
|
||||
// /admin/accounts "Удалить старые" button.
|
||||
const StaleAccountInactivity = 90 * 24 * time.Hour
|
||||
|
||||
// StaleAccountCandidate describes an account that matched the
|
||||
// admin-cleanup eligibility filter but has not yet been deleted. The
|
||||
// snapshot is what the frontend shows in the confirmation dialog.
|
||||
type StaleAccountCandidate struct {
|
||||
AccountID int64 `json:"account_id"`
|
||||
AccountName string `json:"account_name"`
|
||||
UserID int64 `json:"user_id"`
|
||||
UserEmail *string `json:"user_email"`
|
||||
LastActiveAt *time.Time `json:"last_active_at"`
|
||||
}
|
||||
|
||||
// FindStaleAccounts returns the accounts that are eligible for the
|
||||
// admin "Удалить старые" cleanup:
|
||||
//
|
||||
// - the account has zero monitors configured (via group.account_id)
|
||||
// - every user with access to the account has exactly one account
|
||||
// membership total (so removing the account also orphans them)
|
||||
// - every such user's last_active_at is older than
|
||||
// StaleAccountInactivity (3 months). Users who have never logged in
|
||||
// (last_active_at IS NULL) are also eligible.
|
||||
func FindStaleAccounts() ([]StaleAccountCandidate, error) {
|
||||
cutoff := time.Now().Add(-StaleAccountInactivity)
|
||||
|
||||
// Step 1: account IDs that have zero monitors.
|
||||
accountsWithMonitors := make([]int64, 0)
|
||||
if err := DB().
|
||||
Table("monitors").
|
||||
Select("DISTINCT groups.account_id").
|
||||
Joins("JOIN groups ON groups.id = monitors.group_id").
|
||||
Scan(&accountsWithMonitors).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accounts := make([]Account, 0)
|
||||
q := DB().Order("id ASC")
|
||||
if len(accountsWithMonitors) > 0 {
|
||||
q = q.Where("id NOT IN (?)", accountsWithMonitors)
|
||||
}
|
||||
if err := q.Find(&accounts).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]StaleAccountCandidate, 0, len(accounts))
|
||||
for i := range accounts {
|
||||
acc := accounts[i]
|
||||
|
||||
// Step 2: every user with access to this account must have
|
||||
// exactly one account membership total.
|
||||
type userAccessCount struct {
|
||||
UserID int64
|
||||
Cnt int
|
||||
}
|
||||
counts := make([]userAccessCount, 0)
|
||||
err := DB().
|
||||
Table("accesses AS a1").
|
||||
Select("a1.user_id AS user_id, (SELECT COUNT(*) FROM accesses AS a2 WHERE a2.user_id = a1.user_id) AS cnt").
|
||||
Where("a1.account_id = ? AND a1.user_id IS NOT NULL", acc.ID).
|
||||
Group("a1.user_id").
|
||||
Scan(&counts).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
// An account with no user accesses is a config bug
|
||||
// (the owner access should always exist). Skip.
|
||||
continue
|
||||
}
|
||||
allSingle := true
|
||||
for _, c := range counts {
|
||||
if c.Cnt != 1 {
|
||||
allSingle = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allSingle {
|
||||
continue
|
||||
}
|
||||
|
||||
// Step 3: every such user must be inactive beyond the cutoff.
|
||||
userIDs := make([]int64, 0, len(counts))
|
||||
for _, c := range counts {
|
||||
userIDs = append(userIDs, c.UserID)
|
||||
}
|
||||
users := make([]User, 0)
|
||||
if err := DB().Where("id IN (?)", userIDs).Find(&users).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allStale := true
|
||||
for j := range users {
|
||||
u := users[j]
|
||||
if u.LastActiveAt != nil && u.LastActiveAt.After(cutoff) {
|
||||
allStale = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allStale {
|
||||
continue
|
||||
}
|
||||
|
||||
// All gates passed — emit one candidate per user so the UI
|
||||
// can list which specific accounts would be removed.
|
||||
for j := range users {
|
||||
out = append(out, StaleAccountCandidate{
|
||||
AccountID: acc.ID,
|
||||
AccountName: acc.Name,
|
||||
UserID: users[j].ID,
|
||||
UserEmail: users[j].Email,
|
||||
LastActiveAt: users[j].LastActiveAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CleanupStaleAccounts hard-deletes every account eligible for the
|
||||
// admin "Удалить старые" sweep. Returns the number of accounts that
|
||||
// were deleted. The matching users (each of whom only belonged to one
|
||||
// account) are deleted by HardDeleteAccount's cascading access cleanup
|
||||
// only if they no longer have any other account — that final teardown
|
||||
// is done here.
|
||||
func CleanupStaleAccounts() (int, error) {
|
||||
candidates, err := FindStaleAccounts()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
accountIDs := make([]int64, 0, len(candidates))
|
||||
seen := make(map[int64]bool, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if !seen[c.AccountID] {
|
||||
seen[c.AccountID] = true
|
||||
accountIDs = append(accountIDs, c.AccountID)
|
||||
}
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, accountID := range accountIDs {
|
||||
if err := HardDeleteAccount(accountID); err != nil {
|
||||
log.Printf("CleanupStaleAccounts: failed to delete account %d: %v", accountID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
// Users that no longer have any accesses after the cascade are
|
||||
// clearly orphaned — purge them so the auth layer doesn't keep
|
||||
// dangling rows around. We bypass HardDeleteUser here because the
|
||||
// account-level data (monitors, checks, groups, etc.) was already
|
||||
// removed by the HardDeleteAccount loop above, so only the user
|
||||
// row and the dangling identities / contacts need cleanup.
|
||||
if deleted > 0 {
|
||||
orphans := make([]int64, 0)
|
||||
err := DB().
|
||||
Table("users").
|
||||
Select("users.id").
|
||||
Joins("LEFT JOIN accesses ON accesses.user_id = users.id").
|
||||
Where("accesses.id IS NULL").
|
||||
Pluck("users.id", &orphans).Error
|
||||
if err != nil {
|
||||
log.Printf("CleanupStaleAccounts: orphan user scan failed: %v", err)
|
||||
return deleted, nil
|
||||
}
|
||||
for _, userID := range orphans {
|
||||
tx := DB().Begin()
|
||||
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: identities delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Where("user_id = ? AND account_id IS NULL", userID).Delete(&Contact{}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: contacts delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: api_keys delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Delete(&User{}, userID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: user delete failed for %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
log.Printf("CleanupStaleAccounts: commit failed for user %d: %v", userID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if deleted > 0 {
|
||||
log.Printf("CleanupStaleAccounts: hard-deleted %d stale account(s)", deleted)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
352
app/models/deletion_test.go
Обычный файл
352
app/models/deletion_test.go
Обычный файл
@@ -0,0 +1,352 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/icrowley/fake"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
var gormErrRecordNotFound = gorm.ErrRecordNotFound
|
||||
|
||||
func deletionStringPtr(s string) *string { return &s }
|
||||
func deletionBoolPtr(b bool) *bool { return &b }
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// TestUserDeletionPending tests the DeletionPending method
|
||||
func TestUserDeletionPending(t *testing.T) {
|
||||
t.Run("returns false when no deletion requested", func(t *testing.T) {
|
||||
user := models.User{}
|
||||
assert.False(t, user.DeletionPending())
|
||||
})
|
||||
|
||||
t.Run("returns true when deletion requested", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
user := models.User{DeletionRequestedAt: &now}
|
||||
assert.True(t, user.DeletionPending())
|
||||
})
|
||||
}
|
||||
|
||||
// TestUserAsJSONIncludesDeletionField verifies that the AsJSON output
|
||||
// contains the deletion_requested_at field so the frontend can react to it.
|
||||
func TestUserAsJSONIncludesDeletionField(t *testing.T) {
|
||||
email := "test@example.com"
|
||||
now := time.Now()
|
||||
user := models.User{
|
||||
ID: 42,
|
||||
Email: &email,
|
||||
Name: "Test User",
|
||||
DeletionRequestedAt: &now,
|
||||
}
|
||||
|
||||
result := user.AsJSON()
|
||||
assert.NotNil(t, result)
|
||||
assert.Contains(t, result, "deletion_requested_at")
|
||||
assert.Equal(t, &now, result["deletion_requested_at"])
|
||||
}
|
||||
|
||||
// TestRequestUserDeletionNilUser ensures nil-safety
|
||||
func TestRequestUserDeletionNilUser(t *testing.T) {
|
||||
err := models.RequestUserDeletion(nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestCancelUserDeletionNilUser ensures nil-safety
|
||||
func TestCancelUserDeletionNilUser(t *testing.T) {
|
||||
err := models.CancelUserDeletion(nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestCacheInvalidationHook_CalledByRequestDeletion verifies that
|
||||
// RequestUserDeletion calls OnUserCacheInvalidate with the correct user ID.
|
||||
func TestCacheInvalidationHook_CalledByRequestDeletion(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
email := fake.EmailAddress()
|
||||
user := models.User{Email: &email, Name: "CacheHookTest"}
|
||||
assert.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
var calledID int64
|
||||
originalHook := models.OnUserCacheInvalidate
|
||||
models.OnUserCacheInvalidate = func(userID int64) {
|
||||
calledID = userID
|
||||
}
|
||||
defer func() { models.OnUserCacheInvalidate = originalHook }()
|
||||
|
||||
assert.NoError(t, models.RequestUserDeletion(&user))
|
||||
assert.Equal(t, user.ID, calledID)
|
||||
|
||||
models.DB().Unscoped().Delete(&user)
|
||||
}
|
||||
|
||||
// TestCacheInvalidationHook_CalledByCancelUserDeletion verifies that
|
||||
// CancelUserDeletion calls OnUserCacheInvalidate with the correct user ID.
|
||||
func TestCacheInvalidationHook_CalledByCancelUserDeletion(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
email := fake.EmailAddress()
|
||||
now := time.Now()
|
||||
user := models.User{Email: &email, Name: "CacheHookCancel", DeletionRequestedAt: &now}
|
||||
assert.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
var calledID int64
|
||||
originalHook := models.OnUserCacheInvalidate
|
||||
models.OnUserCacheInvalidate = func(userID int64) {
|
||||
calledID = userID
|
||||
}
|
||||
defer func() { models.OnUserCacheInvalidate = originalHook }()
|
||||
|
||||
assert.NoError(t, models.CancelUserDeletion(&user))
|
||||
assert.Equal(t, user.ID, calledID)
|
||||
|
||||
models.DB().Unscoped().Delete(&user)
|
||||
}
|
||||
|
||||
// TestHardDeleteAccount_FKCascadeRegression covers the FK regressions
|
||||
// that surfaced as a series of distinct SQL errors when an operator
|
||||
// purged an account that owned monitors with distributed-worker
|
||||
// activity or per-user contacts:
|
||||
//
|
||||
// 1. check_region_results.check_id → checks(id) had no ON DELETE
|
||||
// CASCADE, so deleting a check while it still had region-result
|
||||
// rows raised 23503.
|
||||
// 2. worker_nodes did not have an account_id column, so the
|
||||
// "Workers scoped to this account" delete raised 42703.
|
||||
// 3. contacts.account_id → accounts(id) had no ON DELETE CASCADE,
|
||||
// and the original delete filter required user_id IS NULL, so
|
||||
// per-user contacts created in CreateAccountForUser (both
|
||||
// account_id and user_id set) survived and blocked the account
|
||||
// delete with 23503.
|
||||
//
|
||||
// The test seeds an account with a monitor, a check with a region
|
||||
// result row, an account-scoped LLM, a per-user contact, and both a
|
||||
// private worker (with account_id) and an operated worker (NULL
|
||||
// account_id), then runs HardDeleteAccount and asserts that the
|
||||
// account, the private worker, the LLM, the monitor/check/region
|
||||
// result, and the per-user contact all disappear, while the operated
|
||||
// worker and an unrelated user-only contact survive.
|
||||
func TestHardDeleteAccount_FKCascadeRegression(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "hard-delete-regression"}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
account := models.Account{Name: "victim", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&account).Error)
|
||||
otherAccount := models.Account{Name: "survivor", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&otherAccount).Error)
|
||||
|
||||
owner := models.User{Name: "owner", Email: deletionStringPtr("owner@example.com"), Timezone: "UTC"}
|
||||
require.NoError(t, models.DB().Create(&owner).Error)
|
||||
stranger := models.User{Name: "stranger", Email: deletionStringPtr("stranger@example.com"), Timezone: "UTC"}
|
||||
require.NoError(t, models.DB().Create(&stranger).Error)
|
||||
|
||||
region := models.Region{}
|
||||
if err := models.DB().Where("code = ?", "test").First(®ion).Error; err != nil {
|
||||
require.NoError(t, models.DB().Create(&models.Region{Code: "test", Name: "test", Enabled: true}).Error)
|
||||
}
|
||||
|
||||
group := models.Group{Name: "g", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{
|
||||
Name: deletionStringPtr("m"),
|
||||
Host: "example.com",
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
check := models.Check{
|
||||
MonitorID: monitor.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
Enabled: deletionBoolPtr(true),
|
||||
State: "UNK",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
|
||||
// Per-user contact that references both the victim account and a
|
||||
// user. CreateAccountForUser writes a contact in this shape, and
|
||||
// the old "user_id IS NULL" filter would let it survive and block
|
||||
// the account delete with contacts_account_id_fkey 23503.
|
||||
ownerContact := models.Contact{
|
||||
AccountID: &account.ID,
|
||||
UserID: &owner.ID,
|
||||
Name: "owner-email",
|
||||
Kind: "email",
|
||||
Value: "owner@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&ownerContact).Error)
|
||||
// Account-only contact (user_id IS NULL) — also tied to the account
|
||||
// via FK and must be removed.
|
||||
accountContact := models.Contact{
|
||||
AccountID: &account.ID,
|
||||
Name: "ops",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&accountContact).Error)
|
||||
notification := models.Notification{AccountID: account.ID, Name: "alerts", Enabled: true}
|
||||
require.NoError(t, models.DB().Create(¬ification).Error)
|
||||
require.NoError(t, models.DB().Model(¬ification).Association("Contacts").Append(&accountContact))
|
||||
message := models.Message{
|
||||
NotificationID: notification.ID,
|
||||
ContactID: accountContact.ID,
|
||||
Kind: "test",
|
||||
State: "OK",
|
||||
CreatedAt: time.Now(),
|
||||
SentAt: time.Now(),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
// User-only contact on a stranger — must survive account deletion.
|
||||
userOnlyContact := models.Contact{
|
||||
UserID: &stranger.ID,
|
||||
Name: "stranger",
|
||||
Kind: "email",
|
||||
Value: "stranger@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&userOnlyContact).Error)
|
||||
|
||||
// Region result row: this is the row that used to trigger the
|
||||
// fk_check_region_results_check FK violation when Check was
|
||||
// deleted. Without the fix the entire HardDeleteAccount would
|
||||
// fail here.
|
||||
privateWorker := &models.WorkerNode{
|
||||
WorkerID: "private-" + uuid.New().String(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "priv-tok-" + uuid.New().String(),
|
||||
AccountID: &account.ID,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(privateWorker).Error)
|
||||
operatedWorker := &models.WorkerNode{
|
||||
WorkerID: "operated-" + uuid.New().String(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "op-tok-" + uuid.New().String(),
|
||||
AccountID: nil,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(operatedWorker).Error)
|
||||
require.NoError(t, models.DB().Create(&models.CheckRegionResult{
|
||||
CheckID: check.ID,
|
||||
RegionCode: "test",
|
||||
WorkerNodeID: &privateWorker.ID,
|
||||
ExecutedAt: time.Now(),
|
||||
State: "OK",
|
||||
}).Error)
|
||||
|
||||
// LLM scoped to the victim account + linked to the private worker.
|
||||
// worker_llms.llm_id has a FK to llms(id) without ON DELETE
|
||||
// CASCADE, so the join row used to block LLM deletion too.
|
||||
llm := models.LLM{
|
||||
AccountID: &account.ID,
|
||||
Name: "private-llm",
|
||||
URL: "https://llm.example.com",
|
||||
ModelName: "gpt-test",
|
||||
APIKey: "secret",
|
||||
Kind: "openai",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&llm).Error)
|
||||
require.NoError(t, models.DB().Exec(
|
||||
"INSERT INTO worker_llms (worker_node_id, llm_id) VALUES (?, ?)",
|
||||
privateWorker.ID, llm.ID,
|
||||
).Error)
|
||||
|
||||
// Inventory entities scoped to the victim account. Each has an
|
||||
// account_id FK to accounts(id) without ON DELETE CASCADE so they
|
||||
// must be removed before the account row goes away. The
|
||||
// shared-infra server belongs to another account and must
|
||||
// survive.
|
||||
victimServer := models.Server{Name: "victim-srv", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&victimServer).Error)
|
||||
victimServerIP := models.ServerIp{ServerID: victimServer.ID, Address: "10.0.0.1"}
|
||||
require.NoError(t, models.DB().Create(&victimServerIP).Error)
|
||||
victimSite := models.Site{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID,
|
||||
Slug: "victim-site", Name: "victim-site", Kind: "production", IsActive: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimSite).Error)
|
||||
victimDeployment := models.Deployment{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
|
||||
Kind: "production", Mode: "compose",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimDeployment).Error)
|
||||
victimDomain := models.Domain{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
|
||||
Name: "victim.example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimDomain).Error)
|
||||
otherServer := models.Server{Name: "shared-srv", AccountID: otherAccount.ID}
|
||||
require.NoError(t, models.DB().Create(&otherServer).Error)
|
||||
|
||||
require.NoError(t, models.HardDeleteAccount(account.ID))
|
||||
|
||||
// Account and all account-scoped rows must be gone.
|
||||
assert.ErrorIs(t, models.DB().First(&models.Account{}, account.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Check{}, check.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Monitor{}, monitor.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Group{}, group.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Notification{}, notification.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Message{}, message.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.LLM{}, llm.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Contact{}, ownerContact.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Contact{}, accountContact.ID).Error, gormErrRecordNotFound)
|
||||
var notificationContactCount int64
|
||||
require.NoError(t, models.DB().Table("notification_contacts").
|
||||
Where("notification_id = ? OR contact_id = ?", notification.ID, accountContact.ID).
|
||||
Count(¬ificationContactCount).Error)
|
||||
assert.Zero(t, notificationContactCount)
|
||||
|
||||
// Private worker must be gone; operated worker must survive.
|
||||
assert.ErrorIs(t, models.DB().First(&models.WorkerNode{}, privateWorker.ID).Error, gormErrRecordNotFound)
|
||||
var stillOperated models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&stillOperated, operatedWorker.ID).Error)
|
||||
assert.Nil(t, stillOperated.AccountID, "operated worker account_id must remain NULL")
|
||||
|
||||
// User-only contact (no account_id) must survive.
|
||||
var stillUserOnly models.Contact
|
||||
require.NoError(t, models.DB().First(&stillUserOnly, userOnlyContact.ID).Error)
|
||||
|
||||
// Inventory entities scoped to the account must be gone.
|
||||
assert.ErrorIs(t, models.DB().First(&models.Server{}, victimServer.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Site{}, victimSite.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Deployment{}, victimDeployment.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Domain{}, victimDomain.ID).Error, gormErrRecordNotFound)
|
||||
var leftover int64
|
||||
require.NoError(t, models.DB().Model(&models.ServerIp{}).
|
||||
Where("server_id = ?", victimServer.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "server_ips for the deleted server must be cleaned up")
|
||||
// Other-account inventory must survive.
|
||||
var stillOtherServer models.Server
|
||||
require.NoError(t, models.DB().First(&stillOtherServer, otherServer.ID).Error)
|
||||
|
||||
// Region-result rows for the deleted check must be gone.
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("check_id = ?", check.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "check_region_results must be cleaned up before checks")
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("worker_node_id = ?", privateWorker.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "check_region_results referencing a private worker must be cleaned up")
|
||||
require.NoError(t, models.DB().Raw(
|
||||
"SELECT COUNT(*) FROM worker_llms WHERE llm_id = ? OR worker_node_id = ?",
|
||||
llm.ID, privateWorker.ID,
|
||||
).Scan(&leftover).Error)
|
||||
assert.Zero(t, leftover, "worker_llms rows for the deleted LLM and private worker must be gone")
|
||||
|
||||
// Sanity: the other account and its data are untouched.
|
||||
var stillOther models.Account
|
||||
require.NoError(t, models.DB().First(&stillOther, otherAccount.ID).Error)
|
||||
}
|
||||
240
app/models/deployment.go
Обычный файл
240
app/models/deployment.go
Обычный файл
@@ -0,0 +1,240 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DeploymentKind is the rstuff-mirrored lifecycle label for a
|
||||
// deployment (production / production_prev / production_next /
|
||||
// internal / staging / old). See
|
||||
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable numeric
|
||||
// mapping (which Postgres stores alphabetically, not numerically).
|
||||
type DeploymentKind string
|
||||
|
||||
// Lifecycle labels for a Deployment row. See DeploymentKind for
|
||||
// the matching rstuff enum. The label set is closed; new values
|
||||
// require adding a Postgres enum value via app/models/migrate.go.
|
||||
const (
|
||||
// DeploymentKindProduction is the customer-facing "live" deployment.
|
||||
DeploymentKindProduction DeploymentKind = "production"
|
||||
DeploymentKindProductionPrev DeploymentKind = "production_prev"
|
||||
DeploymentKindProductionNext DeploymentKind = "production_next"
|
||||
// DeploymentKindInternal is for ops/admin tooling (not customer-facing).
|
||||
DeploymentKindInternal DeploymentKind = "internal"
|
||||
DeploymentKindStaging DeploymentKind = "staging"
|
||||
DeploymentKindOld DeploymentKind = "old"
|
||||
)
|
||||
|
||||
// DeploymentMode is the host-side lifecycle label (kubernetes /
|
||||
// compose / dedicated / vds / user). `dedicated` covers a single
|
||||
// nginx vhost; `compose` covers a Docker Compose project.
|
||||
type DeploymentMode string
|
||||
|
||||
// DeploymentMode values map onto rstuff's Deployment.mode enum.
|
||||
// `dedicated` covers a single nginx vhost; `compose` covers a
|
||||
// Docker Compose project; the others are reserved for future
|
||||
// v2 surfaces (Kubernetes, VDS, user-owned).
|
||||
const (
|
||||
DeploymentModeKubernetes DeploymentMode = "kubernetes"
|
||||
DeploymentModeCompose DeploymentMode = "compose"
|
||||
DeploymentModeDedicated DeploymentMode = "dedicated"
|
||||
DeploymentModeVDS DeploymentMode = "vds"
|
||||
DeploymentModeUser DeploymentMode = "user"
|
||||
)
|
||||
|
||||
// DeploymentAction is the reconciliation state. Updated by the
|
||||
// deploymentd receiver on every POST and by the 90s reconcile sweep
|
||||
// (see app/models/deployment.go ReconcileMissing).
|
||||
type DeploymentAction string
|
||||
|
||||
// DeploymentAction values. Pending/PendingMove/PendingDrop are
|
||||
// transient (operator or receiver-initiated); Deleted/Missing are
|
||||
// sticky until the deployment shows up again on a future POST.
|
||||
const (
|
||||
DeploymentActionOk DeploymentAction = "ok"
|
||||
DeploymentActionPending DeploymentAction = "pending"
|
||||
DeploymentActionPendingMove DeploymentAction = "pending_move"
|
||||
DeploymentActionPendingDrop DeploymentAction = "pending_drop"
|
||||
DeploymentActionDeleted DeploymentAction = "deleted"
|
||||
DeploymentActionMissing DeploymentAction = "missing"
|
||||
)
|
||||
|
||||
// Deployment represents a single host-side binding: one nginx
|
||||
// vhost, one Docker Compose service, or one Kubernetes service. The
|
||||
// shape mirrors rstuff's `deployments` table. See
|
||||
// docs/plans/inventory-management.md §4 / §6.1.
|
||||
type Deployment struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
ServiceName *string `gorm:"size:120" json:"service_name,omitempty"`
|
||||
|
||||
Kind DeploymentKind `gorm:"type:deployment_kind;not null;default:'production'" json:"kind"`
|
||||
Mode DeploymentMode `gorm:"type:deployment_mode;not null;default:'dedicated'" json:"mode"`
|
||||
Action DeploymentAction `gorm:"type:deployment_action;not null;default:'ok'" json:"action"`
|
||||
|
||||
URL *string `gorm:"type:text" json:"url,omitempty"`
|
||||
SSHUser *string `gorm:"size:64" json:"ssh_user,omitempty"`
|
||||
RootPath *string `gorm:"type:text" json:"root_path,omitempty"`
|
||||
ConfigPath *string `gorm:"type:text" json:"config_path,omitempty"`
|
||||
IP *string `gorm:"type:inet" json:"ip,omitempty"`
|
||||
Listen pq.StringArray `gorm:"type:varchar(64)[];not null;default:'{}'" json:"listen"`
|
||||
ServerName pq.StringArray `gorm:"type:varchar(255)[];not null;default:'{}'" json:"server_name"`
|
||||
Auth bool `gorm:"not null;default:false" json:"auth"`
|
||||
IsProxied bool `gorm:"not null;default:false" json:"is_proxied"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Deployment) TableName() string { return "deployments" }
|
||||
|
||||
// Scan implements sql.Scanner so a Postgres enum value can land in
|
||||
// our typed string alias without a code-generation step.
|
||||
func (d *DeploymentKind) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentKind(v)
|
||||
case []byte:
|
||||
*d = DeploymentKind(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_kind: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for the inverse direction.
|
||||
func (d DeploymentKind) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for DeploymentMode.
|
||||
func (d *DeploymentMode) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentMode(v)
|
||||
case []byte:
|
||||
*d = DeploymentMode(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_mode: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for DeploymentMode.
|
||||
func (d DeploymentMode) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for DeploymentAction.
|
||||
func (d *DeploymentAction) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentAction(v)
|
||||
case []byte:
|
||||
*d = DeploymentAction(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_action: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for DeploymentAction.
|
||||
func (d DeploymentAction) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// ReconcileMissingDeployments flips action='missing' on every Deployment for the
|
||||
// given server whose last_seen_at is older than cutoff. Called by
|
||||
// the deploymentd receiver after every successful upsert so the
|
||||
// "missing" badge appears within one POST cycle.
|
||||
//
|
||||
// Idempotent: re-running with the same cutoff is a no-op.
|
||||
func ReconcileMissingDeployments(serverID int64, mode DeploymentMode, cutoff time.Time) (int64, error) {
|
||||
res := DB().Model(&Deployment{}).
|
||||
Where("server_id = ? AND mode = ? AND action NOT IN ?", serverID, mode,
|
||||
[]DeploymentAction{DeploymentActionDeleted, DeploymentActionMissing, DeploymentActionPendingDrop}).
|
||||
Where("last_seen_at IS NULL OR last_seen_at < ?", cutoff).
|
||||
Update("action", DeploymentActionMissing)
|
||||
return res.RowsAffected, res.Error
|
||||
}
|
||||
|
||||
// UpsertNginxDeployment finds or creates a Deployment by
|
||||
// (server_id, config_path) for an nginx vhost. The caller fills in
|
||||
// the lifecycle fields (listen, server_name, etc.) after the upsert
|
||||
// returns. The return value is the row to mutate; the caller MUST
|
||||
// also touch last_seen_at and save.
|
||||
func UpsertNginxDeployment(tx *gorm.DB, accountID int64, serverID int64, configPath string) (*Deployment, error) {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
var d Deployment
|
||||
err := tx.Where("server_id = ? AND config_path = ?", serverID, configPath).First(&d).Error
|
||||
if err == nil {
|
||||
return &d, nil
|
||||
}
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
return nil, err
|
||||
}
|
||||
d = Deployment{
|
||||
AccountID: accountID,
|
||||
ServerID: &serverID,
|
||||
Kind: DeploymentKindProduction,
|
||||
Mode: DeploymentModeDedicated,
|
||||
Action: DeploymentActionOk,
|
||||
ConfigPath: &configPath,
|
||||
}
|
||||
if err := tx.Create(&d).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// RotateServerToken sets a new random token for a server and returns
|
||||
// the plaintext. Called by the operator-only
|
||||
// POST /api/v1/servers/:id/rotate-token endpoint. The plaintext is
|
||||
// returned exactly once — it is not stored anywhere recoverable.
|
||||
func RotateServerToken(tx *gorm.DB, serverID int64, newToken string) error {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
return tx.Model(&Server{}).Where("id = ?", serverID).Update("token", newToken).Error
|
||||
}
|
||||
80
app/models/dns_record.go
Обычный файл
80
app/models/dns_record.go
Обычный файл
@@ -0,0 +1,80 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/netaddr"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DNSRecord provides functionality.
|
||||
type DNSRecord struct {
|
||||
concerns.Model
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
Monitor *Monitor `json:"-"`
|
||||
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Value netaddr.Inet `json:"value" gorm:"type:bytea;"`
|
||||
}
|
||||
|
||||
// SaveIps provides functionality.
|
||||
func SaveIps(m *Monitor, _ Check, ips []NSRecord) { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
// log.Println("save ips for monitor")
|
||||
// spew.Dump(m)
|
||||
var err error
|
||||
tx := DB().Begin()
|
||||
|
||||
currentRecords := make([]DNSRecord, 0)
|
||||
_ = tx.Model(&m).Association("DNSRecords").Find(¤tRecords)
|
||||
|
||||
recordHash := make(map[string]DNSRecord, 0)
|
||||
for _, record := range currentRecords {
|
||||
recordHash[record.Name] = record
|
||||
}
|
||||
nextRecords := make(map[string]bool, 0)
|
||||
|
||||
for _, ip := range ips {
|
||||
if record, ok := recordHash[ip.Name]; ok {
|
||||
// log.Println("old value:", record)
|
||||
record.MonitorID = m.ID
|
||||
record.Value = ip.Value
|
||||
record.Kind = ip.Kind
|
||||
err = tx.Model(&m).Association("DNSRecords").Replace(&record)
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
record = DNSRecord{
|
||||
MonitorID: m.ID,
|
||||
Name: ip.Name,
|
||||
Kind: ip.Kind,
|
||||
Value: ip.Value,
|
||||
}
|
||||
if _, ok := nextRecords[ip.Name]; !ok {
|
||||
nextRecords[ip.Name] = true
|
||||
err = tx.Model(&m).Association("DNSRecords").Append(&record)
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// spew.Dump(ips_hash)
|
||||
|
||||
err = tx.Commit().Error
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// NSRecord provides functionality.
|
||||
type NSRecord struct {
|
||||
Name string
|
||||
Kind string
|
||||
Value netaddr.Inet
|
||||
}
|
||||
32
app/models/domain.go
Обычный файл
32
app/models/domain.go
Обычный файл
@@ -0,0 +1,32 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Domain represents a customer-facing DNS name. The shape mirrors
|
||||
// rstuff's `domains` table — see docs/parity/rstuff-inventory.md §2
|
||||
// and docs/plans/inventory-management.md §4.
|
||||
//
|
||||
// Distinct from RknDomain (the RKN blocklist cache, see
|
||||
// app/models/rkn_domain.go): RknDomain is read-only data about
|
||||
// blocked domains; Domain is the customer-side name→site/server
|
||||
// pointer that monitoring reasons about.
|
||||
type Domain struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
Name string `gorm:"size:255;not null" json:"name"`
|
||||
Env string `gorm:"size:32;not null;default:'production'" json:"env"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Domain) TableName() string { return "domains" }
|
||||
96
app/models/drop.go
Обычный файл
96
app/models/drop.go
Обычный файл
@@ -0,0 +1,96 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
)
|
||||
|
||||
// Drop removes all test data from the database.
|
||||
// It uses TRUNCATE ... CASCADE for join tables and deletes leaf-to-root for main tables.
|
||||
// Safe to call from multiple goroutines within a single test binary; concurrent calls
|
||||
// from separate test binaries are serialized by the advisory lock in Migrate().
|
||||
func Drop() {
|
||||
var dbname string
|
||||
if err := DB().Raw("SELECT current_database()").Scan(&dbname).Error; err != nil {
|
||||
panic(fmt.Sprintf("Drop: cannot read database name: %v", err))
|
||||
}
|
||||
if !strings.Contains(dbname, "test") {
|
||||
panic(fmt.Sprintf(
|
||||
"Drop() refused: database '%s' is not a test database. "+
|
||||
"Set DATABASE_NAME=rsmon_test to run tests safely.", dbname,
|
||||
))
|
||||
}
|
||||
|
||||
// Truncate many2many join tables first to avoid FK violations.
|
||||
DB().Exec("TRUNCATE event_checks, event_messages, notification_contacts, notification_groups RESTART IDENTITY CASCADE")
|
||||
|
||||
// Delete main tables in dependency order (leaf tables first).
|
||||
DB().Where("1=1").Delete(&NotificationDelivery{})
|
||||
DB().Where("1=1").Delete(&TaskReplay{})
|
||||
DB().Where("1=1").Delete(&TelegramBotMessage{})
|
||||
DB().Where("1=1").Delete(&TelegramBotStatus{})
|
||||
DB().Where("1=1").Delete(&Task{})
|
||||
DB().Where("1=1").Delete(&Access{})
|
||||
DB().Where("1=1").Delete(&Invite{})
|
||||
DB().Where("1=1").Delete(&Message{})
|
||||
DB().Where("1=1").Delete(&Event{})
|
||||
// Durable network-diagnostic rows reference checks, monitors, and workers.
|
||||
DB().Where("1=1").Delete(&DiagnosticAuditEvent{})
|
||||
DB().Where("1=1").Delete(&CheckAttempt{})
|
||||
// CheckRegionResult FKs check_id; it must be cleared before Check.
|
||||
DB().Where("1=1").Delete(&CheckRegionResult{})
|
||||
DB().Where("1=1").Delete(&Check{})
|
||||
DB().Where("1=1").Delete(&Notification{})
|
||||
// DNSRecord FKs monitor_id; it must be cleared before Monitor.
|
||||
DB().Where("1=1").Delete(&DNSRecord{})
|
||||
DB().Where("1=1").Delete(&Monitor{})
|
||||
DB().Where("1=1").Delete(&Group{})
|
||||
DB().Where("1=1").Delete(&Contact{})
|
||||
DB().Where("1=1").Delete(&NotificationCredential{})
|
||||
DB().Where("1=1").Delete(&WorkerLogEvent{})
|
||||
DB().Where("1=1").Delete(&WorkerNode{})
|
||||
DB().Where("1=1").Delete(&LLM{})
|
||||
DB().Where("1=1").Delete(&Region{})
|
||||
// Inventory (docs/plans/inventory-management.md §6): leaf tables
|
||||
// (sites, deployments, server_ips) reference accounts/servers, so
|
||||
// they must be cleared before Server is deleted.
|
||||
DB().Where("1=1").Delete(&Deployment{})
|
||||
DB().Where("1=1").Delete(&SiteRepo{})
|
||||
DB().Where("1=1").Delete(&Site{})
|
||||
DB().Where("1=1").Delete(&Repo{})
|
||||
DB().Where("1=1").Delete(&ServerIp{})
|
||||
DB().Where("1=1").Delete(&Domain{})
|
||||
DB().Where("1=1").Delete(&Server{})
|
||||
// Account-scoped tag metadata (account_id FK to accounts(id)).
|
||||
// Cleared before Account so a future cascade change cannot orphan
|
||||
// rows mid-truncate.
|
||||
DB().Where("1=1").Delete(&Tag{})
|
||||
// Status pages (docs/plans/status-pages.md §3.1–3.5). Children
|
||||
// reference status_pages(id) with ON DELETE CASCADE so GORM
|
||||
// ordering would already wipe them, but we delete them
|
||||
// explicitly so the test DB stays clean even if a future model
|
||||
// change drops the cascade.
|
||||
DB().Where("1=1").Delete(&StatusPageDomain{})
|
||||
DB().Unscoped().Where("1=1").Delete(&Maintenance{})
|
||||
DB().Where("1=1").Delete(&StatusPageMaintenance{})
|
||||
DB().Where("1=1").Delete(&StatusPageIncident{})
|
||||
DB().Where("1=1").Delete(&StatusPageDelivery{})
|
||||
DB().Where("1=1").Delete(&StatusPageDigestSchedule{})
|
||||
DB().Where("1=1").Delete(&StatusPageSubscriber{})
|
||||
DB().Unscoped().Where("1=1").Delete(&StatusPage{})
|
||||
DB().Where("1=1").Delete(&SubscriptionEvent{})
|
||||
DB().Where("1=1").Delete(&Subscription{})
|
||||
DB().Where("1=1").Delete(&Account{})
|
||||
|
||||
DB().Unscoped().Where("1=1").Delete(&authidentity.AuthIdentity{})
|
||||
// User has a unique email index, so test fixtures must be physically
|
||||
// removed rather than soft-deleted between tests.
|
||||
DB().Unscoped().Where("1=1").Delete(&User{})
|
||||
// A few audited tables added by feature migrations can retain a user FK that
|
||||
// is intentionally not modeled as an association. CASCADE keeps fixture
|
||||
// cleanup deterministic instead of silently leaving unique emails behind.
|
||||
DB().Exec("TRUNCATE users CASCADE")
|
||||
DB().Exec("TRUNCATE regions CASCADE")
|
||||
}
|
||||
83
app/models/event.go
Обычный файл
83
app/models/event.go
Обычный файл
@@ -0,0 +1,83 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/util"
|
||||
)
|
||||
|
||||
// Event provides functionality.
|
||||
type Event struct {
|
||||
concerns.Model
|
||||
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id"`
|
||||
Monitor *Monitor `json:"monitor,omitempty"`
|
||||
|
||||
StartTime *time.Time `json:"start_time"`
|
||||
EndTime *time.Time `json:"end_time"`
|
||||
Duration int `json:"duration"`
|
||||
Errors int `json:"errors"`
|
||||
Oks int `json:"oks"`
|
||||
State string `gorm:"index" json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
|
||||
Messages []Message `json:"messages" gorm:"many2many:event_messages;"`
|
||||
|
||||
ChecksDown pq.StringArray `gorm:"type:varchar(255)[]" json:"checks_down"`
|
||||
Checks []Check `json:"-" gorm:"many2many:event_checks;"`
|
||||
|
||||
ExpiresAt *time.Time `json:"-"`
|
||||
|
||||
Audited
|
||||
}
|
||||
|
||||
// EventScope provides functionality.
|
||||
func EventScope(q *gorm.DB) *gorm.DB {
|
||||
return q.Where("state IN ('current', 'ended')").
|
||||
Preload("Checks").
|
||||
Preload("Monitor").
|
||||
Preload("Monitor.Group").
|
||||
Preload("Monitor.Group.Notifications").
|
||||
Preload("Monitor.Group.Notifications.Contacts")
|
||||
}
|
||||
|
||||
// GetDuration provides functionality.
|
||||
func (e *Event) GetDuration(tn time.Time) int64 {
|
||||
endTime := e.EndTime
|
||||
if endTime == nil {
|
||||
endTime = &tn
|
||||
}
|
||||
return int64(endTime.Sub(*e.StartTime) / time.Second)
|
||||
}
|
||||
|
||||
// FormatDuration provides functionality.
|
||||
func (e *Event) FormatDuration() string {
|
||||
d := e.GetDuration(time.Now())
|
||||
return util.FormatDuration(d)
|
||||
}
|
||||
|
||||
// Inspect provides functionality.
|
||||
func (e *Event) Inspect() string {
|
||||
var st, et string
|
||||
if e.StartTime != nil {
|
||||
st = e.StartTime.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
if e.EndTime != nil {
|
||||
et = e.EndTime.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"Event<id: %d, monitor_id: %d, start_time: %s, end_time: %s, reason: %s, duration: %d>",
|
||||
e.ID,
|
||||
e.MonitorID,
|
||||
st,
|
||||
et,
|
||||
e.Reason,
|
||||
e.Duration,
|
||||
)
|
||||
}
|
||||
76
app/models/group.go
Обычный файл
76
app/models/group.go
Обычный файл
@@ -0,0 +1,76 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// Group represents a monitor group.
|
||||
type Group struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
IsSystem *bool `json:"is_system" gorm:"default:false"`
|
||||
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
|
||||
Monitors []Monitor `json:"-"`
|
||||
Notifications []Notification `gorm:"many2many:notification_groups;" json:"-"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
Audited
|
||||
}
|
||||
|
||||
// SystemGroups returns all groups marked as system/internal (is_system=true).
|
||||
// These groups are converted to distributed monitors (system checks running
|
||||
// on the distributed worker pool).
|
||||
func SystemGroups() ([]Group, error) {
|
||||
var groups []Group
|
||||
err := DB().Where("is_system = ?", true).Find(&groups).Error
|
||||
return groups, err
|
||||
}
|
||||
|
||||
// GroupIdsForAccountId returns all group IDs for the given account.
|
||||
func GroupIdsForAccountId(accountID int64) []int64 { //nolint:revive // accepted lint exception
|
||||
rows, err := DB().Raw("SELECT id FROM groups WHERE account_id = ?", accountID).Rows()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // accepted lint exception
|
||||
var cid int64
|
||||
groupIDs := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
rows.Scan(&cid) //nolint:errcheck // accepted lint exception
|
||||
groupIDs = append(groupIDs, cid)
|
||||
}
|
||||
return groupIDs
|
||||
}
|
||||
|
||||
// CountGroups counts monitors per group.
|
||||
func CountGroups(groupIDs []int64, groupCount *map[int64]int) { //nolint:gocritic // ptrToRefParam: accepted pattern
|
||||
rows, err := DB().Raw("select group_id, count(id) from monitors where group_id IN (?) group by group_id ", groupIDs).Rows()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // accepted lint exception
|
||||
|
||||
var gid int64
|
||||
var count int
|
||||
for rows.Next() {
|
||||
rows.Scan(&gid, &count) //nolint:errcheck // accepted lint exception
|
||||
(*groupCount)[gid] = count
|
||||
}
|
||||
}
|
||||
|
||||
// GroupsCounts fills MonitorsCount for each group.
|
||||
func GroupsCounts(groups *[]Group) {
|
||||
groupIDs := make([]int64, len(*groups))
|
||||
groupCount := make(map[int64]int, len(*groups))
|
||||
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
|
||||
groupIDs[i] = g.ID
|
||||
groupCount[g.ID] = 0
|
||||
}
|
||||
|
||||
CountGroups(groupIDs, &groupCount)
|
||||
|
||||
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
|
||||
(*groups)[i].MonitorsCount = groupCount[g.ID]
|
||||
}
|
||||
}
|
||||
58
app/models/group_test.go
Обычный файл
58
app/models/group_test.go
Обычный файл
@@ -0,0 +1,58 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestSystemGroups verifies that SystemGroups returns only groups with
|
||||
// is_system=true and ignores groups with is_system=false or nil.
|
||||
func TestSystemGroups(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account := &models.Account{Name: "test-account"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
|
||||
trueVal, falseVal := true, false
|
||||
|
||||
systemGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "system-internal",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
regularGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "regular",
|
||||
IsSystem: &falseVal,
|
||||
}
|
||||
nilSystemGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "nil-system",
|
||||
}
|
||||
|
||||
require.NoError(t, models.DB().Create(systemGroup).Error)
|
||||
require.NoError(t, models.DB().Create(regularGroup).Error)
|
||||
require.NoError(t, models.DB().Create(nilSystemGroup).Error)
|
||||
|
||||
got, err := models.SystemGroups()
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []int64
|
||||
var names []string
|
||||
for _, g := range got {
|
||||
ids = append(ids, g.ID)
|
||||
names = append(names, g.Name)
|
||||
}
|
||||
|
||||
assert.Contains(t, names, "system-internal")
|
||||
assert.NotContains(t, names, "regular")
|
||||
assert.NotContains(t, names, "nil-system")
|
||||
assert.Contains(t, ids, systemGroup.ID)
|
||||
assert.NotContains(t, ids, regularGroup.ID)
|
||||
assert.NotContains(t, ids, nilSystemGroup.ID)
|
||||
}
|
||||
30
app/models/init.go
Обычный файл
30
app/models/init.go
Обычный файл
@@ -0,0 +1,30 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/fatih/structs"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func init() {
|
||||
structs.DefaultTagName = "json"
|
||||
}
|
||||
|
||||
// db Gorm DB
|
||||
var db *gorm.DB
|
||||
|
||||
// DB provides functionality.
|
||||
func DB() *gorm.DB {
|
||||
return db.WithContext(context.TODO())
|
||||
}
|
||||
|
||||
// SetDB provides functionality.
|
||||
func SetDB(newDb *gorm.DB) {
|
||||
db = newDb
|
||||
}
|
||||
|
||||
// IsDBAvailable returns true if the database has been initialized
|
||||
func IsDBAvailable() bool {
|
||||
return db != nil
|
||||
}
|
||||
222
app/models/inventory_test.go
Обычный файл
222
app/models/inventory_test.go
Обычный файл
@@ -0,0 +1,222 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestServer_RotateToken(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "rotate-token-account"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "rotate-target",
|
||||
Slug: "rotate-target",
|
||||
Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
t1 := models.GenerateServerToken()
|
||||
require.NoError(t, models.RotateServerToken(nil, srv.ID, t1))
|
||||
|
||||
got, err := models.FindServerByToken(t1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srv.ID, got.ID)
|
||||
|
||||
t2 := models.GenerateServerToken()
|
||||
require.NoError(t, models.RotateServerToken(nil, srv.ID, t2))
|
||||
|
||||
// Old token no longer matches.
|
||||
_, err = models.FindServerByToken(t1)
|
||||
assert.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
||||
|
||||
got2, err := models.FindServerByToken(t2)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srv.ID, got2.ID)
|
||||
|
||||
// Generated tokens are hex-encoded 32 bytes (64 chars).
|
||||
assert.Len(t, t1, 64)
|
||||
assert.NotEqual(t, t1, t2)
|
||||
}
|
||||
|
||||
func TestServer_InventoryFields_Default(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "inv-defaults"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "fresh-server",
|
||||
Slug: "fresh-server",
|
||||
Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
got := models.Server{}
|
||||
require.NoError(t, models.DB().First(&got, srv.ID).Error)
|
||||
assert.Equal(t, models.ServerKindProduction, got.Kind)
|
||||
assert.Equal(t, 0, got.PriceCents)
|
||||
assert.False(t, got.Paused)
|
||||
}
|
||||
|
||||
func TestServer_KindEnum_OnlyAllowsValidValues(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "inv-enum"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "kinder",
|
||||
Slug: "kinder",
|
||||
Region: "local",
|
||||
Kind: models.ServerKindStaging,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
got := models.Server{}
|
||||
require.NoError(t, models.DB().First(&got, srv.ID).Error)
|
||||
assert.Equal(t, models.ServerKindStaging, got.Kind)
|
||||
|
||||
// Inserting an invalid value via raw SQL fails the enum check.
|
||||
err := models.DB().Exec(
|
||||
"INSERT INTO servers (account_id, name, slug, region, kind) VALUES (?, ?, ?, ?, ?)",
|
||||
acc.ID, "bad-kinder", "bad-kinder", "local", "scrapped",
|
||||
).Error
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid input value for enum")
|
||||
}
|
||||
|
||||
func TestSite_Slugify(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"Cafe", "cafe"}, // cyrillic stripped (latin-only rule, see Slugify)
|
||||
{" Spaces Everywhere ", "spaces-everywhere"},
|
||||
{"dots.and-dashes_and spaces", "dots-and-dashes-and-spaces"},
|
||||
{"", "site"},
|
||||
{"-leading-and-trailing-", "leading-and-trailing"},
|
||||
{"mix_of.dots-dashes spaces", "mix-of-dots-dashes-spaces"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.in, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, models.SiteSlugify(c.in))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSite_FindOrCreateBySlug(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "site-foc"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
got, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, got.ID, "row should be persisted")
|
||||
assert.Equal(t, "my-site", got.Slug)
|
||||
assert.Equal(t, "production", got.Kind)
|
||||
assert.True(t, got.IsActive)
|
||||
|
||||
// Second call returns the same row (idempotent).
|
||||
got2, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, got.ID, got2.ID)
|
||||
}
|
||||
|
||||
func TestDeployment_UpsertNginx_MatchesByConfigPath(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "dep-upsert"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID, Name: "host1", Slug: "host1", Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
d1, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, models.DB().Save(d1).Error)
|
||||
|
||||
// Second call with the same config_path returns the existing row.
|
||||
d2, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, d1.ID, d2.ID, "should reuse the same row on identical config_path")
|
||||
|
||||
// Different config_path creates a new row.
|
||||
d3, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/b.conf")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, d1.ID, d3.ID)
|
||||
}
|
||||
|
||||
func TestDeployment_ReconcileMissing_FlipsAction(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "reconcile"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID, Name: "rec", Slug: "rec", Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
fresh := models.Deployment{
|
||||
AccountID: acc.ID,
|
||||
ServerID: &srv.ID,
|
||||
Kind: models.DeploymentKindProduction,
|
||||
Mode: models.DeploymentModeDedicated,
|
||||
Action: models.DeploymentActionOk,
|
||||
ConfigPath: ptr("/etc/nginx/old.conf"),
|
||||
LastSeenAt: &old,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&fresh).Error)
|
||||
|
||||
recent := models.Deployment{
|
||||
AccountID: acc.ID,
|
||||
ServerID: &srv.ID,
|
||||
Kind: models.DeploymentKindProduction,
|
||||
Mode: models.DeploymentModeDedicated,
|
||||
Action: models.DeploymentActionOk,
|
||||
ConfigPath: ptr("/etc/nginx/recent.conf"),
|
||||
LastSeenAt: ptrTime(time.Now()),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&recent).Error)
|
||||
|
||||
cutoff := time.Now().Add(-90 * time.Second)
|
||||
marked, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1, marked, "only the old row should flip")
|
||||
|
||||
var oldAfter models.Deployment
|
||||
require.NoError(t, models.DB().First(&oldAfter, fresh.ID).Error)
|
||||
assert.Equal(t, models.DeploymentActionMissing, oldAfter.Action)
|
||||
|
||||
var recentAfter models.Deployment
|
||||
require.NoError(t, models.DB().First(&recentAfter, recent.ID).Error)
|
||||
assert.Equal(t, models.DeploymentActionOk, recentAfter.Action, "fresh row stays ok")
|
||||
|
||||
// Re-running with the same cutoff is a no-op.
|
||||
marked2, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 0, marked2)
|
||||
}
|
||||
|
||||
func ptr(s string) *string { return &s }
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
52
app/models/invite.go
Обычный файл
52
app/models/invite.go
Обычный файл
@@ -0,0 +1,52 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Invite provides functionality.
|
||||
type Invite struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
InviterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"inviter_id"`
|
||||
Inviter *User `json:"inviter"`
|
||||
InviteeID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"invitee_id"`
|
||||
Invitee *User `json:"invitee"`
|
||||
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
|
||||
// invite state, FAIL - failed to send, SENT - not regestired, OK - registered
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RegisteredAt time.Time `json:"registered_at"`
|
||||
SentAt time.Time `json:"sent_at"`
|
||||
|
||||
Accesses []Access `json:"accesses" gorm:"foreignkey:invite_id"`
|
||||
|
||||
concerns.HasToken
|
||||
Audited
|
||||
}
|
||||
|
||||
// BeforeCreate runs before creating an Invite record.
|
||||
func (i *Invite) BeforeCreate(_ *gorm.DB) error {
|
||||
i.SetToken()
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillAccesses provides functionality.
|
||||
func (i *Invite) FillAccesses() {
|
||||
for k, a := range i.Accesses { //nolint:gocritic // range copy is acceptable here
|
||||
if a.ID <= 0 {
|
||||
i.Accesses[k].ID = 0
|
||||
}
|
||||
i.Accesses[k].AccountID = i.AccountID
|
||||
}
|
||||
}
|
||||
17
app/models/llm.go
Обычный файл
17
app/models/llm.go
Обычный файл
@@ -0,0 +1,17 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// LLM stores an OpenAI-compatible LLM endpoint available to checks.
|
||||
type LLM struct {
|
||||
concerns.Model
|
||||
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id);index"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
URL string `json:"url" gorm:"not null"`
|
||||
ModelName string `json:"model" gorm:"column:model;not null"`
|
||||
APIKey string `json:"-" gorm:"not null"`
|
||||
Kind string `json:"kind" gorm:"not null;default:'openai'"`
|
||||
Workers []WorkerNode `json:"-" gorm:"many2many:worker_llms;"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
324
app/models/maintenance.go
Обычный файл
324
app/models/maintenance.go
Обычный файл
@@ -0,0 +1,324 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
MaintenanceManual = "manual"
|
||||
MaintenanceSingle = "single"
|
||||
MaintenanceCron = "cron"
|
||||
MaintenanceRecurringInterval = "recurring-interval"
|
||||
MaintenanceRecurringWeekday = "recurring-weekday"
|
||||
MaintenanceRecurringDayOfMonth = "recurring-day-of-month"
|
||||
)
|
||||
|
||||
// Maintenance is account-owned planned downtime. Times are stored as UTC;
|
||||
// Timezone only defines how recurring wall-clock fields are interpreted.
|
||||
type Maintenance struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
Description string `gorm:"type:text;not null;default:''" json:"description"`
|
||||
Strategy string `gorm:"size:32;not null" json:"strategy"`
|
||||
Cron string `gorm:"type:text;not null;default:''" json:"cron"`
|
||||
DurationSec int `gorm:"not null;default:0" json:"duration_sec"`
|
||||
StartDate *time.Time `json:"start_date,omitempty"`
|
||||
EndDate *time.Time `json:"end_date,omitempty"`
|
||||
StartTime string `gorm:"size:5;not null;default:''" json:"start_time"`
|
||||
EndTime string `gorm:"size:5;not null;default:''" json:"end_time"`
|
||||
Weekdays pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"weekdays"`
|
||||
DaysOfMonth pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"days_of_month"`
|
||||
IntervalDay int `gorm:"not null;default:1" json:"interval_day"`
|
||||
Timezone string `gorm:"size:64;not null;default:'UTC'" json:"timezone"`
|
||||
Active bool `gorm:"not null;default:true" json:"active"`
|
||||
LastStartDate *time.Time `json:"last_start_date,omitempty"`
|
||||
LegacyStatusPageMaintenanceID *int64 `gorm:"uniqueIndex" json:"-"`
|
||||
ShowOnAllStatusPages bool `gorm:"not null;default:true" json:"show_on_all_status_pages"`
|
||||
Monitors []Monitor `gorm:"many2many:maintenance_monitors;constraint:OnDelete:CASCADE" json:"monitors,omitempty"`
|
||||
StatusPages []StatusPage `gorm:"many2many:maintenance_status_pages;constraint:OnDelete:CASCADE" json:"status_pages,omitempty"`
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
|
||||
}
|
||||
|
||||
func (Maintenance) TableName() string { return "maintenances" }
|
||||
|
||||
func (m *Maintenance) location() (*time.Location, error) {
|
||||
if m.Timezone == "" || m.Timezone == "SAME_AS_SERVER" {
|
||||
return time.UTC, nil
|
||||
}
|
||||
return time.LoadLocation(m.Timezone)
|
||||
}
|
||||
|
||||
func (m *Maintenance) generatedCron() (string, error) {
|
||||
if m.Strategy == MaintenanceCron {
|
||||
return m.Cron, nil
|
||||
}
|
||||
if m.Strategy == MaintenanceManual || m.Strategy == MaintenanceSingle {
|
||||
return "", nil
|
||||
}
|
||||
parts := strings.Split(m.StartTime, ":")
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("start_time must be HH:MM")
|
||||
}
|
||||
base := parts[1] + " " + parts[0]
|
||||
switch m.Strategy {
|
||||
case MaintenanceRecurringInterval:
|
||||
return "", nil
|
||||
case MaintenanceRecurringWeekday:
|
||||
if len(m.Weekdays) == 0 {
|
||||
return "", fmt.Errorf("at least one weekday is required")
|
||||
}
|
||||
values := make([]string, len(m.Weekdays))
|
||||
for i, day := range m.Weekdays {
|
||||
if day < 0 || day > 6 {
|
||||
return "", fmt.Errorf("weekday must be 0 through 6")
|
||||
}
|
||||
values[i] = fmt.Sprint(day)
|
||||
}
|
||||
return base + " * * " + strings.Join(values, ","), nil
|
||||
case MaintenanceRecurringDayOfMonth:
|
||||
if len(m.DaysOfMonth) == 0 {
|
||||
return "", fmt.Errorf("at least one day of month is required")
|
||||
}
|
||||
values := make([]string, 0, len(m.DaysOfMonth))
|
||||
for _, day := range m.DaysOfMonth {
|
||||
if day == "lastDay1" {
|
||||
values = append(values, "28-31")
|
||||
} else {
|
||||
values = append(values, day)
|
||||
}
|
||||
}
|
||||
return base + " " + strings.Join(values, ",") + " * *", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate normalizes generated schedules and rejects ambiguous or invalid
|
||||
// input before it can reach the scheduler.
|
||||
func (m *Maintenance) Validate() error {
|
||||
m.Title = strings.TrimSpace(m.Title)
|
||||
if m.Title == "" || len(m.Title) > 200 {
|
||||
return fmt.Errorf("title is required and must be at most 200 characters")
|
||||
}
|
||||
if _, err := m.location(); err != nil {
|
||||
return fmt.Errorf("invalid timezone: %w", err)
|
||||
}
|
||||
switch m.Strategy {
|
||||
case MaintenanceManual:
|
||||
return nil
|
||||
case MaintenanceSingle:
|
||||
if m.StartDate == nil || m.EndDate == nil || !m.EndDate.After(*m.StartDate) {
|
||||
return fmt.Errorf("single maintenance requires end_date after start_date")
|
||||
}
|
||||
m.DurationSec = int(m.EndDate.Sub(*m.StartDate).Seconds())
|
||||
return nil
|
||||
case MaintenanceRecurringInterval:
|
||||
if m.DurationSec <= 0 || m.IntervalDay <= 0 || (m.IntervalDay > 1 && m.StartDate == nil) {
|
||||
return fmt.Errorf("recurring interval requires positive duration_sec and interval_day; intervals over one day require start_date")
|
||||
}
|
||||
if _, _, err := parseMaintenanceTime(m.StartTime); err != nil {
|
||||
return err
|
||||
}
|
||||
m.Cron = ""
|
||||
return nil
|
||||
case MaintenanceCron, MaintenanceRecurringWeekday, MaintenanceRecurringDayOfMonth:
|
||||
if m.DurationSec <= 0 {
|
||||
return fmt.Errorf("duration_sec must be positive")
|
||||
}
|
||||
cronText, err := m.generatedCron()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := cron.ParseStandard(cronText); err != nil {
|
||||
return fmt.Errorf("invalid cron: %w", err)
|
||||
}
|
||||
m.Cron = cronText
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Maintenance) BeforeSave(_ *gorm.DB) error { return m.Validate() }
|
||||
|
||||
// IsUnderMaintenance evaluates durable data only. This intentionally avoids
|
||||
// scheduler-owned state so a process restart and multiple web pods agree.
|
||||
func (m *Maintenance) IsUnderMaintenance(now time.Time) bool {
|
||||
if !m.Active {
|
||||
return false
|
||||
}
|
||||
if m.Strategy == MaintenanceManual {
|
||||
return true
|
||||
}
|
||||
if m.Strategy == MaintenanceSingle {
|
||||
return m.StartDate != nil && m.EndDate != nil && !now.Before(*m.StartDate) && now.Before(*m.EndDate)
|
||||
}
|
||||
if m.Strategy == MaintenanceRecurringInterval {
|
||||
return m.isUnderInterval(now)
|
||||
}
|
||||
if m.DurationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
schedule, err := cron.ParseStandard(m.Cron)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// Ask cron for each candidate since the earliest possible active start.
|
||||
// Cron is minute-granular, hence the extra minute catches exact boundaries.
|
||||
from := now.In(loc).Add(-time.Duration(m.DurationSec)*time.Second - time.Minute)
|
||||
to := now.In(loc)
|
||||
for candidate := schedule.Next(from); !candidate.After(to); candidate = schedule.Next(candidate) {
|
||||
if !m.allowsRecurringCandidate(candidate.In(loc)) {
|
||||
continue
|
||||
}
|
||||
start := candidate.UTC()
|
||||
if !now.Before(start) && now.Before(start.Add(time.Duration(m.DurationSec)*time.Second)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMaintenanceTime(value string) (int, int, error) {
|
||||
parsed, err := time.Parse("15:04", value)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("start_time must be HH:MM")
|
||||
}
|
||||
return parsed.Hour(), parsed.Minute(), nil
|
||||
}
|
||||
|
||||
func (m *Maintenance) intervalStartOn(date time.Time, loc *time.Location) (time.Time, bool) {
|
||||
if m.IntervalDay <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
hour, minute, err := parseMaintenanceTime(m.StartTime)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if m.IntervalDay == 1 && m.StartDate == nil {
|
||||
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
|
||||
}
|
||||
if m.StartDate == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
anchor := m.StartDate.In(loc)
|
||||
anchorDay := time.Date(anchor.Year(), anchor.Month(), anchor.Day(), 0, 0, 0, 0, loc)
|
||||
candidateDay := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc)
|
||||
// Compare civil dates rather than elapsed hours: a local day can be 23 or
|
||||
// 25 hours when the maintenance timezone crosses a DST boundary.
|
||||
days := civilDaysBetween(anchorDay, candidateDay)
|
||||
if days < 0 || days%m.IntervalDay != 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
|
||||
}
|
||||
|
||||
func civilDaysBetween(from, to time.Time) int {
|
||||
fromDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
|
||||
toDay := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC)
|
||||
return int(toDay.Sub(fromDay) / (24 * time.Hour))
|
||||
}
|
||||
|
||||
func (m *Maintenance) isUnderInterval(now time.Time) bool {
|
||||
loc, err := m.location()
|
||||
if err != nil || m.DurationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
for day := 0; day <= int(time.Duration(m.DurationSec)/24/time.Hour)+1; day++ {
|
||||
start, ok := m.intervalStartOn(localNow.AddDate(0, 0, -day), loc)
|
||||
if ok && !now.Before(start.UTC()) && now.Before(start.UTC().Add(time.Duration(m.DurationSec)*time.Second)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// robfig/cron cannot express "last day". The generated 28-31 range is only
|
||||
// a candidate generator; this final predicate makes lastDay1 exact.
|
||||
func (m *Maintenance) allowsRecurringCandidate(candidate time.Time) bool {
|
||||
if m.Strategy != MaintenanceRecurringDayOfMonth {
|
||||
return true
|
||||
}
|
||||
lastDay := candidate.AddDate(0, 0, 1).Month() != candidate.Month()
|
||||
for _, value := range m.DaysOfMonth {
|
||||
if value == "lastDay1" && lastDay {
|
||||
return true
|
||||
}
|
||||
if value == fmt.Sprint(candidate.Day()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Maintenance) NextRun(now time.Time) *time.Time {
|
||||
if !m.Active || m.Strategy == MaintenanceManual {
|
||||
return nil
|
||||
}
|
||||
if m.Strategy == MaintenanceSingle {
|
||||
if m.StartDate != nil && m.StartDate.After(now) {
|
||||
return m.StartDate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if m.Strategy == MaintenanceRecurringInterval {
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
for day := 0; day <= m.IntervalDay; day++ {
|
||||
if next, ok := m.intervalStartOn(localNow.AddDate(0, 0, day), loc); ok && next.After(localNow) {
|
||||
result := next.UTC()
|
||||
return &result
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
s, err := cron.ParseStandard(m.Cron)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for candidate := s.Next(now.In(loc)); ; candidate = s.Next(candidate) {
|
||||
if m.allowsRecurringCandidate(candidate.In(loc)) {
|
||||
next := candidate.UTC()
|
||||
return &next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MonitorUnderMaintenance is the notifier/public-page lookup.
|
||||
func MonitorUnderMaintenance(monitorID int64, now time.Time) (bool, error) {
|
||||
var rows []Maintenance
|
||||
err := DB().Joins("JOIN maintenance_monitors mm ON mm.maintenance_id = maintenances.id").Where("mm.monitor_id = ? AND maintenances.active = TRUE", monitorID).Find(&rows).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for i := range rows {
|
||||
if rows[i].IsUnderMaintenance(now) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
33
app/models/maintenance_migration_test.go
Обычный файл
33
app/models/maintenance_migration_test.go
Обычный файл
@@ -0,0 +1,33 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMigratePreservesEveryLegacyStatusPageMaintenance(t *testing.T) {
|
||||
Drop()
|
||||
Migrate()
|
||||
plan := Plan{Name: "legacy migration plan"}
|
||||
require.NoError(t, DB().Create(&plan).Error)
|
||||
account := Account{Name: "legacy migration account", PlanID: &plan.ID}
|
||||
require.NoError(t, DB().Create(&account).Error)
|
||||
page := StatusPage{AccountID: account.ID, Slug: "legacy-maintenance-migration", Name: "Legacy"}
|
||||
require.NoError(t, DB().Create(&page).Error)
|
||||
start := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
|
||||
for i := 0; i < 2; i++ {
|
||||
row := StatusPageMaintenance{StatusPageID: page.ID, Title: "same", StartsAt: start, EndsAt: start.Add(time.Hour), MonitorIDs: pq.Int64Array{}}
|
||||
require.NoError(t, DB().Create(&row).Error)
|
||||
}
|
||||
|
||||
Migrate()
|
||||
var migrated []Maintenance
|
||||
require.NoError(t, DB().Where("legacy_status_page_maintenance_id IS NOT NULL").Find(&migrated).Error)
|
||||
require.Len(t, migrated, 2)
|
||||
var joins int64
|
||||
require.NoError(t, DB().Table("maintenance_status_pages").Where("status_page_id = ?", page.ID).Count(&joins).Error)
|
||||
require.EqualValues(t, 2, joins)
|
||||
}
|
||||
44
app/models/maintenance_notifications.go
Обычный файл
44
app/models/maintenance_notifications.go
Обычный файл
@@ -0,0 +1,44 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const maintenanceStartNotificationPrefix = "maintenance:%d:start:"
|
||||
|
||||
// MaintenanceStartNotificationKey uniquely identifies one contact's warning
|
||||
// for one schedule revision and occurrence. Updated windows get a new revision
|
||||
// while concurrent scheduler replicas share the same key.
|
||||
func MaintenanceStartNotificationKey(maintenanceID int64, revision, startsAt time.Time, timezone string, notificationID, contactID int64) string {
|
||||
loc, err := time.LoadLocation(timezone)
|
||||
if err != nil || timezone == "SAME_AS_SERVER" || timezone == "" {
|
||||
loc = time.UTC
|
||||
}
|
||||
// A fall-back hour can have two UTC instants for one wall-clock occurrence.
|
||||
// Warnings are once per civil occurrence, matching the recurrence editor.
|
||||
civilOccurrence := startsAt.In(loc).Format("200601021504")
|
||||
return fmt.Sprintf("maintenance:%d:start:%d:%s:notification:%d:contact:%d", maintenanceID, revision.UnixNano(), civilOccurrence, notificationID, contactID)
|
||||
}
|
||||
|
||||
// CancelMaintenanceStartNotificationsTx prevents queued warnings from being
|
||||
// delivered after an operator pauses, changes, or deletes the maintenance.
|
||||
// Leased work may already be executing and cannot be recalled from a worker.
|
||||
func CancelMaintenanceStartNotificationsTx(tx *gorm.DB, maintenanceID int64, reason string) error {
|
||||
prefix := fmt.Sprintf(maintenanceStartNotificationPrefix, maintenanceID) + "%"
|
||||
var tasks []Task
|
||||
if err := tx.Clauses(SkipLockedClause).Where("idempotency_key LIKE ? AND state IN ?", prefix, []string{TaskStateQueued, TaskStateFailedRetry}).Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range tasks {
|
||||
if err := tx.Model(&Task{}).Where("id = ? AND state IN ?", tasks[i].ID, []string{TaskStateQueued, TaskStateFailedRetry}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "canceled", "canceled: "+reason); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
105
app/models/maintenance_test.go
Обычный файл
105
app/models/maintenance_test.go
Обычный файл
@@ -0,0 +1,105 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestMaintenanceValidateGeneratesRecurringCron(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
m models.Maintenance
|
||||
want string
|
||||
}{
|
||||
{"interval", models.Maintenance{Title: "interval", Strategy: models.MaintenanceRecurringInterval, StartDate: maintenanceTimePtr(time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)), StartTime: "02:30", IntervalDay: 3, DurationSec: 60, Timezone: "UTC"}, ""},
|
||||
{"weekday", models.Maintenance{Title: "weekdays", Strategy: models.MaintenanceRecurringWeekday, StartTime: "02:30", Weekdays: []int64{1, 5}, DurationSec: 60, Timezone: "UTC"}, "30 02 * * 1,5"},
|
||||
{"month", models.Maintenance{Title: "month", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "02:30", DaysOfMonth: []string{"1", "lastDay1"}, DurationSec: 60, Timezone: "UTC"}, "30 02 1,28-31 * *"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) { require.NoError(t, test.m.Validate()); assert.Equal(t, test.want, test.m.Cron) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceIsUnderMaintenanceBoundariesAndTimezone(t *testing.T) {
|
||||
start := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
end := start.Add(time.Hour)
|
||||
single := models.Maintenance{Title: "single", Strategy: models.MaintenanceSingle, StartDate: &start, EndDate: &end, Active: true, Timezone: "UTC"}
|
||||
assert.True(t, single.IsUnderMaintenance(start))
|
||||
assert.True(t, single.IsUnderMaintenance(end.Add(-time.Nanosecond)))
|
||||
assert.False(t, single.IsUnderMaintenance(end))
|
||||
cron := models.Maintenance{Title: "moscow", Strategy: models.MaintenanceCron, Cron: "0 12 * * *", DurationSec: 3600, Active: true, Timezone: "Europe/Moscow"}
|
||||
require.NoError(t, cron.Validate())
|
||||
assert.True(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 9, 30, 0, 0, time.UTC)), "12:30 Moscow is 09:30 UTC in July")
|
||||
assert.False(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)))
|
||||
}
|
||||
|
||||
func TestMaintenanceValidationRejectsInvalidSchedules(t *testing.T) {
|
||||
badCron := models.Maintenance{Title: "bad", Strategy: models.MaintenanceCron, Cron: "not cron", DurationSec: 1, Timezone: "UTC"}
|
||||
badSingle := models.Maintenance{Title: "bad", Strategy: models.MaintenanceSingle, Timezone: "UTC"}
|
||||
assert.Error(t, badCron.Validate())
|
||||
assert.Error(t, badSingle.Validate())
|
||||
}
|
||||
|
||||
func TestMaintenanceLastDayIsNotEveryDayInExpandedCronRange(t *testing.T) {
|
||||
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 3600, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 3, 28, 12, 30, 0, 0, time.UTC)))
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 31, 12, 30, 0, 0, time.UTC)))
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalUsesAnchorAndIntervalDay(t *testing.T) {
|
||||
anchor := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
m := models.Maintenance{Title: "every three days", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "12:00", IntervalDay: 3, DurationSec: 3600, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 7, 4, 12, 30, 0, 0, time.UTC)))
|
||||
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 7, 5, 12, 30, 0, 0, time.UTC)))
|
||||
next := m.NextRun(time.Date(2026, 7, 2, 13, 0, 0, 0, time.UTC))
|
||||
require.NotNil(t, next)
|
||||
assert.Equal(t, time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC), *next)
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalRequiresAnchorForMultiDayAndPositiveInterval(t *testing.T) {
|
||||
m := models.Maintenance{Title: "invalid", Strategy: models.MaintenanceRecurringInterval, StartTime: "12:00", IntervalDay: 2, DurationSec: 60, Timezone: "UTC"}
|
||||
assert.Error(t, m.Validate())
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalKeepsCivilDayAcrossDST(t *testing.T) {
|
||||
loc, err := time.LoadLocation("Europe/Berlin")
|
||||
require.NoError(t, err)
|
||||
// March 29, 2026 is the spring-forward day in Berlin. The second run is
|
||||
// still two civil days after the anchor, not one because a day was 23h.
|
||||
anchor := time.Date(2026, 3, 27, 0, 0, 0, 0, loc)
|
||||
m := models.Maintenance{Title: "DST", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "03:30", IntervalDay: 2, DurationSec: 3600, Active: true, Timezone: "Europe/Berlin"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 29, 4, 0, 0, 0, loc).UTC()))
|
||||
}
|
||||
|
||||
func TestMaintenanceStartNotificationKeyIncludesRevisionAndOccurrence(t *testing.T) {
|
||||
revision := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
start := revision.Add(5 * time.Minute)
|
||||
key := models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13)
|
||||
assert.Equal(t, key, models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13))
|
||||
assert.NotEqual(t, key, models.MaintenanceStartNotificationKey(7, revision.Add(time.Second), start, "Europe/Berlin", 11, 13))
|
||||
}
|
||||
|
||||
func TestMaintenanceStartNotificationKeyDeduplicatesDSTFallbackCivilOccurrence(t *testing.T) {
|
||||
revision := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
first := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC)
|
||||
second := first.Add(time.Hour)
|
||||
assert.Equal(t, models.MaintenanceStartNotificationKey(7, revision, first, "Europe/Berlin", 11, 13), models.MaintenanceStartNotificationKey(7, revision, second, "Europe/Berlin", 11, 13))
|
||||
}
|
||||
|
||||
func TestMaintenanceNextRunSkipsNonFinalLastDayCandidates(t *testing.T) {
|
||||
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 60, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
next := m.NextRun(time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC))
|
||||
require.NotNil(t, next)
|
||||
assert.Equal(t, time.Date(2026, 4, 30, 12, 0, 0, 0, time.UTC), *next)
|
||||
}
|
||||
|
||||
func maintenanceTimePtr(value time.Time) *time.Time { return &value }
|
||||
48
app/models/message.go
Обычный файл
48
app/models/message.go
Обычный файл
@@ -0,0 +1,48 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Message info about performed notification
|
||||
type Message struct {
|
||||
concerns.Model
|
||||
|
||||
NotificationID int64 `json:"notification_id"`
|
||||
Notification *Notification `json:"notification,omitempty"`
|
||||
|
||||
ContactID int64 `gorm:"index;type:bigint REFERENCES contacts(id)" json:"contact_id"`
|
||||
Contact *Contact `json:"contact,omitempty"`
|
||||
|
||||
// Events are for up/down messages
|
||||
Events []Event `json:"events" gorm:"many2many:event_messages;"`
|
||||
|
||||
// Checks are for expires messages
|
||||
CheckID *int64 `gorm:"index;type:bigint REFERENCES checks(id)" json:"check_id"`
|
||||
Check *Check `json:"check,omitempty"`
|
||||
|
||||
Kind string `json:"kind"`
|
||||
State string `json:"state"`
|
||||
Error *string `json:"error"`
|
||||
Response *string `json:"response"`
|
||||
Tries int `json:"-"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
SentAt time.Time `json:"sent_at"`
|
||||
}
|
||||
|
||||
// MessageScope provides functionality.
|
||||
func MessageScope(q *gorm.DB) *gorm.DB {
|
||||
return q.
|
||||
Preload("Events").
|
||||
Preload("Events.Checks").
|
||||
Preload("Events.Monitor").
|
||||
Preload("Notification").
|
||||
Preload("Contact").
|
||||
Preload("Check").
|
||||
Preload("Check.Monitor")
|
||||
}
|
||||
750
app/models/migrate.go
Обычный файл
750
app/models/migrate.go
Обычный файл
@@ -0,0 +1,750 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
)
|
||||
|
||||
// isTypeExistsError returns true if the error is a Postgres "type already exists" error.
|
||||
// This happens when AutoMigrate is called concurrently from multiple test processes.
|
||||
func isTypeExistsError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
// SQLSTATE 42710 = duplicate_object (type already exists)
|
||||
return strings.Contains(s, "42710") ||
|
||||
strings.Contains(s, "already exists") ||
|
||||
strings.Contains(s, "pg_type_typname_nsp_index")
|
||||
}
|
||||
|
||||
// migrateOrIgnoreTypeExists runs AutoMigrate and ignores "type already exists" errors
|
||||
// that can occur when parallel test processes both try to create the same Postgres types.
|
||||
func migrateOrIgnoreTypeExists(models ...interface{}) {
|
||||
err := DB().AutoMigrate(models...)
|
||||
if err != nil && !isTypeExistsError(err) {
|
||||
panic(err)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("migrate: ignoring type-exists error (expected during parallel test runs): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ensureSingleCurrentEventInvariant pins cleanup and index creation to one
|
||||
// transaction/connection. The global migration lock is session-scoped through a
|
||||
// pool, so it is not sufficient for this multi-statement invariant by itself.
|
||||
func ensureSingleCurrentEventInvariant() error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", int64(1234567892)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`WITH ranked AS (
|
||||
SELECT id, row_number() OVER (PARTITION BY monitor_id ORDER BY start_time DESC NULLS LAST, id DESC) AS n
|
||||
FROM events WHERE state = 'current'
|
||||
) UPDATE events SET state = 'ended', end_time = COALESCE(end_time, now())
|
||||
FROM ranked WHERE events.id = ranked.id AND ranked.n > 1`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS events_one_current_per_monitor
|
||||
ON events (monitor_id) WHERE state = 'current'`).Error
|
||||
})
|
||||
}
|
||||
|
||||
// Migrate run db migration
|
||||
var migrateMu sync.Mutex
|
||||
|
||||
func Migrate() {
|
||||
migrateMu.Lock()
|
||||
defer migrateMu.Unlock()
|
||||
withMigrationAdvisoryLock(migrateLocked)
|
||||
}
|
||||
|
||||
// withMigrationAdvisoryLock holds the session lock on a dedicated connection
|
||||
// while migration work uses GORM's normal pool. Reusing the lock connection
|
||||
// for GORM transactions can leave its *sql.Conn closed after commit.
|
||||
func withMigrationAdvisoryLock(migrate func()) {
|
||||
const migrateAdvisoryLock = int64(1234567890)
|
||||
sqlDB, err := DB().DB()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("migrate: database handle: %v", err))
|
||||
}
|
||||
ctx := context.Background()
|
||||
conn, err := sqlDB.Conn(ctx)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("migrate: lock connection: %v", err))
|
||||
}
|
||||
defer conn.Close() //nolint:errcheck // closing releases the session lock after a migration panic
|
||||
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", migrateAdvisoryLock); err != nil {
|
||||
panic(fmt.Sprintf("migrate: advisory lock: %v", err))
|
||||
}
|
||||
unlocked := false
|
||||
defer func() {
|
||||
if unlocked {
|
||||
return
|
||||
}
|
||||
if _, unlockErr := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); unlockErr != nil {
|
||||
log.Printf("migrate: unlock after failure: %v", unlockErr)
|
||||
}
|
||||
}()
|
||||
|
||||
migrate()
|
||||
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); err != nil {
|
||||
panic(fmt.Sprintf("migrate: unlock: %v", err))
|
||||
}
|
||||
unlocked = true
|
||||
}
|
||||
|
||||
func migrateLocked() {
|
||||
// M0 replaces the old flat plans table without rewriting historical rows.
|
||||
// Rename before AutoMigrate so GORM creates the canonical table rather than
|
||||
// adding columns to the incompatible legacy schema.
|
||||
if err := prepareCanonicalPlansTable(); err != nil {
|
||||
panic(fmt.Sprintf("migrate: prepare canonical plans: %v", err))
|
||||
}
|
||||
|
||||
// Step 0: create inventory Postgres enum types FIRST. The DO/EXCEPTION
|
||||
// blocks are idempotent so concurrent migrateOrIgnoreTypeExists
|
||||
// reruns from parallel test binaries are safe (the type already
|
||||
// exists → duplicate_object is swallowed). The enum types MUST
|
||||
// exist before any AutoMigrate below because GORM emits
|
||||
// `kind server_kind` literals in CREATE TABLE for the Server
|
||||
// struct (referenced transitively from Monitor.Site → Site → Server).
|
||||
for _, ddl := range []string{
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE server_kind AS ENUM ('production','staging','old');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_kind AS ENUM
|
||||
('production','production_prev','production_next','internal','staging','old');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_mode AS ENUM
|
||||
('kubernetes','compose','dedicated','vds','user');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_action AS ENUM
|
||||
('ok','pending','pending_move','pending_drop','deleted','missing');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
} {
|
||||
if err := DB().Exec(ddl).Error; err != nil {
|
||||
panic(fmt.Sprintf("migrate: enum creation: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Step 1: Migrate core models (User, Plan, Account, ApiKey, AuthIdentity, Invite)
|
||||
// Note: Access is moved to Step 2 because it has FKs to groups/monitors
|
||||
// Seed and remap plans before Account migration recreates its plan FK.
|
||||
migrateOrIgnoreTypeExists(&Plan{})
|
||||
if err := seedCanonicalPlansAndBackfill(true); err != nil {
|
||||
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
|
||||
}
|
||||
migrateOrIgnoreTypeExists(
|
||||
&User{},
|
||||
&Account{},
|
||||
&ApiKey{},
|
||||
&authidentity.AuthIdentity{}, // After User (has FK to users)
|
||||
&Invite{}, // After Account/User (has FKs to accounts/users)
|
||||
&Subscription{}, // After Account/Plan
|
||||
&SubscriptionEvent{}, // After Subscription
|
||||
)
|
||||
|
||||
// Monitor and metric rows reference worker_nodes, while worker_nodes itself
|
||||
// references servers. Create the two roots without their associations before
|
||||
// migrating Monitor/ServerMetric on a fresh database.
|
||||
migrateOrIgnoreTypeExists(&Region{}, &LLM{})
|
||||
if err = DB().Omit("Monitors", "Workers").AutoMigrate(&Server{}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
migrateOrIgnoreTypeExists(&WorkerNode{})
|
||||
|
||||
// Step 2: Migrate Group, Monitor, and Access (which has FKs to groups/monitors)
|
||||
// This ensures the groups table exists when GORM creates foreign keys
|
||||
migrateOrIgnoreTypeExists(
|
||||
&Group{},
|
||||
&Monitor{},
|
||||
&Check{},
|
||||
// Server is a customer-facing logical host, distinct from the
|
||||
// WorkerNode executor. Keep the join/cache models here so a fresh
|
||||
// database gets the complete server metrics schema in one migration.
|
||||
&Server{},
|
||||
&AccountMCPToken{},
|
||||
&MonitorServer{},
|
||||
&ServerMetric{},
|
||||
&ServerAlertRule{},
|
||||
&ServerAlertEvent{},
|
||||
&RknIP{},
|
||||
&RknDomain{},
|
||||
&DNSRecord{},
|
||||
&Contact{},
|
||||
&Whois{},
|
||||
&Payment{},
|
||||
&Message{},
|
||||
&TelegramBotMessage{},
|
||||
&TelegramBotStatus{},
|
||||
&Event{},
|
||||
&SelfCheck{},
|
||||
&Notification{}, // After Group/Monitor so notification_groups FK works
|
||||
&Access{}, // After Group/Monitor so access FKs work
|
||||
&NotificationCredential{}, // No FKs to other domain tables; safe here.
|
||||
)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS servers_account_slug_unique ON servers (account_id, slug)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS monitor_servers_position_idx ON monitor_servers (server_id, position)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_server_recent_idx ON server_metrics (server_id, id DESC)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_worker_idx ON server_metrics (worker_id)`)
|
||||
|
||||
// Step 3: Clean up orphaned references (now that all tables exist)
|
||||
// Fix typo in old table name (only present on DBs migrated from older versions)
|
||||
err = DB().Exec("ALTER TABLE IF EXISTS envent_checks RENAME TO event_checks;").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec("DROP TABLE IF EXISTS envent_messages;").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
// Defense-in-depth: make sure the columns that the in-process notifier
|
||||
// scheduler eagerly queries at startup exist, even if AutoMigrate above
|
||||
// was skipped or the column was dropped by a manual operation. Without
|
||||
// these, a fresh restore from a pre-soft-delete production dump will
|
||||
// panic the first time RunExp preloads Contacts or
|
||||
// ProcessPendingDeletions queries Users (see internal/notifier for the
|
||||
// defensive recover() that catches the resulting query errors).
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT TRUE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS disabled BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS blocked BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE users ADD COLUMN IF NOT EXISTS deletion_requested_at TIMESTAMPTZ",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS webhook_token VARCHAR(128)",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
var telegramCreds []NotificationCredential
|
||||
if err = DB().Where("kind = ? AND (webhook_token IS NULL OR webhook_token = '')", CredentialKindTelegram).Find(&telegramCreds).Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
for i := range telegramCreds {
|
||||
telegramCreds[i].EnsureWebhookToken()
|
||||
if err = DB().Save(&telegramCreds[i]).Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
err = DB().Exec("DROP INDEX IF EXISTS idx_notification_credentials_webhook_token").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_credentials_webhook_token ON notification_credentials (webhook_token) WHERE webhook_token IS NOT NULL AND webhook_token <> ''",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
// Replace the early M0 per-event-id index with provider-scoped webhook
|
||||
// idempotency: PSP event IDs are only unique inside a provider.
|
||||
if err = DB().Exec("DROP INDEX IF EXISTS idx_subscription_events_provider_event_id").Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if err = DB().Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_subscription_events_provider_event ON subscription_events (provider, provider_event_id) WHERE provider_event_id IS NOT NULL").Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned event_checks
|
||||
err = DB().Exec("DELETE FROM event_checks where event_id NOT IN (select id from events)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM event_checks where check_id NOT IN (select id from checks)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned event_messages
|
||||
err = DB().Exec("DELETE FROM event_messages where event_id NOT IN (select id from events)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM event_messages where message_id NOT IN (select id from messages)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned notification_contacts
|
||||
err = DB().Exec("DELETE FROM notification_contacts where notification_id NOT IN (select id from notifications)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM notification_contacts where contact_id NOT IN (select id from contacts)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned notification_groups (now safe - groups table exists)
|
||||
err = DB().Exec("DELETE FROM notification_groups where notification_id NOT IN (select id from notifications)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM notification_groups where group_id NOT IN (select id from groups)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS not_old_events on events (monitor_id, id) where state != 'old'")
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS current_events ON event (monitor_id, start_time) WHERE state = 'current'")
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS ended_events ON event (monitor_id, start_time) WHERE state = 'ended'")
|
||||
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS queued_messages ON message (id) WHERE state = 'queued'")
|
||||
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, account_id)")
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, group_id)")
|
||||
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS invite_email ON invites (account_id, email)")
|
||||
|
||||
// RKN indexes — see app/models/rkn_ip.go EnsureRknIndexes. GORM
|
||||
// AutoMigrate above declared the uniqueIndex on RknDomain.Domain
|
||||
// and the cidr column type on RknIP, but GiST on rkn_ips.network
|
||||
// is not expressible via the GORM tag language; we add it here so
|
||||
// the (>>=) containment operator used by IsRknIPBlocked has an
|
||||
// index to back it.
|
||||
if err := EnsureRknIndexes(); err != nil {
|
||||
log.Printf("migrate: EnsureRknIndexes failed: %v", err)
|
||||
}
|
||||
|
||||
// Distributed worker models
|
||||
migrateOrIgnoreTypeExists(
|
||||
&WorkerLogEvent{},
|
||||
&CheckAttempt{},
|
||||
&DiagnosticAuditEvent{},
|
||||
&CheckRegionResult{},
|
||||
&Task{},
|
||||
&TaskReplay{},
|
||||
&NotificationDelivery{},
|
||||
&Tag{},
|
||||
)
|
||||
|
||||
// Tags — (account_id, name) is the unique key so a single account
|
||||
// cannot register two metadata rows for the same tag string. The
|
||||
// tag name itself is also the join key against monitors.tags, so
|
||||
// uniqueness is enforced at the table level (not just on the
|
||||
// metadata row).
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tags_account_name_unique ON tags (account_id, name)`)
|
||||
|
||||
// Status pages use globally unique public slugs. A public URL has no
|
||||
// account component, so account-scoped uniqueness would make /status/:slug
|
||||
// ambiguous. Soft-deleted rows do not reserve their slug.
|
||||
// subscriber email index uses lower(email) for case-insensitive
|
||||
// matching (the codebase does not adopt citext). All five tables
|
||||
// are created together so M0 ships a consistent schema baseline
|
||||
// regardless of which milestone first writes rows.
|
||||
migrateOrIgnoreTypeExists(
|
||||
&StatusPage{},
|
||||
&StatusPageSubscriber{},
|
||||
&StatusPageIncident{},
|
||||
&StatusPageMaintenance{},
|
||||
&StatusPageDomain{},
|
||||
&StatusPageDelivery{},
|
||||
&StatusPageDigestSchedule{},
|
||||
&Maintenance{},
|
||||
)
|
||||
DB().Exec(`DROP INDEX IF EXISTS status_pages_account_slug_unique`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_pages_slug_unique
|
||||
ON status_pages (slug) WHERE deleted_at IS NULL`)
|
||||
if err := ensureSingleCurrentEventInvariant(); err != nil {
|
||||
panic(fmt.Sprintf("migrate: current event invariant: %v", err))
|
||||
}
|
||||
DB().Exec(
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS status_page_subscribers_active_email
|
||||
ON status_page_subscribers (status_page_id, lower(email))
|
||||
WHERE unsubscribed_at IS NULL`,
|
||||
)
|
||||
// Existing installations can already have subscriber rows. Keep the legacy
|
||||
// token column during the nullable transition: outstanding links remain
|
||||
// valid, while each resend/confirmation rotates it into a hash.
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS confirm_token_hash varchar(64)`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS token_expires_at timestamptz`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS unsubscribe_token_hash varchar(64)`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS contact_id bigint REFERENCES contacts(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`UPDATE status_page_subscribers SET token_expires_at = created_at + interval '24 hours' WHERE token_expires_at IS NULL`)
|
||||
for {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := DB().Where("confirm_token_hash IS NULL AND confirm_token IS NOT NULL").Limit(500).Find(&subscribers).Error; err != nil || len(subscribers) == 0 {
|
||||
break
|
||||
}
|
||||
for i := range subscribers {
|
||||
subscribers[i].ConfirmTokenHash = HashStatusPageToken(*subscribers[i].LegacyConfirmToken)
|
||||
_ = DB().Model(&subscribers[i]).Update("confirm_token_hash", subscribers[i].ConfirmTokenHash).Error
|
||||
}
|
||||
}
|
||||
DB().Exec(`UPDATE status_page_subscribers SET confirm_token_hash = '' WHERE confirm_token_hash IS NULL`)
|
||||
DB().Exec(`UPDATE status_page_subscribers SET unsubscribe_token_hash = '' WHERE unsubscribe_token_hash IS NULL`)
|
||||
// A page may expose more than one verified hostname; older schema reserved
|
||||
// only one domain per page.
|
||||
DB().Exec(`DROP INDEX IF EXISTS idx_status_page_domains_status_page_id`)
|
||||
DB().Exec(
|
||||
`CREATE INDEX IF NOT EXISTS status_page_incidents_started
|
||||
ON status_page_incidents (status_page_id, started_at DESC)`,
|
||||
)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_page_incidents_event_unique
|
||||
ON status_page_incidents (status_page_id, event_id) WHERE event_id IS NOT NULL`)
|
||||
DB().Exec(
|
||||
`CREATE INDEX IF NOT EXISTS status_page_maintenance_starts
|
||||
ON status_page_maintenance (status_page_id, starts_at DESC)`,
|
||||
)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS maintenances_account_active ON maintenances (account_id) WHERE deleted_at IS NULL`)
|
||||
// Legacy status-page rows predate account-scoped maintenance. Keep their
|
||||
// source IDs so equal title/start rows remain distinct and reruns can safely
|
||||
// preserve every window and each of its joins.
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS maintenances_legacy_status_page_maintenance_unique
|
||||
ON maintenances (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL`)
|
||||
DB().Exec(`INSERT INTO maintenances (account_id, title, description, strategy, duration_sec, start_date, end_date, timezone, active, show_on_all_status_pages, legacy_status_page_maintenance_id, created_at, updated_at)
|
||||
SELECT sp.account_id, old.title, old.description, 'single', EXTRACT(EPOCH FROM (old.ends_at - old.starts_at))::int, old.starts_at, old.ends_at, 'UTC', TRUE, FALSE, old.id, old.created_at, old.updated_at
|
||||
FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id
|
||||
ON CONFLICT (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL DO NOTHING`)
|
||||
DB().Exec(`INSERT INTO maintenance_status_pages (maintenance_id, status_page_id)
|
||||
SELECT m.id, old.status_page_id FROM status_page_maintenance old JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id
|
||||
ON CONFLICT DO NOTHING`)
|
||||
DB().Exec(`INSERT INTO maintenance_monitors (maintenance_id, monitor_id)
|
||||
SELECT m.id, monitor_id FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id,
|
||||
LATERAL unnest(CASE WHEN cardinality(old.monitor_ids) > 0 THEN old.monitor_ids ELSE sp.monitor_ids END) AS monitor_id
|
||||
ON CONFLICT DO NOTHING`)
|
||||
|
||||
// Worker-driven task queue indexes.
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_queued_due ON tasks (kind, not_before) WHERE state = 'queued'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_leased_expires ON tasks (lease_expires_at) WHERE state = 'leased'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_failed_retry_due ON tasks (not_before) WHERE state = 'failed_retry'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_dead_kind ON tasks (kind, account_id) WHERE state = 'dead'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS check_attempts_worker_finished ON check_attempts (worker_node_id, finished_at DESC)`)
|
||||
err = DB().Exec("ALTER TABLE worker_nodes ALTER COLUMN concurrency SET DEFAULT 10").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET DEFAULT '{}'::jsonb").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("UPDATE checks SET settings = '{}'::jsonb WHERE settings IS NULL").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET NOT NULL").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Inventory models (docs/plans/inventory-management.md §6, §10 M0).
|
||||
// Enum types are created at Step 0 above so they exist before any
|
||||
// AutoMigrate. The numeric labels of each enum value are taken
|
||||
// verbatim from rstuff (`/data/int/rstuff/app/models/*.rb`) so a
|
||||
// future sync layer does not need a value-mapping table — see
|
||||
// docs/parity/rstuff-inventory.md §6.1.
|
||||
|
||||
// Servers — extend with the rstuff-shaped inventory fields. All
|
||||
// statements are IF NOT EXISTS so existing rows keep working
|
||||
// untouched (ext_id/token/price_cents default sensibly; meta gets
|
||||
// an empty jsonb).
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS ext_id VARCHAR(64)`)
|
||||
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_ext_id_unique ON servers (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS kind server_kind NOT NULL DEFAULT 'production'`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS token VARCHAR(64)`)
|
||||
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_token_unique ON servers (token) WHERE token IS NOT NULL`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS price_cents INTEGER NOT NULL DEFAULT 0`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS comment TEXT`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS meta JSONB NOT NULL DEFAULT '{}'::jsonb`)
|
||||
|
||||
// WorkerNodes — optional server_id join for inventory correlation.
|
||||
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS server_id BIGINT REFERENCES servers(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_server_idx ON worker_nodes (server_id)`)
|
||||
|
||||
// WorkerNodes — optional account_id FK marking a private (customer-
|
||||
// operated) worker per docs/distributed/private-workers.md. NULL
|
||||
// rows are platform-operated workers eligible to serve any account;
|
||||
// non-NULL rows are pinned to a single account and are removed by
|
||||
// HardDeleteAccount before the account row itself is dropped.
|
||||
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_account_idx ON worker_nodes (account_id)`)
|
||||
|
||||
// Notification credentials are either platform-managed (account_id IS NULL)
|
||||
// or owned by one account. Replace the legacy global name constraint with
|
||||
// scope-aware unique indexes.
|
||||
DB().Exec(`ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE CASCADE`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS notification_credentials_account_idx ON notification_credentials (account_id)`)
|
||||
DB().Exec(`DROP INDEX IF EXISTS cred_kind_name`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_system_kind_name_unique ON notification_credentials (kind, name) WHERE account_id IS NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_account_kind_name_unique ON notification_credentials (account_id, kind, name) WHERE account_id IS NOT NULL`)
|
||||
|
||||
// Inventory entities. Order matters: server_ips before sites
|
||||
// (FK), sites before deployments (FK), deployments before
|
||||
// domains (FK). Audit columns (creator_id/updater_id) use the
|
||||
// Audited mixin via concerns.Timestamped + Audited.
|
||||
migrateOrIgnoreTypeExists(
|
||||
&ServerIp{},
|
||||
&Repo{},
|
||||
&Site{},
|
||||
&Deployment{},
|
||||
&SiteRepo{},
|
||||
&Domain{},
|
||||
)
|
||||
// Monitors is migrated earlier for historical FK ordering. Add the optional
|
||||
// site reference only after sites exists on fresh databases.
|
||||
DB().Exec(`ALTER TABLE monitors ADD COLUMN IF NOT EXISTS site_id BIGINT REFERENCES sites(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS monitors_site_idx ON monitors (site_id)`)
|
||||
|
||||
// Indexes — kept here so the AutoMigrate path stays the single
|
||||
// source of truth. IF NOT EXISTS guards the rerun case.
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_ips_server_idx ON server_ips (server_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS server_ips_address_unique ON server_ips (server_id, address)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_account_idx ON sites (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_server_idx ON sites (server_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_ext_id_unique ON sites (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_account_slug_unique ON sites (account_id, slug)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_ext_id_unique ON repos (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_gitlab_id_unique ON repos (gitlab_id) WHERE gitlab_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_account_idx ON deployments (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_server_idx ON deployments (server_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_site_idx ON deployments (site_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_ext_id_unique ON deployments (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_account_idx ON domains (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_server_idx ON domains (server_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_site_idx ON domains (site_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS domains_name_unique ON domains (name)`)
|
||||
// Dedupe by (server_id, config_path) for nginx sites and by
|
||||
// (server_id, site_id, service_name) for compose services — matches
|
||||
// the upsert keys in app/controllers/api/inventory_deploymentd.go.
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_nginx
|
||||
ON deployments (server_id, config_path) WHERE mode = 'dedicated' AND config_path IS NOT NULL`)
|
||||
DB().Exec(`DROP INDEX IF EXISTS deployments_unique_compose`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_compose
|
||||
ON deployments (server_id, site_id, service_name) WHERE mode = 'compose' AND site_id IS NOT NULL AND service_name IS NOT NULL`)
|
||||
|
||||
if err := seedCanonicalPlansAndBackfill(false); err != nil {
|
||||
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
|
||||
}
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS plans_active_code_unique ON plans (code) WHERE archived = FALSE`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS subscriptions_provider_external_unique ON subscriptions (provider, provider_subscription_id) WHERE provider_subscription_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS subscriptions_account_status_idx ON subscriptions (account_id, status)`)
|
||||
DB().Exec(`DO $$ BEGIN
|
||||
ALTER TABLE plans ADD CONSTRAINT plans_price_nonnegative CHECK (price_monthly_minor >= 0 AND price_annual_minor >= 0);
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
|
||||
DB().Exec(`DO $$ BEGIN
|
||||
ALTER TABLE plans ADD CONSTRAINT plans_limits_valid CHECK (monitor_cap >= 0 AND interval_min_seconds >= 30);
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
|
||||
|
||||
// Seed default region. Use defaultRegionCode (declared in check_jobs.go)
|
||||
// so the literal does not appear three times in the package.
|
||||
localRegion := Region{}
|
||||
DB().Where("code = ?", defaultRegionCode).First(&localRegion)
|
||||
if localRegion.ID == 0 {
|
||||
DB().Create(&Region{Code: defaultRegionCode, Name: "Local (default)", Enabled: true, Priority: 100})
|
||||
}
|
||||
|
||||
DB().Exec(`
|
||||
WITH t AS (
|
||||
select u.id as user_id, i.encrypted_password as encrypted_password
|
||||
from users as u
|
||||
join identities as i on u.id = i.user_id
|
||||
where u.encrypted_password is NULL
|
||||
)
|
||||
UPDATE users
|
||||
SET encrypted_password = t.encrypted_password
|
||||
from t
|
||||
where users.id = t.user_id
|
||||
`)
|
||||
|
||||
log.Println("migrated DB.")
|
||||
}
|
||||
|
||||
func prepareCanonicalPlansTable() error {
|
||||
if !DB().Migrator().HasTable("plans") || DB().Migrator().HasColumn("plans", "code") {
|
||||
return nil
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return tx.Exec(`ALTER TABLE plans RENAME TO plans_legacy`).Error
|
||||
})
|
||||
}
|
||||
|
||||
func planForeignKeyReferences(tx *gorm.DB, table, target string) (bool, error) {
|
||||
var references bool
|
||||
err := tx.Raw(`SELECT EXISTS (
|
||||
SELECT 1 FROM pg_constraint c
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.contype = 'f' AND c.conrelid = to_regclass(?)
|
||||
AND c.confrelid = to_regclass(?) AND a.attname = 'plan_id'
|
||||
)`, table, target).Scan(&references).Error
|
||||
return references, err
|
||||
}
|
||||
|
||||
func seedCanonicalPlansAndBackfill(remapAccounts bool) error {
|
||||
plans := []Plan{
|
||||
{Code: "free", NameRU: "Бесплатный", NameEN: "Free", Currency: "RUB", MonitorCap: 50, IntervalMinSeconds: 300, StatusPagesCap: 1, MaintenanceCap: -1, LoginSeatsIncluded: 3, NotifySeatsIncluded: 0, Integrations: []string{"email", "telegram"}, CheckKinds: []string{"http", "ssl", "dns", "whois", "ping"}, DataRetentionMonths: 3, IsDefault: true},
|
||||
{Code: "solo", NameRU: "Соло", NameEN: "Solo", PriceMonthlyMinor: 74900, PriceAnnualMinor: 64900, Currency: "RUB", MonitorCap: 10, IntervalMinSeconds: 60, StatusPagesCap: 3, MaintenanceCap: 5, LoginSeatsIncluded: 5, NotifySeatsIncluded: 3, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping"}, DataRetentionMonths: 12, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 60},
|
||||
{Code: "team", NameRU: "Команда", NameEN: "Team", PriceMonthlyMinor: 299000, PriceAnnualMinor: 254900, Currency: "RUB", MonitorCap: 100, IntervalMinSeconds: 60, StatusPagesCap: 100, MaintenanceCap: 50, LoginSeatsIncluded: 5, NotifySeatsIncluded: 5, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 24, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 45},
|
||||
{Code: "enterprise", NameRU: "Предприятие", NameEN: "Enterprise", PriceMonthlyMinor: 549000, PriceAnnualMinor: 464900, Currency: "RUB", MonitorCap: 200, IntervalMinSeconds: 30, StatusPagesCap: 0, MaintenanceCap: 0, LoginSeatsIncluded: 0, NotifySeatsIncluded: 0, UnlimitedSeats: true, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook", "sso_saml"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 36, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 30, SOC2: true, GDPRDPA: true},
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
for i := range plans {
|
||||
var existing Plan
|
||||
err := tx.Where("code = ?", plans[i].Code).First(&existing).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
if err := tx.Create(&plans[i]).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Catalog rows already existed before maintenance_cap was introduced.
|
||||
// Update this new entitlement only; do not overwrite customer-adjusted
|
||||
// prices or other existing plan attributes during a normal migration.
|
||||
if err := tx.Exec(`UPDATE plans SET maintenance_cap = CASE code
|
||||
WHEN 'free' THEN -1 WHEN 'solo' THEN 5 WHEN 'team' THEN 50 WHEN 'enterprise' THEN 0 ELSE maintenance_cap END
|
||||
WHERE code IN ('free', 'solo', 'team', 'enterprise')`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
hasLegacy := tx.Migrator().HasTable("plans_legacy")
|
||||
hasMigratedLegacy := tx.Migrator().HasTable("plans_legacy_migrated")
|
||||
if !hasLegacy && !hasMigratedLegacy {
|
||||
return nil
|
||||
}
|
||||
hasAccounts := tx.Migrator().HasTable("accounts")
|
||||
accountsNeedRemap := false
|
||||
if remapAccounts && hasLegacy && hasAccounts {
|
||||
referencesCanonical, err := planForeignKeyReferences(tx, "accounts", "plans")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
accountsNeedRemap = !referencesCanonical
|
||||
if accountsNeedRemap {
|
||||
if err := tx.Exec(`UPDATE accounts SET plan_id = CASE
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
hasSubscriptions := tx.Migrator().HasTable("subscriptions")
|
||||
subscriptionsNeedRemap := false
|
||||
if remapAccounts && hasLegacy && hasSubscriptions {
|
||||
referencesCanonical, err := planForeignKeyReferences(tx, "subscriptions", "plans")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subscriptionsNeedRemap = !referencesCanonical
|
||||
if subscriptionsNeedRemap {
|
||||
if err := tx.Exec(`UPDATE subscriptions SET plan_id = CASE
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if remapAccounts && hasLegacy && (accountsNeedRemap || subscriptionsNeedRemap) && tx.Migrator().HasTable("subscription_events") {
|
||||
if err := tx.Exec(`UPDATE subscription_events e SET from_plan_id = CASE
|
||||
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
FROM plans_legacy l WHERE e.from_plan_id = l.id`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`UPDATE subscription_events e SET to_plan_id = CASE
|
||||
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
FROM plans_legacy l WHERE e.to_plan_id = l.id`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !remapAccounts && hasAccounts && hasSubscriptions {
|
||||
now := time.Now().UTC()
|
||||
if err := tx.Exec(`INSERT INTO subscriptions (account_id, plan_id, provider, status, billing_cycle, current_period_start, current_period_end, currency, amount_minor, metadata_json, created_at, updated_at)
|
||||
SELECT a.id, a.plan_id, 'manual', 'active', 'monthly', ?, ?, p.currency, p.price_monthly_minor, '{}'::jsonb, ?, ?
|
||||
FROM accounts a JOIN plans p ON p.id = a.plan_id
|
||||
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.account_id = a.id)`, now, now.AddDate(0, 0, 30), now, now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !remapAccounts || !hasLegacy {
|
||||
return nil
|
||||
}
|
||||
if hasAccounts {
|
||||
// AutoMigrate names this association fk_accounts_plan, while an older
|
||||
// migration used accounts_plan_id_fkey. Either name may still point at
|
||||
// plans_legacy after the table rename, so replace both deterministically.
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts ADD CONSTRAINT fk_accounts_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if hasSubscriptions {
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions ADD CONSTRAINT fk_subscriptions_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Exec("ALTER TABLE plans_legacy RENAME TO plans_legacy_migrated").Error
|
||||
})
|
||||
}
|
||||
162
app/models/migrate_test.go
Обычный файл
162
app/models/migrate_test.go
Обычный файл
@@ -0,0 +1,162 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestMigrateUpgradesLegacyPlansWithoutClosingLockConnection(t *testing.T) {
|
||||
var databaseName string
|
||||
require.NoError(t, DB().Raw("SELECT current_database()").Scan(&databaseName).Error)
|
||||
require.Contains(t, databaseName, "test")
|
||||
|
||||
original := db
|
||||
schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
require.NoError(t, original.Exec("CREATE SCHEMA "+schema).Error)
|
||||
var isolatedSQLDB interface{ Close() error }
|
||||
t.Cleanup(func() {
|
||||
SetDB(original)
|
||||
if isolatedSQLDB != nil {
|
||||
_ = isolatedSQLDB.Close()
|
||||
}
|
||||
original.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE")
|
||||
})
|
||||
|
||||
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable search_path=%s",
|
||||
testDatabaseEnv("DATABASE_HOST", "POSTGRES_HOST", "localhost"),
|
||||
testDatabaseEnv("DATABASE_PORT", "POSTGRES_PORT", "5432"),
|
||||
testDatabaseEnv("DATABASE_USER", "POSTGRES_USER", "rsmon"),
|
||||
testDatabaseEnv("DATABASE_PASSWORD", "POSTGRES_PASSWORD", "rsmon"),
|
||||
databaseName,
|
||||
schema,
|
||||
)
|
||||
isolated, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
sqlDB, err := isolated.DB()
|
||||
require.NoError(t, err)
|
||||
isolatedSQLDB = sqlDB
|
||||
RegisterCallbacks(isolated)
|
||||
SetDB(isolated.Set("gorm:association_autoupdate", false))
|
||||
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE plans (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
price BIGINT NOT NULL DEFAULT 0,
|
||||
total_monitors BIGINT NOT NULL DEFAULT 0,
|
||||
"default" BOOLEAN NOT NULL DEFAULT FALSE
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE accounts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
plan_id BIGINT REFERENCES plans(id)
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE subscriptions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
account_id BIGINT NOT NULL REFERENCES accounts(id),
|
||||
plan_id BIGINT NOT NULL REFERENCES plans(id),
|
||||
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
|
||||
status VARCHAR(24) NOT NULL DEFAULT 'active',
|
||||
billing_cycle VARCHAR(8) NOT NULL DEFAULT 'monthly',
|
||||
currency VARCHAR(3) NOT NULL DEFAULT 'RUB',
|
||||
amount_minor BIGINT NOT NULL DEFAULT 0,
|
||||
metadata_json JSONB NOT NULL DEFAULT '{}'
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE subscription_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
subscription_id BIGINT NOT NULL,
|
||||
account_id BIGINT NOT NULL,
|
||||
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
|
||||
kind VARCHAR(32) NOT NULL,
|
||||
from_plan_id BIGINT,
|
||||
to_plan_id BIGINT,
|
||||
payload_json JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`INSERT INTO plans (id, name, price, total_monitors, "default")
|
||||
VALUES (42, 'Legacy free', 0, 10, TRUE)`).Error)
|
||||
require.NoError(t, DB().Exec("INSERT INTO accounts (id, name, plan_id) VALUES (7, 'Legacy account', 42)").Error)
|
||||
require.NoError(t, DB().Exec("INSERT INTO subscriptions (id, account_id, plan_id) VALUES (9, 7, 42)").Error)
|
||||
require.NoError(t, DB().Exec(`INSERT INTO subscription_events
|
||||
(id, subscription_id, account_id, kind, from_plan_id, to_plan_id)
|
||||
VALUES (11, 9, 7, 'legacy_change', 42, 42)`).Error)
|
||||
|
||||
Migrate()
|
||||
|
||||
require.True(t, DB().Migrator().HasColumn("plans", "code"))
|
||||
require.False(t, DB().Migrator().HasTable("plans_legacy"))
|
||||
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
|
||||
var codes []string
|
||||
require.NoError(t, DB().Table("plans").Order("code").Pluck("code", &codes).Error)
|
||||
require.ElementsMatch(t, CanonicalPlanCodes(), codes)
|
||||
var accountPlanCode string
|
||||
require.NoError(t, DB().Table("accounts").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
|
||||
Scan(&accountPlanCode).Error)
|
||||
require.Equal(t, "free", accountPlanCode)
|
||||
var subscriptionPlanCode string
|
||||
require.NoError(t, DB().Table("subscriptions").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = subscriptions.plan_id").Where("subscriptions.id = 9").
|
||||
Scan(&subscriptionPlanCode).Error)
|
||||
require.Equal(t, "free", subscriptionPlanCode)
|
||||
var eventPlanCodes struct {
|
||||
FromCode string
|
||||
ToCode string
|
||||
}
|
||||
require.NoError(t, DB().Table("subscription_events e").
|
||||
Select("fp.code AS from_code, tp.code AS to_code").
|
||||
Joins("JOIN plans fp ON fp.id = e.from_plan_id").
|
||||
Joins("JOIN plans tp ON tp.id = e.to_plan_id").
|
||||
Where("e.id = 11").Scan(&eventPlanCodes).Error)
|
||||
require.Equal(t, "free", eventPlanCodes.FromCode)
|
||||
require.Equal(t, "free", eventPlanCodes.ToCode)
|
||||
require.NoError(t, DB().Exec("SELECT 1").Error)
|
||||
|
||||
// The previous implementation retained this name after remapping. Its
|
||||
// canonical FKs must prevent a retry from remapping the same rows again.
|
||||
require.NoError(t, DB().Exec("ALTER TABLE plans_legacy_migrated RENAME TO plans_legacy").Error)
|
||||
require.NotPanics(t, Migrate)
|
||||
accountPlanCode = ""
|
||||
require.NoError(t, DB().Table("accounts").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
|
||||
Scan(&accountPlanCode).Error)
|
||||
require.Equal(t, "free", accountPlanCode)
|
||||
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
|
||||
}
|
||||
|
||||
func TestMigrationAdvisoryLockIsReleasedAfterPanic(t *testing.T) {
|
||||
sqlDB, err := DB().DB()
|
||||
require.NoError(t, err)
|
||||
otherConn, err := sqlDB.Conn(context.Background())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = otherConn.Close() })
|
||||
|
||||
require.PanicsWithValue(t, "migration failed", func() {
|
||||
withMigrationAdvisoryLock(func() { panic("migration failed") })
|
||||
})
|
||||
|
||||
const migrateAdvisoryLock = int64(1234567890)
|
||||
var acquired bool
|
||||
require.NoError(t, otherConn.QueryRowContext(context.Background(),
|
||||
"SELECT pg_try_advisory_lock($1)", migrateAdvisoryLock).Scan(&acquired))
|
||||
require.True(t, acquired)
|
||||
_, err = otherConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func testDatabaseEnv(primary, fallback, defaultValue string) string {
|
||||
if value := os.Getenv(primary); value != "" {
|
||||
return value
|
||||
}
|
||||
if value := os.Getenv(fallback); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
534
app/models/monitor.go
Обычный файл
534
app/models/monitor.go
Обычный файл
@@ -0,0 +1,534 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
stateOK = "OK"
|
||||
stateERR = "ERR"
|
||||
stateWARN = "WARN"
|
||||
stateFail = "FAIL"
|
||||
stateDegraded = "DEGRADED"
|
||||
|
||||
// Event states.
|
||||
stateEnded = "ended"
|
||||
|
||||
// Check column names used in map[string]interface{} GORM updates. Defining
|
||||
// them as constants keeps GORM column references in sync with model fields.
|
||||
colLastStart = "last_start"
|
||||
colLastEnd = "last_end"
|
||||
colState = "state"
|
||||
colWarnings = "warnings"
|
||||
colInfos = "infos"
|
||||
|
||||
// Check kinds. Used to avoid sprinkling magic strings across the codebase.
|
||||
kindHTTP = "http"
|
||||
kindSSL = "ssl"
|
||||
kindSSH = "ssh"
|
||||
kindFTP = "ftp"
|
||||
kindDNS = "dns"
|
||||
kindWhois = "whois"
|
||||
kindRKN = "rkn"
|
||||
kindBSSL = "bssl"
|
||||
kindLLM = "llm"
|
||||
kindLLMHTTP = "llm-http"
|
||||
kindPing = "ping"
|
||||
kindTCP = "tcp"
|
||||
kindUDP = "udp"
|
||||
)
|
||||
|
||||
// Monitor monitor
|
||||
type Monitor struct {
|
||||
concerns.Model
|
||||
// activity status
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
|
||||
// check state, OK - all green, ERR - some checks have failed, UNK - new or not run, FAIL - unable to check
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
ConfirmState string `gorm:"size:32;not null;default:'none';index" json:"confirm_state"`
|
||||
ConfirmAt *time.Time `json:"confirm_at,omitempty"`
|
||||
ConfirmedByWorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"confirmed_by_worker_id,omitempty"`
|
||||
|
||||
// Group ID
|
||||
GroupID int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty" validate:"required"`
|
||||
Group *Group `json:"group,omitempty"`
|
||||
|
||||
// Optional inventory Site join (see docs/plans/inventory-management.md §6.3).
|
||||
// Lets the operator navigate monitor → site → deployments → server in one query.
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
|
||||
// Tags for monitor grouping/searching
|
||||
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
|
||||
|
||||
// PreferredRegions is the optional whitelist of region codes a distributed
|
||||
// worker must be in to receive this monitor's checks. Empty/NULL means
|
||||
// "no preference" — any worker can pick it up. Used by Phase 2 of
|
||||
// docs/plans/worker-notifier-mvp.md (regional job routing); see
|
||||
// app/models/check_jobs.go applyRegionRouting.
|
||||
PreferredRegions pq.StringArray `gorm:"type:varchar(64)[]" json:"preferred_regions,omitempty"`
|
||||
|
||||
// RegionMode controls how PreferredRegions is interpreted by the
|
||||
// distributed-worker job router. Defaults to "any" so monitors without
|
||||
// explicit routing still match every worker — backwards-compatible with
|
||||
// Phase 1 deployments.
|
||||
// "any" — no region filter; legacy behavior (default)
|
||||
// "specific" — only workers whose region_code is in PreferredRegions
|
||||
// "all" — Phase 3 placeholder; today behaves like "any". The
|
||||
// multi-region quorum aggregation is not implemented yet,
|
||||
// see docs/todo.md Phase 3.
|
||||
RegionMode string `gorm:"size:16;not null;default:'any'" json:"region_mode" validate:"omitempty,oneof=any specific all"`
|
||||
|
||||
// Monitor name
|
||||
Name *string `json:"name,omitempty"`
|
||||
|
||||
// Host to monitor
|
||||
Host string `json:"host" validate:"required"`
|
||||
|
||||
// UserID specify user for this monitor (info field)
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
|
||||
User *User `json:"user,omitempty"`
|
||||
|
||||
// Comment (info field)
|
||||
Comment *string `json:"comment"`
|
||||
|
||||
Checks []Check `json:"checks,omitempty"`
|
||||
DNSRecords []DNSRecord `json:"-"`
|
||||
|
||||
StatsData `gorm:"-:all" sql:"-" json:"stats"`
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// KINDS Check kinds
|
||||
var KINDS = []string{kindHTTP, kindSSL, kindSSH, kindFTP, kindDNS, kindWhois, kindRKN, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
|
||||
|
||||
// ValidCheckKind is the single canonical allow-list for user supplied check
|
||||
// kinds. Keep it beside the kind constants so every transport validates the
|
||||
// same set before a check reaches a worker.
|
||||
func ValidCheckKind(kind string) bool {
|
||||
for _, candidate := range KINDS {
|
||||
if kind == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Region routing mode constants used by Monitor.RegionMode and
|
||||
// app/models/check_jobs.go applyRegionRouting. Centralized so the literal
|
||||
// values are not sprinkled through the codebase.
|
||||
const (
|
||||
// RegionModeAny keeps the legacy behavior: every worker is eligible,
|
||||
// PreferredRegions is ignored. Default for newly-created monitors.
|
||||
RegionModeAny = "any"
|
||||
|
||||
// RegionModeSpecific limits eligible workers to those whose RegionCode
|
||||
// is contained in PreferredRegions. Empty PreferredRegions falls back to
|
||||
// RegionModeAny so the field is safe to leave blank in the UI.
|
||||
RegionModeSpecific = "specific"
|
||||
|
||||
// RegionModeAll is the Phase 3 placeholder: a monitor pinned to all of
|
||||
// its preferred regions for quorum aggregation. Phase 2 treats it as
|
||||
// RegionModeAny and logs a TODO marker so it is easy to grep for.
|
||||
RegionModeAll = "all"
|
||||
)
|
||||
|
||||
// RegionCodesFromSlice is a convenience wrapper so callers (mainly tests
|
||||
// and HTTP handlers) can pass a plain []string and get the pq.StringArray
|
||||
// type the model expects. nil/empty input is preserved as a nil slice so
|
||||
// the GORM column writes a SQL NULL instead of an empty array, matching
|
||||
// the column default.
|
||||
func RegionCodesFromSlice(in []string) pq.StringArray {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(pq.StringArray, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
// Int64ArrayFromSlice mirrors RegionCodesFromSlice for bigint[] columns
|
||||
// such as status_pages.monitor_ids and status_page_maintenance.monitor_ids.
|
||||
// The GORM pq.Int64Array driver expects a non-nil slice for ordered
|
||||
// inserts; callers that always have a non-empty list (the dashboard list
|
||||
// filter, the maintenance form) can rely on this to write a stable shape.
|
||||
func Int64ArrayFromSlice(in []int64) pq.Int64Array {
|
||||
if len(in) == 0 {
|
||||
return pq.Int64Array{}
|
||||
}
|
||||
out := make(pq.Int64Array, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
// ValidateRegionMode returns an error when RegionMode is not one of the
|
||||
// documented values ("any", "specific", "all"). Empty strings are treated as
|
||||
// "any" for backwards compatibility with monitors persisted before the field
|
||||
// existed; the DB column also defaults to "any".
|
||||
func (m *Monitor) ValidateRegionMode() error {
|
||||
switch m.RegionMode {
|
||||
case "", RegionModeAny, RegionModeSpecific, RegionModeAll:
|
||||
return nil
|
||||
default:
|
||||
return errors.Errorf("invalid region_mode %q (expected any|specific|all)", m.RegionMode)
|
||||
}
|
||||
}
|
||||
|
||||
// WantsRegion returns true when the monitor should be routed to a worker
|
||||
// operating in the given region code. Callers use this in
|
||||
// app/models/check_jobs.go to filter the eligible worker pool per check.
|
||||
//
|
||||
// - RegionModeAny: always true (no preference).
|
||||
// - RegionModeAll (Phase 3 placeholder): behaves like Any today; returns
|
||||
// true unconditionally so every region sees the check.
|
||||
// - RegionModeSpecific: true when code is contained in PreferredRegions,
|
||||
// or when PreferredRegions is empty (fall-back to Any).
|
||||
func (m *Monitor) WantsRegion(code string) bool {
|
||||
switch m.RegionMode {
|
||||
case RegionModeSpecific:
|
||||
if len(m.PreferredRegions) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, r := range m.PreferredRegions {
|
||||
if r == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
case RegionModeAll:
|
||||
// TODO(phase3): enumerate PreferredRegions and emit one assignment
|
||||
// per region so the result aggregator can do quorum. Today we
|
||||
// behave like Any so existing workers keep getting checks.
|
||||
return true
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// GetLabel provides functionality.
|
||||
func (m *Monitor) GetLabel() string {
|
||||
if m.Name != nil {
|
||||
return *m.Name
|
||||
}
|
||||
return m.Host
|
||||
}
|
||||
|
||||
// ProcessChecks provides functionality.
|
||||
func (m *Monitor) ProcessChecks(tx *gorm.DB) error {
|
||||
log.Println("process checks")
|
||||
checks := make([]Check, 0)
|
||||
for _, c := range m.Checks { //nolint:gocritic // range copy is acceptable here
|
||||
log.Println("maybe delete check", c.ID, c.Deleted, c.IsNew)
|
||||
if c.Deleted {
|
||||
if !c.IsNew {
|
||||
log.Println("delete check", c.ID)
|
||||
err := tx.Exec("delete from event_checks where check_id = ?", c.ID).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// First, find all message IDs for this check
|
||||
var messageIDs []int64
|
||||
err = tx.Model(&Message{}).Where("check_id = ?", c.ID).Pluck("id", &messageIDs).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete event_messages (join table) first to avoid FK constraint violation
|
||||
if len(messageIDs) > 0 {
|
||||
err = tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Now delete the messages
|
||||
err = tx.Where("check_id = ?", c.ID).Delete(Message{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.Where("id = ? AND monitor_id = ?", c.ID, m.ID).Delete(Check{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if c.IsNew {
|
||||
c.ID = 0
|
||||
}
|
||||
err := c.ValidateSettings()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "check validation error")
|
||||
}
|
||||
checks = append(checks, c)
|
||||
}
|
||||
m.Checks = checks
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveEvent provides functionality.
|
||||
func (m *Monitor) ActiveEvent() Event {
|
||||
evt := Event{}
|
||||
DB().Where("monitor_id = ? AND state != 'old'", m.ID).First(&evt)
|
||||
if evt.ID != 0 {
|
||||
evt.MonitorID = m.ID
|
||||
t := time.Now()
|
||||
evt.StartTime = &t
|
||||
}
|
||||
return evt
|
||||
}
|
||||
|
||||
var mutex sync.Mutex
|
||||
|
||||
// checkSeverityRank assigns an ordinal to each check state so the monitor
|
||||
// aggregator can pick the highest-severity child deterministically.
|
||||
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
|
||||
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
|
||||
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
|
||||
// real check state takes precedence over them.
|
||||
func checkSeverityRank(state string) int {
|
||||
switch state {
|
||||
case stateFail:
|
||||
return 5
|
||||
case stateERR:
|
||||
return 4
|
||||
case stateDegraded:
|
||||
return 3
|
||||
case stateWARN:
|
||||
return 2
|
||||
case stateOK:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateStatusFromChecks updates the monitor status based on its checks.
|
||||
func (m *Monitor) UpdateStatusFromChecks() {
|
||||
mutex.Lock()
|
||||
checks := make([]Check, 0)
|
||||
|
||||
tx := DB().Begin()
|
||||
var locked Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&locked, m.ID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
mutex.Unlock()
|
||||
log.Println("UpdateStatusFromChecks lock monitor", err)
|
||||
return
|
||||
}
|
||||
m.State = locked.State
|
||||
|
||||
_ = tx.Model(m).Association("Checks").Find(&checks)
|
||||
prevState := m.State
|
||||
m.State = stateOK
|
||||
|
||||
// Pick the highest-severity enabled check. The previous implementation
|
||||
// inlined three if-statements with non-obvious precedence (a WARN that
|
||||
// appeared AFTER an ERR in the iteration would never downgrade back,
|
||||
// but a FAIL after ERR would silently get clobbered). Using a single
|
||||
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
|
||||
// independent of slice ordering — the same rule Phase 3 introduces
|
||||
// for DEGRADED, applied uniformly to the existing states too.
|
||||
bestRank := checkSeverityRank(stateOK)
|
||||
bestState := stateOK
|
||||
hasChecks := false
|
||||
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
||||
if check.Enabled == nil || !*check.Enabled {
|
||||
continue
|
||||
}
|
||||
hasChecks = true
|
||||
if r := checkSeverityRank(check.State); r > bestRank {
|
||||
bestRank = r
|
||||
bestState = check.State
|
||||
}
|
||||
}
|
||||
m.State = bestState
|
||||
|
||||
if !hasChecks {
|
||||
m.State = stateWARN
|
||||
}
|
||||
|
||||
if m.State != prevState {
|
||||
err := tx.Model(&m).UpdateColumn("state", m.State).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks fail update state", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
evt := Event{}
|
||||
if err := tx.Where("monitor_id = ? AND state = ?", m.ID, "current").First(&evt).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
tx.Rollback()
|
||||
mutex.Unlock()
|
||||
log.Println("UpdateStatusFromChecks active event", err)
|
||||
return
|
||||
}
|
||||
// log.Println("state", m.State, "active event:", evt.ID)
|
||||
|
||||
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
||||
if check.State == stateERR || check.State == stateFail {
|
||||
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
|
||||
evt.Checks = append(evt.Checks, check)
|
||||
if check.Error != nil {
|
||||
evt.Reason = *check.Error
|
||||
} else {
|
||||
evt.Reason = "unknown error"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch m.State {
|
||||
case stateOK, stateWARN, stateDegraded:
|
||||
// DEGRADED is treated like OK/WARN for the event lifecycle: we do
|
||||
// NOT open a new "current" outage event for a partial regional
|
||||
// failure. Operators see DEGRADED on the check detail page and
|
||||
// the monitor list, but the existing notifier pipeline (down /
|
||||
// restore events) only fires for full ERR/FAIL. A future
|
||||
// improvement can add a separate "degraded" message kind.
|
||||
if evt.ID != 0 {
|
||||
upd := map[string]interface{}{
|
||||
"duration": time.Since(*evt.StartTime).Seconds(),
|
||||
"state": stateEnded,
|
||||
"oks": evt.Oks + 1,
|
||||
}
|
||||
if evt.EndTime == nil {
|
||||
upd["end_time"] = time.Now()
|
||||
}
|
||||
err := tx.Model(&evt).UpdateColumns(upd).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks fail update to ended", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
case stateERR, stateFail:
|
||||
if evt.ID == 0 {
|
||||
tn := time.Now()
|
||||
evt.StartTime = &tn
|
||||
evt.Duration = 0
|
||||
evt.State = "current"
|
||||
evt.MonitorID = m.ID
|
||||
err := tx.Save(&evt).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
spew.Dump(evt)
|
||||
log.Println("UpdateStatusFromChecks fail create", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
} else {
|
||||
upd := map[string]interface{}{
|
||||
"end_time": nil,
|
||||
"state": "current",
|
||||
"errors": evt.Errors + 1,
|
||||
}
|
||||
if evt.StartTime == nil {
|
||||
upd["start_time"] = time.Now()
|
||||
upd["duration"] = 0
|
||||
} else {
|
||||
upd["duration"] = time.Since(*evt.StartTime).Seconds()
|
||||
}
|
||||
|
||||
err := tx.Model(&evt).UpdateColumns(upd).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
spew.Dump(evt)
|
||||
spew.Dump(upd)
|
||||
log.Println("UpdateStatusFromChecks fail update to current", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := m.syncStatusPageIncidentsTx(tx, &evt); err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks status page incident", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
err := tx.Commit().Error
|
||||
|
||||
mutex.Unlock()
|
||||
|
||||
if err != nil {
|
||||
log.Println("UpdateStatusFromChecks commit fail", err)
|
||||
return
|
||||
}
|
||||
if m.State != prevState {
|
||||
m.invalidateStatusPages()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Monitor) invalidateStatusPages() {
|
||||
var ids []int64
|
||||
if err := DB().Model(&StatusPage{}).Where("? = ANY(monitor_ids)", m.ID).Pluck("id", &ids).Error; err != nil {
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
InvalidateStatusPageCache(id)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
|
||||
if event == nil || event.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
var pages []StatusPage
|
||||
if err := tx.Where("auto_open_incidents = TRUE AND ? = ANY(monitor_ids)", m.ID).Find(&pages).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range pages {
|
||||
page := &pages[i]
|
||||
var incident StatusPageIncident
|
||||
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
|
||||
if m.State == stateERR || m.State == "FAIL" {
|
||||
if err != nil {
|
||||
// The database uniqueness constraint makes concurrent state updates idempotent.
|
||||
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
|
||||
if err := tx.Create(&incident).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "opened"); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if incident.BodyMD != event.Reason {
|
||||
if err := tx.Model(&incident).Update("body_md", event.Reason).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
incident.BodyMD, incident.UpdatedAt = event.Reason, time.Now()
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "updated"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if err == nil && incident.ResolvedAt == nil {
|
||||
now := time.Now()
|
||||
if err := tx.Model(&incident).Update("resolved_at", now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
incident.ResolvedAt, incident.UpdatedAt = &now, now
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "resolved"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
215
app/models/monitor_state_test.go
Обычный файл
215
app/models/monitor_state_test.go
Обычный файл
@@ -0,0 +1,215 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// monitorStateTestWorld bundles the account/group/monitor fixture the
|
||||
// monitor-state tests need. It deliberately re-creates the row in each
|
||||
// test rather than sharing, because Monitor.UpdateStatusFromChecks
|
||||
// mutates the row in place and the per-test assertions need a clean
|
||||
// baseline.
|
||||
type monitorStateTestWorld struct {
|
||||
plan models.Plan
|
||||
account models.Account
|
||||
group models.Group
|
||||
monitor models.Monitor
|
||||
}
|
||||
|
||||
// seedMonitorStateWorld provisions one plan/account/group/monitor with
|
||||
// the requested initial state. The monitor is enabled so
|
||||
// UpdateStatusFromChecks treats its checks as live.
|
||||
func seedMonitorStateWorld(t *testing.T, initialState string) monitorStateTestWorld {
|
||||
t.Helper()
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "ms-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
account := models.Account{Name: "ms-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&account).Error)
|
||||
|
||||
group := models.Group{AccountID: account.ID, Name: "ms"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrAgg("mon.test"),
|
||||
Host: "mon.test",
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
State: initialState,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
return monitorStateTestWorld{
|
||||
plan: plan,
|
||||
account: account,
|
||||
group: group,
|
||||
monitor: mon,
|
||||
}
|
||||
}
|
||||
|
||||
// attachCheck creates an enabled check on the monitor with the given
|
||||
// state. Returns the persisted check so the test can re-load it.
|
||||
func attachCheck(t *testing.T, monitorID int64, kind string, state string) models.Check {
|
||||
t.Helper()
|
||||
enTrue := true
|
||||
c := models.Check{
|
||||
MonitorID: monitorID,
|
||||
Kind: kind,
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: state,
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&c).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
// reloadMonitor pulls the latest monitor state from the DB so the test
|
||||
// can compare against the post-UpdateStatusFromChecks row.
|
||||
func reloadMonitor(t *testing.T, id int64) models.Monitor {
|
||||
t.Helper()
|
||||
var m models.Monitor
|
||||
require.NoError(t, models.DB().First(&m, id).Error)
|
||||
return m
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 3 severity rules: FAIL > ERR > DEGRADED > WARN > OK.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestMonitorStateFromChecks_Degraded: monitor with 1 OK + 1 DEGRADED
|
||||
// child → monitor.State = DEGRADED. The new severity rank must
|
||||
// correctly promote DEGRADED above OK.
|
||||
func TestMonitorStateFromChecks_Degraded(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"DEGRADED child must promote monitor above OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DegradedWithError: monitor with 1 ERR +
|
||||
// 1 DEGRADED → monitor.State = ERR. ERR beats DEGRADED in the
|
||||
// severity order.
|
||||
func TestMonitorStateFromChecks_DegradedWithError(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "ERR")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "ERR", got.State,
|
||||
"ERR must beat DEGRADED — the order is FAIL > ERR > DEGRADED > WARN > OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DegradedOnlyOK covers the single-DEGRADED
|
||||
// case explicitly so a regression that treats DEGRADED as "WARN-ish"
|
||||
// would flip this assertion.
|
||||
func TestMonitorStateFromChecks_DegradedOnlyOK(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"mixed OK+DEGRADED monitor must be DEGRADED, not OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_SeverityOrderingTable is a table-driven
|
||||
// sweep of the FAIL > ERR > DEGRADED > WARN > OK ladder. The case set
|
||||
// is intentionally small — every pair that could reveal a wrong
|
||||
// winner under the new severity rank. Keeping it table-driven makes
|
||||
// it trivial to add more cases if a future state is introduced.
|
||||
func TestMonitorStateFromChecks_SeverityOrderingTable(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
checks []string
|
||||
wantMon string
|
||||
}{
|
||||
{"all_ok", []string{"OK", "OK"}, "OK"},
|
||||
{"all_warn", []string{"WARN", "WARN"}, "WARN"},
|
||||
{"ok_with_warn", []string{"OK", "WARN"}, "WARN"},
|
||||
{"warn_with_ok", []string{"WARN", "OK"}, "WARN"}, // order independence
|
||||
{"ok_with_degraded", []string{"OK", "DEGRADED"}, "DEGRADED"},
|
||||
{"warn_with_degraded", []string{"WARN", "DEGRADED"}, "DEGRADED"},
|
||||
{"degraded_with_warn", []string{"DEGRADED", "WARN"}, "DEGRADED"},
|
||||
{"err_with_degraded", []string{"ERR", "DEGRADED"}, "ERR"},
|
||||
{"degraded_with_err", []string{"DEGRADED", "ERR"}, "ERR"},
|
||||
{"fail_with_err", []string{"FAIL", "ERR"}, "FAIL"},
|
||||
{"err_with_fail", []string{"ERR", "FAIL"}, "FAIL"}, // order independence
|
||||
{"fail_alone", []string{"FAIL"}, "FAIL"},
|
||||
{"all_degraded", []string{"DEGRADED", "DEGRADED"}, "DEGRADED"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
for i, st := range c.checks {
|
||||
attachCheck(t, world.monitor.ID,
|
||||
"http-"+string(rune('a'+i)), st)
|
||||
}
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, c.wantMon, got.State,
|
||||
"severity winner for child states %v must be %s", c.checks, c.wantMon)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_NoChecksIsWarn pins the existing "no
|
||||
// enabled checks → WARN" behavior — the new severity rank must not
|
||||
// accidentally produce OK for an empty monitor.
|
||||
func TestMonitorStateFromChecks_NoChecksIsWarn(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "WARN", got.State,
|
||||
"an enabled monitor with zero checks must remain WARN")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DisabledCheckIgnored verifies that a
|
||||
// disabled check is not folded into the severity decision. Otherwise
|
||||
// a stuck-in-ERR check that has been disabled would keep tripping the
|
||||
// monitor forever.
|
||||
func TestMonitorStateFromChecks_DisabledCheckIgnored(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
|
||||
enFalse := false
|
||||
disabled := models.Check{
|
||||
MonitorID: world.monitor.ID,
|
||||
Kind: "http-disabled",
|
||||
Interval: 60,
|
||||
Enabled: &enFalse,
|
||||
State: "ERR",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&disabled).Error)
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "OK", got.State,
|
||||
"disabled check must be ignored — only the enabled OK check counts")
|
||||
}
|
||||
|
||||
// guard against time import being pruned by an editor when individual
|
||||
// test bodies stop referencing it directly.
|
||||
var _ = time.Second
|
||||
94
app/models/monitor_transfer_test.go
Обычный файл
94
app/models/monitor_transfer_test.go
Обычный файл
@@ -0,0 +1,94 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestMonitorTransferGroupSwap verifies that swapping a monitor's
|
||||
// group_id between two accounts correctly re-homes the monitor without
|
||||
// touching any other monitor data. This is the database primitive that
|
||||
// POST /api/v1/monitors/:id/transfer relies on.
|
||||
func TestMonitorTransferGroupSwap(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
// Two accounts, each with their own default group.
|
||||
accA := models.Account{Name: "A", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&accA).Error)
|
||||
accB := models.Account{Name: "B", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&accB).Error)
|
||||
|
||||
groupA := models.Group{AccountID: accA.ID, Name: "A-default"}
|
||||
require.NoError(t, models.DB().Create(&groupA).Error)
|
||||
groupB := models.Group{AccountID: accB.ID, Name: "B-default"}
|
||||
require.NoError(t, models.DB().Create(&groupB).Error)
|
||||
|
||||
// Monitor lives in account A with one HTTP check.
|
||||
monitor := models.Monitor{
|
||||
GroupID: groupA.ID,
|
||||
Host: "example-a.test",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
check := models.Check{
|
||||
MonitorID: monitor.ID,
|
||||
Kind: "http",
|
||||
URL: ptrString("https://example-a.test/"),
|
||||
Interval: 300,
|
||||
Settings: datatypes.JSON([]byte("{}")),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
|
||||
// Simulate the controller-side update.
|
||||
require.NoError(t, models.DB().
|
||||
Model(&models.Monitor{}).
|
||||
Where("id = ?", monitor.ID).
|
||||
Update("group_id", groupB.ID).Error)
|
||||
|
||||
// Monitor now lives in B; check follows by FK on monitor_id.
|
||||
var reloaded models.Monitor
|
||||
require.NoError(t, models.DB().Preload("Group").First(&reloaded, monitor.ID).Error)
|
||||
assert.Equal(t, groupB.ID, reloaded.GroupID, "monitor group_id should now point at account B's group")
|
||||
assert.Equal(t, accB.ID, reloaded.Group.AccountID, "preloaded group should belong to account B")
|
||||
|
||||
var checkCount int64
|
||||
require.NoError(t, models.DB().Model(&models.Check{}).
|
||||
Where("monitor_id = ?", monitor.ID).Count(&checkCount).Error)
|
||||
assert.Equal(t, int64(1), checkCount, "check rows must follow the monitor across the move")
|
||||
}
|
||||
|
||||
// TestMonitorTransferSameAccountGuard documents the early-return path: the
|
||||
// controller must not silently no-op when the caller picks the current
|
||||
// account, and the test exercises that the DB stays untouched.
|
||||
func TestMonitorTransferSameAccountGuard(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "only", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := models.Group{AccountID: acc.ID, Name: "only"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "x.test"}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
// No update is issued because the controller rejects same-account moves
|
||||
// before the SQL UPDATE. Verify the row is unchanged.
|
||||
var reloaded models.Monitor
|
||||
require.NoError(t, models.DB().First(&reloaded, monitor.ID).Error)
|
||||
assert.Equal(t, group.ID, reloaded.GroupID)
|
||||
}
|
||||
|
||||
func ptrString(s string) *string { return &s }
|
||||
528
app/models/network_diagnostics.go
Обычный файл
528
app/models/network_diagnostics.go
Обычный файл
@@ -0,0 +1,528 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfirmStateNone = "none"
|
||||
ConfirmStatePending = "pending"
|
||||
ConfirmStateConfirmed = "confirmed" // A different worker reproduced the failure.
|
||||
ConfirmStateTimeout = "confirmed_by_timeout"
|
||||
AttemptKindRegular = "regular"
|
||||
AttemptKindConfirm = "confirmation"
|
||||
AttemptStateQueued = "queued"
|
||||
AttemptStateLeased = "leased"
|
||||
AttemptStateFinished = "finished"
|
||||
diagnosticSoft = "soft"
|
||||
diagnosticHard = "hard"
|
||||
diagnosticRecovery = "recovery"
|
||||
)
|
||||
|
||||
type diagnosticSettings struct {
|
||||
confirmTimeout time.Duration
|
||||
healthWindow time.Duration
|
||||
healthRate float64
|
||||
healthMin int64
|
||||
}
|
||||
|
||||
func settingsForAccount(account *Account) diagnosticSettings {
|
||||
settings := diagnosticSettings{confirmTimeout: 90 * time.Second, healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
|
||||
if account == nil || account.Plan == nil {
|
||||
return settings
|
||||
}
|
||||
plan := account.Plan
|
||||
if plan.ConfirmTimeoutSec > 0 {
|
||||
settings.confirmTimeout = time.Duration(plan.ConfirmTimeoutSec) * time.Second
|
||||
}
|
||||
if plan.HealthWindowSec > 0 {
|
||||
settings.healthWindow = time.Duration(plan.HealthWindowSec) * time.Second
|
||||
}
|
||||
if plan.HealthRateThreshold > 0 && plan.HealthRateThreshold <= 1 {
|
||||
settings.healthRate = plan.HealthRateThreshold
|
||||
}
|
||||
if plan.HealthMinAttempts > 0 {
|
||||
settings.healthMin = int64(plan.HealthMinAttempts)
|
||||
}
|
||||
if !plan.Confirmations {
|
||||
return settings
|
||||
}
|
||||
if account.ConfirmTimeoutSec != nil && *account.ConfirmTimeoutSec >= 15 {
|
||||
settings.confirmTimeout = time.Duration(*account.ConfirmTimeoutSec) * time.Second
|
||||
}
|
||||
if account.HealthWindowSec != nil && *account.HealthWindowSec >= 60 {
|
||||
settings.healthWindow = time.Duration(*account.HealthWindowSec) * time.Second
|
||||
}
|
||||
if account.HealthRateThreshold != nil && *account.HealthRateThreshold > 0 && *account.HealthRateThreshold <= 1 {
|
||||
settings.healthRate = *account.HealthRateThreshold
|
||||
}
|
||||
if account.HealthMinAttempts != nil && *account.HealthMinAttempts > 0 {
|
||||
settings.healthMin = int64(*account.HealthMinAttempts)
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
// CheckAttempt is the durable worker-attribution record. Unlike check state,
|
||||
// it is append-only and therefore remains useful after a worker is deweighted.
|
||||
type CheckAttempt struct {
|
||||
concerns.Model
|
||||
JobID string `gorm:"uniqueIndex;size:64;not null" json:"job_id"`
|
||||
CheckID int64 `gorm:"index;not null" json:"check_id"`
|
||||
MonitorID int64 `gorm:"index;not null" json:"monitor_id"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
|
||||
WorkerNode *WorkerNode `json:"worker_node,omitempty"`
|
||||
SourceWorkerNodeID *int64 `gorm:"index" json:"source_worker_node_id,omitempty"`
|
||||
Kind string `gorm:"size:32;not null" json:"kind"`
|
||||
State string `gorm:"size:32;not null" json:"state"`
|
||||
ResultState string `gorm:"size:16" json:"result_state"`
|
||||
Result datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"result"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
LeaseToken string `gorm:"size:64" json:"-"`
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
Deweighted bool `gorm:"not null;default:false" json:"deweighted"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// DiagnosticAuditEvent is a compact, queryable control-plane audit record.
|
||||
type DiagnosticAuditEvent struct {
|
||||
concerns.Model
|
||||
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
|
||||
Kind string `gorm:"size:64;index;not null" json:"kind"`
|
||||
Metadata datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"metadata"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func auditDiagnostic(tx *gorm.DB, kind string, monitorID, workerID *int64, metadata map[string]interface{}) {
|
||||
b, _ := json.Marshal(metadata)
|
||||
_ = tx.Create(&DiagnosticAuditEvent{MonitorID: monitorID, WorkerNodeID: workerID, Kind: kind, Metadata: b}).Error
|
||||
}
|
||||
|
||||
// AuditNetworkRecovery is used by the admin incident-response endpoint.
|
||||
func AuditNetworkRecovery(tx *gorm.DB, workerID int64) {
|
||||
auditDiagnostic(tx, "worker.network_problem_force_recover", nil, &workerID, nil)
|
||||
}
|
||||
|
||||
// enqueueDiagnosticDelivery creates the message and its durable delivery task in
|
||||
// the transition transaction. A transaction advisory lock prevents concurrent
|
||||
// result frames from leaving duplicate messages when the task dedupe wins.
|
||||
func enqueueDiagnosticDelivery(tx *gorm.DB, monitor *Monitor, account *Account, tier string, now time.Time) error {
|
||||
if account == nil || account.Plan == nil {
|
||||
return nil
|
||||
}
|
||||
if tier == diagnosticHard && !account.Plan.AllowHardAlerts {
|
||||
return nil
|
||||
}
|
||||
var notifications []Notification
|
||||
if err := tx.Joins("JOIN notification_groups ON notification_groups.notification_id = notifications.id").
|
||||
Where("notifications.account_id = ? AND notifications.enabled AND notification_groups.group_id = ?", account.ID, monitor.GroupID).
|
||||
Preload("Contacts", "enabled = ?", true).Find(¬ifications).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range notifications {
|
||||
for j := range notifications[i].Contacts {
|
||||
contact := notifications[i].Contacts[j]
|
||||
method := diagnosticContactMethod(contact.Kind)
|
||||
if method == "" || (tier == diagnosticSoft && method != "email" && method != "telegram") {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("diagnostic:%d:%s:%d:%d", monitor.ID, tier, notifications[i].ID, contact.ID)
|
||||
if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", key).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var existing Task
|
||||
if err := tx.Where("idempotency_key = ?", key).First(&existing).Error; err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
message := Message{NotificationID: notifications[i].ID, ContactID: contact.ID, Kind: "diagnostic_" + tier, State: TaskStateQueued}
|
||||
if err := tx.Create(&message).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
monitorID, messageID := monitor.ID, message.ID
|
||||
payload, err := json.Marshal(wire.NotificationTask{AccountID: account.ID, MessageID: messageID, NotificationID: notifications[i].ID, MonitorID: &monitorID, Method: method, Contact: wire.NotificationContact{ID: contact.ID, Kind: contact.Kind, Value: contact.Value, Name: contact.Name}, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notifications[i].ID, ContactID: contact.ID, MessageID: &messageID, MonitorID: &monitorID, Method: method, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind, NotBefore: now, Payload: payload, IdempotencyKey: key}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func diagnosticContactMethod(kind string) string {
|
||||
switch kind {
|
||||
case "email":
|
||||
return "email"
|
||||
case "telegram_private", "telegram_group":
|
||||
return "telegram"
|
||||
case "webhook", "mattermost", "sms", "voice":
|
||||
return kind
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func diagnosticSubject(monitor *Monitor, tier string) string {
|
||||
return fmt.Sprintf("Monitor %s: %s", monitor.Host, tier)
|
||||
}
|
||||
|
||||
func diagnosticBody(monitor *Monitor, tier string) string {
|
||||
return fmt.Sprintf("Network diagnostic %s for monitor %s.", tier, monitor.Host)
|
||||
}
|
||||
|
||||
// ConfirmationJobsForWorker atomically leases confirmation jobs assigned to this
|
||||
// worker or left unassigned by an expired lease. Unassigned attempts still retain
|
||||
// SourceWorkerNodeID, so the original failing worker can never claim them.
|
||||
func ConfirmationJobsForWorker(worker *WorkerNode, kinds []string, limit int) ([]wire.CheckJob, error) {
|
||||
if worker == nil || worker.AccountID != nil || !worker.SupportsTaskEnvelope() || worker.NetworkProblemActive(time.Now()) || limit < 1 {
|
||||
return nil, nil
|
||||
}
|
||||
var jobs []wire.CheckJob
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var attempts []CheckAttempt
|
||||
if err := tx.Clauses(SkipLockedClause).Where("(worker_node_id = ? OR worker_node_id IS NULL) AND kind = ? AND state = ?", worker.ID, AttemptKindConfirm, AttemptStateQueued).Order("id").Limit(limit).Find(&attempts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range attempts {
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor").First(&check, attempts[i].CheckID).Error; err != nil {
|
||||
continue
|
||||
}
|
||||
if check.Monitor == nil || !check.Monitor.Enabled {
|
||||
continue
|
||||
}
|
||||
if attempts[i].SourceWorkerNodeID != nil && *attempts[i].SourceWorkerNodeID == worker.ID {
|
||||
continue
|
||||
}
|
||||
if !containsString(kinds, check.Kind) || !containsString(worker.CheckTypes(), check.Kind) {
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(DefaultTaskLeaseTTL)
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateLeased, "started_at": now, "lease_token": leaseToken, "lease_expires_at": leaseUntil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
jobs = append(jobs, wire.CheckJob{JobID: attempts[i].JobID, LeaseToken: leaseToken, CheckID: check.ID, MonitorID: check.MonitorID, Kind: check.Kind, Host: check.Monitor.Host, URL: check.URL, Interval: check.Interval, Settings: json.RawMessage(check.Settings)})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return jobs, err
|
||||
}
|
||||
|
||||
// StartConfirmation creates exactly one targeted confirmation for a new outage.
|
||||
func StartConfirmation(checkID, sourceWorkerID int64, now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return StartConfirmationTx(tx, checkID, sourceWorkerID, now)
|
||||
})
|
||||
}
|
||||
|
||||
// StartConfirmationTx is StartConfirmation's transaction-aware form.
|
||||
func StartConfirmationTx(tx *gorm.DB, checkID, sourceWorkerID int64, now time.Time) error {
|
||||
if tx == nil {
|
||||
return errors.New("start confirmation: nil transaction")
|
||||
}
|
||||
{
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Confirmations are a paid distributed-check entitlement. Free accounts
|
||||
// retain the legacy direct soft alert path and do not consume worker budget.
|
||||
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil || check.Monitor.Group.Account.Plan == nil || !check.Monitor.Group.Account.Plan.Confirmations {
|
||||
return nil
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor.ConfirmState == ConfirmStatePending || monitor.ConfirmState == ConfirmStateConfirmed {
|
||||
return nil
|
||||
}
|
||||
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, 0, now)
|
||||
if err != nil {
|
||||
monitor.ConfirmState, monitor.ConfirmAt = ConfirmStateTimeout, &now
|
||||
auditDiagnostic(tx, "check.confirm_unavailable", &monitor.ID, &sourceWorkerID, nil)
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticHard, now)
|
||||
}
|
||||
monitor.ConfirmState, monitor.ConfirmAt, monitor.ConfirmedByWorkerID = ConfirmStatePending, &now, &worker.ID
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticSoft, now); err != nil {
|
||||
return err
|
||||
}
|
||||
attempt := CheckAttempt{JobID: uuid.NewString(), CheckID: check.ID, MonitorID: monitor.ID, WorkerNodeID: &worker.ID, SourceWorkerNodeID: &sourceWorkerID, Kind: AttemptKindConfirm, State: AttemptStateQueued}
|
||||
if err := tx.Create(&attempt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_assign", &monitor.ID, &worker.ID, map[string]interface{}{"exclude_worker_id": sourceWorkerID, "job_id": attempt.JobID})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// confirmationWorkerTx selects an independent active platform worker that can
|
||||
// execute this exact kind. Capability filtering is deliberately performed in
|
||||
// Go because the JSON capability format also supports legacy rows safely.
|
||||
func confirmationWorkerTx(tx *gorm.DB, checkKind string, sourceWorkerID, excludeWorkerID int64, now time.Time) (*WorkerNode, error) {
|
||||
var workers []WorkerNode
|
||||
if err := tx.Where("id <> ? AND id <> ? AND account_id IS NULL AND status = 'active' AND (network_problems = FALSE OR network_problems_until <= ? OR network_problems_until IS NULL)", sourceWorkerID, excludeWorkerID, now).Order("id").Find(&workers).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range workers {
|
||||
if workers[i].SupportsTaskEnvelope() && containsString(workers[i].CheckTypes(), checkKind) {
|
||||
return &workers[i], nil
|
||||
}
|
||||
}
|
||||
return nil, gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
// ApplyDiagnosticResult resolves a targeted attempt once. Duplicate reports are ignored.
|
||||
func ApplyDiagnosticResult(report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
|
||||
if report.JobID == "" || worker == nil {
|
||||
return false, nil
|
||||
}
|
||||
handled := true
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
return ApplyDiagnosticResultTx(tx, report, worker, now, &handled)
|
||||
})
|
||||
return handled, err
|
||||
}
|
||||
|
||||
// ApplyDiagnosticResultTx resolves a diagnostic attempt within the caller's
|
||||
// transaction. handled distinguishes a normal check result from a diagnostic.
|
||||
func ApplyDiagnosticResultTx(tx *gorm.DB, report wire.CheckResultReport, worker *WorkerNode, now time.Time, handled *bool) error {
|
||||
if tx == nil {
|
||||
return errors.New("apply diagnostic: nil transaction")
|
||||
}
|
||||
if handled == nil {
|
||||
return errors.New("apply diagnostic: nil handled result")
|
||||
}
|
||||
*handled = true
|
||||
{
|
||||
var attempt CheckAttempt
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("job_id = ?", report.JobID).First(&attempt).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if attempt.Kind != AttemptKindConfirm {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
if attempt.State == AttemptStateFinished {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
if attempt.State != AttemptStateLeased || attempt.WorkerNodeID == nil || *attempt.WorkerNodeID != worker.ID {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
if attempt.LeaseToken == "" || report.LeaseToken == "" || report.LeaseToken != attempt.LeaseToken || attempt.LeaseExpiresAt == nil || !attempt.LeaseExpiresAt.After(now) {
|
||||
return errors.New("apply diagnostic: lease token is invalid or expired")
|
||||
}
|
||||
payload, _ := json.Marshal(report)
|
||||
deweighted := worker.NetworkProblemActive(now)
|
||||
if err := tx.Model(&attempt).Where("state = ? AND lease_token = ? AND lease_expires_at > ?", AttemptStateLeased, report.LeaseToken, now).Updates(map[string]interface{}{"state": AttemptStateFinished, "result_state": report.State, "result": payload, "finished_at": now, "lease_token": "", "lease_expires_at": nil, "deweighted": deweighted}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, attempt.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if attempt.Kind == AttemptKindConfirm && monitor.ConfirmState == ConfirmStatePending {
|
||||
if report.State == stateERR || report.State == stateFail {
|
||||
monitor.ConfirmState = ConfirmStateConfirmed
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticHard, now); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
monitor.ConfirmState = ConfirmStateNone
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticRecovery, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_result", &monitor.ID, &worker.ID, map[string]interface{}{"state": report.State, "deweighted": deweighted})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func monitorAccount(tx *gorm.DB, monitor *Monitor) *Account {
|
||||
var group Group
|
||||
if err := tx.Preload("Account.Plan").First(&group, monitor.GroupID).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return group.Account
|
||||
}
|
||||
|
||||
// RecoverDiagnostic clears a completed hard escalation only once and queues the
|
||||
// corresponding recovery tasks in the same transaction.
|
||||
func RecoverDiagnostic(checkID int64, now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return RecoverDiagnosticTx(tx, checkID, now)
|
||||
})
|
||||
}
|
||||
|
||||
// RecoverDiagnosticTx is RecoverDiagnostic's transaction-aware form.
|
||||
func RecoverDiagnosticTx(tx *gorm.DB, checkID int64, now time.Time) error {
|
||||
if tx == nil {
|
||||
return errors.New("recover diagnostic: nil transaction")
|
||||
}
|
||||
{
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil {
|
||||
return nil
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor.ConfirmState != ConfirmStateConfirmed && monitor.ConfirmState != ConfirmStateTimeout {
|
||||
return nil
|
||||
}
|
||||
monitor.ConfirmState = ConfirmStateNone
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_recovery", &monitor.ID, nil, nil)
|
||||
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticRecovery, now)
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkDiagnosticsTick expires confirmations and derives worker health from durable attempts.
|
||||
func NetworkDiagnosticsTick(now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := reapExpiredConfirmationAttemptsTx(tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
var monitors []Monitor
|
||||
if err := tx.Clauses(SkipLockedClause).Preload("Group.Account.Plan").Where("confirm_state = ?", ConfirmStatePending).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range monitors {
|
||||
settings := settingsForAccount(monitors[i].Group.Account)
|
||||
if monitors[i].ConfirmAt == nil || monitors[i].ConfirmAt.After(now.Add(-settings.confirmTimeout)) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Model(&monitors[i]).Update("confirm_state", ConfirmStateTimeout).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_timeout", &monitors[i].ID, nil, nil)
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitors[i], monitors[i].Group.Account, diagnosticHard, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var workers []WorkerNode
|
||||
if err := tx.Find(&workers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range workers {
|
||||
var total, failed int64
|
||||
// Operated workers serve accounts on different plans. Use the most
|
||||
// sensitive entitled setting among their recent attempts.
|
||||
settings := diagnosticSettings{healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
|
||||
q := tx.Model(&CheckAttempt{}).Where("worker_node_id = ? AND finished_at >= ?", workers[i].ID, now.Add(-settings.healthWindow))
|
||||
q.Count(&total)
|
||||
q.Where("result_state IN ?", []string{stateERR, stateFail}).Count(&failed)
|
||||
flagged := total >= settings.healthMin && float64(failed)/float64(total) >= settings.healthRate
|
||||
updates := map[string]interface{}{"last_total_count": total, "last_failure_count": failed}
|
||||
if flagged {
|
||||
updates["network_problems"] = true
|
||||
updates["network_problems_until"] = now.Add(10 * time.Minute)
|
||||
}
|
||||
if workers[i].NetworkProblems && workers[i].NetworkProblemsUntil != nil && workers[i].NetworkProblemsUntil.Before(now) && !flagged {
|
||||
updates["network_problems"] = false
|
||||
updates["network_problems_until"] = nil
|
||||
auditDiagnostic(tx, "worker.network_problem_unflag", nil, &workers[i].ID, nil)
|
||||
}
|
||||
if flagged && !workers[i].NetworkProblems {
|
||||
auditDiagnostic(tx, "worker.network_problem_flag", nil, &workers[i].ID, map[string]interface{}{"failures": failed, "total": total})
|
||||
}
|
||||
if err := tx.Model(&workers[i]).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
_ = influx.WriteOne("worker_health", map[string]string{"worker_id": workers[i].WorkerID}, map[string]interface{}{"failures": failed, "total": total, "failure_rate": float64(failed) / float64(maxInt64(total, 1))})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func reapExpiredConfirmationAttemptsTx(tx *gorm.DB, now time.Time) error {
|
||||
var attempts []CheckAttempt
|
||||
if err := tx.Clauses(SkipLockedClause).Where("kind = ? AND state = ? AND lease_expires_at <= ?", AttemptKindConfirm, AttemptStateLeased, now).Find(&attempts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range attempts {
|
||||
var check Check
|
||||
if err := tx.First(&check, attempts[i].CheckID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
oldWorkerID := int64(0)
|
||||
if attempts[i].WorkerNodeID != nil {
|
||||
oldWorkerID = *attempts[i].WorkerNodeID
|
||||
}
|
||||
sourceWorkerID := int64(0)
|
||||
if attempts[i].SourceWorkerNodeID != nil {
|
||||
sourceWorkerID = *attempts[i].SourceWorkerNodeID
|
||||
}
|
||||
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, oldWorkerID, now)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
// No replacement is available now. Remove the stale assignment so a
|
||||
// later capable independent worker can claim this queued attempt.
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": nil, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", nil).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func maxInt64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
193
app/models/notification.go
Обычный файл
193
app/models/notification.go
Обычный файл
@@ -0,0 +1,193 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/workdays"
|
||||
)
|
||||
|
||||
// Notification provides functionality.
|
||||
type Notification struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
ContactIDs []int64 `gorm:"-:all" json:"contact_ids"`
|
||||
Contacts []Contact `gorm:"many2many:notification_contacts;" json:"contacts,omitempty"`
|
||||
GroupIDs []int64 `gorm:"-:all" json:"group_ids"`
|
||||
Groups []Group `gorm:"many2many:notification_groups;" json:"-"`
|
||||
AlertDelay *int64 `json:"alert_delay,omitempty"`
|
||||
// RepeatAlert *int64 `json:"repeat_alert,omitempty"`
|
||||
BeforeExpiration *int64 `json:"before_expiration,omitempty"`
|
||||
|
||||
NotifyDown bool `gorm:"default:true" json:"notify_down"`
|
||||
NotifyRestore bool `gorm:"default:true" json:"notify_restore"`
|
||||
|
||||
NotifyWHOIS bool `gorm:"default:true" json:"notify_whois"`
|
||||
NotifySSL bool `gorm:"default:true" json:"notify_ssl"`
|
||||
|
||||
NotifyDays *int `json:"notify_days"`
|
||||
NotifyDayStart *int `json:"notify_day_start"`
|
||||
NotifyDayEnd *int `json:"notify_day_end"`
|
||||
NotifyHolidays bool `gorm:"default:true" json:"notify_holidays"`
|
||||
|
||||
Messages []Message `json:"-"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
const notificationDebug = false
|
||||
|
||||
// EnabledNow checks if the notification is enabled at the given time.
|
||||
func (n *Notification) EnabledNow(tn *time.Time) bool {
|
||||
weekday := int(tn.Weekday())
|
||||
// делаем из 0-воскр 1-пн 6-сб вариант 0-пн 6-воскр
|
||||
if weekday == 0 {
|
||||
weekday = 7
|
||||
}
|
||||
weekday--
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, "check enabled now at", tn, "for day", weekday)
|
||||
}
|
||||
|
||||
if !n.NotifyHolidays {
|
||||
c := workdays.GetCalendar()
|
||||
if !c.IsWorkday(*tn) {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, "is not enabled on holiday", tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
minusOneDay := false
|
||||
notifyFromDay := true
|
||||
|
||||
if n.NotifyDayStart != nil && n.NotifyDayEnd != nil {
|
||||
bod := BeginningOfDay(*tn)
|
||||
secondsToday := int(tn.Sub(bod) / time.Second)
|
||||
|
||||
ds := *n.NotifyDayStart
|
||||
de := *n.NotifyDayEnd
|
||||
// вариант 9 утра - 2 часа ночи
|
||||
if ds == de { //nolint:gocritic // complex condition chain
|
||||
notifyFromDay = true
|
||||
} else if ds > de {
|
||||
// с 0 до DayEnd
|
||||
if secondsToday < de { //nolint:gocritic // complex condition chain
|
||||
minusOneDay = true
|
||||
notifyFromDay = true
|
||||
} else if secondsToday < ds {
|
||||
// с DayEnd до DayStart
|
||||
notifyFromDay = false
|
||||
} else {
|
||||
notifyFromDay = true
|
||||
}
|
||||
} else {
|
||||
if secondsToday < ds { //nolint:gocritic // complex condition chain
|
||||
// с 0 до DayStart
|
||||
notifyFromDay = false
|
||||
} else if secondsToday > de {
|
||||
notifyFromDay = false
|
||||
} else {
|
||||
notifyFromDay = true
|
||||
}
|
||||
}
|
||||
// log.Println(secondsToday)
|
||||
}
|
||||
|
||||
if !notifyFromDay {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled as notifyFromDay", tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Если время 0-dayStart считаем что это прошлый день
|
||||
if minusOneDay {
|
||||
weekday--
|
||||
if weekday < 0 {
|
||||
weekday = 6
|
||||
}
|
||||
}
|
||||
|
||||
if n.NotifyDays != nil {
|
||||
if !HasBit(*n.NotifyDays, uint(weekday)) {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled on weekday", tn.Weekday(), weekday, tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is enabled on weekday", tn.Weekday(), weekday, tn)
|
||||
}
|
||||
}
|
||||
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is enabled", tn)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// NotificationLoadIDs provides functionality.
|
||||
func NotificationLoadIDs(notifications *[]Notification) {
|
||||
for i, n := range *notifications { //nolint:gocritic // range copy is acceptable here
|
||||
cids := make([]int64, len(n.Contacts))
|
||||
for i, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
|
||||
cids[i] = c.ID
|
||||
}
|
||||
(*notifications)[i].ContactIDs = cids
|
||||
|
||||
gids := make([]int64, len(n.Groups))
|
||||
for j, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
gids[j] = g.ID
|
||||
}
|
||||
(*notifications)[i].GroupIDs = gids
|
||||
}
|
||||
}
|
||||
|
||||
// PersistRelations provides functionality.
|
||||
func (n *Notification) PersistRelations() error {
|
||||
cts := make([]Contact, len(n.ContactIDs))
|
||||
for i, c := range n.ContactIDs {
|
||||
ct := Contact{}
|
||||
ct.ID = c
|
||||
cts[i] = ct
|
||||
}
|
||||
err := DB().Model(&n).Association("Contacts").Replace(cts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
grp := make([]Group, len(n.GroupIDs))
|
||||
for i, g := range n.GroupIDs {
|
||||
gr := Group{}
|
||||
gr.ID = g
|
||||
grp[i] = gr
|
||||
}
|
||||
err = DB().Model(&n).Association("Groups").Replace(grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetContacts returns the contacts associated with this notification.
|
||||
// Errors are logged and an empty slice is returned instead of panicking so
|
||||
// that a single misconfigured notification cannot kill the scheduler goroutine
|
||||
// that processes expiry alerts (see internal/notifier.RunExp).
|
||||
func (n *Notification) GetContacts() []Contact {
|
||||
contacts := make([]Contact, 0)
|
||||
err := DB().Model(*n).Where("enabled = ?", true).Association("Contacts").Find(&contacts)
|
||||
if err != nil {
|
||||
log.Printf("notification %d: GetContacts failed: %v", n.ID, err)
|
||||
return contacts
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
167
app/models/notification_credential.go
Обычный файл
167
app/models/notification_credential.go
Обычный файл
@@ -0,0 +1,167 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// NotificationCredential kinds stored in the database.
|
||||
const (
|
||||
CredentialKindSMTP = "smtp"
|
||||
CredentialKindTelegram = "telegram"
|
||||
)
|
||||
|
||||
// NotificationCredential stores per-method delivery credentials (SMTP login,
|
||||
// Telegram bot token, etc.) encrypted at rest. Credentials are pushed to
|
||||
// workers through the init/config websocket refresh (see docs/worker-protocol.md
|
||||
// "Credentials Push").
|
||||
type NotificationCredential struct {
|
||||
concerns.Model
|
||||
// AccountID is nil for platform-managed credentials and set for credentials
|
||||
// owned by one customer account.
|
||||
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;index" json:"account_id,omitempty"`
|
||||
Kind string `gorm:"not null" json:"kind"`
|
||||
Name string `gorm:"not null" json:"name"`
|
||||
|
||||
// SMTP-specific
|
||||
Server *string `json:"server,omitempty"`
|
||||
Port *int `json:"port,omitempty"`
|
||||
Login *string `json:"login,omitempty"`
|
||||
FromName *string `json:"from_name,omitempty"`
|
||||
FromAddr *string `json:"from_address,omitempty"`
|
||||
InsecureSkipVerify bool `gorm:"default:false" json:"insecure_skip_verify"`
|
||||
|
||||
// Telegram-specific
|
||||
BotName *string `json:"bot_name,omitempty"`
|
||||
APIURL *string `json:"api_url,omitempty"`
|
||||
WebhookToken string `gorm:"size:128" json:"webhook_token,omitempty"`
|
||||
|
||||
// SecretEnc holds the encrypted (or "plain:"-prefixed fallback) secret
|
||||
// value — SMTP password or Telegram bot token. Decrypt via GetSecret.
|
||||
SecretEnc string `gorm:"column:secret;type:text" json:"-"`
|
||||
SecretMasked string `gorm:"-" json:"secret_masked,omitempty"`
|
||||
|
||||
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;default:'{}'" json:"meta,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// SetSecret encrypts and stores the plaintext secret value.
|
||||
func (c *NotificationCredential) SetSecret(plaintext string) error {
|
||||
enc, err := encryptSecret(plaintext)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.SecretEnc = enc
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSecret decrypts and returns the secret value.
|
||||
func (c *NotificationCredential) GetSecret() (string, error) {
|
||||
return decryptSecret(c.SecretEnc)
|
||||
}
|
||||
|
||||
// FillSecretMasked populates SecretMasked with a display-safe version of the credential secret.
|
||||
func (c *NotificationCredential) FillSecretMasked() {
|
||||
secret, err := c.GetSecret()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.SecretMasked = maskSecret(secret)
|
||||
}
|
||||
|
||||
// EnsureWebhookToken creates the per-bot webhook URL token if it is missing.
|
||||
func (c *NotificationCredential) EnsureWebhookToken() {
|
||||
if c.WebhookToken != "" {
|
||||
return
|
||||
}
|
||||
c.WebhookToken = base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
|
||||
}
|
||||
|
||||
func maskSecret(secret string) string {
|
||||
if secret == "" {
|
||||
return ""
|
||||
}
|
||||
if len(secret) == 1 {
|
||||
return secret
|
||||
}
|
||||
return secret[:1] + "***" + secret[len(secret)-1:]
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (NotificationCredential) TableName() string {
|
||||
return "notification_credentials"
|
||||
}
|
||||
|
||||
// AllNotificationCredentials returns all credentials ordered by kind and name.
|
||||
func AllNotificationCredentials() ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id IS NULL").Order("kind ASC, name ASC").Find(&creds).Error
|
||||
for i := range creds {
|
||||
creds[i].FillSecretMasked()
|
||||
}
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// AccountNotificationCredentials returns credentials owned by accountID.
|
||||
func AccountNotificationCredentials(accountID int64) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id = ?", accountID).Order("kind ASC, name ASC").Find(&creds).Error
|
||||
for i := range creds {
|
||||
creds[i].FillSecretMasked()
|
||||
}
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// EnabledCredentialsByKind returns enabled credentials of the given kind.
|
||||
func EnabledCredentialsByKind(kind string) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id IS NULL AND kind = ? AND enabled = ?", kind, true).Order("name ASC").Find(&creds).Error
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// EnabledCredentialsByAccountAndKind returns only enabled credentials owned by
|
||||
// accountID. It never falls back to platform credentials.
|
||||
func EnabledCredentialsByAccountAndKind(accountID int64, kind string) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id = ? AND kind = ? AND enabled = ?", accountID, kind, true).Order("name ASC").Find(&creds).Error
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// FindCredential returns a credential by id.
|
||||
func FindCredential(id int64) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().First(&c, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.FillSecretMasked()
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// FindCredentialByName returns a credential by kind and name.
|
||||
func FindCredentialByName(kind, name string) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().Where("kind = ? AND name = ?", kind, name).First(&c).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// FindTelegramCredentialByWebhookToken returns an enabled Telegram credential by webhook token.
|
||||
func FindTelegramCredentialByWebhookToken(token string) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().Where("kind = ? AND webhook_token = ? AND enabled = ?", CredentialKindTelegram, token, true).First(&c).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// DeleteCredential removes a credential by id.
|
||||
func DeleteCredential(id int64) error {
|
||||
return DB().Delete(&NotificationCredential{}, id).Error
|
||||
}
|
||||
186
app/models/notification_credential_test.go
Обычный файл
186
app/models/notification_credential_test.go
Обычный файл
@@ -0,0 +1,186 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// TestNotificationCredential_EncryptDecryptRoundTrip verifies that SetSecret
|
||||
// followed by GetSecret returns the original plaintext regardless of which
|
||||
// encryption mode (AES-GCM or plain: fallback) is active.
|
||||
func TestNotificationCredential_EncryptDecryptRoundTrip(t *testing.T) {
|
||||
c := &models.NotificationCredential{}
|
||||
plaintext := "super-secret-smtp-password"
|
||||
|
||||
require.NoError(t, c.SetSecret(plaintext))
|
||||
|
||||
got, err := c.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, plaintext, got)
|
||||
}
|
||||
|
||||
// TestNotificationCredential_EncryptDecryptWithKey verifies that when
|
||||
// RSMON_CRED_KEY is configured the stored value is prefixed with "enc:" and
|
||||
// can still be decrypted back to the original plaintext.
|
||||
func TestNotificationCredential_EncryptDecryptWithKey(t *testing.T) {
|
||||
t.Setenv("RSMON_CRED_KEY", "test-key-rotate-me-1234567890")
|
||||
|
||||
c := &models.NotificationCredential{}
|
||||
plaintext := "bot-token-9876543210:ABCDEFG"
|
||||
|
||||
require.NoError(t, c.SetSecret(plaintext))
|
||||
assert.True(t, strings.HasPrefix(c.SecretEnc, "enc:"),
|
||||
"expected SecretEnc to start with 'enc:' prefix, got %q", c.SecretEnc)
|
||||
assert.NotEqual(t, plaintext, c.SecretEnc, "encrypted value must not equal plaintext")
|
||||
|
||||
got, err := c.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, plaintext, got)
|
||||
}
|
||||
|
||||
// TestNotificationCredential_CRUD exercises create / find-by-id / find-by-name
|
||||
// / delete against the test database.
|
||||
func TestNotificationCredential_CRUD(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
server := "smtp.example.com"
|
||||
port := 587
|
||||
login := "alerts@example.com"
|
||||
enabled := true
|
||||
c := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindSMTP,
|
||||
Name: "primary",
|
||||
Server: &server,
|
||||
Port: &port,
|
||||
Login: &login,
|
||||
Enabled: &enabled,
|
||||
}
|
||||
require.NoError(t, c.SetSecret("smtp-password-xyz"))
|
||||
require.NoError(t, models.DB().Create(c).Error)
|
||||
require.NotZero(t, c.ID, "expected ID to be assigned after Create")
|
||||
|
||||
found, err := models.FindCredential(c.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "primary", found.Name)
|
||||
assert.Equal(t, models.CredentialKindSMTP, found.Kind)
|
||||
require.NotNil(t, found.Server)
|
||||
assert.Equal(t, "smtp.example.com", *found.Server)
|
||||
|
||||
gotSecret, err := found.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "smtp-password-xyz", gotSecret)
|
||||
assert.Equal(t, "s***z", found.SecretMasked)
|
||||
|
||||
byName, err := models.FindCredentialByName(models.CredentialKindSMTP, "primary")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.ID, byName.ID)
|
||||
|
||||
require.NoError(t, models.DeleteCredential(c.ID))
|
||||
|
||||
_, err = models.FindCredential(c.ID)
|
||||
assert.Error(t, err, "FindCredential should fail after delete")
|
||||
}
|
||||
|
||||
// TestNotificationCredential_UniqueKindName verifies that two credentials with
|
||||
// the same (kind, name) pair violate the unique index.
|
||||
func TestNotificationCredential_UniqueKindName(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
enTrue := true
|
||||
first := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: "main-bot",
|
||||
Enabled: &enTrue,
|
||||
}
|
||||
require.NoError(t, first.SetSecret("token-a"))
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
second := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: "main-bot",
|
||||
Enabled: &enTrue,
|
||||
}
|
||||
require.NoError(t, second.SetSecret("token-b"))
|
||||
|
||||
err := models.DB().Create(second).Error
|
||||
require.Error(t, err, "expected unique constraint violation for duplicate (kind, name)")
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"expected error mentioning unique/duplicate, got: %v", err)
|
||||
}
|
||||
|
||||
// TestEnabledCredentialsByKind verifies the kind+enabled filter.
|
||||
func TestEnabledCredentialsByKind(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
enTrue := true
|
||||
enFalse := false
|
||||
enabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "enabled-1", Enabled: &enTrue}
|
||||
disabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "disabled-1", Enabled: &enFalse}
|
||||
require.NoError(t, models.DB().Create(enabled).Error)
|
||||
require.NoError(t, models.DB().Create(disabled).Error)
|
||||
|
||||
got, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
|
||||
var names []string
|
||||
for _, c := range got {
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
assert.Contains(t, names, "enabled-1")
|
||||
assert.NotContains(t, names, "disabled-1")
|
||||
}
|
||||
|
||||
func TestNotificationCredentialsAreScopedToSystemOrAccount(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
accountA := &models.Account{Name: "credential-a"}
|
||||
accountB := &models.Account{Name: "credential-b"}
|
||||
require.NoError(t, models.DB().Create(accountA).Error)
|
||||
require.NoError(t, models.DB().Create(accountB).Error)
|
||||
enabled := true
|
||||
system := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "system", Enabled: &enabled}
|
||||
ownedA := &models.NotificationCredential{AccountID: &accountA.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
|
||||
ownedB := &models.NotificationCredential{AccountID: &accountB.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
|
||||
require.NoError(t, models.DB().Create(system).Error)
|
||||
require.NoError(t, models.DB().Create(ownedA).Error)
|
||||
require.NoError(t, models.DB().Create(ownedB).Error)
|
||||
|
||||
systemCreds, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, systemCreds, 1)
|
||||
assert.Equal(t, system.ID, systemCreds[0].ID)
|
||||
|
||||
accountCreds, err := models.EnabledCredentialsByAccountAndKind(accountA.ID, models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accountCreds, 1)
|
||||
assert.Equal(t, ownedA.ID, accountCreds[0].ID)
|
||||
}
|
||||
|
||||
func TestNotificationCredentialSecretMasked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
cred := models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "smtp"}
|
||||
require.NoError(t, cred.SetSecret("password"))
|
||||
require.NoError(t, models.DB().Create(&cred).Error)
|
||||
|
||||
loaded, err := models.FindCredential(cred.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "p***d", loaded.SecretMasked)
|
||||
}
|
||||
146
app/models/notification_get_contacts_test.go
Обычный файл
146
app/models/notification_get_contacts_test.go
Обычный файл
@@ -0,0 +1,146 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/spec/factories"
|
||||
)
|
||||
|
||||
// TestNotificationGetContactsIncludesSystemContact exercises the regression
|
||||
// reported when the production dump was restored into dev: the
|
||||
// contacts.is_system column was missing and the GORM preload panicked on
|
||||
// GetContacts. The fix has two layers:
|
||||
//
|
||||
// 1. AutoMigrate must add is_system (and deletion_requested_at) before the
|
||||
// notifier scheduler starts running.
|
||||
// 2. GetContacts itself must not panic on a query error so a single bad row
|
||||
// cannot tear down the scheduler goroutine.
|
||||
//
|
||||
// This test verifies both layers by:
|
||||
// - asserting that the schema post-Migrate includes is_system, so the
|
||||
// production-like scenario no longer panics; and
|
||||
// - building a notification that contains a contact flagged is_system=true
|
||||
// and checking GetContacts returns it.
|
||||
func TestNotificationGetContactsIncludesSystemContact(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
// Column must exist after Migrate; otherwise GetContacts would fail
|
||||
// with the same panic we saw in production.
|
||||
assertColumnExists(t, "contacts", "is_system")
|
||||
|
||||
account := &models.Account{Name: "acct-get-contacts"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
accountID := account.ID
|
||||
|
||||
trueVal := true
|
||||
contact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "system-admin",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(contact).Error)
|
||||
|
||||
notification := factories.PersistedNotification(
|
||||
account, []int64{contact.ID}, nil, 300, false,
|
||||
)
|
||||
|
||||
// Reload so the model has its persisted ID; the factory's PersistRelations
|
||||
// may have left ContactIDs empty on the returned value depending on the
|
||||
// GORM version, so fetch fresh.
|
||||
require.NoError(t, models.DB().
|
||||
Preload("Contacts").
|
||||
First(¬ification, notification.ID).Error)
|
||||
|
||||
got := notification.GetContacts()
|
||||
|
||||
ids := make([]int64, 0, len(got))
|
||||
for _, c := range got {
|
||||
ids = append(ids, c.ID)
|
||||
}
|
||||
assert.Contains(t, ids, contact.ID, "GetContacts must include the is_system contact")
|
||||
}
|
||||
|
||||
// TestUserDeletionRequestedAtColumnAndRoundTrip verifies the second missing
|
||||
// column reported by the panic: users.deletion_requested_at. It asserts that
|
||||
// AutoMigrate creates the column and that the field round-trips through the
|
||||
// DB correctly. ProcessPendingDeletions (the consumer of this column) relies
|
||||
// on it being present and queryable.
|
||||
func TestUserDeletionRequestedAtColumnAndRoundTrip(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
assertColumnExists(t, "users", "deletion_requested_at")
|
||||
|
||||
user := factories.PersistedUser("deletion-roundtrip@test.ru", "secret")
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
user.DeletionRequestedAt = &now
|
||||
|
||||
require.NoError(t, models.DB().Save(&user).Error)
|
||||
|
||||
reloaded := models.User{}
|
||||
require.NoError(t, models.DB().First(&reloaded, user.ID).Error)
|
||||
|
||||
require.NotNil(t, reloaded.DeletionRequestedAt, "deletion_requested_at must round-trip via Save/First")
|
||||
assert.True(t, reloaded.DeletionRequestedAt.Equal(now),
|
||||
"deletion_requested_at must preserve the timestamp value (got %v, want %v)",
|
||||
reloaded.DeletionRequestedAt, now)
|
||||
|
||||
// ProcessPendingDeletions should not panic on the populated schema and
|
||||
// must respect the cutoff: a recently-set deletion_requested_at is
|
||||
// still inside the 7-day grace period, so no hard-delete must occur.
|
||||
deleted, err := models.ProcessPendingDeletions()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, deleted, "users within the 7-day grace period must not be hard-deleted")
|
||||
}
|
||||
|
||||
// TestProcessPendingDeletionsQueriesMissingColumnGracefully asserts that even
|
||||
// if the deletion_requested_at column were missing, ProcessPendingDeletions
|
||||
// would not panic (the panic-on-error pattern was historically present in
|
||||
// other notifier helpers). We force the failure by renaming the column back,
|
||||
// calling ProcessPendingDeletions, then restoring the column.
|
||||
func TestProcessPendingDeletionsQueriesMissingColumnGracefully(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
assertColumnExists(t, "users", "deletion_requested_at")
|
||||
|
||||
// Simulate the production-missing-column scenario in a contained way:
|
||||
// rename the column so the SELECT against deletion_requested_at fails.
|
||||
require.NoError(t, models.DB().
|
||||
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at TO deletion_requested_at_hidden").Error)
|
||||
t.Cleanup(func() {
|
||||
// Restore so subsequent tests in this package keep working.
|
||||
_ = models.DB().
|
||||
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at_hidden TO deletion_requested_at").Error
|
||||
})
|
||||
|
||||
// Must not panic; must return an error.
|
||||
assert.NotPanics(t, func() {
|
||||
_, err := models.ProcessPendingDeletions()
|
||||
assert.Error(t, err, "missing column must surface as an error, not a panic")
|
||||
})
|
||||
}
|
||||
|
||||
// assertColumnExists checks that the given table has the given column by
|
||||
// querying information_schema. It is the canary for the AutoMigrate step
|
||||
// ordering bug: if the column is missing, every test that touches it will
|
||||
// panic with SQLSTATE 42703.
|
||||
func assertColumnExists(t *testing.T, table, column string) {
|
||||
t.Helper()
|
||||
var n int
|
||||
err := models.DB().Raw(
|
||||
`SELECT COUNT(*) FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = ? AND column_name = ?`,
|
||||
table, column,
|
||||
).Scan(&n).Error
|
||||
require.NoError(t, err, "information_schema query must succeed")
|
||||
assert.Equal(t, 1, n, "table %q must have column %q after Migrate()", table, column)
|
||||
}
|
||||
128
app/models/notification_test.go
Обычный файл
128
app/models/notification_test.go
Обычный файл
@@ -0,0 +1,128 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
mondayFriday int
|
||||
allDays int
|
||||
)
|
||||
|
||||
var time1, time3, time8, time9, time10, time23 time.Time
|
||||
|
||||
func init() {
|
||||
// database.ConfigFile = "." + database.ConfigFile
|
||||
// database.Init()
|
||||
// Migrate()
|
||||
// Drop()
|
||||
|
||||
mondayFriday = 0
|
||||
mondayFriday = SetBit(mondayFriday, 0)
|
||||
mondayFriday = SetBit(mondayFriday, 1)
|
||||
mondayFriday = SetBit(mondayFriday, 2)
|
||||
mondayFriday = SetBit(mondayFriday, 3)
|
||||
mondayFriday = SetBit(mondayFriday, 4)
|
||||
|
||||
allDays = 0
|
||||
allDays = SetBit(allDays, 0)
|
||||
allDays = SetBit(allDays, 1)
|
||||
allDays = SetBit(allDays, 2)
|
||||
allDays = SetBit(allDays, 3)
|
||||
allDays = SetBit(allDays, 4)
|
||||
allDays = SetBit(allDays, 5)
|
||||
allDays = SetBit(allDays, 6)
|
||||
|
||||
time1 = time.Date(2019, time.May, 26, 1, 0, 0, 0, time.Local)
|
||||
time3 = time.Date(2019, time.May, 26, 3, 0, 0, 0, time.Local)
|
||||
time8 = time.Date(2019, time.May, 26, 8, 0, 0, 0, time.Local)
|
||||
time9 = time.Date(2019, time.May, 26, 9, 0, 0, 0, time.Local)
|
||||
time10 = time.Date(2019, time.May, 26, 10, 0, 0, 0, time.Local)
|
||||
time23 = time.Date(2019, time.May, 26, 23, 0, 0, 0, time.Local)
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowHolidays(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
|
||||
n.ID = 1
|
||||
n.NotifyHolidays = true
|
||||
holiday := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.Local)
|
||||
assert.Equal(t, true, n.EnabledNow(&holiday), "notification by default should be enabled on holidays")
|
||||
|
||||
n.ID = 2
|
||||
n.NotifyHolidays = false
|
||||
assert.Equal(t, false, n.EnabledNow(&holiday), "notification with NotifyHolidays=fasle should not be enabled on holiday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowMondayFriday(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyHolidays = true
|
||||
|
||||
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
|
||||
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
|
||||
|
||||
n.ID = 3
|
||||
n.NotifyDays = &mondayFriday
|
||||
assert.Equal(t, false, n.EnabledNow(&weekend), "notification mon-fri should not be enabled on sunday")
|
||||
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-fri should be enabled on monday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowAllDays(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyHolidays = true
|
||||
|
||||
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
|
||||
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
|
||||
|
||||
n.ID = 4
|
||||
n.NotifyDays = &allDays
|
||||
assert.Equal(t, true, n.EnabledNow(&weekend), "notification mon-sat should be enabled on sunday")
|
||||
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-sat should be enabled on monday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowNormal(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
n.NotifyHolidays = true
|
||||
|
||||
// 9am - 18pm
|
||||
n.ID = 5
|
||||
start := 9 * 3600
|
||||
end := 18 * 3600
|
||||
n.NotifyDayStart = &start
|
||||
n.NotifyDayEnd = &end
|
||||
|
||||
assert.Equal(t, false, n.EnabledNow(&time1), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time23), "notification should not be enabled outside day")
|
||||
}
|
||||
|
||||
// TestNotificationEnabledNowRollover tests time range that crosses midnight
|
||||
//
|
||||
//nolint:dupl // Test structure similar to TestNotificationEnabledNowNormal but tests different behavior (rollover vs normal time range)
|
||||
func TestNotificationEnabledNowRollover(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
n.NotifyHolidays = true
|
||||
|
||||
// 9am - 2am
|
||||
n.ID = 6
|
||||
start := 9 * 3600
|
||||
end := 2 * 3600
|
||||
n.NotifyDayStart = &start
|
||||
n.NotifyDayEnd = &end
|
||||
|
||||
assert.Equal(t, true, n.EnabledNow(&time1), "notification should be enabled inside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time23), "notification should be enabled inside day")
|
||||
}
|
||||
18
app/models/payment.go
Обычный файл
18
app/models/payment.go
Обычный файл
@@ -0,0 +1,18 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// Payment provides functionality.
|
||||
type Payment struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `json:"account_id"`
|
||||
Account User `json:"-"`
|
||||
|
||||
Kind string
|
||||
ExtID string
|
||||
Amount int
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
108
app/models/plan.go
Обычный файл
108
app/models/plan.go
Обычный файл
@@ -0,0 +1,108 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
var canonicalPlanCodes = []string{"free", "solo", "team", "enterprise"}
|
||||
|
||||
// CanonicalPlanCodes returns the supported public catalog codes.
|
||||
func CanonicalPlanCodes() []string {
|
||||
return append([]string(nil), canonicalPlanCodes...)
|
||||
}
|
||||
|
||||
// Plan is a versioned-by-code catalog entry. A zero cap means unlimited.
|
||||
type Plan struct {
|
||||
concerns.Model
|
||||
|
||||
Code string `gorm:"size:32;not null" json:"code"`
|
||||
NameRU string `gorm:"size:64;not null" json:"name_ru"`
|
||||
NameEN string `gorm:"size:64;not null" json:"name_en"`
|
||||
PriceMonthlyMinor int64 `gorm:"not null;default:0" json:"price_monthly_minor"`
|
||||
PriceAnnualMinor int64 `gorm:"not null;default:0" json:"price_annual_minor"`
|
||||
Currency string `gorm:"size:3;not null;default:'RUB'" json:"currency"`
|
||||
MonitorCap int64 `gorm:"not null;default:0" json:"monitor_cap"`
|
||||
IntervalMinSeconds int `gorm:"not null;default:30" json:"interval_min_seconds"`
|
||||
StatusPagesCap int64 `gorm:"not null;default:0" json:"status_pages_cap"`
|
||||
MaintenanceCap int64 `gorm:"not null;default:0" json:"maintenance_cap"`
|
||||
LoginSeatsIncluded int64 `gorm:"not null;default:0" json:"login_seats_included"`
|
||||
NotifySeatsIncluded int64 `gorm:"not null;default:0" json:"notify_seats_included"`
|
||||
UnlimitedSeats bool `gorm:"not null;default:false" json:"unlimited_seats"`
|
||||
Integrations pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"integrations"`
|
||||
CheckKinds pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"check_kinds"`
|
||||
DataRetentionMonths int `gorm:"not null;default:3" json:"data_retention_months"`
|
||||
DistributedWorkers bool `gorm:"not null;default:false" json:"distributed_workers"`
|
||||
Confirmations bool `gorm:"not null;default:false" json:"confirmations"`
|
||||
AllowHardAlerts bool `gorm:"not null;default:false" json:"allow_hard_alerts"`
|
||||
ConfirmTimeoutSec int `gorm:"not null;default:90" json:"confirm_timeout_sec"`
|
||||
HealthWindowSec int `gorm:"not null;default:300" json:"health_window_sec"`
|
||||
HealthRateThreshold float64 `gorm:"not null;default:0.5" json:"health_rate_threshold"`
|
||||
HealthMinAttempts int `gorm:"not null;default:10" json:"health_min_attempts"`
|
||||
SOC2 bool `gorm:"not null;default:false" json:"soc2"`
|
||||
GDPRDPA bool `gorm:"not null;default:false" json:"gdpr_dpa"`
|
||||
IsDefault bool `gorm:"not null;default:false" json:"is_default"`
|
||||
Archived bool `gorm:"not null;default:false" json:"archived"`
|
||||
|
||||
// Deprecated source-compatibility fields. The legacy plans table is retained
|
||||
// as plans_legacy; these values are never written to the canonical catalog.
|
||||
Default bool `gorm:"-" json:"Default,omitempty"`
|
||||
Name string `gorm:"-" json:"Name,omitempty"`
|
||||
HTTPMonitors *int64 `gorm:"-" json:"HTTPMonitors,omitempty"`
|
||||
DNSMonitors *int64 `gorm:"-" json:"DNSMonitors,omitempty"`
|
||||
WHOISMonitors *int64 `gorm:"-" json:"WHOISMonitors,omitempty"`
|
||||
TotalMonitors *int64 `gorm:"-" json:"TotalMonitors,omitempty"`
|
||||
Price int `gorm:"-" json:"Price,omitempty"`
|
||||
TrialPeriod int `gorm:"-" json:"TrialPeriod,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
func (p *Plan) BeforeCreate(_ *gorm.DB) error {
|
||||
if p.Code == "" {
|
||||
p.Code = "legacy-" + uuid.NewString()[:24]
|
||||
}
|
||||
if p.NameRU == "" {
|
||||
p.NameRU = p.Name
|
||||
}
|
||||
if p.NameEN == "" {
|
||||
p.NameEN = p.NameRU
|
||||
}
|
||||
if p.Currency == "" {
|
||||
p.Currency = "RUB"
|
||||
}
|
||||
if p.IntervalMinSeconds == 0 {
|
||||
p.IntervalMinSeconds = 30
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plan) AfterFind(_ *gorm.DB) error {
|
||||
p.Name = p.NameRU
|
||||
p.Price = int(p.PriceMonthlyMinor / 100)
|
||||
p.Default = p.IsDefault
|
||||
p.TotalMonitors = &p.MonitorCap
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultPlan returns the canonical free plan.
|
||||
func DefaultPlan() Plan {
|
||||
pl := Plan{}
|
||||
if err := DB().Where("code = ? AND archived = FALSE", "free").First(&pl).Error; err != nil {
|
||||
log.Println("unable to find default plan")
|
||||
panic(err)
|
||||
}
|
||||
return pl
|
||||
}
|
||||
|
||||
// AllowsDistributed keeps legacy callers working while using the canonical
|
||||
// entitlement flag for catalog plans.
|
||||
func (p *Plan) AllowsDistributed() bool {
|
||||
return p != nil && (p.DistributedWorkers || p.Price > 0)
|
||||
}
|
||||
20
app/models/region.go
Обычный файл
20
app/models/region.go
Обычный файл
@@ -0,0 +1,20 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DefaultRegionCode is the historical default region code seeded by
|
||||
// Migrate() (see app/models/migrate.go). Centralized here so admin
|
||||
// endpoints and check_jobs.go agree on the literal.
|
||||
const DefaultRegionCode = "local"
|
||||
|
||||
// Region represents a geographic region where distributed workers can run
|
||||
type Region struct {
|
||||
concerns.Model
|
||||
Code string `gorm:"uniqueIndex;size:20;not null" json:"code"` // e.g. "ru-msk", "us-east", "eu-west"
|
||||
Name string `gorm:"not null" json:"name"` // "Moscow, Russia"
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
Priority int `gorm:"not null;default:0" json:"priority"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
38
app/models/repo.go
Обычный файл
38
app/models/repo.go
Обычный файл
@@ -0,0 +1,38 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Repo is a source repository shared by one or more account-scoped sites.
|
||||
// Repositories themselves are global rstuff mirrors; SiteRepo supplies the
|
||||
// account boundary through its Site.
|
||||
type Repo struct {
|
||||
concerns.Model
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
GitlabID *int64 `json:"gitlab_id,omitempty"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Namespace *string `gorm:"size:120" json:"namespace,omitempty"`
|
||||
Path *string `gorm:"size:255" json:"path,omitempty"`
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the repository table name.
|
||||
func (Repo) TableName() string { return "repos" }
|
||||
|
||||
// SiteRepo is the explicit site/repository join. Role is intentionally data,
|
||||
// rather than an enum, to preserve the rstuff contract as it evolves.
|
||||
type SiteRepo struct {
|
||||
SiteID int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE CASCADE;primaryKey" json:"site_id"`
|
||||
RepoID int64 `gorm:"type:bigint REFERENCES repos(id) ON DELETE CASCADE;primaryKey" json:"repo_id"`
|
||||
Role string `gorm:"size:32;not null;default:'primary'" json:"role"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the repository assignment table name.
|
||||
func (SiteRepo) TableName() string { return "site_repos" }
|
||||
135
app/models/rkn_domain.go
Обычный файл
135
app/models/rkn_domain.go
Обычный файл
@@ -0,0 +1,135 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// RknDomain is one row of the locally cached ru-blocked domains list. The
|
||||
// `Domain` field carries a lowercased domain string and is the unique key.
|
||||
type RknDomain struct {
|
||||
concerns.Model
|
||||
Domain string `gorm:"size:255;uniqueIndex;not null" json:"domain"`
|
||||
}
|
||||
|
||||
// TableName pins the underlying table name so GORM migrations and raw SQL
|
||||
// (used by IsRknDomainBlocked and the rkn updater) agree.
|
||||
func (RknDomain) TableName() string { return "rkn_domains" }
|
||||
|
||||
// ReplaceRknDomains deletes every existing row and inserts the given domains
|
||||
// in batches. Caller-supplied domains are lower-cased and de-duplicated, and
|
||||
// blank entries are dropped. The whole operation runs in a single
|
||||
// transaction so readers see either the old set or the new set — there is no
|
||||
// in-between state where the table is half-flushed.
|
||||
//
|
||||
// Designed to be called once per parser-updater tick (default every 6h); the
|
||||
// uniqueIndex on Domain guarantees idempotent re-runs even when the caller
|
||||
// forgets to filter.
|
||||
func ReplaceRknDomains(domains []string) error {
|
||||
if domains == nil {
|
||||
domains = []string{}
|
||||
}
|
||||
deduped := make([]string, 0, len(domains))
|
||||
seen := make(map[string]struct{}, len(domains))
|
||||
for _, d := range domains {
|
||||
d = strings.ToLower(strings.TrimSpace(d))
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[d]; ok {
|
||||
continue
|
||||
}
|
||||
seen[d] = struct{}{}
|
||||
deduped = append(deduped, d)
|
||||
}
|
||||
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
// Step 1 — wipe the existing cache. Using a scoped Where("1 = 1") Delete
|
||||
// instead of TRUNCATE so the advisory lock taken by Migrate() does not
|
||||
// become a bottleneck and so any in-flight foreign-key checks against
|
||||
// rkn_domains remain satisfied (the table has no FKs today, but this
|
||||
// matches the convention used by Drop()).
|
||||
if err := tx.Where("1 = 1").Delete(&RknDomain{}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
// Step 2 — bulk-insert in chunks of 1000 rows. CreateInBatches runs N
|
||||
// multi-row INSERT statements, which for the typical ~30k ru-blocked
|
||||
// entries is ~3ms per batch — significantly cheaper than per-row
|
||||
// Create() in tight loops (the previous AddRknDomain implementation).
|
||||
const batchSize = 1000
|
||||
for start := 0; start < len(deduped); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(deduped) {
|
||||
end = len(deduped)
|
||||
}
|
||||
rows := make([]RknDomain, 0, end-start)
|
||||
for _, d := range deduped[start:end] {
|
||||
rows = append(rows, RknDomain{Domain: d})
|
||||
}
|
||||
if err := tx.CreateInBatches(rows, batchSize).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// IsRknDomainBlocked returns true iff `domain` (or its root label, or any
|
||||
// parent suffix already recorded as `*.parent.tld`) is present in the
|
||||
// rkn_domains table.
|
||||
//
|
||||
// Matching rules — see checks/crkn/rkn_init.go for the original logic we
|
||||
// consolidate here:
|
||||
// 1. Exact match against the stored domain string.
|
||||
// 2. Root-domain match (last two labels of the input) — covers the case
|
||||
// where the user passed a subdomain but the upstream only lists the
|
||||
// apex.
|
||||
// 3. Suffix match (`stored LIKE '%' || input || ?`) — covers the case
|
||||
// where the user passed the apex (or a higher-level label) but the
|
||||
// upstream lists a child subdomain.
|
||||
//
|
||||
// All three checks are combined into a single SQL statement via OR so the
|
||||
// table is scanned at most once and the SQL planner can pick a single
|
||||
// index access path.
|
||||
func IsRknDomainBlocked(domain string) (bool, error) {
|
||||
domain = strings.ToLower(strings.TrimSpace(domain))
|
||||
if domain == "" {
|
||||
return false, nil
|
||||
}
|
||||
rootDomain := rootDomainOf(domain)
|
||||
|
||||
// Build the suffix patterns once. Note: every ".X" entry in the table
|
||||
// (i.e. a domain that begins with a dot) matches any subdomain whose
|
||||
// suffix is domain.
|
||||
suffixPattern := "%." + domain
|
||||
|
||||
var count int64
|
||||
err := DB().Raw(
|
||||
"SELECT COUNT(*) FROM rkn_domains WHERE domain = ? OR domain = ? OR domain LIKE ?",
|
||||
domain, rootDomain, suffixPattern,
|
||||
).Scan(&count).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// rootDomainOf returns the last two labels of `domain` (e.g. "a.b.c" → "b.c").
|
||||
// Returns `domain` unchanged when it has fewer than three labels, because a
|
||||
// one- or two-label input IS already the apex/root domain.
|
||||
func rootDomainOf(domain string) string {
|
||||
i := strings.LastIndex(domain, ".")
|
||||
if i < 0 {
|
||||
return domain
|
||||
}
|
||||
j := strings.LastIndex(domain[:i], ".")
|
||||
if j < 0 {
|
||||
return domain
|
||||
}
|
||||
return domain[j+1:]
|
||||
}
|
||||
90
app/models/rkn_domain_test.go
Обычный файл
90
app/models/rkn_domain_test.go
Обычный файл
@@ -0,0 +1,90 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestReplaceRknDomains_BulkAndDedup(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []string{
|
||||
"Foo.example",
|
||||
"foo.example", // dup after lowercasing
|
||||
" bar.example ",
|
||||
"",
|
||||
"baz.example",
|
||||
"qux.example",
|
||||
"qux.example", // dup within input
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknDomains(input))
|
||||
|
||||
got := []string{}
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &got).Error)
|
||||
|
||||
want := []string{"bar.example", "baz.example", "foo.example", "qux.example"}
|
||||
sort.Strings(want)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
func TestReplaceRknDomains_ReplacesExisting(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknDomains([]string{"old1.example", "old2.example"}))
|
||||
|
||||
var n int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(2), n)
|
||||
|
||||
// Second call wipes and replaces — no overlap with old set.
|
||||
require.NoError(t, models.ReplaceRknDomains([]string{"new1.example", "new2.example", "new3.example"}))
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(3), n)
|
||||
|
||||
var domains []string
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &domains).Error)
|
||||
assert.Equal(t, []string{"new1.example", "new2.example", "new3.example"}, domains)
|
||||
}
|
||||
|
||||
func TestIsRknDomainBlocked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []string{
|
||||
"example.com",
|
||||
"foo.bar",
|
||||
"sub.test",
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknDomains(input))
|
||||
|
||||
cases := []struct {
|
||||
host string
|
||||
want bool
|
||||
}{
|
||||
{"example.com", true}, // exact
|
||||
{"EXAMPLE.com", true}, // case insensitive (caller lowercases)
|
||||
{"www.example.com", true}, // stored has apex; query apex → root-domain match
|
||||
{"deep.nested.example.com", true}, // suffix match via LIKE '%.X'
|
||||
{"foo.bar", true},
|
||||
{"sub.test", true},
|
||||
{"a.sub.test", true},
|
||||
{"unrelated.org", false},
|
||||
{"two.labels", false}, // 2-label input not in list → must not collapse to "labels"
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.host, func(t *testing.T) {
|
||||
got, err := models.IsRknDomainBlocked(c.host)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
193
app/models/rkn_ip.go
Обычный файл
193
app/models/rkn_ip.go
Обычный файл
@@ -0,0 +1,193 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/netaddr"
|
||||
)
|
||||
|
||||
// RknIP stores a single CIDR from the ru-blocked IP list. The cidr column
|
||||
// uses Postgres's native cidr type so the (>>) containment operator works
|
||||
// directly inside IsRknIPBlocked queries — GiST index recommended for any
|
||||
// table large enough to amortize the planner cost; see EnsureRknIndexes.
|
||||
type RknIP struct {
|
||||
concerns.Model
|
||||
Network *netaddr.Cidr `json:"network" gorm:"type:cidr;"`
|
||||
}
|
||||
|
||||
// TableName pins the underlying table name.
|
||||
func (RknIP) TableName() string { return "rkn_ips" }
|
||||
|
||||
// PanicOnErr panics with a spew-formatted error dump if err is non-nil.
|
||||
// Kept for callers that still use the old add-by-row InsertInBatches path
|
||||
// (none after this commit, but kept in case external scripts reference it).
|
||||
func PanicOnErr(err error) {
|
||||
if err != nil {
|
||||
spew.Dump(err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// FindRknIP returns true iff the IP inside `ip` (treated as a /32 or /128
|
||||
// host) falls inside any CIDR already stored in rkn_ips. Kept for callers
|
||||
// that already construct an internal/netaddr.Inet.
|
||||
func FindRknIP(ip netaddr.Inet) (bool, error) {
|
||||
ipstr := ip.Inet.String()
|
||||
if !strings.Contains(ipstr, "/") {
|
||||
if ip.Inet.To4() != nil {
|
||||
ipstr += "/32"
|
||||
} else {
|
||||
ipstr += "/128"
|
||||
}
|
||||
}
|
||||
_, network, err := net.ParseCIDR(ipstr)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
|
||||
var matched []RknIP
|
||||
if err := DB().Raw("SELECT id FROM rkn_ips WHERE network >>= ?", &cidr).Scan(&matched).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return len(matched) > 0, nil
|
||||
}
|
||||
|
||||
// AddRknIP is the legacy per-row insert path. Deprecated: callers should
|
||||
// invoke ReplaceRknIPs from the rkn updater. Kept around so existing cron
|
||||
// scripts that import the symbol keep compiling.
|
||||
func AddRknIP(data []string, count int) {
|
||||
tx := DB().Begin()
|
||||
for k, dataIP := range data {
|
||||
if count > 0 && k > count-1 {
|
||||
break
|
||||
}
|
||||
if !strings.Contains(dataIP, "/") {
|
||||
dataIP += "/32"
|
||||
}
|
||||
_, network, err := net.ParseCIDR(dataIP)
|
||||
PanicOnErr(err)
|
||||
|
||||
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
|
||||
var rknIPs []RknIP
|
||||
err = tx.Raw("SELECT id FROM rkn_ips where network >>= ?", &cidr).Scan(&rknIPs).Error
|
||||
PanicOnErr(err)
|
||||
if len(rknIPs) == 0 {
|
||||
rknIPs = []RknIP{}
|
||||
err = tx.Raw("SELECT id FROM rkn_ips where network << ?", &cidr).Scan(&rknIPs).Error
|
||||
PanicOnErr(err)
|
||||
if len(rknIPs) > 0 {
|
||||
for _, r := range rknIPs {
|
||||
PanicOnErr(tx.Delete(&r).Error)
|
||||
}
|
||||
}
|
||||
PanicOnErr(tx.Create(&RknIP{Network: &cidr}).Error)
|
||||
}
|
||||
log.Println("created:", dataIP)
|
||||
}
|
||||
PanicOnErr(tx.Commit().Error)
|
||||
}
|
||||
|
||||
// ReplaceRknIPs deletes every existing row and bulk-inserts the given CIDRs
|
||||
// in batches of 1000. Whole operation runs in a single transaction so a
|
||||
// partially-applied update can never leave the table in a hybrid state.
|
||||
//
|
||||
// CIDR.parseCIDR-loop uses net.ParseCIDR to canonicalise the address —
|
||||
// upstream .dat files occasionally contain range/mask pairs that aren't
|
||||
// already reduced (e.g. 192.168.0.0/16 spelled as 192.168.5.0/16); the
|
||||
// Postgres cidr type normalises on insert.
|
||||
func ReplaceRknIPs(nets []*net.IPNet) error {
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
|
||||
rows := make([]RknIP, 0, len(nets))
|
||||
seen := make(map[string]struct{}, len(nets))
|
||||
for _, n := range nets {
|
||||
if n == nil || n.IP == nil {
|
||||
continue
|
||||
}
|
||||
// Canonicalise by routing through net.ParseCIDR. This drops the
|
||||
// host bits (a common bug in upstream dumps where a /24 range
|
||||
// is written with the .5 host bit set) and stamps the right
|
||||
// address family flag for Postgres.
|
||||
canonical := n.String()
|
||||
if _, parsed, err := net.ParseCIDR(canonical); err == nil {
|
||||
n = parsed
|
||||
canonical = parsed.String()
|
||||
}
|
||||
if _, ok := seen[canonical]; ok {
|
||||
continue
|
||||
}
|
||||
seen[canonical] = struct{}{}
|
||||
cidr := netaddr.Cidr{Cidr: *n, Valid: true}
|
||||
rows = append(rows, RknIP{Network: &cidr})
|
||||
}
|
||||
|
||||
// Wipe + re-insert in one transaction.
|
||||
if err := tx.Where("1 = 1").Delete(&RknIP{}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
const batchSize = 1000
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
if err := tx.CreateInBatches(rows[start:end], batchSize).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// IsRknIPBlocked returns true iff `ip` (any textual form ParseCIDR accepts)
|
||||
// falls inside any CIDR stored in the rkn_ips table. The query uses the
|
||||
// cidr >>= inet containment operator — see EnsureRknIndexes for the GiST
|
||||
// index that makes this fast at scale.
|
||||
func IsRknIPBlocked(ip string) (bool, error) {
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip == "" {
|
||||
return false, nil
|
||||
}
|
||||
var count int64
|
||||
if err := DB().Raw("SELECT COUNT(*) FROM rkn_ips WHERE network >>= ?::inet", ip).Scan(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// EnsureRknIndexes creates the indexes that AutoMigrate cannot express —
|
||||
// the GiST index on rkn_ips.network uses cidr >>= cidr containment (the
|
||||
// expression index `network` already covers equality and prefix ranges,
|
||||
// but the planner benefits from a GiST for `network >>= <other cidr>`
|
||||
// queries against ~30k rows). The unique index on rkn_domains.domain is
|
||||
// also declared in the GORM tag, this function only adds what GORM can't
|
||||
// (GiST) and is idempotent so it's safe to call repeatedly during boot
|
||||
// or migration.
|
||||
func EnsureRknIndexes() error {
|
||||
// GiST on cidr requires the btree_gist contrib — its `cidr_ops`
|
||||
// opclass exposes cidr to GiST. CREATE EXTENSION IF NOT EXISTS is
|
||||
// idempotent.
|
||||
if err := DB().Exec("CREATE EXTENSION IF NOT EXISTS btree_gist").Error; err != nil {
|
||||
return fmt.Errorf("ensure btree_gist: %w", err)
|
||||
}
|
||||
// rkn_ips GiST index on the cidr column supports the >>= containment
|
||||
// operator that IsRknIPBlocked uses. Without it a 30k-row table makes
|
||||
// every IP check a sequential scan; with it each check is an index
|
||||
// probe.
|
||||
if err := DB().Exec(
|
||||
"CREATE INDEX IF NOT EXISTS idx_rkn_ips_network ON rkn_ips USING gist (network)",
|
||||
).Error; err != nil {
|
||||
return fmt.Errorf("ensure idx_rkn_ips_network: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
114
app/models/rkn_ip_test.go
Обычный файл
114
app/models/rkn_ip_test.go
Обычный файл
@@ -0,0 +1,114 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func mustIPNet(t *testing.T, cidr string) *net.IPNet {
|
||||
t.Helper()
|
||||
_, n, err := net.ParseCIDR(cidr)
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
}
|
||||
|
||||
func TestReplaceRknIPs_BulkAndDedup(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []*net.IPNet{
|
||||
mustIPNet(t, "10.0.0.0/8"),
|
||||
mustIPNet(t, "10.5.0.0/8"), // same canonical /8 — dedup
|
||||
mustIPNet(t, "192.168.1.0/24"),
|
||||
mustIPNet(t, "2001:db8::/32"),
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknIPs(input))
|
||||
|
||||
// The uniqueIndex on rkn_ips.network ensures the dedup actually drops
|
||||
// duplicates; ReplaceRknIPs does an in-memory dedup, but the DB-level
|
||||
// constraint is the guarantee.
|
||||
var count int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&count).Error)
|
||||
assert.Equal(t, int64(3), count, "expected dedup to 3 unique CIDRs")
|
||||
}
|
||||
|
||||
func TestReplaceRknIPs_ReplacesExisting(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "8.8.8.0/24"),
|
||||
}))
|
||||
|
||||
var n int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(1), n)
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "1.0.0.0/8"),
|
||||
mustIPNet(t, "2.0.0.0/8"),
|
||||
}))
|
||||
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(2), n)
|
||||
}
|
||||
|
||||
func TestIsRknIPBlocked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "10.0.0.0/8"),
|
||||
mustIPNet(t, "192.168.1.0/24"),
|
||||
mustIPNet(t, "2001:db8::/32"),
|
||||
}))
|
||||
|
||||
cases := []struct {
|
||||
ip string
|
||||
want bool
|
||||
}{
|
||||
{"10.5.7.9", true},
|
||||
{"10.255.255.255", true},
|
||||
{"192.168.1.42", true},
|
||||
{"11.0.0.1", false},
|
||||
{"8.8.8.8", false},
|
||||
{"2001:db8::1", true},
|
||||
{"2001:db9::1", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.ip, func(t *testing.T) {
|
||||
got, err := models.IsRknIPBlocked(c.ip)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.want, got, "ip=%s", c.ip)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureRknIndexes_Idempotent(t *testing.T) {
|
||||
// Calling EnsureRknIndexes twice must not error — it's used both by
|
||||
// Migrate() and could be called from boot scripts.
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.EnsureRknIndexes())
|
||||
require.NoError(t, models.EnsureRknIndexes())
|
||||
|
||||
// GiST index must exist on rkn_ips.network.
|
||||
var exists bool
|
||||
err := models.DB().Raw(`
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE schemaname='public' AND tablename='rkn_ips'
|
||||
AND indexname='idx_rkn_ips_network'
|
||||
)
|
||||
`).Scan(&exists).Error
|
||||
require.NoError(t, err)
|
||||
assert.True(t, exists, "idx_rkn_ips_network must exist")
|
||||
}
|
||||
55
app/models/selfcheck.go
Обычный файл
55
app/models/selfcheck.go
Обычный файл
@@ -0,0 +1,55 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// SelfCheck provides functionality.
|
||||
type SelfCheck struct {
|
||||
concerns.Model
|
||||
Kind string `gorm:"not null;uniqueIndex:selfchecks" json:"kind"`
|
||||
Server *string `gorm:"uniqueIndex:selfchecks" json:"server"`
|
||||
Info string `json:"info"`
|
||||
LastCheck time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// LogCheck provides functionality.
|
||||
func LogCheck(kind string) error {
|
||||
m := SelfCheck{
|
||||
Kind: kind,
|
||||
Server: nil,
|
||||
}
|
||||
DB().FirstOrInit(&m, m)
|
||||
|
||||
m.LastCheck = time.Now()
|
||||
|
||||
return DB().Save(&m).Error
|
||||
}
|
||||
|
||||
// IsOk checks if the selfcheck for the given kind ran recently.
|
||||
func IsOk(kind string) (bool, string, error) {
|
||||
m := SelfCheck{
|
||||
Kind: kind,
|
||||
Server: nil,
|
||||
}
|
||||
DB().First(&m, m)
|
||||
|
||||
if m.ID == 0 {
|
||||
return false, "", errors.New("not run")
|
||||
}
|
||||
|
||||
var ago time.Time
|
||||
if kind == "exp" {
|
||||
ago = time.Now().Add(-3 * time.Hour)
|
||||
} else {
|
||||
ago = time.Now().Add(-15 * time.Minute)
|
||||
}
|
||||
|
||||
isOk := m.LastCheck.After(ago)
|
||||
|
||||
return isOk, m.LastCheck.Format(time.RFC3339), nil
|
||||
}
|
||||
388
app/models/server.go
Обычный файл
388
app/models/server.go
Обычный файл
@@ -0,0 +1,388 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql/driver"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// AccountMCPToken stores a one-way verifier. MCP tokens are bearer credentials
|
||||
// and must remain valid even when the deployment has no encryption key.
|
||||
type AccountMCPToken struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
TokenEnc string `gorm:"column:token;type:char(64);not null;index" json:"-"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (AccountMCPToken) TableName() string { return "account_mcp_tokens" }
|
||||
|
||||
func (t *AccountMCPToken) SetToken(token string) error {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
t.TokenEnc = hex.EncodeToString(sum[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *AccountMCPToken) TokenMatches(token string) bool {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
encoded := hex.EncodeToString(sum[:])
|
||||
return subtle.ConstantTimeCompare([]byte(t.TokenEnc), []byte(encoded)) == 1
|
||||
}
|
||||
|
||||
func GenerateMCPToken() string {
|
||||
return "mcp_" + base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
|
||||
}
|
||||
|
||||
// Server health states — worst-of-monitor-states rollup; see
|
||||
// docs/plans/servers-and-hardware-metrics.md §5.1 for the data model.
|
||||
const (
|
||||
ServerHealthDown = "down"
|
||||
ServerHealthWarn = "warn"
|
||||
ServerHealthUp = "up"
|
||||
ServerHealthPaused = "paused"
|
||||
ServerHealthUnknown = "unknown"
|
||||
)
|
||||
|
||||
// ServerEnvironments is the allow-list for Server.Environment.
|
||||
// New environments require an explicit edit so they are visible in
|
||||
// tests.
|
||||
var ServerEnvironments = []string{"production", "staging", "dev", "test"}
|
||||
|
||||
// ServerKind is the rstuff-mirrored lifecycle label for a Server
|
||||
// (production / staging / old). See
|
||||
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable mapping.
|
||||
type ServerKind string
|
||||
|
||||
// ServerKind values match rstuff's Server.kind enum exactly.
|
||||
// New values require adding a Postgres enum value via
|
||||
// app/models/migrate.go.
|
||||
const (
|
||||
ServerKindProduction ServerKind = "production"
|
||||
ServerKindStaging ServerKind = "staging"
|
||||
ServerKindOld ServerKind = "old"
|
||||
)
|
||||
|
||||
// Scan implements sql.Scanner for ServerKind.
|
||||
func (k *ServerKind) Scan(src any) error {
|
||||
if src == nil {
|
||||
*k = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*k = ServerKind(v)
|
||||
case []byte:
|
||||
*k = ServerKind(string(v))
|
||||
default:
|
||||
return fmt.Errorf("server_kind: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for ServerKind.
|
||||
func (k ServerKind) Value() (driver.Value, error) {
|
||||
if k == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(k), nil
|
||||
}
|
||||
|
||||
// Server represents a customer-facing logical host (e.g. "prod-web-01").
|
||||
//
|
||||
// One Server can host many WorkerNodes (HA after a VM migration). Each
|
||||
// WorkerNode carries a nullable ServerID so legacy "no server assigned"
|
||||
// rows keep working. The 1:N relation is stored as a nullable FK on
|
||||
// worker_nodes.server_id. The N:M relation to Monitor is the
|
||||
// monitor_servers join table. See
|
||||
// docs/plans/servers-and-hardware-metrics.md §3 for the layer model
|
||||
// and §5.1 for the schema.
|
||||
//
|
||||
// The inventory fields (ExtID, Kind, Token, PriceCents, Comment,
|
||||
// Meta) are added per docs/plans/inventory-management.md §6.2 so
|
||||
// rstuff can push the same row in via Valkey Streams and
|
||||
// deploymentd can authenticate via Token.
|
||||
type Server struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Slug string `gorm:"size:120;not null;index" json:"slug"`
|
||||
Description *string `gorm:"type:text" json:"description"`
|
||||
Region string `gorm:"size:64;not null;index" json:"region"`
|
||||
Environment string `gorm:"size:32;not null;default:'production'" json:"environment"`
|
||||
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
|
||||
Icon *string `gorm:"size:16" json:"icon"`
|
||||
Color *string `gorm:"size:16" json:"color"`
|
||||
Paused bool `gorm:"not null;default:false" json:"paused"`
|
||||
|
||||
// Inventory fields (rstuff mirror; see inventory-management.md §6.2).
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
Kind ServerKind `gorm:"type:server_kind;not null;default:'production'" json:"kind"`
|
||||
// Token is omitted from JSON because it is a bearer credential.
|
||||
// Read it back only via /api/v1/servers/:id/token (operator-only)
|
||||
// and never echoed in list/show responses.
|
||||
Token *string `gorm:"size:64" json:"-"`
|
||||
PriceCents int `gorm:"not null;default:0" json:"price_cents"`
|
||||
Comment *string `gorm:"type:text" json:"comment,omitempty"`
|
||||
// Meta is rstuff-style free-form jsonb; serialized via JSON
|
||||
// encoding (gin renders it as a nested object).
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
HealthState string `gorm:"size:16;not null;default:'unknown';index" json:"health_state"`
|
||||
LastCheckAt *time.Time `json:"last_check_at"`
|
||||
Uptime24h float64 `gorm:"not null;default:1.0" json:"uptime_24h"`
|
||||
Uptime30d float64 `gorm:"not null;default:1.0" json:"uptime_30d"`
|
||||
|
||||
Monitors []Monitor `gorm:"many2many:monitor_servers;joinForeignKey:server_id;joinReferences:monitor_id;" json:"monitors,omitempty"`
|
||||
Workers []WorkerNode `gorm:"foreignKey:ServerID" json:"workers,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Server) TableName() string { return "servers" }
|
||||
|
||||
// MonitorServer is the join row for the N:M relation between Monitor and
|
||||
// Server. One Monitor can be hosted on many Servers (multi-region
|
||||
// failover); one Server can host many Monitors.
|
||||
type MonitorServer struct {
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id) ON DELETE CASCADE;primaryKey" json:"monitor_id"`
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;primaryKey" json:"server_id"`
|
||||
Position int `gorm:"not null;default:0" json:"position"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (MonitorServer) TableName() string { return "monitor_servers" }
|
||||
|
||||
// ServerMetric is the last-N-point cache written alongside VictoriaMetrics.
|
||||
// The full time-series lives in TSDB; Postgres only keeps the most recent
|
||||
// row per (server, source) for fast health badges and "last seen" cells.
|
||||
// See docs/plans/servers-and-hardware-metrics.md §5.3.
|
||||
type ServerMetric struct {
|
||||
concerns.Model
|
||||
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
WorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"worker_id"`
|
||||
Source string `gorm:"size:32;not null;default:'worker'" json:"source"`
|
||||
CPUPercent *float64 `json:"cpu_percent"`
|
||||
MemUsed *int64 `json:"mem_used"`
|
||||
MemTotal *int64 `json:"mem_total"`
|
||||
DiskUsed *int64 `json:"disk_used"`
|
||||
DiskTotal *int64 `json:"disk_total"`
|
||||
NetRx *int64 `json:"net_rx"`
|
||||
NetTx *int64 `json:"net_tx"`
|
||||
HostUptimeSec *int64 `json:"host_uptime_sec"`
|
||||
Load1 *float64 `json:"load1"`
|
||||
Load5 *float64 `json:"load5"`
|
||||
Load15 *float64 `json:"load15"`
|
||||
ProcessCount *int `json:"process_count"`
|
||||
Processes datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"processes"`
|
||||
Networks datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"networks"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// ServerAlertRule defines one account-owned threshold for a server metric.
|
||||
// ClearThreshold implements hysteresis: a firing rule only recovers after the
|
||||
// value drops below it, avoiding alert flapping around Threshold.
|
||||
type ServerAlertRule struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
Metric string `gorm:"size:32;not null" json:"metric"`
|
||||
Threshold float64 `gorm:"not null" json:"threshold"`
|
||||
ClearThreshold float64 `gorm:"not null" json:"clear_threshold"`
|
||||
DurationSec int `gorm:"not null;default:300" json:"duration_sec"`
|
||||
NotificationID int64 `gorm:"type:bigint REFERENCES notifications(id) ON DELETE CASCADE;not null" json:"notification_id"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
State string `gorm:"size:16;not null;default:'ok'" json:"state"`
|
||||
BreachSince *time.Time `json:"breach_since"`
|
||||
LastValue *float64 `json:"last_value"`
|
||||
LastMetricID *int64 `json:"last_metric_id"`
|
||||
LastFiredAt *time.Time `json:"last_fired_at"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (ServerAlertRule) TableName() string { return "server_alert_rules" }
|
||||
|
||||
// ServerAlertEvent is the durable dedupe/audit record for threshold changes.
|
||||
type ServerAlertEvent struct {
|
||||
concerns.Model
|
||||
RuleID int64 `gorm:"type:bigint REFERENCES server_alert_rules(id) ON DELETE CASCADE;not null;index" json:"rule_id"`
|
||||
State string `gorm:"size:16;not null" json:"state"`
|
||||
Value float64 `gorm:"not null" json:"value"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (ServerAlertEvent) TableName() string { return "server_alert_events" }
|
||||
|
||||
// TableName provides functionality.
|
||||
func (ServerMetric) TableName() string { return "server_metrics" }
|
||||
|
||||
// ValidateEnvironment returns nil iff env is in the allow list.
|
||||
func ValidateEnvironment(env string) error {
|
||||
for _, e := range ServerEnvironments {
|
||||
if env == e {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("invalid environment")
|
||||
}
|
||||
|
||||
// Slugify turns a server name into a URL-safe slug.
|
||||
func Slugify(name string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(name))
|
||||
var b strings.Builder
|
||||
for _, r := range slug {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == ' ', r == '_', r == '-', r == '.':
|
||||
b.WriteByte('-')
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "-")
|
||||
if out == "" {
|
||||
out = "server"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AssignMonitors replaces the full set of monitors for a server. Saves
|
||||
// the join table explicitly because gorm:association_autoupdate is
|
||||
// disabled globally (CLAUDE.md GORM Conventions).
|
||||
func (s *Server) AssignMonitors(tx *gorm.DB, monitorIDs []uint) error {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
if err := tx.Exec("DELETE FROM monitor_servers WHERE server_id = ?", s.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(monitorIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[uint]struct{}, len(monitorIDs))
|
||||
for i, mid := range monitorIDs {
|
||||
if _, ok := seen[mid]; ok {
|
||||
continue
|
||||
}
|
||||
seen[mid] = struct{}{}
|
||||
row := MonitorServer{ServerID: s.ID, MonitorID: int64(mid), Position: i}
|
||||
if err := tx.Create(&row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RollupHealthState computes the worst-of-monitor-states. Returns one of
|
||||
// ServerHealth{Down,Warn,Up,Paused,Unknown}.
|
||||
func (s *Server) RollupHealthState(monitors []Monitor) string {
|
||||
if s.Paused {
|
||||
return ServerHealthPaused
|
||||
}
|
||||
if len(monitors) == 0 {
|
||||
return ServerHealthUnknown
|
||||
}
|
||||
allPaused := true
|
||||
worst := ServerHealthUp
|
||||
for i := range monitors {
|
||||
m := &monitors[i]
|
||||
if m.Enabled {
|
||||
allPaused = false
|
||||
}
|
||||
switch m.State {
|
||||
case stateERR, stateFail:
|
||||
return ServerHealthDown
|
||||
case stateWARN:
|
||||
worst = ServerHealthWarn
|
||||
}
|
||||
}
|
||||
if allPaused {
|
||||
return ServerHealthPaused
|
||||
}
|
||||
return worst
|
||||
}
|
||||
|
||||
// SortedTagList returns tags sorted ascending; helper for stable JSON.
|
||||
func (s *Server) SortedTagList() []string {
|
||||
out := make([]string, len(s.Tags))
|
||||
copy(out, s.Tags)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// HealthForServer recomputes and persists health_state + last_check_at
|
||||
// for one server. Called by the distworker health ticker.
|
||||
func HealthForServer(serverID int64) error {
|
||||
server := Server{}
|
||||
if err := DB().First(&server, serverID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var monitors []Monitor
|
||||
if err := DB().Joins("JOIN monitor_servers ms ON ms.monitor_id = monitors.id").
|
||||
Where("ms.server_id = ?", serverID).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
state := server.RollupHealthState(monitors)
|
||||
now := time.Now()
|
||||
updates := map[string]interface{}{"health_state": state}
|
||||
if len(monitors) > 0 {
|
||||
updates["last_check_at"] = &now
|
||||
}
|
||||
return DB().Model(&server).Updates(updates).Error
|
||||
}
|
||||
|
||||
// LatestServerMetric returns the newest accepted worker snapshot for a server.
|
||||
func LatestServerMetric(serverID int64) (*ServerMetric, error) {
|
||||
metric := ServerMetric{}
|
||||
err := DB().Where("server_id = ?", serverID).Order("id DESC").First(&metric).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &metric, nil
|
||||
}
|
||||
|
||||
// FindServerByToken returns the Server whose token column equals
|
||||
// the given hex value, or nil with gorm.ErrRecordNotFound when no
|
||||
// row matches. Used by the deploymentd receiver middleware.
|
||||
func FindServerByToken(token string) (*Server, error) {
|
||||
var s Server
|
||||
err := DB().Where("token = ?", token).First(&s).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// GenerateServerToken returns a 32-byte random hex string. Caller
|
||||
// stores the plaintext exactly once (via /servers/:id/rotate-token)
|
||||
// and updates Server.Token; the old value is no longer recoverable.
|
||||
func GenerateServerToken() string {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
// crypto/rand should not fail on Linux; panic keeps the
|
||||
// contract simple for callers in the rare fatal case.
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
43
app/models/server_health_ticker.go
Обычный файл
43
app/models/server_health_ticker.go
Обычный файл
@@ -0,0 +1,43 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HealthTickInterval is exported so focused tests can exercise the same
|
||||
// lifecycle with a short interval.
|
||||
var HealthTickInterval = time.Minute
|
||||
|
||||
var serverHealthTickerOnce sync.Once
|
||||
|
||||
// StartServerHealthTicker periodically refreshes derived server health. The
|
||||
// first production tick is delayed to keep CLI migration paths side-effect
|
||||
// free; assignment/pause paths recompute synchronously.
|
||||
func StartServerHealthTicker(parent context.Context) {
|
||||
serverHealthTickerOnce.Do(func() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(HealthTickInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-parent.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
var ids []int64
|
||||
if err := DB().Model(&Server{}).Pluck("id", &ids).Error; err != nil {
|
||||
log.Printf("server health: list: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, id := range ids {
|
||||
if err := HealthForServer(id); err != nil {
|
||||
log.Printf("server health %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
36
app/models/server_ip.go
Обычный файл
36
app/models/server_ip.go
Обычный файл
@@ -0,0 +1,36 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// ServerIp is a single IP (v4 or v6) bound to a Server. The
|
||||
// canonical source for these rows is the deploymentd server-inventory
|
||||
// receiver (M1) and the network-diagnostics partial plan; today the
|
||||
// only writer is operator-entered via /api/v1/servers/:id/ips.
|
||||
//
|
||||
// `address` is Postgres `inet` so range queries (`<<` / `>>`) work
|
||||
// without parsing text. One row per (server_id, address). The
|
||||
// `is_primary` flag is set when more than one IP exists and the
|
||||
// deploymentd payload signals a primary; otherwise the first row wins.
|
||||
//
|
||||
//nolint:revive // ServerIP rename deferred to M3; rstuff schema uses ServerIp verbatim and parity test depends on it.
|
||||
type ServerIp struct {
|
||||
concerns.Model
|
||||
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
Server *Server `json:"-"`
|
||||
// Address is mapped to Postgres inet via raw SQL in Migrate(); the
|
||||
// GORM `type:` tag is not enough because gorm.io/driver/postgres
|
||||
// does not register `inet` in its type map. Field is stored as a
|
||||
// string and validated by the API layer (see
|
||||
// app/controllers/api/server.go ServerIPsAdd).
|
||||
Address string `gorm:"type:inet;not null" json:"address"`
|
||||
IsPrimary bool `gorm:"not null;default:false" json:"is_primary"`
|
||||
RelatedSitesCount int `gorm:"not null;default:0" json:"related_sites_count"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (ServerIp) TableName() string { return "server_ips" }
|
||||
98
app/models/site.go
Обычный файл
98
app/models/site.go
Обычный файл
@@ -0,0 +1,98 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Site represents a customer-facing website or app. One Site can be
|
||||
// hosted on zero or one Server (server_id nullable) and exposes one
|
||||
// or more Deployments (compose services or nginx vhosts) plus zero
|
||||
// or more Repos (via site_repos). The RSMon slice mirrors rstuff's
|
||||
// `sites` table verbatim — see docs/parity/rstuff-inventory.md §2
|
||||
// and docs/plans/inventory-management.md §4.
|
||||
type Site struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Slug string `gorm:"size:120;not null;index" json:"slug"`
|
||||
URL *string `gorm:"type:text" json:"url,omitempty"`
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
// Kind is a free-text label (not the PG enum) so we can absorb
|
||||
// rstuff additions without a migration. Default "production".
|
||||
Kind string `gorm:"size:32;not null;default:'production'" json:"kind"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
Deployments []Deployment `gorm:"foreignKey:SiteID" json:"deployments,omitempty"`
|
||||
Repos []Repo `gorm:"many2many:site_repos;" json:"repos,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName returns the table name used for Site. Matches rstuff's
|
||||
// `sites` plural exactly so the parity test stays trivial.
|
||||
func (Site) TableName() string { return "sites" }
|
||||
|
||||
// SiteSlugify turns a name into a URL-safe slug. Mirrors the rules
|
||||
// in app/models/server.go:Slugify so /sites/:slug looks the same as
|
||||
// /servers/:slug. Consecutive separators collapse to a single dash;
|
||||
// non-ASCII letters are stripped (the same rule as Slugify — we
|
||||
// don't transliterate in v1, see docs/plans/inventory-management.md
|
||||
// §11 for transliteration as a future hardening item).
|
||||
func SiteSlugify(name string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(name))
|
||||
var b strings.Builder
|
||||
prevDash := false
|
||||
for _, r := range slug {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
prevDash = false
|
||||
case r == ' ', r == '_', r == '-', r == '.':
|
||||
if !prevDash && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
prevDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "-")
|
||||
if out == "" {
|
||||
out = "site"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindOrCreateSiteBySlug returns the site with the given slug for the
|
||||
// given account, creating an empty row (Name=slug, Kind=production,
|
||||
// IsActive=true) when no match exists. The caller's tx wraps the
|
||||
// operation so docker payload ingestion stays atomic.
|
||||
// See docs/plans/inventory-management.md §7.2 — Docker receiver.
|
||||
func FindOrCreateSiteBySlug(tx *gorm.DB, accountID int64, slug string) (*Site, error) {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
var site Site
|
||||
err := tx.Where("account_id = ? AND slug = ?", accountID, slug).First(&site).Error
|
||||
if err == nil {
|
||||
return &site, nil
|
||||
}
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
return nil, err
|
||||
}
|
||||
site = Site{AccountID: accountID, Slug: slug, Name: slug, Kind: "production", IsActive: true}
|
||||
if err := tx.Create(&site).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &site, nil
|
||||
}
|
||||
120
app/models/stats_data.go
Обычный файл
120
app/models/stats_data.go
Обычный файл
@@ -0,0 +1,120 @@
|
||||
package models
|
||||
|
||||
// StatsData provides functionality.
|
||||
type StatsData struct {
|
||||
MonitorID *int64 `json:"monitor_id,omitempty"`
|
||||
Up30d *float64 `json:"up_30d"`
|
||||
Up7d *float64 `json:"up_7d"`
|
||||
Up1d *float64 `json:"up_1d"`
|
||||
}
|
||||
|
||||
// Process provides functionality.
|
||||
func (data *StatsData) Process() {
|
||||
if data.Up1d == nil {
|
||||
dv := 100.0
|
||||
data.Up1d = &dv
|
||||
}
|
||||
if data.Up7d == nil {
|
||||
dv := 100.0
|
||||
data.Up7d = &dv
|
||||
}
|
||||
if data.Up30d == nil {
|
||||
dv := 100.0
|
||||
data.Up30d = &dv
|
||||
}
|
||||
}
|
||||
|
||||
// 86400 seconds / 1d
|
||||
// 604800 seconds / 7d
|
||||
// 2592000 seconds / 30d
|
||||
|
||||
// UptimeSQL provides functionality.
|
||||
const UptimeSQL = `
|
||||
SELECT
|
||||
monitors.id AS monitor_id,
|
||||
round(up1.up::numeric, 3) AS up1d,
|
||||
round(up7.up::numeric, 3) AS up7d,
|
||||
round(up30.up::numeric, 3) AS up30d
|
||||
FROM monitors
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '1' day
|
||||
group by monitor_id
|
||||
) up1 ON monitors.id = up1.monitor_id
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '7' day
|
||||
group by monitor_id
|
||||
) up7 ON monitors.id = up7.monitor_id
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '30' day
|
||||
group by monitor_id
|
||||
) up30 ON monitors.id = up30.monitor_id
|
||||
`
|
||||
|
||||
// UptimeAllSQL provides functionality.
|
||||
const UptimeAllSQL = `
|
||||
SELECT
|
||||
round(up1.up::numeric, 3) AS up1d,
|
||||
round(up7.up::numeric, 3) AS up7d,
|
||||
round(up30.up::numeric, 3) AS up30d
|
||||
FROM (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '1' day AND events.monitor_id IN (?)
|
||||
) up1, (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '7' day AND events.monitor_id IN (?)
|
||||
) up7, (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '30' day AND events.monitor_id IN (?)
|
||||
) up30
|
||||
`
|
||||
|
||||
// MonitorStats provides functionality.
|
||||
func MonitorStats(monitors *[]Monitor) error {
|
||||
ids := make([]int64, len(*monitors))
|
||||
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
|
||||
ids[i] = m.ID
|
||||
}
|
||||
|
||||
rows, err := DB().Raw(UptimeSQL+"WHERE monitors.id IN (?)", ids).Rows()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats := make(map[int64]StatsData, 0)
|
||||
|
||||
defer rows.Close() //nolint:errcheck
|
||||
for rows.Next() {
|
||||
data := StatsData{}
|
||||
_ = DB().ScanRows(rows, &data)
|
||||
data.Process()
|
||||
|
||||
stats[*data.MonitorID] = data
|
||||
}
|
||||
|
||||
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
|
||||
(*monitors)[i].StatsData = stats[m.ID]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
461
app/models/status_page.go
Обычный файл
461
app/models/status_page.go
Обычный файл
@@ -0,0 +1,461 @@
|
||||
// Package models — status page subsystem (docs/plans/status-pages.md).
|
||||
//
|
||||
// M0 ships the schema for status_pages and its five related tables
|
||||
// (subscribers, incidents, maintenance, domains). The M0 milestone is
|
||||
// read-only at the dashboard level — no editor and no public render yet —
|
||||
// but landing the schema now lets downstream milestones wire public
|
||||
// routes, editor flows, and the notifier→subscriber bridge without
|
||||
// further ALTER TABLE churn. M5 (custom domain) only fills in
|
||||
// status_page_domains rows; the table itself is reserved here so the
|
||||
// M5 migration is just data, not DDL.
|
||||
//
|
||||
// All tables follow the existing RSMon conventions: concerns.Model +
|
||||
// concerns.Timestamped + Audited mixins, gorm.DeletedAt for soft delete
|
||||
// on the top-level status_pages row, pq.Int64Array for the monitor_ids
|
||||
// bigint[] join columns (same shape as sites and the Check.Warnings
|
||||
// slice). Partial-unique indexes (slug, subscriber email) are added via
|
||||
// raw SQL in app/models/migrate.go because the GORM tag language cannot
|
||||
// express a WHERE deleted_at IS NULL predicate.
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"golang.org/x/net/idna"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/config/credis"
|
||||
)
|
||||
|
||||
// Status page color defaults. Matches the existing landing-page primary
|
||||
// green and the accent blue used in the /settings UI, so a freshly
|
||||
// created page already blends in with the rest of the app.
|
||||
const (
|
||||
statusPageDefaultPrimaryColor = "#62c600"
|
||||
statusPageDefaultAccentColor = "#1a73e8"
|
||||
statusPageDefaultHistoryDays = 90
|
||||
statusPageMaxSlugLen = 64
|
||||
statusPageMaxNameLen = 120
|
||||
)
|
||||
|
||||
// Hex color regex — accepts #RGB and #RRGGBB. Centralized so the
|
||||
// controller/model validation agree on the same shape.
|
||||
var hexColorRegex = regexp.MustCompile(`^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$`)
|
||||
|
||||
// StatusPageSlugRegex mirrors the slug format enforced in
|
||||
// StatusPage.NormalizeSlug / ValidateSlug. Lowercase alphanumerics and
|
||||
// dashes, must start and end with an alphanumeric. Length is checked
|
||||
// separately so the regex stays readable.
|
||||
var statusPageSlugRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`)
|
||||
|
||||
// StatusPage represents a public status page owned by an account. One
|
||||
// account may own many pages (gated by plan in M2+); slugs are globally
|
||||
// unique because public URLs do not contain the account ID. Each page exposes
|
||||
// a curated subset of the account's monitors and a recent-incidents
|
||||
// feed. Soft-deleted rows remain in the table so the partial-unique
|
||||
// index on (account_id, slug) WHERE deleted_at IS NULL still rejects
|
||||
// duplicate slugs against historical records — see the comment on
|
||||
// StatusPagesAccountSlugUnique in migrate.go.
|
||||
type StatusPage struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
|
||||
Slug string `gorm:"size:64;not null;index" json:"slug"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
LogoURL *string `gorm:"size:255" json:"logo_url,omitempty"`
|
||||
|
||||
PrimaryColor string `gorm:"size:7;not null;default:'#62c600'" json:"primary_color"`
|
||||
AccentColor string `gorm:"size:7;not null;default:'#1a73e8'" json:"accent_color"`
|
||||
|
||||
// MonitorIDs is the curated subset of account monitors the page
|
||||
// exposes. Order is preserved so the dashboard list and the public
|
||||
// render show the same ordering. Stored as bigint[] to keep
|
||||
// monitor-to-page mapping lookup-free on the read path; M2 will
|
||||
// add a UI to maintain this set.
|
||||
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
|
||||
|
||||
ShowUptimeBars bool `gorm:"not null;default:true" json:"show_uptime_bars"`
|
||||
ShowResponseTimes bool `gorm:"not null;default:true" json:"show_response_times"`
|
||||
ShowHistoryDays int `gorm:"not null;default:90" json:"show_history_days"`
|
||||
|
||||
// PasswordHash is populated in M4 only. Stored at length 255 so
|
||||
// a future bcrypt cost bump does not need a column resize.
|
||||
PasswordHash *string `gorm:"size:255" json:"-"`
|
||||
// GATrackingID — Google Analytics 4 measurement ID; emitted by
|
||||
// the public renderer in M4.
|
||||
GATrackingID *string `gorm:"size:32" json:"ga_tracking_id,omitempty"`
|
||||
|
||||
// NoIndex emits <meta name="robots" content="noindex"> so the
|
||||
// page can be staged without polluting search indexes.
|
||||
NoIndex bool `gorm:"not null;default:false" json:"no_index"`
|
||||
|
||||
// IsPublished gates the public /status/:slug render. Until M2
|
||||
// ships the editor the default value keeps M0 pages invisible.
|
||||
IsPublished bool `gorm:"not null;default:false" json:"is_published"`
|
||||
// AutoOpenIncidents is opt-in so publishing a page does not change
|
||||
// existing alert behavior until an owner explicitly enables it.
|
||||
AutoOpenIncidents bool `gorm:"not null;default:false" json:"auto_open_incidents"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
|
||||
// DeletedAt is the GORM soft-delete marker. Using gorm.DeletedAt
|
||||
// rather than concerns.SoftDelete because the latter adds a
|
||||
// DeleterID users(id) FK that we do not yet need on status_pages.
|
||||
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
|
||||
}
|
||||
|
||||
// TableName returns the explicit table name so GORM does not try to
|
||||
// pluralize to "status_pages" via inflection. The plural is already
|
||||
// correct; we declare it anyway for clarity.
|
||||
func (StatusPage) TableName() string { return "status_pages" }
|
||||
|
||||
// StatusPageSubscriberKind — values stored in status_page_subscribers.kind.
|
||||
// "alert" subscribes to incident-driven notifications; "digest_daily"
|
||||
// receives the morning summary (M3). New kinds should be appended so
|
||||
// the JSON serializations stay stable.
|
||||
const (
|
||||
StatusPageSubscriberKindAlert = "alert"
|
||||
StatusPageSubscriberKindDigestDaily = "digest_daily"
|
||||
)
|
||||
|
||||
// StatusPageSubscriber is a row in status_page_subscribers. Email is
|
||||
// stored verbatim (no citext) because the codebase already persists
|
||||
// contact emails as-is; case-insensitive uniqueness is enforced via
|
||||
// the partial unique index in migrate.go using lower(email).
|
||||
type StatusPageSubscriber struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
// ContactID is an internal delivery endpoint. It is never returned by public
|
||||
// subscription APIs; tasks use it to preserve the normal worker email path.
|
||||
ContactID *int64 `gorm:"type:bigint REFERENCES contacts(id) ON DELETE SET NULL;index" json:"-"`
|
||||
|
||||
Email string `gorm:"size:255;not null" json:"email"`
|
||||
Kind string `gorm:"size:16;not null;default:'alert'" json:"kind"`
|
||||
ConfirmTokenHash string `gorm:"size:64" json:"-"`
|
||||
// LegacyConfirmToken is retained only for confirmation links issued before
|
||||
// token hashing shipped. It is cleared on first use or resend.
|
||||
LegacyConfirmToken *string `gorm:"column:confirm_token;size:255" json:"-"`
|
||||
TokenExpiresAt time.Time `json:"-"`
|
||||
UnsubscribeTokenHash string `gorm:"size:64;default:''" json:"-"`
|
||||
ConfirmedAt *time.Time `json:"confirmed_at,omitempty"`
|
||||
UnsubscribedAt *time.Time `json:"unsubscribed_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_subscribers table name.
|
||||
func (StatusPageSubscriber) TableName() string { return "status_page_subscribers" }
|
||||
|
||||
// StatusPageIncident severity values. info = heads-up notices, warn =
|
||||
// degradation, crit = full outage. Used for color-coding in the public
|
||||
// render (M1) and for filtering in the dashboard list (M0).
|
||||
const (
|
||||
StatusPageIncidentSeverityInfo = "info"
|
||||
StatusPageIncidentSeverityWarn = "warn"
|
||||
StatusPageIncidentSeverityCrit = "crit"
|
||||
)
|
||||
|
||||
// StatusPageIncident represents a single incident entry on a status
|
||||
// page. event_id is a soft link back to the existing Event model so
|
||||
// "auto-open on monitor error" can be wired later without a second
|
||||
// migration. posted_by_user_id is nullable so external integrations
|
||||
// can write incidents anonymously.
|
||||
type StatusPageIncident struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
EventID *int64 `gorm:"type:bigint REFERENCES events(id) ON DELETE SET NULL;index" json:"event_id,omitempty"`
|
||||
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
BodyMD string `gorm:"type:text" json:"body_md,omitempty"`
|
||||
Severity string `gorm:"size:16;not null;default:'info'" json:"severity"`
|
||||
|
||||
StartedAt time.Time `gorm:"not null;index" json:"started_at"`
|
||||
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
|
||||
|
||||
PostedByUserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"posted_by_user_id,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_incidents table name.
|
||||
func (StatusPageIncident) TableName() string { return "status_page_incidents" }
|
||||
|
||||
// StatusPageMaintenance is a scheduled maintenance window. The
|
||||
// monitor_ids column is the set of monitors the window covers; empty
|
||||
// means "all monitors on the page".
|
||||
type StatusPageMaintenance struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
Description string `gorm:"type:text" json:"description,omitempty"`
|
||||
|
||||
StartsAt time.Time `gorm:"not null;index" json:"starts_at"`
|
||||
EndsAt time.Time `gorm:"not null" json:"ends_at"`
|
||||
|
||||
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
|
||||
|
||||
NotifySubscribers bool `gorm:"not null;default:true" json:"notify_subscribers"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_maintenance table name.
|
||||
func (StatusPageMaintenance) TableName() string { return "status_page_maintenance" }
|
||||
|
||||
// StatusPageDomain is the M5 custom-domain mapping. Reserved in M0 so
|
||||
// the table does not need to be created at M5 — only rows are written
|
||||
// then. domain is unique globally (CNAMEs are hostnames, they cannot
|
||||
// be reused across pages), txt_token is the value the user adds as a
|
||||
// DNS TXT record to prove ownership.
|
||||
type StatusPageDomain struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
Domain string `gorm:"size:255;not null;uniqueIndex" json:"domain"`
|
||||
VerifiedAt *time.Time `json:"verified_at,omitempty"`
|
||||
TXTToken string `gorm:"size:64;not null" json:"txt_token,omitempty"`
|
||||
VerifyError string `gorm:"type:text" json:"verify_error,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_domains table name.
|
||||
func (StatusPageDomain) TableName() string { return "status_page_domains" }
|
||||
|
||||
// NormalizeStatusPageDomain accepts a hostname only. URLs, ports, IP literals,
|
||||
// wildcard names, and invalid IDNA are deliberately rejected before DNS work.
|
||||
func NormalizeStatusPageDomain(in string) (string, error) {
|
||||
domain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(in)), ".")
|
||||
if domain == "" || len(domain) > 253 || strings.ContainsAny(domain, "/:@") || net.ParseIP(domain) != nil {
|
||||
return "", errStatusPage("domain must be a hostname")
|
||||
}
|
||||
ascii, err := idna.Lookup.ToASCII(domain)
|
||||
if err != nil || ascii == "" || len(ascii) > 253 || !strings.Contains(ascii, ".") {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
for _, label := range strings.Split(ascii, ".") {
|
||||
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
for _, r := range label {
|
||||
if r != '-' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
}
|
||||
}
|
||||
return ascii, nil
|
||||
}
|
||||
|
||||
// HashStatusPageToken keeps bearer-style subscription URLs out of the database.
|
||||
func HashStatusPageToken(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return fmt.Sprintf("%x", sum[:])
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) TokenMatches(token string) bool {
|
||||
return s != nil && s.TokenExpiresAt.After(time.Now()) && s.ConfirmTokenHash == HashStatusPageToken(token)
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) UnsubscribeTokenMatches(token string) bool {
|
||||
return s != nil && s.UnsubscribeTokenHash != "" && s.UnsubscribeTokenHash == HashStatusPageToken(token)
|
||||
}
|
||||
|
||||
// StatusPagePlatformDomain and StatusPagePublicIPs are deployment-owned DNS
|
||||
// targets. Customer domains must point here before they can become routable.
|
||||
func StatusPagePlatformDomain() string {
|
||||
return strings.TrimSuffix(strings.ToLower(os.Getenv("STATUS_PAGE_PLATFORM_DOMAIN")), ".")
|
||||
}
|
||||
|
||||
func StatusPagePublicIPs() []string {
|
||||
return strings.FieldsFunc(os.Getenv("STATUS_PAGE_PUBLIC_IPS"), func(r rune) bool { return r == ',' || r == ' ' })
|
||||
}
|
||||
|
||||
var (
|
||||
statusPageLookupCNAME = net.LookupCNAME
|
||||
statusPageLookupHost = net.LookupHost
|
||||
statusPageLookupTXT = net.LookupTXT
|
||||
)
|
||||
|
||||
// VerifyStatusPageDomain performs the BYO-DNS preflight. A customer must keep
|
||||
// the ownership TXT record and point either a CNAME at the platform hostname or
|
||||
// an A/AAAA record at one of the explicitly configured public addresses.
|
||||
func VerifyStatusPageDomain(domain *StatusPageDomain) error {
|
||||
if domain == nil {
|
||||
return errStatusPage("domain is required")
|
||||
}
|
||||
want, err := NormalizeStatusPageDomain(domain.Domain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
platform := StatusPagePlatformDomain()
|
||||
publicIPs := StatusPagePublicIPs()
|
||||
if platform == "" && len(publicIPs) == 0 {
|
||||
return errStatusPage("custom domain verification is not configured")
|
||||
}
|
||||
matchedTarget := false
|
||||
if platform != "" {
|
||||
if cname, lookupErr := statusPageLookupCNAME(want); lookupErr == nil {
|
||||
matchedTarget = strings.TrimSuffix(strings.ToLower(cname), ".") == platform
|
||||
}
|
||||
}
|
||||
if !matchedTarget && len(publicIPs) > 0 {
|
||||
if hosts, lookupErr := statusPageLookupHost(want); lookupErr == nil {
|
||||
for _, host := range hosts {
|
||||
for _, allowed := range publicIPs {
|
||||
if host == allowed {
|
||||
matchedTarget = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matchedTarget {
|
||||
return errStatusPage("DNS must contain the configured CNAME or public A/AAAA address")
|
||||
}
|
||||
txt, lookupErr := statusPageLookupTXT(want)
|
||||
if lookupErr != nil {
|
||||
return errStatusPage("ownership TXT record was not found")
|
||||
}
|
||||
for _, value := range txt {
|
||||
if value == "rsmon-verify="+domain.TXTToken {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errStatusPage("ownership TXT record does not match")
|
||||
}
|
||||
|
||||
// NormalizeStatusPageSlug lowercases and trims a candidate slug so the
|
||||
// global partial unique index on slug is satisfied
|
||||
// regardless of how the caller capitalizes the input. Returns the
|
||||
// empty string when the result would be unusable as a URL path;
|
||||
// callers should fall back to a name-derived slug in that case.
|
||||
func NormalizeStatusPageSlug(in string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(in))
|
||||
return slug
|
||||
}
|
||||
|
||||
// ValidateStatusPageSlug enforces the slug format we expose to users:
|
||||
// lowercase alphanumeric plus dash, must start and end with an
|
||||
// alphanumeric, max length 64. Used by the editor before save (M2).
|
||||
// Returns nil when the slug is acceptable.
|
||||
func ValidateStatusPageSlug(slug string) error {
|
||||
if slug == "" {
|
||||
return errStatusPage("slug is required")
|
||||
}
|
||||
if len(slug) > statusPageMaxSlugLen {
|
||||
return errStatusPage("slug is too long")
|
||||
}
|
||||
if !statusPageSlugRegex.MatchString(slug) {
|
||||
return errStatusPage("slug must be lowercase alphanumeric with dashes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateStatusPageColors returns an error if either color is set but
|
||||
// not a valid CSS hex string. Empty strings fall back to the model
|
||||
// defaults when written via BeforeSave hooks.
|
||||
func ValidateStatusPageColors(primary, accent string) error {
|
||||
if primary != "" && !hexColorRegex.MatchString(primary) {
|
||||
return errStatusPage("primary_color must be #RGB or #RRGGBB")
|
||||
}
|
||||
if accent != "" && !hexColorRegex.MatchString(accent) {
|
||||
return errStatusPage("accent_color must be #RGB or #RRGGBB")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BeforeSave is the GORM hook that fills in the canonical defaults
|
||||
// (colors, history days) so callers can pass an empty struct and still
|
||||
// get a usable page. Hook is also the single place where slugs are
|
||||
// normalized, so the unique index never has to chase trailing spaces.
|
||||
// The gorm.DB parameter is required by the hook signature but unused —
|
||||
// the validation here is purely local to the model.
|
||||
func (p *StatusPage) BeforeSave(_ *gorm.DB) error {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
p.Slug = NormalizeStatusPageSlug(p.Slug)
|
||||
if err := ValidateStatusPageSlug(p.Slug); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.PrimaryColor == "" {
|
||||
p.PrimaryColor = statusPageDefaultPrimaryColor
|
||||
}
|
||||
if p.AccentColor == "" {
|
||||
p.AccentColor = statusPageDefaultAccentColor
|
||||
}
|
||||
if p.ShowHistoryDays == 0 {
|
||||
p.ShowHistoryDays = statusPageDefaultHistoryDays
|
||||
}
|
||||
return ValidateStatusPageColors(p.PrimaryColor, p.AccentColor)
|
||||
}
|
||||
|
||||
// IsPublishedNow reports whether the page is publicly visible. M0
|
||||
// always returns false because the editor (M2) is the only thing that
|
||||
// flips IsPublished to true; this helper centralizes that contract.
|
||||
func (p *StatusPage) IsPublishedNow() bool {
|
||||
return p != nil && p.IsPublished && !p.DeletedAt.Valid
|
||||
}
|
||||
|
||||
// errStatusPage builds a validation error carrying the message. The
|
||||
// returned error is a plain error; controllers translate it into a
|
||||
// 422 response.
|
||||
func errStatusPage(msg string) error {
|
||||
if msg == "" {
|
||||
return errors.New("status_page: invalid")
|
||||
}
|
||||
return fmt.Errorf("status_page: %s", msg)
|
||||
}
|
||||
|
||||
// IsActiveSubscriber returns true when the subscriber has confirmed and
|
||||
// has not unsubscribed. Used by the M3 incident-notification loop.
|
||||
func (s *StatusPageSubscriber) IsActiveSubscriber() bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
return s.ConfirmedAt != nil && s.UnsubscribedAt == nil
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) BeforeCreate(_ *gorm.DB) error {
|
||||
if s.TokenExpiresAt.IsZero() {
|
||||
s.TokenExpiresAt = time.Now().Add(24 * time.Hour)
|
||||
}
|
||||
if s.ConfirmTokenHash == "" {
|
||||
s.ConfirmTokenHash = HashStatusPageToken(fmt.Sprintf("legacy-%d-%s", time.Now().UnixNano(), s.Email))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InvalidateStatusPageCache removes the public HTML cache without making
|
||||
// Redis availability part of the monitor or management write path.
|
||||
func InvalidateStatusPageCache(pageID int64) {
|
||||
if credis.Redis != nil {
|
||||
_ = credis.Redis.Del(context.Background(), "statuspage:html:"+strconv.FormatInt(pageID, 10)).Err()
|
||||
}
|
||||
}
|
||||
408
app/models/status_page_delivery.go
Обычный файл
408
app/models/status_page_delivery.go
Обычный файл
@@ -0,0 +1,408 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
const (
|
||||
statusPageDigestHour = 9
|
||||
statusPageOutboxPending = "pending"
|
||||
statusPageOutboxDispatched = "dispatched"
|
||||
statusPageOutboxCanceled = "canceled"
|
||||
)
|
||||
|
||||
// StatusPageDelivery is an outbox row. It does not retain email addresses;
|
||||
// confirmation links are the only transient body data and are redacted on
|
||||
// cancellation. The subscriber/contact data is read while dispatching.
|
||||
type StatusPageDelivery struct {
|
||||
ID int64 `gorm:"primarykey"`
|
||||
StatusPageID int64 `gorm:"not null;index"`
|
||||
SubscriberID int64 `gorm:"not null;index"`
|
||||
IncidentID *int64 `gorm:"index"`
|
||||
Kind string `gorm:"size:32;not null"`
|
||||
Version string `gorm:"size:64;not null"`
|
||||
LocalDate string `gorm:"size:10"`
|
||||
State string `gorm:"size:16;not null;index"`
|
||||
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
|
||||
MessageID *int64 `gorm:"index"`
|
||||
TaskID *int64 `gorm:"index"`
|
||||
LastError string `gorm:"type:text"`
|
||||
Subject string `gorm:"type:text"`
|
||||
BodyText string `gorm:"type:text"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
|
||||
|
||||
type StatusPageDigestSchedule struct {
|
||||
ID int64 `gorm:"primarykey"`
|
||||
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
|
||||
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
|
||||
DueAt time.Time `gorm:"not null;index"`
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
|
||||
|
||||
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
|
||||
if subscriber.ContactID != nil {
|
||||
return nil
|
||||
}
|
||||
accountID := page.AccountID
|
||||
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
|
||||
if err := tx.Create(contact).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subscriber.ContactID = &contact.ID
|
||||
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
|
||||
}
|
||||
|
||||
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
|
||||
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
|
||||
}
|
||||
|
||||
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
|
||||
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
|
||||
}
|
||||
|
||||
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
|
||||
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
|
||||
}
|
||||
|
||||
func valueOrZero(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
|
||||
// transaction, so an incident can never become visible without its delivery
|
||||
// intent being recoverable by the dispatcher.
|
||||
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
||||
for _, subscriber := range subscribers {
|
||||
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
|
||||
var notification Notification
|
||||
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(¬ification).Error
|
||||
if err == nil {
|
||||
return ¬ification, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
|
||||
return ¬ification, tx.Create(¬ification).Error
|
||||
}
|
||||
|
||||
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
|
||||
// Message and Task rows in the same transaction, and preserves a pending row on
|
||||
// transient worker/capability failure for the next tick.
|
||||
func DispatchStatusPageDeliveries(now time.Time) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, row := range rows {
|
||||
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func dispatchStatusPageDelivery(id int64, now time.Time) error {
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var outbox StatusPageDelivery
|
||||
if err := tx.First(&outbox, id).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var subscriber StatusPageSubscriber
|
||||
var page StatusPage
|
||||
// Subscriber then outbox is the global lock order shared with
|
||||
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
|
||||
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
|
||||
}
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if outbox.State != statusPageOutboxPending {
|
||||
return nil
|
||||
}
|
||||
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
|
||||
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
|
||||
}
|
||||
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
|
||||
return err
|
||||
}
|
||||
notification, err := statusPageNotificationTx(tx, page.AccountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, text := outbox.Subject, outbox.BodyText
|
||||
if subject == "" {
|
||||
subject, text = "["+page.Name+"] status update", "Status update"
|
||||
}
|
||||
if outbox.IncidentID != nil {
|
||||
var incident StatusPageIncident
|
||||
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subject = "[" + page.Name + "] " + incident.Title
|
||||
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
|
||||
} else if outbox.BodyText == "" {
|
||||
text = "Open incidents daily digest"
|
||||
}
|
||||
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
|
||||
if err := tx.Create(message).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
|
||||
})
|
||||
if err != nil {
|
||||
// This update intentionally runs after rollback: diagnostic state must not
|
||||
// disappear with the failed message/task transaction.
|
||||
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
|
||||
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
|
||||
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if outbox.TaskID != nil {
|
||||
// Redact every task, including terminal audit rows. State is preserved for
|
||||
// terminal rows, while the payload can no longer disclose the address.
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if outbox.MessageID != nil {
|
||||
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
|
||||
var subscriber StatusPageSubscriber
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
|
||||
// retained status-page lineage without touching unrelated account messages.
|
||||
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
row := &rows[i]
|
||||
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if row.TaskID != nil {
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if row.MessageID != nil {
|
||||
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if rows[i].TaskID != nil {
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if rows[i].MessageID != nil {
|
||||
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
|
||||
// endpoint inert and irreversibly removing subscriber PII.
|
||||
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
for i := range subscribers {
|
||||
s := &subscribers[i]
|
||||
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if s.ContactID != nil {
|
||||
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return tx.Delete(page).Error
|
||||
}
|
||||
|
||||
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
|
||||
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
|
||||
func EnqueueStatusPageDailyDigests(now time.Time) error {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for i := range subscribers {
|
||||
s := &subscribers[i]
|
||||
if s.StatusPage == nil || s.StatusPage.Account == nil {
|
||||
continue
|
||||
}
|
||||
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
|
||||
if err != nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
|
||||
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
|
||||
// Only persist today's actual due date. Older pending outbox rows are
|
||||
// retried by DispatchStatusPageDeliveries; never invent history from a
|
||||
// current incident snapshot after downtime.
|
||||
if now.Before(due) || s.ConfirmedAt.After(due) {
|
||||
continue
|
||||
}
|
||||
var incidents []StatusPageIncident
|
||||
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if len(incidents) == 0 {
|
||||
continue
|
||||
} // No digest is better than a misleading outage summary.
|
||||
lines := make([]string, 0, len(incidents))
|
||||
for _, incident := range incidents {
|
||||
lines = append(lines, "- "+incident.Title)
|
||||
}
|
||||
err = DB().Transaction(func(tx *gorm.DB) error {
|
||||
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
|
||||
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := DispatchStatusPageDeliveries(now); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func StartStatusPageDigestScheduler(ctx context.Context) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
|
||||
log.Printf("status-page delivery dispatch: %v", err)
|
||||
}
|
||||
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
|
||||
log.Printf("status-page digest: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
401
app/models/status_page_test.go
Обычный файл
401
app/models/status_page_test.go
Обычный файл
@@ -0,0 +1,401 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// statusPageSeedAcc returns a fresh account for status-page tests.
|
||||
func statusPageSeedAcc(t *testing.T) *models.Account {
|
||||
t.Helper()
|
||||
acc := &models.Account{Name: "status-page-" + strings.ReplaceAll(t.Name(), "/", "_")}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
return acc
|
||||
}
|
||||
|
||||
// TestStatusPage_Defaults asserts that creating a StatusPage with the
|
||||
// minimum required fields fills in the documented defaults (colors,
|
||||
// history days, soft-delete marker) and leaves IsPublished false.
|
||||
func TestStatusPage_Defaults(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
page := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "acme-status",
|
||||
Name: "Acme Status",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
got := models.StatusPage{}
|
||||
require.NoError(t, models.DB().First(&got, page.ID).Error)
|
||||
|
||||
assert.Equal(t, "#62c600", got.PrimaryColor, "default primary color")
|
||||
assert.Equal(t, "#1a73e8", got.AccentColor, "default accent color")
|
||||
assert.Equal(t, 90, got.ShowHistoryDays, "default history days")
|
||||
assert.True(t, got.ShowUptimeBars, "show_uptime_bars defaults true")
|
||||
assert.True(t, got.ShowResponseTimes, "show_response_times defaults true")
|
||||
assert.False(t, got.IsPublished, "is_published defaults false")
|
||||
assert.False(t, got.IsPublishedNow(), "IsPublishedNow() returns false until publish")
|
||||
assert.True(t, got.DeletedAt.Valid == false, "no soft-delete timestamp on fresh row")
|
||||
}
|
||||
|
||||
// TestStatusPage_SlugUniquenessSoftDelete verifies the partial unique
|
||||
// index allows recreating a slug after soft-deleting the previous row.
|
||||
func TestStatusPage_SlugUniquenessSoftDelete(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
first := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
// Second live row with the same slug must fail.
|
||||
dup := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout copy",
|
||||
}
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err, "duplicate slug on live rows must fail")
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"unexpected error: %v", err,
|
||||
)
|
||||
|
||||
// Soft-delete the first row. After that the slug is reusable.
|
||||
require.NoError(t, models.DB().Delete(first).Error)
|
||||
|
||||
again := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout v2",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(again).Error,
|
||||
"recreating a slug after a soft-delete must succeed (partial index)")
|
||||
assert.NotEqual(t, first.ID, again.ID)
|
||||
}
|
||||
|
||||
// TestStatusPage_SlugIsGlobal verifies public URLs are unambiguous: the
|
||||
// account is not part of /status/:slug, so another account cannot claim it.
|
||||
func TestStatusPage_SlugIsGlobal(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
accA := statusPageSeedAcc(t)
|
||||
accB := &models.Account{Name: "status-page-other"}
|
||||
require.NoError(t, models.DB().Create(accB).Error)
|
||||
|
||||
a := &models.StatusPage{AccountID: accA.ID, Slug: "shared", Name: "A"}
|
||||
require.NoError(t, models.DB().Create(a).Error)
|
||||
|
||||
b := &models.StatusPage{AccountID: accB.ID, Slug: "shared", Name: "B"}
|
||||
require.Error(t, models.DB().Create(b).Error,
|
||||
"different accounts must not be able to reuse a public slug")
|
||||
}
|
||||
|
||||
func TestStatusPageIncidentDeliveryIsIdempotentAndAccountScoped(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: account.ID, Slug: "delivery-page", Name: "Delivery"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
now := time.Now()
|
||||
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "subscriber@example.test", Kind: models.StatusPageSubscriberKindAlert, ConfirmedAt: &now}
|
||||
require.NoError(t, models.DB().Create(subscriber).Error)
|
||||
incident := &models.StatusPageIncident{StatusPageID: page.ID, Title: "API unavailable", Severity: models.StatusPageIncidentSeverityCrit, StartedAt: now}
|
||||
require.NoError(t, models.DB().Create(incident).Error)
|
||||
|
||||
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
|
||||
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
|
||||
}))
|
||||
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
|
||||
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
|
||||
}))
|
||||
var deliveries []models.StatusPageDelivery
|
||||
require.NoError(t, models.DB().Where("idempotency_key LIKE ?", "status-page:opened:%").Find(&deliveries).Error)
|
||||
require.Len(t, deliveries, 1)
|
||||
assert.Equal(t, page.ID, deliveries[0].StatusPageID)
|
||||
}
|
||||
|
||||
func TestStatusPageDailyDigestIsTimezoneDateIdempotent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account := statusPageSeedAcc(t)
|
||||
account.Timezone = "UTC"
|
||||
require.NoError(t, models.DB().Save(account).Error)
|
||||
page := &models.StatusPage{AccountID: account.ID, Slug: "digest-page", Name: "Digest"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
now := time.Date(2026, time.July, 13, 9, 15, 0, 0, time.UTC)
|
||||
confirmedAt := now.Add(-time.Hour)
|
||||
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "digest@example.test", Kind: models.StatusPageSubscriberKindDigestDaily, ConfirmedAt: &confirmedAt}
|
||||
require.NoError(t, models.DB().Create(subscriber).Error)
|
||||
require.NoError(t, models.DB().Create(&models.StatusPageIncident{StatusPageID: page.ID, Title: "Still open", Severity: models.StatusPageIncidentSeverityWarn, StartedAt: now}).Error)
|
||||
|
||||
_ = models.EnqueueStatusPageDailyDigests(now)
|
||||
_ = models.EnqueueStatusPageDailyDigests(now.Add(30 * time.Second))
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.StatusPageDelivery{}).Where("idempotency_key LIKE ?", "status-page:digest:%").Count(&count).Error)
|
||||
assert.GreaterOrEqual(t, count, int64(1), "catch-up may include earlier local due dates")
|
||||
}
|
||||
|
||||
// TestStatusPage_NormalizeSlugAndValidate exercises the slug normalizer
|
||||
// and validator — uppercase input becomes lowercase, and an invalid
|
||||
// slug (leading dash) is rejected on save via BeforeSave.
|
||||
func TestStatusPage_NormalizeSlugAndValidate(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
assert.Equal(t, "lower-case", models.NormalizeStatusPageSlug(" Lower-Case "))
|
||||
assert.NoError(t, models.ValidateStatusPageSlug("acme-status"))
|
||||
|
||||
bad := []string{"", "-leading-dash", "trailing-dash-", "UPPER", "with spaces", "with_underscore"}
|
||||
for _, slug := range bad {
|
||||
err := models.ValidateStatusPageSlug(slug)
|
||||
assert.Error(t, err, "expected error for slug %q", slug)
|
||||
}
|
||||
|
||||
// Save enforces the same rules — uppercase gets normalized, invalid
|
||||
// characters reject.
|
||||
good := &models.StatusPage{AccountID: acc.ID, Slug: "ACME-Status", Name: "Acme"}
|
||||
require.NoError(t, models.DB().Create(good).Error)
|
||||
assert.Equal(t, "acme-status", good.Slug)
|
||||
|
||||
bad2 := &models.StatusPage{AccountID: acc.ID, Slug: "-bad", Name: "Bad"}
|
||||
err := models.DB().Create(bad2).Error
|
||||
require.Error(t, err, "invalid slug should be rejected by BeforeSave")
|
||||
}
|
||||
|
||||
// TestStatusPage_ColorValidation rejects non-hex colors at save time.
|
||||
func TestStatusPage_ColorValidation(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
bad := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "badcolor",
|
||||
Name: "BadColor",
|
||||
PrimaryColor: "not-a-color",
|
||||
}
|
||||
err := models.DB().Create(bad).Error
|
||||
require.Error(t, err, "non-hex color must be rejected")
|
||||
|
||||
good := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "goodcolor",
|
||||
Name: "GoodColor",
|
||||
PrimaryColor: "#0a1b2c",
|
||||
AccentColor: "#abc",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(good).Error)
|
||||
|
||||
reloaded := models.StatusPage{}
|
||||
require.NoError(t, models.DB().First(&reloaded, good.ID).Error)
|
||||
assert.Equal(t, "#0a1b2c", reloaded.PrimaryColor)
|
||||
assert.Equal(t, "#abc", reloaded.AccentColor)
|
||||
}
|
||||
|
||||
// TestStatusPageSubscriber_UniqueActive verifies the partial unique
|
||||
// index on (status_page_id, lower(email)) only fires while a
|
||||
// subscriber is not unsubscribed.
|
||||
func TestStatusPageSubscriber_UniqueActive(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "sub",
|
||||
Name: "Sub",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
mk := func(email string, unsub *time.Time) *models.StatusPageSubscriber {
|
||||
s := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: email,
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-" + email),
|
||||
}
|
||||
s.UnsubscribedAt = unsub
|
||||
return s
|
||||
}
|
||||
|
||||
first := mk("Alice@Example.com", nil)
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
// Same email with mixed case must conflict (lower(email) is the
|
||||
// unique key), so a true case-insensitive uniqueness is in place.
|
||||
dup := mk("alice@example.com", nil)
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err, "active subscriber with same lowercased email must conflict")
|
||||
|
||||
// Soft-unsubscribe the original then re-add a fresh row with the
|
||||
// same email — this must succeed because the partial index
|
||||
// excludes unsubscribed rows.
|
||||
now := time.Now()
|
||||
first.UnsubscribedAt = &now
|
||||
require.NoError(t, models.DB().Save(first).Error)
|
||||
|
||||
resurrected := mk("alice@example.com", nil)
|
||||
require.NoError(t, models.DB().Create(resurrected).Error,
|
||||
"creating a fresh subscriber after the previous one unsubscribed must succeed")
|
||||
assert.NotEqual(t, first.ID, resurrected.ID)
|
||||
|
||||
// ActiveSubscriber reflects both confirmed and unsubscribed flags.
|
||||
confirmed := time.Now()
|
||||
active := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: "bob@example.com",
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-bob"),
|
||||
ConfirmedAt: &confirmed,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(active).Error)
|
||||
assert.True(t, active.IsActiveSubscriber(), "confirmed and not unsubscribed → active")
|
||||
|
||||
pending := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: "carol@example.com",
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-carol"),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(pending).Error)
|
||||
assert.False(t, pending.IsActiveSubscriber(), "unconfirmed → not active")
|
||||
}
|
||||
|
||||
// TestStatusPageIncident_SeverityAndEventFK checks that the FK to
|
||||
// events is wired correctly and that severity defaults to "info".
|
||||
func TestStatusPageIncident_SeverityAndEventFK(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "inc", Name: "Inc"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
now := time.Now()
|
||||
inc := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "API slowdown",
|
||||
BodyMD: "Investigating.",
|
||||
Severity: models.StatusPageIncidentSeverityWarn,
|
||||
StartedAt: now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(inc).Error)
|
||||
|
||||
got := models.StatusPageIncident{}
|
||||
require.NoError(t, models.DB().First(&got, inc.ID).Error)
|
||||
assert.Equal(t, "warn", got.Severity)
|
||||
|
||||
// Default severity is "info" when omitted.
|
||||
auto := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "heads up",
|
||||
StartedAt: now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(auto).Error)
|
||||
gotAuto := models.StatusPageIncident{}
|
||||
require.NoError(t, models.DB().First(&gotAuto, auto.ID).Error)
|
||||
assert.Equal(t, models.StatusPageIncidentSeverityInfo, gotAuto.Severity)
|
||||
|
||||
// EventID stays nullable and the FK tolerates a NULL event.
|
||||
var nilEvt *int64
|
||||
noEvt := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "no event",
|
||||
Severity: models.StatusPageIncidentSeverityCrit,
|
||||
StartedAt: now,
|
||||
EventID: nilEvt,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(noEvt).Error)
|
||||
}
|
||||
|
||||
// TestStatusPageMaintenance_BigintArray ensures the monitor_ids bigint[]
|
||||
// column round-trips through the GORM pq.Int64Array driver correctly.
|
||||
func TestStatusPageMaintenance_BigintArray(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "mt", Name: "Maintenance"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
now := time.Now()
|
||||
maint := &models.StatusPageMaintenance{
|
||||
StatusPageID: page.ID,
|
||||
Title: "DB upgrade",
|
||||
Description: "Rolling upgrade.",
|
||||
StartsAt: now,
|
||||
EndsAt: now.Add(2 * time.Hour),
|
||||
MonitorIDs: models.Int64ArrayFromSlice([]int64{1, 7, 42}),
|
||||
NotifySubscribers: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(maint).Error)
|
||||
|
||||
got := models.StatusPageMaintenance{}
|
||||
require.NoError(t, models.DB().First(&got, maint.ID).Error)
|
||||
assert.Equal(t, []int64{1, 7, 42}, []int64(got.MonitorIDs))
|
||||
assert.True(t, got.NotifySubscribers)
|
||||
}
|
||||
|
||||
// TestStatusPageDomain_TableReservation verifies the M5 table is
|
||||
// created with the right unique constraints even though M0 does not
|
||||
// yet populate it.
|
||||
func TestStatusPageDomain_TableReservation(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "dom", Name: "Domain"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
d := &models.StatusPageDomain{
|
||||
StatusPageID: page.ID,
|
||||
Domain: "status.example.com",
|
||||
TXTToken: "rsmon-verify=abc123",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(d).Error)
|
||||
require.NoError(t, models.DB().Create(&models.StatusPageDomain{
|
||||
StatusPageID: page.ID,
|
||||
Domain: "www.status.example.com",
|
||||
TXTToken: "rsmon-verify=second",
|
||||
}).Error, "one status page may own multiple domains")
|
||||
|
||||
// Second page claiming the same domain must fail because domain
|
||||
// is globally unique.
|
||||
other := &models.StatusPage{AccountID: acc.ID, Slug: "dom2", Name: "Domain2"}
|
||||
require.NoError(t, models.DB().Create(other).Error)
|
||||
dup := &models.StatusPageDomain{
|
||||
StatusPageID: other.ID,
|
||||
Domain: "status.example.com",
|
||||
TXTToken: "rsmon-verify=xyz789",
|
||||
}
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err)
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"expected unique-constraint error, got %v", err)
|
||||
}
|
||||
|
||||
func TestNormalizeStatusPageDomain(t *testing.T) {
|
||||
got, err := models.NormalizeStatusPageDomain(" Status.Example.COM. ")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "status.example.com", got)
|
||||
for _, input := range []string{"https://example.com", "example.com:443", "127.0.0.1", "*.example.com", "-bad.example.com"} {
|
||||
_, err := models.NormalizeStatusPageDomain(input)
|
||||
assert.Error(t, err, input)
|
||||
}
|
||||
}
|
||||
61
app/models/subscription.go
Обычный файл
61
app/models/subscription.go
Обычный файл
@@ -0,0 +1,61 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
SubscriptionStatusTrialing = "trialing"
|
||||
SubscriptionStatusActive = "active"
|
||||
SubscriptionStatusPastDue = "past_due"
|
||||
SubscriptionStatusPaused = "paused"
|
||||
SubscriptionStatusCanceled = "canceled"
|
||||
SubscriptionStatusExpired = "expired"
|
||||
)
|
||||
|
||||
// Subscription is the current billable entitlement for one account. M0 only
|
||||
// writes manual subscriptions; provider flows are deliberately deferred.
|
||||
type Subscription struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
PlanID int64 `gorm:"not null;index" json:"plan_id"`
|
||||
Plan *Plan `json:"plan,omitempty"`
|
||||
Provider string `gorm:"size:16;not null;default:'manual'" json:"provider"`
|
||||
ProviderSubscriptionID *string `gorm:"size:128" json:"provider_subscription_id,omitempty"`
|
||||
ProviderCustomerID *string `gorm:"size:128" json:"provider_customer_id,omitempty"`
|
||||
Status string `gorm:"size:24;not null;default:'active';index" json:"status"`
|
||||
BillingCycle string `gorm:"size:8;not null;default:'monthly'" json:"billing_cycle"`
|
||||
CurrentPeriodStart *time.Time `json:"current_period_start,omitempty"`
|
||||
CurrentPeriodEnd *time.Time `gorm:"index" json:"current_period_end,omitempty"`
|
||||
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
|
||||
CancelAtPeriodEnd bool `gorm:"not null;default:false" json:"cancel_at_period_end"`
|
||||
CanceledAt *time.Time `json:"canceled_at,omitempty"`
|
||||
Currency string `gorm:"size:3;not null" json:"currency"`
|
||||
AmountMinor int64 `gorm:"not null;default:0" json:"amount_minor"`
|
||||
MetadataJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"metadata_json"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// SubscriptionEvent is an append-only audit record for entitlement changes
|
||||
// and provider deliveries. ProviderEventID makes future webhook processing
|
||||
// idempotent without coupling the ledger to a specific PSP.
|
||||
type SubscriptionEvent struct {
|
||||
concerns.Model
|
||||
SubscriptionID int64 `gorm:"not null;index" json:"subscription_id"`
|
||||
AccountID int64 `gorm:"not null;index" json:"account_id"`
|
||||
Provider string `gorm:"size:16;not null;default:'manual';uniqueIndex:idx_subscription_events_provider_event,priority:1" json:"provider"`
|
||||
Kind string `gorm:"size:32;not null" json:"kind"`
|
||||
FromPlanID *int64 `json:"from_plan_id,omitempty"`
|
||||
ToPlanID *int64 `json:"to_plan_id,omitempty"`
|
||||
AmountMinor *int64 `json:"amount_minor,omitempty"`
|
||||
Currency string `gorm:"size:3" json:"currency,omitempty"`
|
||||
ActorUserID *int64 `json:"actor_user_id,omitempty"`
|
||||
ProviderEventID *string `gorm:"size:128;uniqueIndex:idx_subscription_events_provider_event,priority:2" json:"provider_event_id,omitempty"`
|
||||
PayloadJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"payload_json"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
47
app/models/tag.go
Обычный файл
47
app/models/tag.go
Обычный файл
@@ -0,0 +1,47 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Tag stores metadata for a single tag name within an account: the
|
||||
// user-chosen color (hex string like "#FF5733") and icon (FontAwesome
|
||||
// icon name like "faTag"). The (account_id, name) pair is unique.
|
||||
//
|
||||
// Tags are still attached to monitors via the monitors.tags text[]
|
||||
// column (see Monitor.Tags). This table only holds the per-tag display
|
||||
// metadata so the same tag renders consistently across the SPA — it
|
||||
// does not affect monitor scoping or filtering.
|
||||
//
|
||||
// Rows are created lazily: the SPA POSTs a Tag the first time a user
|
||||
// customizes its color/icon, and GET /tags?with_counts=1 LEFT JOINs
|
||||
// the unnested monitors.tags array against this table to enrich the
|
||||
// (name, count) pairs with display metadata. A Tag row may exist with
|
||||
// zero monitors using it (count=0) — that happens when a user creates
|
||||
// a tag from /settings/tags/new but has not yet applied it.
|
||||
type Tag struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"index;not null" json:"account_id"`
|
||||
Account *Account `gorm:"foreignKey:AccountID" json:"account,omitempty"`
|
||||
Name string `gorm:"type:varchar(255);not null" json:"name"`
|
||||
Color string `gorm:"type:varchar(32);not null;default:'#6c757d'" json:"color"`
|
||||
Icon string `gorm:"type:varchar(64);not null;default:'faTag'" json:"icon"`
|
||||
Audited
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TagInfo is a tag with the number of monitors using it plus optional
|
||||
// display metadata (color, icon) coming from the Tag table.
|
||||
type TagInfo struct {
|
||||
Name string `json:"name"`
|
||||
Count int64 `json:"count"`
|
||||
Color string `json:"color"`
|
||||
Icon string `json:"icon"`
|
||||
}
|
||||
|
||||
// DefaultTagColor is the hex color used when a Tag has no metadata row.
|
||||
const DefaultTagColor = "#6c757d"
|
||||
|
||||
// DefaultTagIcon is the FontAwesome icon name used when a Tag has no
|
||||
// metadata row. Matches the icon the SPA renders by default.
|
||||
const DefaultTagIcon = "faTag"
|
||||
135
app/models/tags_test.go
Обычный файл
135
app/models/tags_test.go
Обычный файл
@@ -0,0 +1,135 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// seedTagsMonitors creates one group with four monitors:
|
||||
//
|
||||
// a.test [prod, web]
|
||||
// b.test [prod, api]
|
||||
// c.test [staging]
|
||||
// d.test [] (NULL tags)
|
||||
//
|
||||
// and returns the group id. It mirrors the exact SQL the /tags rename, delete
|
||||
// and with_counts endpoints run, pinning the contract.
|
||||
func seedTagsMonitors(t *testing.T) int64 {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "A", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := models.Group{AccountID: acc.ID, Name: "A-default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
monitors := []models.Monitor{
|
||||
{GroupID: group.ID, Host: "a.test", Tags: []string{"prod", "web"}},
|
||||
{GroupID: group.ID, Host: "b.test", Tags: []string{"prod", "api"}},
|
||||
{GroupID: group.ID, Host: "c.test", Tags: []string{"staging"}},
|
||||
{GroupID: group.ID, Host: "d.test"},
|
||||
}
|
||||
for i := range monitors {
|
||||
require.NoError(t, models.DB().Create(&monitors[i]).Error)
|
||||
}
|
||||
return group.ID
|
||||
}
|
||||
|
||||
func TestTagsRenameSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = (
|
||||
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
|
||||
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", "production", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected, "only a.test and b.test contain prod")
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"production", "web"}, []string(a.Tags))
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.ElementsMatch(t, []string{"production", "api"}, []string(b.Tags))
|
||||
|
||||
var c models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "c.test").First(&c).Error)
|
||||
assert.Equal(t, []string{"staging"}, []string(c.Tags), "staging-only monitor untouched")
|
||||
}
|
||||
|
||||
func TestTagsRenameSQL_Dedup(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
// Rename prod -> web. a.test has both prod and web: must collapse to a
|
||||
// single "web" entry (array_agg DISTINCT), not [web, web].
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = (
|
||||
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
|
||||
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", "web", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected)
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"web"}, []string(a.Tags), "duplicate must be deduped")
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.ElementsMatch(t, []string{"web", "api"}, []string(b.Tags))
|
||||
}
|
||||
|
||||
func TestTagsDeleteSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = array_remove(tags, ?)
|
||||
WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected)
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"web"}, []string(a.Tags))
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.Equal(t, []string{"api"}, []string(b.Tags))
|
||||
}
|
||||
|
||||
func TestTagsCountSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
type tagInfo struct {
|
||||
Name string
|
||||
Count int64
|
||||
}
|
||||
var tags []tagInfo
|
||||
err := models.DB().Raw(
|
||||
`SELECT tag AS name, COUNT(*) AS count
|
||||
FROM (SELECT unnest(tags) AS tag FROM monitors
|
||||
WHERE tags IS NOT NULL AND group_id IN (?)) sub
|
||||
GROUP BY tag
|
||||
ORDER BY tag`,
|
||||
[]int64{groupID},
|
||||
).Scan(&tags).Error
|
||||
require.NoError(t, err)
|
||||
|
||||
got := map[string]int64{}
|
||||
for _, tg := range tags {
|
||||
got[tg.Name] = tg.Count
|
||||
}
|
||||
assert.Equal(t, map[string]int64{"prod": 2, "web": 1, "api": 1, "staging": 1}, got)
|
||||
}
|
||||
145
app/models/task.go
Обычный файл
145
app/models/task.go
Обычный файл
@@ -0,0 +1,145 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Task kinds stored in the tasks.kind column. The plan (docs/plans/worker-notifier-mvp.md
|
||||
// section 4.1) reserves the same enum for checks and notifications; this phase only
|
||||
// emits notification rows but the enum is shared so the selector can stay a single
|
||||
// function.
|
||||
const (
|
||||
TaskKindNotification = "notification"
|
||||
TaskKindCheck = "check"
|
||||
TaskKindServerMetric = "server_metric"
|
||||
)
|
||||
|
||||
// Task states for the durable task envelope.
|
||||
const (
|
||||
TaskStateQueued = "queued"
|
||||
TaskStateLeased = "leased"
|
||||
TaskStateSucceeded = "succeeded"
|
||||
TaskStateFailedRetry = "failed_retry"
|
||||
TaskStateFailedPerm = "failed_perm"
|
||||
TaskStateDead = "dead"
|
||||
)
|
||||
|
||||
// Notification result statuses reported by the worker (mirrors the wire enum so the
|
||||
// result handler can decode without re-typing the constants).
|
||||
const (
|
||||
NotificationResultDelivered = "delivered"
|
||||
NotificationResultRetryable = "retryable"
|
||||
NotificationResultPermanent = "permanent"
|
||||
NotificationResultPartial = "partial"
|
||||
)
|
||||
|
||||
// SkipLockedClause is the SELECT ... FOR UPDATE SKIP LOCKED clause used by
|
||||
// every worker-pool selector (checks in check_jobs.ChecksForWorker and
|
||||
// tasks in task_selector.TasksForWorker / TasksForWorkerNotification).
|
||||
// Sharing the value keeps the SQL identical across selectors so goconst
|
||||
// does not flag the literal, and a future change (e.g. NOWAIT) only has
|
||||
// to touch one place.
|
||||
var SkipLockedClause = clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}
|
||||
|
||||
// Task is the durable envelope for both check and notification work executed by the
|
||||
// distributed worker pool. Selection uses FOR UPDATE SKIP LOCKED per worker poll so
|
||||
// a single primary key or sequence never becomes the bottleneck.
|
||||
//
|
||||
// Phase 1 (this commit) only emits notification tasks. The `kind` discriminator and
|
||||
// capability filters are designed to accept checks in phase 2 without a schema change.
|
||||
type Task struct {
|
||||
concerns.Model
|
||||
|
||||
JobID string `gorm:"uniqueIndex;size:64" json:"job_id"`
|
||||
Kind string `gorm:"size:32;index" json:"kind"`
|
||||
State string `gorm:"size:32;index" json:"state"`
|
||||
LastError string `gorm:"type:text" json:"last_error"`
|
||||
|
||||
// Tenancy + audit anchor. AccountID is required for the capability match in
|
||||
// TasksForWorker; monitor_id / message_id / contact_id are denormalized for
|
||||
// fast admin queries.
|
||||
AccountID int64 `gorm:"index" json:"account_id"`
|
||||
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
|
||||
CheckID *int64 `json:"check_id,omitempty"`
|
||||
MessageID *int64 `json:"message_id,omitempty"`
|
||||
ContactID *int64 `json:"contact_id,omitempty"`
|
||||
|
||||
// Payload is the kind-specific blob the worker needs to execute. For
|
||||
// notifications the producer pre-renders subject/body so the worker does not
|
||||
// need templating context (see RenderNotificationContent in internal/notifier).
|
||||
Payload datatypes.JSON `gorm:"type:jsonb" json:"payload"`
|
||||
|
||||
// Scheduling + retry envelope. NotBefore is set to NOW() by the producer and
|
||||
// bumped by the result handler on retryable failures. Deadline is a soft cap
|
||||
// the selector can use to skip stale tasks.
|
||||
NotBefore time.Time `json:"not_before"`
|
||||
Deadline *time.Time `json:"deadline,omitempty"`
|
||||
|
||||
// LeaseOwner + LeaseExpiresAt are owned by the selector while the task is
|
||||
// in state=leased. The reaper clears them when the lease expires.
|
||||
LeaseOwner string `gorm:"size:128" json:"lease_owner"`
|
||||
LeaseToken string `gorm:"size:64" json:"-"`
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
|
||||
Attempts int `json:"attempts"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
|
||||
// IdempotencyKey is unique per logical event so a retry of the producer's
|
||||
// enqueue never produces a second Task row. See EnqueueNotificationTask.
|
||||
IdempotencyKey string `gorm:"uniqueIndex;size:255" json:"idempotency_key"`
|
||||
|
||||
// Result holds the most recent worker result body (NotificationResultReport or
|
||||
// CheckResultReport shape, depending on Kind). It is JSONB so the admin UI can
|
||||
// pretty-print without a separate result table for transient lookups.
|
||||
Result datatypes.JSON `gorm:"type:jsonb" json:"result"`
|
||||
SucceededAt *time.Time `json:"succeeded_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TaskReplay records the single operator-initiated replay of a dead task.
|
||||
// OriginalTaskID is unique, making repeated clicks/API retries idempotent.
|
||||
type TaskReplay struct {
|
||||
concerns.Model
|
||||
OriginalTaskID int64 `gorm:"uniqueIndex" json:"original_task_id"`
|
||||
RequeuedTaskID int64 `gorm:"uniqueIndex" json:"requeued_task_id"`
|
||||
OperatorUserID int64 `gorm:"index" json:"operator_user_id"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName overrides the default table name so pluralization stays consistent
|
||||
// with the rest of the schema (tasks, not "task" or "taskses").
|
||||
func (Task) TableName() string {
|
||||
return "tasks"
|
||||
}
|
||||
|
||||
// NotificationDelivery is the per-attempt audit row required by section 7.5 of the
|
||||
// plan ("Audit rows: each successful or failed delivery writes a row in a new
|
||||
// notification_deliveries table"). The result handler appends one row per result
|
||||
// frame, which lets support answer "did the customer ever get this alert" without
|
||||
// scanning application logs.
|
||||
type NotificationDelivery struct {
|
||||
concerns.Model
|
||||
|
||||
MessageID int64 `gorm:"index" json:"message_id"`
|
||||
WorkerID string `gorm:"size:128;index" json:"worker_id"`
|
||||
TaskID int64 `gorm:"index" json:"task_id"`
|
||||
|
||||
Status string `gorm:"size:32" json:"status"`
|
||||
Error string `gorm:"type:text" json:"error"`
|
||||
|
||||
DurationMs int `json:"duration_ms"`
|
||||
ProviderResponse string `gorm:"type:text" json:"provider_response"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName mirrors the plan's preferred lowercase plural.
|
||||
func (NotificationDelivery) TableName() string {
|
||||
return "notification_deliveries"
|
||||
}
|
||||
136
app/models/task_reaper.go
Обычный файл
136
app/models/task_reaper.go
Обычный файл
@@ -0,0 +1,136 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ReapExpiredTasks is the periodic cleanup function described in
|
||||
// docs/plans/worker-notifier-mvp.md section 8.5:
|
||||
//
|
||||
// - tasks in state='leased' whose lease_expires_at is past are returned to
|
||||
// state='queued' and have their lease_owner cleared, so the next selector
|
||||
// poll can pick them up.
|
||||
// - tasks in state='failed_retry' whose not_before is past AND
|
||||
// attempts >= max_attempts are moved to state='dead' so they show up on
|
||||
// the admin dead-letter page and stop consuming selector bandwidth.
|
||||
//
|
||||
// It returns the number of rows it touched so the caller can log a metric.
|
||||
// Cheap enough to run from the web process every 30s.
|
||||
func ReapExpiredTasks() (reaped int, deaded int, err error) {
|
||||
now := time.Now()
|
||||
err = DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := expireQueuedNotificationTasksTx(tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
var expired []Task
|
||||
if err := tx.Where("state = ? AND lease_expires_at IS NOT NULL AND lease_expires_at < ?", TaskStateLeased, now).Find(&expired).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range expired {
|
||||
if expired[i].Attempts >= expired[i].MaxAttempts {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateDead, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "last_error": "lease expired after max attempts", "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 1 {
|
||||
deaded++
|
||||
if err := FinalizeNotificationTaskTx(tx, &expired[i], "dead", "lease expired after max attempts"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateQueued, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
reaped += int(result.RowsAffected)
|
||||
}
|
||||
var exhausted []Task
|
||||
if err := tx.Where("state = ? AND not_before <= ? AND attempts >= max_attempts", TaskStateFailedRetry, now).Find(&exhausted).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range exhausted {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", exhausted[i].ID, TaskStateFailedRetry).Updates(map[string]interface{}{"state": TaskStateDead, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 1 {
|
||||
deaded++
|
||||
if err := FinalizeNotificationTaskTx(tx, &exhausted[i], "dead", exhausted[i].LastError); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return reaped, deaded, err
|
||||
}
|
||||
|
||||
// FinalizeNotificationTaskTx makes a terminal notification task customer-visible
|
||||
// and auditable. The caller owns the task state transition in this transaction.
|
||||
func FinalizeNotificationTaskTx(tx *gorm.DB, task *Task, status, reason string) error {
|
||||
if task == nil || task.Kind != TaskKindNotification || task.MessageID == nil {
|
||||
return nil
|
||||
}
|
||||
if err := tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *task.MessageID, []string{"sent", "error"}).Updates(map[string]interface{}{"state": "error", "error": reason}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Create(&NotificationDelivery{MessageID: *task.MessageID, TaskID: task.ID, Status: status, Error: reason}).Error
|
||||
}
|
||||
|
||||
func expireQueuedNotificationTasksTx(tx *gorm.DB, now time.Time) error {
|
||||
var tasks []Task
|
||||
if err := tx.Where("state = ? AND kind = ? AND deadline IS NOT NULL AND deadline <= ?", TaskStateQueued, TaskKindNotification, now).Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range tasks {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", tasks[i].ID, TaskStateQueued).Updates(map[string]interface{}{
|
||||
"state": TaskStateDead, "last_error": "notification deadline expired", "updated_at": now,
|
||||
})
|
||||
if result.Error != nil || result.RowsAffected == 0 {
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "expired", "notification deadline expired"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartTaskReaper launches a goroutine that runs ReapExpiredTasks on the given
|
||||
// interval. It honors ctx.Done() so the caller can wind it down without
|
||||
// leaking. The function is safe to call once per process; the control plane
|
||||
// runs the reaper from main.init() so only one ticker ever exists in a single
|
||||
// web process.
|
||||
func StartTaskReaper(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
reaped, deaded, err := ReapExpiredTasks()
|
||||
if err != nil {
|
||||
log.Printf("task_reaper: error: %v", err)
|
||||
continue
|
||||
}
|
||||
if reaped > 0 || deaded > 0 {
|
||||
log.Printf("task_reaper: reaped=%d dead=%d", reaped, deaded)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
391
app/models/task_selector.go
Обычный файл
391
app/models/task_selector.go
Обычный файл
@@ -0,0 +1,391 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// DefaultCheckTaskLeaseTTL is how long a leased check remains owned before the reaper
|
||||
// returns it to the queue. It is deliberately larger than the worker's heartbeat
|
||||
// (10s) so a healthy worker can finish a 30s check without the reaper stealing
|
||||
// the lease, and deliberately smaller than the heartbeat timeout (2m) so a
|
||||
// crashed worker sees its tasks reaped within one watchdog cycle.
|
||||
const DefaultCheckTaskLeaseTTL = 60 * time.Second
|
||||
|
||||
// DefaultTaskLeaseTTL remains the check-task default for existing confirmation
|
||||
// callers. Generic task selection must use TaskLeaseTTL so notification work is
|
||||
// not reclaimed during its longer execution window.
|
||||
const DefaultTaskLeaseTTL = DefaultCheckTaskLeaseTTL
|
||||
|
||||
// Notification execution is bounded by the worker runner at five minutes. The
|
||||
// additional minute covers result serialization, websocket/HTTP transport, and
|
||||
// a control-plane scheduling delay before the persisted lease may be reaped.
|
||||
const (
|
||||
DefaultNotificationExecutionTimeout = 5 * time.Minute
|
||||
NotificationTaskReportMargin = time.Minute
|
||||
DefaultNotificationTaskLeaseTTL = DefaultNotificationExecutionTimeout + NotificationTaskReportMargin
|
||||
)
|
||||
|
||||
// TaskLeaseTTL returns the persisted lease lifetime for a task kind.
|
||||
func TaskLeaseTTL(kind string) time.Duration {
|
||||
if kind == TaskKindNotification {
|
||||
return DefaultNotificationTaskLeaseTTL
|
||||
}
|
||||
return DefaultCheckTaskLeaseTTL
|
||||
}
|
||||
|
||||
// DefaultTaskMaxAttempts is the retry budget for a task before it moves to dead.
|
||||
const DefaultTaskMaxAttempts = 5
|
||||
|
||||
// DefaultNotificationTaskDeadline is assigned to manually replayed notification
|
||||
// dead letters. Normal producer tasks may be deadline-free, but a replay must
|
||||
// never inherit an already-expired deadline.
|
||||
const DefaultNotificationTaskDeadline = 15 * time.Minute
|
||||
|
||||
// ErrNotificationMethodNotAuthorized is returned by EnqueueNotificationTask
|
||||
// when the producer can prove no worker in the pool is authorized for the
|
||||
// (method, account) pair. The caller may skip the enqueue or log + continue.
|
||||
var ErrNotificationMethodNotAuthorized = errors.New("no worker authorized for method/account")
|
||||
|
||||
// EnqueueNotificationTaskInput is the pre-rendered envelope produced by the
|
||||
// notifier producer. All slices are required; the selector never reads them.
|
||||
type EnqueueNotificationTaskInput struct {
|
||||
AccountID int64
|
||||
NotificationID int64
|
||||
ContactID int64
|
||||
MessageID *int64
|
||||
MonitorID *int64
|
||||
CheckID *int64
|
||||
EventIDs []int64
|
||||
Method string // "email", "telegram", "webhook", "mattermost", "sms", "voice"
|
||||
Subject string
|
||||
BodyText string
|
||||
BodyHTML string
|
||||
BodyMarkdown string
|
||||
Language string
|
||||
MessageKind string // "down", "up", "exp", "test"
|
||||
NotBefore time.Time
|
||||
Deadline *time.Time
|
||||
MaxAttempts int
|
||||
Payload []byte // marshaled task-specific data
|
||||
IdempotencyKey string // optional; manual test tasks use a unique key and do not have event IDs
|
||||
}
|
||||
|
||||
// EnqueueNotificationTask writes one Task row keyed by a stable idempotency key.
|
||||
// A second call with the same key (same notification/contact/event triple) is a
|
||||
// no-op so the producer is safe to call more than once per pass.
|
||||
//
|
||||
// The capability precheck uses the same NotificationMethods + NotificationAccounts
|
||||
// rule the selector does, so the producer can skip enqueueing work that no
|
||||
// operated worker could ever pick up (sms/voice until phase 4).
|
||||
func EnqueueNotificationTask(input *EnqueueNotificationTaskInput) (*Task, error) {
|
||||
return EnqueueNotificationTaskTx(DB(), input)
|
||||
}
|
||||
|
||||
// EnqueueNotificationTaskTx is the transactional form used by state machines
|
||||
// that must commit their transition, audit event, message, and task together.
|
||||
func EnqueueNotificationTaskTx(tx *gorm.DB, input *EnqueueNotificationTaskInput) (*Task, error) {
|
||||
if tx == nil {
|
||||
return nil, errors.New("enqueue: nil transaction")
|
||||
}
|
||||
if input.AccountID == 0 || input.ContactID == 0 {
|
||||
return nil, errors.New("enqueue: account_id and contact_id are required")
|
||||
}
|
||||
|
||||
idempotencyKey := input.IdempotencyKey
|
||||
if idempotencyKey == "" {
|
||||
if input.NotificationID == 0 || len(input.EventIDs) == 0 {
|
||||
return nil, errors.New("enqueue: notification_id and event_ids are required without explicit idempotency_key")
|
||||
}
|
||||
idempotencyKey = notificationIdempotencyKey(input.NotificationID, input.ContactID, input.EventIDs[0])
|
||||
}
|
||||
|
||||
// Fast path: row already exists from a previous producer tick. Returning
|
||||
// the existing row is the idempotency guarantee — second calls return the
|
||||
// same id, no second INSERT.
|
||||
var existing Task
|
||||
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
|
||||
return &existing, nil
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxAttempts := input.MaxAttempts
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = DefaultTaskMaxAttempts
|
||||
}
|
||||
if !anyWorkerCanDeliver(input.Method, input.AccountID) {
|
||||
return nil, fmt.Errorf("%w: method=%s account=%d", ErrNotificationMethodNotAuthorized, input.Method, input.AccountID)
|
||||
}
|
||||
|
||||
notBefore := input.NotBefore
|
||||
if notBefore.IsZero() {
|
||||
notBefore = time.Now()
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
task := &Task{
|
||||
JobID: uuid.New().String(),
|
||||
Kind: TaskKindNotification,
|
||||
State: TaskStateQueued,
|
||||
AccountID: input.AccountID,
|
||||
MessageID: input.MessageID,
|
||||
ContactID: &input.ContactID,
|
||||
MonitorID: input.MonitorID,
|
||||
CheckID: input.CheckID,
|
||||
NotBefore: notBefore,
|
||||
Deadline: input.Deadline,
|
||||
Attempts: 0,
|
||||
MaxAttempts: maxAttempts,
|
||||
IdempotencyKey: idempotencyKey,
|
||||
}
|
||||
if len(input.Payload) > 0 {
|
||||
task.Payload = input.Payload
|
||||
}
|
||||
task.CreatedAt = now
|
||||
task.UpdatedAt = now
|
||||
|
||||
// ON CONFLICT DO NOTHING so a concurrent producer tick racing with us on
|
||||
// the same idempotency_key loses the race but does not duplicate the row.
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(task).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task.ID == 0 {
|
||||
// Lost the race. Re-read and return the winner.
|
||||
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &existing, nil
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// NotificationIdempotencyKey exposes the producer's idempotency key so the
|
||||
// result handler and admin tooling can match a Task row back to the logical
|
||||
// (notification, contact, event) tuple without re-deriving the format.
|
||||
func NotificationIdempotencyKey(notificationID, contactID, eventID int64) string {
|
||||
return notificationIdempotencyKey(notificationID, contactID, eventID)
|
||||
}
|
||||
|
||||
func notificationIdempotencyKey(notificationID, contactID, eventID int64) string {
|
||||
return fmt.Sprintf("notif:%d:contact:%d:event:%d", notificationID, contactID, eventID)
|
||||
}
|
||||
|
||||
// anyWorkerCanDeliver returns true if at least one active worker in the pool is
|
||||
// authorized to deliver the given (method, account) pair. Used by the producer
|
||||
// to skip enqueues no worker could ever pick up.
|
||||
func anyWorkerCanDeliver(method string, accountID int64) bool {
|
||||
var nodes []WorkerNode
|
||||
if err := DB().Where("status = ? AND last_seen > ?", "active", time.Now().Add(-WorkerHeartbeatFreshness)).Find(&nodes).Error; err != nil {
|
||||
log.Printf("task_selector: cannot list workers: %v", err)
|
||||
// Be permissive on lookup failure: the selector's own filter would still
|
||||
// hold the lease back, so the worst case is a queued task nobody picks
|
||||
// up — which the reaper eventually dead-letters.
|
||||
return true
|
||||
}
|
||||
for i := range nodes {
|
||||
if nodes[i].SupportsTaskEnvelope() && nodes[i].CanDeliverNotification(method, accountID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TasksForWorker leases up to `limit` due tasks for the worker. The selection
|
||||
// is one transaction so the FOR UPDATE SKIP LOCKED + UPDATE that flips state
|
||||
// from queued to leased is atomic. Notification tasks are filtered by the worker's
|
||||
// notification_methods + notification_accounts capability set; check tasks are
|
||||
// filtered by check_types in their payload.
|
||||
//
|
||||
// The function is safe to call from multiple goroutines for different workers.
|
||||
// Two workers that hit the DB at the same time will see disjoint task sets.
|
||||
func TasksForWorker(worker *WorkerNode, limit int) ([]Task, error) {
|
||||
if worker == nil {
|
||||
return nil, errors.New("TasksForWorker: worker is nil")
|
||||
}
|
||||
if !worker.SupportsTaskEnvelope() {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 1
|
||||
}
|
||||
|
||||
notifMethods := worker.NotificationMethods()
|
||||
notifAccounts := worker.AccessibleAccountIDs()
|
||||
hasNotif := len(notifMethods) > 0
|
||||
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return nil, tx.Error
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_ = tx.Rollback().Error
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
now := time.Now()
|
||||
var out []Task
|
||||
|
||||
// First pass: notification tasks the worker is authorized to deliver. We
|
||||
// also bump attempts and flip state to leased in the same row so the
|
||||
// outer selector+lease is atomic. The method filter is a JSONB extract on
|
||||
// payload->>'method' so a single worker query can target one method list.
|
||||
if hasNotif {
|
||||
notifQuery := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'method' IN (?)", notifMethods).
|
||||
Where("payload->>'method' <> ''")
|
||||
if len(notifAccounts) > 0 {
|
||||
notifQuery = notifQuery.Where("account_id IN (?)", notifAccounts)
|
||||
}
|
||||
|
||||
var picked []Task
|
||||
if err := notifQuery.Limit(limit).Find(&picked).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range picked {
|
||||
row := picked[i]
|
||||
newAttempts := row.Attempts + 1
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
|
||||
if err := tx.Model(&row).Updates(map[string]interface{}{
|
||||
colState: TaskStateLeased,
|
||||
"lease_owner": worker.WorkerID,
|
||||
"lease_expires_at": leaseUntil,
|
||||
"attempts": newAttempts,
|
||||
"lease_token": leaseToken,
|
||||
"updated_at": now,
|
||||
}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
row.State = TaskStateLeased
|
||||
row.LeaseOwner = worker.WorkerID
|
||||
row.LeaseExpiresAt = &leaseUntil
|
||||
row.Attempts = newAttempts
|
||||
row.LeaseToken = leaseToken
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
|
||||
remaining := limit - len(out)
|
||||
if checkTypes := worker.CheckTypes(); remaining > 0 && len(checkTypes) > 0 {
|
||||
checkQuery := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindCheck).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'kind' IN (?)", checkTypes)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
checkQuery = checkQuery.Where("account_id IN (?)", accounts)
|
||||
}
|
||||
var picked []Task
|
||||
if err := checkQuery.Limit(remaining).Find(&picked).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
for i := range picked {
|
||||
row := picked[i]
|
||||
newAttempts := row.Attempts + 1
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
|
||||
if err := tx.Model(&row).Updates(map[string]interface{}{
|
||||
colState: TaskStateLeased, "lease_owner": worker.WorkerID,
|
||||
"lease_expires_at": leaseUntil, "attempts": newAttempts, "lease_token": leaseToken, "updated_at": now,
|
||||
}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
row.State, row.LeaseOwner, row.LeaseExpiresAt, row.Attempts, row.LeaseToken = TaskStateLeased, worker.WorkerID, &leaseUntil, newAttempts, leaseToken
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AvailableWorkerTaskCapacity returns unoccupied local worker slots. Durable
|
||||
// unexpired leases and the worker's heartbeat-reported active/queued workload
|
||||
// describe the same work from different sides, so the larger value is used to
|
||||
// avoid both over-dispatching and double-counting a healthy worker.
|
||||
func AvailableWorkerTaskCapacity(worker *WorkerNode) (int, error) {
|
||||
if worker == nil {
|
||||
return 0, errors.New("worker capacity: worker is nil")
|
||||
}
|
||||
concurrency := worker.Concurrency
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
var leased int64
|
||||
if err := DB().Model(&Task{}).Where("state = ? AND lease_owner = ? AND lease_expires_at > ?", TaskStateLeased, worker.WorkerID, time.Now()).Count(&leased).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var confirmationLeases int64
|
||||
if err := DB().Model(&CheckAttempt{}).Where("worker_node_id = ? AND kind = ? AND state = ? AND lease_expires_at > ?", worker.ID, AttemptKindConfirm, AttemptStateLeased, time.Now()).Count(&confirmationLeases).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
used := int(leased + confirmationLeases)
|
||||
if reported := worker.ReportedWorkload(); reported > used {
|
||||
used = reported
|
||||
}
|
||||
if used >= concurrency {
|
||||
return 0, nil
|
||||
}
|
||||
return concurrency - used, nil
|
||||
}
|
||||
|
||||
// GetTaskByJobID returns one task row keyed by its unique job_id. The result
|
||||
// handler uses this to validate that the incoming JobID exists and matches the
|
||||
// calling worker before it mutates state.
|
||||
func GetTaskByJobID(jobID string) (*Task, error) {
|
||||
if jobID == "" {
|
||||
return nil, errors.New("GetTaskByJobID: empty job_id")
|
||||
}
|
||||
var task Task
|
||||
if err := DB().Where("job_id = ?", jobID).First(&task).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &task, nil
|
||||
}
|
||||
|
||||
// TasksForWorkerTx is the variant exposed for tests so a single SELECT inside a
|
||||
// caller-provided transaction can be inspected without the auto-commit wrapper.
|
||||
// Production code should use TasksForWorker.
|
||||
func TasksForWorkerTx(tx *gorm.DB, worker *WorkerNode, limit int) ([]Task, error) {
|
||||
if tx == nil {
|
||||
return nil, errors.New("TasksForWorkerTx: nil tx")
|
||||
}
|
||||
notifMethods := worker.NotificationMethods()
|
||||
if len(notifMethods) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
now := time.Now()
|
||||
var out []Task
|
||||
// Use SKIP LOCKED to avoid contention between workers (mirror of ChecksForWorker).
|
||||
q := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'method' IN (?)", notifMethods)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Where("account_id IN (?)", accounts)
|
||||
}
|
||||
if err := q.Limit(limit).Find(&out).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
681
app/models/task_test.go
Обычный файл
681
app/models/task_test.go
Обычный файл
@@ -0,0 +1,681 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// seedRegion creates a Region row before a worker fixture inserts, so the FK
|
||||
// from worker_nodes -> regions holds. Idempotent: Drop() cleans up.
|
||||
func seedRegion(t *testing.T, code string) models.Region {
|
||||
t.Helper()
|
||||
r := models.Region{}
|
||||
err := models.DB().Where("code = ?", code).First(&r).Error
|
||||
if err == nil {
|
||||
return r
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("seed region lookup: %v", err)
|
||||
}
|
||||
r = models.Region{Code: code, Name: code, Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&r).Error)
|
||||
return r
|
||||
}
|
||||
|
||||
// seedAccountUserPlan returns an account with a default plan and the first user
|
||||
// for the FK chain that contacts/notifications walk. Cleanup is the caller's
|
||||
// responsibility (Drop() at end of test).
|
||||
func seedAccountUserPlan(t *testing.T) (models.Account, models.User) {
|
||||
t.Helper()
|
||||
plan := models.Plan{Name: "test-plan", Default: false}
|
||||
if err := models.DB().Create(&plan).Error; err != nil {
|
||||
t.Fatalf("seed plan: %v", err)
|
||||
}
|
||||
|
||||
user := models.User{Name: "test-user", Email: taskStringPtr("test-" + uuid.NewString() + "@example.com"), Timezone: "UTC"}
|
||||
if err := models.DB().Create(&user).Error; err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
|
||||
account := models.Account{Name: "test-account", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
if err := models.DB().Create(&account).Error; err != nil {
|
||||
t.Fatalf("seed account: %v", err)
|
||||
}
|
||||
return account, user
|
||||
}
|
||||
|
||||
func seedNotification(t *testing.T, accountID int64) models.Notification {
|
||||
t.Helper()
|
||||
n := models.Notification{Name: "default", AccountID: accountID, Enabled: true, NotifyDown: true, NotifyRestore: true}
|
||||
require.NoError(t, models.DB().Create(&n).Error)
|
||||
return n
|
||||
}
|
||||
|
||||
func seedEmailContact(t *testing.T, accountID int64) models.Contact {
|
||||
t.Helper()
|
||||
c := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &accountID}
|
||||
require.NoError(t, models.DB().Create(&c).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
func taskStringPtr(s string) *string { return &s }
|
||||
|
||||
func TestTaskSchemaMigration(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.True(t, models.DB().Migrator().HasTable(&models.Task{}), "tasks table must exist after Migrate()")
|
||||
require.True(t, models.DB().Migrator().HasTable(&models.NotificationDelivery{}), "notification_deliveries table must exist after Migrate()")
|
||||
}
|
||||
|
||||
func TestEnqueueNotificationTask_Idempotency(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
|
||||
// Producer precheck requires at least one worker authorized for the
|
||||
// (method, account) pair. Add an operated-style email worker.
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-idempotency-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
LastSeen: &now,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
payload := []byte(`{"method":"email","subject":"[rsmon] x is down","body_text":"down","body_html":"<p>down</p>","body_markdown":"**down**","language":"en","message_kind":"down"}`)
|
||||
|
||||
input := models.EnqueueNotificationTaskInput{
|
||||
AccountID: account.ID,
|
||||
NotificationID: notification.ID,
|
||||
ContactID: contact.ID,
|
||||
Method: "email",
|
||||
Subject: "[rsmon] x is down",
|
||||
BodyText: "down",
|
||||
BodyHTML: "<p>down</p>",
|
||||
Language: "en",
|
||||
MessageKind: "down",
|
||||
EventIDs: []int64{42},
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
first, err := models.EnqueueNotificationTask(&input)
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, first.ID)
|
||||
|
||||
// Second call with the same logical event must not create a duplicate row.
|
||||
second, err := models.EnqueueNotificationTask(&input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first.ID, second.ID, "idempotency: second enqueue should return the same row")
|
||||
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Task{}).Where("idempotency_key = ?", first.IdempotencyKey).Count(&count).Error)
|
||||
assert.EqualValues(t, 1, count, "exactly one Task row per idempotency key")
|
||||
}
|
||||
|
||||
// TestEnqueueNotificationTask_AuthorizationSkip makes sure the producer can
|
||||
// observe ErrNotificationMethodNotAuthorized when no worker is eligible for
|
||||
// the (method, account) pair. The producer uses this to avoid enqueueing work
|
||||
// no operated worker could ever pick up.
|
||||
func TestEnqueueNotificationTask_AuthorizationSkip(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
// Register a worker that only knows telegram. An email task must fail the
|
||||
// precheck.
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-tg-only-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"telegram"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
_, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
||||
AccountID: account.ID,
|
||||
NotificationID: notification.ID,
|
||||
ContactID: contact.ID,
|
||||
Method: "email",
|
||||
EventIDs: []int64{1},
|
||||
Payload: []byte(`{"method":"email"}`),
|
||||
})
|
||||
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
||||
}
|
||||
|
||||
func TestEnqueueNotificationTask_RequiresTaskEnvelopeWorker(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
now := time.Now()
|
||||
worker := &models.WorkerNode{WorkerID: "legacy-notify-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"}, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
||||
}))}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
require.NoError(t, models.DB().Model(&models.WorkerNode{}).Where("id <> ?", worker.ID).Update("status", "dead").Error)
|
||||
input := &models.EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notification.ID, ContactID: contact.ID, Method: "email", EventIDs: []int64{77}, Payload: []byte(`{"method":"email"}`)}
|
||||
_, err := models.EnqueueNotificationTask(input)
|
||||
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Task{}).Count(&count).Error)
|
||||
assert.Zero(t, count)
|
||||
worker.Capabilities = datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"}, "task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
||||
}))
|
||||
require.NoError(t, models.DB().Save(worker).Error)
|
||||
task, err := models.EnqueueNotificationTask(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, models.TaskStateQueued, task.State)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-email-only-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
LastSeen: &now,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
// Enqueue 3 tasks of different methods; only the email ones should be
|
||||
// picked up by the worker. We use the raw helper because the producer's
|
||||
// precheck would refuse the telegram row when no worker handles telegram —
|
||||
// the selector test must exercise the SELECT-side filter, not the
|
||||
// producer-side authorization.
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{1})
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "telegram", []int64{2})
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{3})
|
||||
|
||||
picked, err := models.TasksForWorker(w, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 2, "only email tasks should be leased")
|
||||
|
||||
for _, p := range picked {
|
||||
assert.Equal(t, models.TaskStateLeased, p.State)
|
||||
assert.Equal(t, w.WorkerID, p.LeaseOwner)
|
||||
assert.NotNil(t, p.LeaseExpiresAt)
|
||||
assert.Equal(t, 1, p.Attempts)
|
||||
}
|
||||
|
||||
// A second call must not return the same rows.
|
||||
picked2, err := models.TasksForWorker(w, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked2, "second selector poll should see an empty queue while leased")
|
||||
}
|
||||
|
||||
func TestEnqueueDueCheckTasks_UsesGenericEnvelope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
group := models.Group{Name: "checks", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "example.com", Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
enabled := true
|
||||
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "generic-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
|
||||
require.NoError(t, models.EnqueueDueCheckTasks(worker, []string{"http"}, 1))
|
||||
picked, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 1)
|
||||
assert.Equal(t, models.TaskKindCheck, picked[0].Kind)
|
||||
assert.Equal(t, models.TaskStateLeased, picked[0].State)
|
||||
assert.Equal(t, check.ID, *picked[0].CheckID)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_ChecksRespectPrivateAccountScope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
accountA, _ := seedAccountUserPlan(t)
|
||||
accountB, _ := seedAccountUserPlan(t)
|
||||
accountID := accountA.ID
|
||||
private := &models.WorkerNode{
|
||||
WorkerID: "private-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
platform := &models.WorkerNode{
|
||||
WorkerID: "platform-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(),
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(private).Error)
|
||||
require.NoError(t, models.DB().Create(platform).Error)
|
||||
task := models.Task{JobID: uuid.NewString(), Kind: models.TaskKindCheck, State: models.TaskStateQueued, AccountID: accountB.ID, Payload: datatypes.JSON([]byte(`{"kind":"http"}`)), NotBefore: time.Now().Add(-time.Second), MaxAttempts: 5, IdempotencyKey: "cross-account-" + uuid.NewString()}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
picked, err := models.TasksForWorker(private, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked)
|
||||
picked, err = models.TasksForWorker(platform, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 1)
|
||||
assert.Equal(t, task.ID, picked[0].ID)
|
||||
}
|
||||
|
||||
func TestChecksForWorker_RespectsPrivateAccountScope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
accountA, _ := seedAccountUserPlan(t)
|
||||
accountB, _ := seedAccountUserPlan(t)
|
||||
makeCheck := func(accountID int64, host string) models.Check {
|
||||
group := models.Group{Name: host, AccountID: accountID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: host, Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
enabled := true
|
||||
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return check
|
||||
}
|
||||
owned := makeCheck(accountA.ID, "owned.example")
|
||||
_ = makeCheck(accountB.ID, "other.example")
|
||||
accountID := accountA.ID
|
||||
private := &models.WorkerNode{
|
||||
WorkerID: "private-legacy-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
||||
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(private).Error)
|
||||
checks := models.ChecksForWorker(private, []string{"http"}, 10)
|
||||
require.Len(t, checks, 1)
|
||||
assert.Equal(t, owned.ID, checks[0].ID)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_SkipsExpiredDeadline(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
now := time.Now()
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "worker-deadline-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
|
||||
expired := time.Now().Add(-time.Second)
|
||||
task := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{91})
|
||||
require.NoError(t, models.DB().Model(&task).Update("deadline", expired).Error)
|
||||
picked, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked)
|
||||
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, stored.State)
|
||||
assert.Equal(t, 0, stored.Attempts)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasksTerminatesExpiredQueuedNotification(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
deadline := time.Now().Add(-time.Minute)
|
||||
// This reaper test intentionally has no eligible worker; insert directly
|
||||
// so it tests deadline handling rather than producer capability validation.
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{92})
|
||||
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{"message_id": message.ID, "deadline": deadline}).Error)
|
||||
|
||||
_, _, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, stored.State)
|
||||
assert.Equal(t, "notification deadline expired", stored.LastError)
|
||||
var storedMessage models.Message
|
||||
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
||||
assert.Equal(t, "error", storedMessage.State)
|
||||
var auditCount int64
|
||||
require.NoError(t, models.DB().Model(&models.NotificationDelivery{}).Where("task_id = ? AND status = ?", task.ID, "expired").Count(&auditCount).Error)
|
||||
assert.EqualValues(t, 1, auditCount)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasks_RecyclesLeasesAndDeadsExhaustedRetries(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-reap-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
LastSeen: &now,
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
// 1) A leased task whose lease expired — should go back to queued.
|
||||
expiredLease := time.Now().Add(-time.Minute)
|
||||
leased := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{10})
|
||||
require.NoError(t, models.DB().Model(&leased).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateLeased,
|
||||
"lease_owner": w.WorkerID,
|
||||
"lease_expires_at": expiredLease,
|
||||
"attempts": 1,
|
||||
}).Error)
|
||||
|
||||
// 2) A failed_retry task past its not_before and at max_attempts — should move to dead.
|
||||
failedRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{11})
|
||||
require.NoError(t, models.DB().Model(&failedRetry).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateFailedRetry,
|
||||
"attempts": 5,
|
||||
"max_attempts": 5,
|
||||
"not_before": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
|
||||
// 3) A failed_retry task past not_before but attempts < max_attempts — must stay failed_retry.
|
||||
pendingRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{12})
|
||||
require.NoError(t, models.DB().Model(&pendingRetry).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateFailedRetry,
|
||||
"attempts": 2,
|
||||
"max_attempts": 5,
|
||||
"not_before": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
|
||||
reaped, deaded, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped, "one expired lease should be returned to queue")
|
||||
assert.Equal(t, 1, deaded, "one exhausted retry should move to dead")
|
||||
|
||||
var leasedRow models.Task
|
||||
require.NoError(t, models.DB().First(&leasedRow, leased.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, leasedRow.State)
|
||||
assert.Empty(t, leasedRow.LeaseOwner)
|
||||
assert.Nil(t, leasedRow.LeaseExpiresAt)
|
||||
|
||||
var deadRow models.Task
|
||||
require.NoError(t, models.DB().First(&deadRow, failedRetry.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, deadRow.State)
|
||||
|
||||
var pendingRow models.Task
|
||||
require.NoError(t, models.DB().First(&pendingRow, pendingRetry.ID).Error)
|
||||
assert.Equal(t, models.TaskStateFailedRetry, pendingRow.State)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasks_ExpiredNotificationLeaseExhaustionFinalizesMessage(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{99})
|
||||
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{
|
||||
"message_id": message.ID, "state": models.TaskStateLeased, "attempts": 5, "max_attempts": 5,
|
||||
"lease_expires_at": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
_, deaded, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, deaded)
|
||||
var storedTask models.Task
|
||||
require.NoError(t, models.DB().First(&storedTask, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, storedTask.State)
|
||||
assert.Equal(t, "lease expired after max attempts", storedTask.LastError)
|
||||
var storedMessage models.Message
|
||||
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
||||
assert.Equal(t, "error", storedMessage.State)
|
||||
require.NotNil(t, storedMessage.Error)
|
||||
assert.Equal(t, "lease expired after max attempts", *storedMessage.Error)
|
||||
var delivery models.NotificationDelivery
|
||||
require.NoError(t, models.DB().Where("task_id = ?", task.ID).First(&delivery).Error)
|
||||
assert.Equal(t, "dead", delivery.Status)
|
||||
assert.Equal(t, "lease expired after max attempts", delivery.Error)
|
||||
}
|
||||
|
||||
func TestNotificationTaskLeaseOutlivesExecutionTimeoutAndReapsAfterExpiry(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "notification-lease-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{101})
|
||||
leased, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, leased, 1)
|
||||
require.NotNil(t, leased[0].LeaseExpiresAt)
|
||||
assert.GreaterOrEqual(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationTaskLeaseTTL-time.Second)
|
||||
assert.Greater(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationExecutionTimeout)
|
||||
|
||||
withinExecution := time.Now().Add(models.DefaultNotificationExecutionTimeout)
|
||||
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", withinExecution).Error)
|
||||
reaped, _, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reaped)
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateLeased, stored.State)
|
||||
|
||||
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error)
|
||||
reaped, _, err = models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, stored.State)
|
||||
}
|
||||
|
||||
func mustEnqueue(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
||||
t.Helper()
|
||||
payload := []byte(`{"method":"` + method + `"}`)
|
||||
task, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
||||
AccountID: accountID,
|
||||
NotificationID: notificationID,
|
||||
ContactID: contactID,
|
||||
Method: method,
|
||||
EventIDs: eventIDs,
|
||||
Payload: payload,
|
||||
NotBefore: time.Now().Add(-time.Second),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, task.ID)
|
||||
return *task
|
||||
}
|
||||
|
||||
// mustEnqueueRaw inserts a Task row directly without going through the producer
|
||||
// precheck. The selector test deliberately mixes methods (email + telegram) on a
|
||||
// worker that only handles email; the producer would refuse the telegram row,
|
||||
// which is the wrong thing to assert about in a selector test.
|
||||
func mustEnqueueRaw(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
||||
t.Helper()
|
||||
payload := datatypes.JSON([]byte(`{"method":"` + method + `"}`))
|
||||
contact := contactID
|
||||
task := models.Task{
|
||||
JobID: uuid.New().String(),
|
||||
Kind: models.TaskKindNotification,
|
||||
State: models.TaskStateQueued,
|
||||
AccountID: accountID,
|
||||
ContactID: &contact,
|
||||
Payload: payload,
|
||||
NotBefore: time.Now().Add(-time.Second),
|
||||
Attempts: 0,
|
||||
MaxAttempts: 5,
|
||||
IdempotencyKey: models.NotificationIdempotencyKey(notificationID, contactID, eventIDs[0]),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
require.NotZero(t, task.ID)
|
||||
return task
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v interface{}) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestWorkerNodeCapabilities_NotificationFlags confirms that the JSON-backed
|
||||
// capabilities blob correctly exposes the notification_methods and
|
||||
// notification_accounts arrays that the selector and credential push depend on.
|
||||
func TestWorkerNodeCapabilities_NotificationFlags(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "caps-worker-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"email", "telegram"},
|
||||
"notification_accounts": []int64{7, 8},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
got := models.WorkerNode{}
|
||||
require.NoError(t, models.DB().First(&got, w.ID).Error)
|
||||
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
||||
assert.Equal(t, []int64{7, 8}, got.NotificationAccounts())
|
||||
assert.True(t, got.CanDeliverNotification("email", 7))
|
||||
assert.False(t, got.CanDeliverNotification("email", 9), "account 9 is not in the allowed list")
|
||||
assert.False(t, got.CanDeliverNotification("mattermost", 7), "method not authorized")
|
||||
|
||||
// Operated-style worker: empty accounts list means "all accounts".
|
||||
w2 := &models.WorkerNode{
|
||||
WorkerID: "ops-worker-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w2).Error)
|
||||
|
||||
got2 := models.WorkerNode{}
|
||||
require.NoError(t, models.DB().First(&got2, w2.ID).Error)
|
||||
assert.True(t, got2.CanDeliverNotification("email", 9999), "empty accounts list means all accounts")
|
||||
}
|
||||
|
||||
func TestWorkerNodeReportedWorkloadUsesDisjointHeartbeatFields(t *testing.T) {
|
||||
w := &models.WorkerNode{Capabilities: datatypes.JSON([]byte(`{"active_checks":2,"queue_depth":3,"active_notifications":5,"notification_queue_depth":7}`))}
|
||||
assert.Equal(t, 17, w.ReportedWorkload())
|
||||
}
|
||||
|
||||
// TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults verifies that
|
||||
// EnsureConfiguredWorkerNode (the in-cluster worker provisioner) populates
|
||||
// notification_methods / notification_accounts on the JSON blob so the new
|
||||
// selector and the new credential push work out of the box for the bundled
|
||||
// worker.
|
||||
func TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
t.Setenv("WORKER_AUTH_TOKEN", "secret-token-for-test-xyz")
|
||||
t.Setenv("DEPLOY_ENV", "test-env")
|
||||
t.Setenv("RSMON_WORKER_ID", "worker-test")
|
||||
t.Setenv("NOTIFICATION_METHODS", "email,telegram")
|
||||
t.Setenv("NOTIFICATION_ACCOUNTS", "11,22")
|
||||
|
||||
models.EnsureConfiguredWorkerNode()
|
||||
|
||||
var got models.WorkerNode
|
||||
require.NoError(t, models.DB().Where("worker_id = ?", "worker-test").First(&got).Error)
|
||||
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
||||
assert.Equal(t, []int64{11, 22}, got.NotificationAccounts())
|
||||
}
|
||||
|
||||
// TestNotificationIdempotencyKeyFormat pins the producer-side key shape so the
|
||||
// result handler can re-derive it for matching without depending on internal
|
||||
// package state.
|
||||
func TestNotificationIdempotencyKeyFormat(t *testing.T) {
|
||||
key := models.NotificationIdempotencyKey(7, 9, 13)
|
||||
assert.Equal(t, "notif:7:contact:9:event:13", key)
|
||||
}
|
||||
|
||||
// guard against uuid being accidentally dropped from the imports.
|
||||
var _ = uuid.New
|
||||
77
app/models/telegram_bot.go
Обычный файл
77
app/models/telegram_bot.go
Обычный файл
@@ -0,0 +1,77 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
// TelegramBotMessageReceived marks inbound bot messages.
|
||||
TelegramBotMessageReceived = "received"
|
||||
// TelegramBotMessageSent marks outbound bot replies.
|
||||
TelegramBotMessageSent = "sent"
|
||||
// TelegramBotStatusMain is the singleton status row name for the bot.
|
||||
TelegramBotStatusMain = "main"
|
||||
)
|
||||
|
||||
// TelegramBotMessage stores a Telegram bot chat message for admin history.
|
||||
type TelegramBotMessage struct {
|
||||
concerns.Model
|
||||
|
||||
Direction string `gorm:"size:16;index" json:"direction"`
|
||||
ChatID int64 `gorm:"index" json:"chat_id"`
|
||||
ChatType string `gorm:"size:32" json:"chat_type"`
|
||||
Username string `gorm:"size:255" json:"username"`
|
||||
Text string `gorm:"type:text" json:"text"`
|
||||
Command string `gorm:"size:64" json:"command"`
|
||||
ContactID *int64 `gorm:"index" json:"contact_id,omitempty"`
|
||||
Error string `gorm:"type:text" json:"error,omitempty"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (TelegramBotMessage) TableName() string {
|
||||
return "telegram_bot_messages"
|
||||
}
|
||||
|
||||
// TelegramBotStatus stores the current Telegram bot heartbeat/status.
|
||||
type TelegramBotStatus struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `gorm:"uniqueIndex;size:64;not null" json:"name"`
|
||||
Username string `gorm:"size:255" json:"username"`
|
||||
Online bool `gorm:"not null;default:false" json:"online"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty"`
|
||||
LastError string `gorm:"type:text" json:"last_error,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (TelegramBotStatus) TableName() string {
|
||||
return "telegram_bot_statuses"
|
||||
}
|
||||
|
||||
// RecentTelegramBotMessages returns the latest Telegram bot messages, capped to a safe limit.
|
||||
func RecentTelegramBotMessages(limit int) ([]TelegramBotMessage, error) {
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
messages := []TelegramBotMessage{}
|
||||
err := DB().Order("id DESC").Limit(limit).Find(&messages).Error
|
||||
return messages, err
|
||||
}
|
||||
|
||||
// TelegramBotCurrentStatus returns the singleton Telegram bot status row.
|
||||
func TelegramBotCurrentStatus() (*TelegramBotStatus, error) {
|
||||
status := TelegramBotStatus{}
|
||||
if err := DB().Where("name = ?", TelegramBotStatusMain).First(&status).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status.LastSeen == nil || time.Since(*status.LastSeen) > 2*time.Minute {
|
||||
status.Online = false
|
||||
}
|
||||
return &status, nil
|
||||
}
|
||||
24
app/models/telegram_bot_test.go
Обычный файл
24
app/models/telegram_bot_test.go
Обычный файл
@@ -0,0 +1,24 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestTelegramBotCurrentStatusMarksStaleOffline(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seen := time.Now().Add(-3 * time.Minute)
|
||||
status := models.TelegramBotStatus{Name: models.TelegramBotStatusMain, Online: true, LastSeen: &seen}
|
||||
require.NoError(t, models.DB().Create(&status).Error)
|
||||
|
||||
got, err := models.TelegramBotCurrentStatus()
|
||||
require.NoError(t, err)
|
||||
assert.False(t, got.Online)
|
||||
}
|
||||
233
app/models/user.go
Обычный файл
233
app/models/user.go
Обычный файл
@@ -0,0 +1,233 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// User represents a platform user.
|
||||
//
|
||||
// A User is a global identity that may belong to many tenants via the
|
||||
// Access join table (see Access). Contacts created by or assigned to a
|
||||
// User carry that UserID; admins reach them via the cross-tenant
|
||||
// /admin/users page, while per-account management is via
|
||||
// /settings/users (which shows Accesses preloaded with User + Invite).
|
||||
//
|
||||
// Authentication state (password, confirmation, lock, recover) lives
|
||||
// here, not on Access, because those attributes are account-independent.
|
||||
type User struct {
|
||||
// concerns.Model
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
|
||||
Email *string `gorm:"uniqueIndex;size:255" json:"email" validate:"required"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"-"`
|
||||
// Operator grants platform-wide operational access. Account ownership alone
|
||||
// must never grant cross-tenant task inspection or replay.
|
||||
Operator bool `gorm:"not null;default:false" json:"operator"`
|
||||
Timezone string `json:"timezone"`
|
||||
Language string `gorm:"default:ru" json:"language"`
|
||||
// Settings holds small per-user UI preferences. It intentionally stays
|
||||
// separate from account settings because the sidebar is a personal view.
|
||||
Settings datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"settings"`
|
||||
Phone string `gorm:"index,size:255" json:"phone"`
|
||||
TelegramID *int64 `gorm:"uniqueIndex" json:"telegram_id"`
|
||||
TelegramUsername string `gorm:"size:255" json:"telegram_username"`
|
||||
|
||||
Accesses []Access `json:"-"`
|
||||
Contacts []Contact `json:"-"`
|
||||
|
||||
LastActiveAt *time.Time `json:"last_active_at"`
|
||||
LastActiveIP *string `json:"last_active_ip"`
|
||||
|
||||
EncryptedPassword *string `json:"-"`
|
||||
PasswordSetAt *time.Time `json:"-"`
|
||||
|
||||
// Confirm
|
||||
ConfirmationToken *string `json:"-"`
|
||||
Confirmed bool `json:"confirmed"`
|
||||
|
||||
// Lock
|
||||
AttemptCount int `json:"-"`
|
||||
LastAttempt *time.Time `json:"-"`
|
||||
LockedAt *time.Time `json:"-"`
|
||||
|
||||
// Recover
|
||||
RecoverToken *string `json:"-"`
|
||||
RecoverTokenAt *time.Time `json:"-"`
|
||||
|
||||
RememberTokens pq.StringArray `gorm:"index;type:varchar(100)[]" json:"-"`
|
||||
|
||||
// DeletionRequestedAt is set when the user requests account deletion.
|
||||
// During the 7-day grace period the user can cancel the deletion
|
||||
// (clearing this field). After 7 days the user and all related data
|
||||
// are hard-deleted by a scheduled job.
|
||||
DeletionRequestedAt *time.Time `json:"deletion_requested_at"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
}
|
||||
|
||||
// DeletionPending returns true if the user has requested deletion and is
|
||||
// still within the 7-day grace period.
|
||||
func (u *User) DeletionPending() bool {
|
||||
return u.DeletionRequestedAt != nil
|
||||
}
|
||||
|
||||
// GetLabel returns info label for user.
|
||||
func (u *User) GetLabel() string {
|
||||
return u.DisplayName()
|
||||
}
|
||||
|
||||
// DisplayName implements qor.CurrentUser for admin.
|
||||
func (u *User) DisplayName() string {
|
||||
if u.Email != nil {
|
||||
return u.Name + " " + *u.Email
|
||||
}
|
||||
return u.Name
|
||||
}
|
||||
|
||||
// AfterSocialLogin is a callback after social login.
|
||||
func (u *User) AfterSocialLogin(inviteID int64) (*User, error) {
|
||||
oldUser := User{}
|
||||
DB().Where("email = ?", u.Email).Where("id != ?", u.ID).First(&oldUser)
|
||||
if oldUser.ID > 0 {
|
||||
log.Println("new user", u.ID, "has same email", u.Email, "as old user", oldUser.ID, "so replacing")
|
||||
err := DB().Model(&authidentity.AuthIdentity{}).Where("user_id = ?", u.ID).Updates(
|
||||
authidentity.Basic{
|
||||
UserID: &oldUser.ID,
|
||||
},
|
||||
).Error
|
||||
|
||||
return &oldUser, err
|
||||
}
|
||||
err := u.AfterRegister(inviteID)
|
||||
return u, err
|
||||
}
|
||||
|
||||
// AfterInvite is a callback after invite acceptance.
|
||||
func (u *User) AfterInvite(invite *Invite) error {
|
||||
invite.InviteeID = &u.ID
|
||||
|
||||
err := DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to add accesses to invited user")
|
||||
}
|
||||
|
||||
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{
|
||||
"confirmed_at": time.Now(),
|
||||
}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed set user as confirmed")
|
||||
}
|
||||
|
||||
eml := invite.Email
|
||||
u.Email = &eml
|
||||
if invite.Name != "" {
|
||||
u.Name = invite.Name
|
||||
}
|
||||
err = DB().Save(&u).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to save user")
|
||||
}
|
||||
|
||||
if invite.Name == "" {
|
||||
log.Println("set name", u.Name)
|
||||
invite.Name = u.Name
|
||||
}
|
||||
|
||||
invite.State = stateOK
|
||||
|
||||
err = DB().Save(&invite).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to save invite")
|
||||
}
|
||||
|
||||
invite.Invitee = u
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AfterRegister is a callback after registration.
|
||||
func (u *User) AfterRegister(inviteID int64) error {
|
||||
var err error
|
||||
|
||||
if inviteID > 0 {
|
||||
invite := Invite{}
|
||||
err := DB().First(&invite, inviteID).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to find invite")
|
||||
}
|
||||
|
||||
err = u.AfterInvite(&invite)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = CreateAccountForUser("", u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AfterLogin is a callback after login.
|
||||
func (u *User) AfterLogin(inviteID int64) error {
|
||||
var err error
|
||||
if inviteID > 0 {
|
||||
invite := Invite{}
|
||||
err = DB().First(&invite, inviteID).Error
|
||||
if err == nil {
|
||||
invite.InviteeID = &u.ID
|
||||
invite.State = stateOK
|
||||
err = DB().Save(&invite).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to save invite")
|
||||
}
|
||||
err = DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to add accesses to invited user")
|
||||
}
|
||||
|
||||
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{ //nolint:lll
|
||||
"confirmed_at": time.Now(),
|
||||
}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed set user as confirmed")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Gravatar returns the Gravatar URL for the user.
|
||||
func (u *User) Gravatar(size int) string {
|
||||
if u.Email == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
hash := md5.Sum([]byte(*u.Email))
|
||||
return fmt.Sprintf("https://www.gravatar.com/avatar/%x?s=%d&d=blank", hash, size)
|
||||
}
|
||||
|
||||
// AsJSON returns a JSON representation of user.
|
||||
func (u User) AsJSON() map[string]interface{} { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
r := map[string]interface{}{
|
||||
"id": u.ID,
|
||||
"email": u.Email,
|
||||
"avatar": u.Gravatar(32),
|
||||
"deletion_requested_at": u.DeletionRequestedAt,
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
23
app/models/whois.go
Обычный файл
23
app/models/whois.go
Обычный файл
@@ -0,0 +1,23 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Whois provides functionality.
|
||||
type Whois struct {
|
||||
concerns.Model
|
||||
|
||||
MonitorID *int64
|
||||
Monitor *Monitor
|
||||
|
||||
Tld string
|
||||
Nameservers pq.StringArray `gorm:"type:varchar(255)[]"`
|
||||
Expires *time.Time
|
||||
UpdatedAt *time.Time
|
||||
Data string
|
||||
}
|
||||
31
app/models/worker_log_event.go
Обычный файл
31
app/models/worker_log_event.go
Обычный файл
@@ -0,0 +1,31 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// WorkerLogEvent stores critical log events sent by worker nodes. The same
|
||||
// payload may also be forwarded to VictoriaLogs; Postgres keeps a compact audit
|
||||
// copy so the control plane can show recent critical events even if external log
|
||||
// storage is temporarily unavailable.
|
||||
type WorkerLogEvent struct {
|
||||
concerns.Model
|
||||
|
||||
WorkerID int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL;index" json:"worker_id"`
|
||||
WorkerNodeID string `gorm:"size:100;not null;index" json:"worker_node_id"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Level string `gorm:"size:16;not null;index" json:"level"`
|
||||
Message string `gorm:"type:text;not null" json:"message"`
|
||||
Source string `gorm:"size:64" json:"source"`
|
||||
Payload datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"payload"`
|
||||
OccurredAt time.Time `gorm:"not null;index" json:"occurred_at"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (WorkerLogEvent) TableName() string { return "worker_log_events" }
|
||||
452
app/models/worker_node.go
Обычный файл
452
app/models/worker_node.go
Обычный файл
@@ -0,0 +1,452 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// WorkerNode represents a distributed monitoring worker
|
||||
type WorkerNode struct {
|
||||
concerns.Model
|
||||
WorkerID string `gorm:"uniqueIndex;size:100;not null" json:"worker_id"` // UUID or configured ID
|
||||
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
|
||||
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
|
||||
Status string `gorm:"not null;default:'registered'" json:"status"` // registered, active, inactive, dead
|
||||
AuthToken string `gorm:"uniqueIndex;size:64;not null" json:"-"`
|
||||
LastSeen *time.Time `json:"last_seen"`
|
||||
Version string `gorm:"size:50" json:"version"`
|
||||
URL string `gorm:"size:500" json:"url"` //nolint:lll // publicly-advertised URL; may differ from bind host:port when behind Traefik
|
||||
Capabilities datatypes.JSON `gorm:"not null;default:'{}'" json:"capabilities"` // {"check_types": ["http","ssl","dns",...]}
|
||||
Concurrency int `gorm:"not null;default:20" json:"concurrency"`
|
||||
NetworkProblems bool `gorm:"not null;default:false;index" json:"network_problems"`
|
||||
NetworkProblemsUntil *time.Time `json:"network_problems_until,omitempty"`
|
||||
LastFailureCount int `gorm:"not null;default:0" json:"last_failure_count"`
|
||||
LastTotalCount int `gorm:"not null;default:0" json:"last_total_count"`
|
||||
// Capability flags (see docs/plans/inventory-management.md §3).
|
||||
// All default to true so an existing worker row that pre-dates
|
||||
// this migration keeps running checks. Toggle from the admin UI
|
||||
// or POST a boolean to /api/v1/workers to disable any one of
|
||||
// them; the distworker client re-reads these flags on every
|
||||
// task poll.
|
||||
//
|
||||
// Pointer types so GORM can distinguish "client didn't supply
|
||||
// the key, fall back to the DB default" from "client explicitly
|
||||
// set false". A plain `bool` would be silently re-overwritten
|
||||
// by the column default on Save (default:true kicks in when
|
||||
// GORM sees the zero value, regardless of whether the handler
|
||||
// asked for false). See the controller tests for the
|
||||
// partial-update case.
|
||||
RunChecks *bool `gorm:"not null;default:true" json:"run_checks"`
|
||||
CollectMetrics *bool `gorm:"not null;default:true" json:"collect_metrics"`
|
||||
DetectProjects *bool `gorm:"not null;default:true" json:"detect_projects"`
|
||||
// ServerID is the optional inventory Server this worker daemon
|
||||
// is running on (see docs/plans/servers-and-hardware-metrics.md
|
||||
// §3 and docs/plans/inventory-management.md §1). Nullable so
|
||||
// legacy "no server assigned" rows keep working. Indexed because
|
||||
// the distworker health ticker joins servers→workers frequently.
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
// AccountID scopes a worker to a single customer account (private
|
||||
// worker per docs/distributed/private-workers.md). NULL means a
|
||||
// platform-operated worker eligible to serve any account; non-NULL
|
||||
// is a customer-operated worker pinned to one account. ON DELETE
|
||||
// SET NULL keeps an operated worker valid if its account row is
|
||||
// ever removed without an explicit private-worker cleanup.
|
||||
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE SET NULL;index" json:"account_id,omitempty"`
|
||||
LLMs []LLM `json:"llms,omitempty" gorm:"many2many:worker_llms;"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
const WorkerHeartbeatFreshness = 2 * time.Minute
|
||||
|
||||
func (w *WorkerNode) NetworkProblemActive(now time.Time) bool {
|
||||
return w != nil && w.NetworkProblems && (w.NetworkProblemsUntil == nil || w.NetworkProblemsUntil.After(now))
|
||||
}
|
||||
|
||||
// WorkerStatuses provides functionality.
|
||||
// WorkerStatus represents the possible worker statuses
|
||||
var WorkerStatuses = []string{"registered", "active", "inactive", "dead"}
|
||||
|
||||
// AllWorkerCheckKinds is the complete distributed-worker capability set.
|
||||
func AllWorkerCheckKinds() []string {
|
||||
return []string{kindHTTP, kindSSL, kindDNS, kindSSH, kindFTP, kindWhois, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
|
||||
}
|
||||
|
||||
// NormalizeWorkerCapabilities expands aliases and removes unknown or duplicate capabilities.
|
||||
func NormalizeWorkerCapabilities(capabilities []string) []string {
|
||||
allowed := make(map[string]bool)
|
||||
for _, kind := range AllWorkerCheckKinds() {
|
||||
allowed[kind] = true
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
normalized := make([]string, 0, len(capabilities))
|
||||
for _, capability := range capabilities {
|
||||
capability = strings.TrimSpace(strings.ToLower(capability))
|
||||
if capability == "all" || capability == "*" {
|
||||
return AllWorkerCheckKinds()
|
||||
}
|
||||
if !allowed[capability] || seen[capability] {
|
||||
continue
|
||||
}
|
||||
seen[capability] = true
|
||||
normalized = append(normalized, capability)
|
||||
}
|
||||
if len(normalized) == 0 {
|
||||
return AllWorkerCheckKinds()
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// IsAlive returns true if the worker is considered alive based on last_seen
|
||||
func (w *WorkerNode) IsAlive() bool {
|
||||
if w.LastSeen == nil {
|
||||
return false
|
||||
}
|
||||
// Worker is considered dead if no heartbeat for 2 minutes
|
||||
return w.LastSeen.After(time.Now().Add(-WorkerHeartbeatFreshness))
|
||||
}
|
||||
|
||||
// NotificationMethods returns the notification methods the worker is authorized
|
||||
// to deliver (e.g. ["email", "telegram"]). Empty slice means "no notification
|
||||
// delivery authorized". See docs/plans/worker-notifier-mvp.md section 4.3.
|
||||
func (w *WorkerNode) NotificationMethods() []string {
|
||||
caps := w.capabilitiesMap()
|
||||
if caps == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := caps["notification_methods"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return parseStringList(raw)
|
||||
}
|
||||
|
||||
// NotificationAccounts returns the account IDs the worker is authorized to
|
||||
// serve for notifications. Empty slice means "owned by RSMon, all accounts".
|
||||
// Customer-hosted workers (phase 4) ship a non-empty slice to scope credentials.
|
||||
func (w *WorkerNode) NotificationAccounts() []int64 {
|
||||
caps := w.capabilitiesMap()
|
||||
if caps == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := caps["notification_accounts"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return parseInt64List(raw)
|
||||
}
|
||||
|
||||
// AccessibleAccountIDs returns the accounts this worker may access. Empty means
|
||||
// RSMon-operated/global worker. It combines the legacy single AccountID field
|
||||
// with the newer notification_accounts capability list.
|
||||
func (w *WorkerNode) AccessibleAccountIDs() []int64 {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if w.AccountID != nil && *w.AccountID > 0 {
|
||||
// A private worker cannot widen its account scope through a mutable
|
||||
// capability JSON blob.
|
||||
return []int64{*w.AccountID}
|
||||
}
|
||||
seen := map[int64]bool{}
|
||||
out := []int64{}
|
||||
for _, id := range w.NotificationAccounts() {
|
||||
if id > 0 && !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CanDeliverNotification returns true when the worker is allowed to deliver
|
||||
// the given method for the given account. An empty NotificationAccounts slice
|
||||
// means the worker is RSMon-operated and may serve any account.
|
||||
func (w *WorkerNode) CanDeliverNotification(method string, accountID int64) bool {
|
||||
methods := w.NotificationMethods()
|
||||
if len(methods) == 0 {
|
||||
return false
|
||||
}
|
||||
if !containsString(methods, method) {
|
||||
return false
|
||||
}
|
||||
accounts := w.AccessibleAccountIDs()
|
||||
if len(accounts) == 0 {
|
||||
return true
|
||||
}
|
||||
return containsInt64(accounts, accountID)
|
||||
}
|
||||
|
||||
// CheckTypes returns the check kinds this worker may execute.
|
||||
func (w *WorkerNode) CheckTypes() []string {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return nil
|
||||
}
|
||||
return parseStringList(capabilities["check_types"])
|
||||
}
|
||||
|
||||
// SupportsTaskEnvelope is an explicit protocol capability. Version labels are
|
||||
// build metadata (and may be "latest" or a commit SHA), not a wire contract.
|
||||
// Rows created before this capability existed intentionally remain v1.
|
||||
func (w *WorkerNode) SupportsTaskEnvelope() bool {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return false
|
||||
}
|
||||
supported, _ := capabilities["task_envelope"].(bool)
|
||||
return supported
|
||||
}
|
||||
|
||||
// ReportedWorkload is the worker's local active and queued work across both
|
||||
// checks and notifications. It is advisory; durable leases remain authoritative.
|
||||
func (w *WorkerNode) ReportedWorkload() int {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return 0
|
||||
}
|
||||
keys := []string{"active_checks", "queue_depth", "active_notifications", "notification_queue_depth"}
|
||||
total := 0
|
||||
for _, key := range keys {
|
||||
switch value := capabilities[key].(type) {
|
||||
case float64:
|
||||
if value > 0 {
|
||||
total += int(value)
|
||||
}
|
||||
case int:
|
||||
if value > 0 {
|
||||
total += value
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (w *WorkerNode) capabilitiesMap() map[string]interface{} {
|
||||
if w == nil || len(w.Capabilities) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out map[string]interface{}
|
||||
if err := json.Unmarshal(w.Capabilities, &out); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseStringList(raw interface{}) []string {
|
||||
switch v := raw.(type) {
|
||||
case []interface{}:
|
||||
out := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
out := make([]string, 0, len(v))
|
||||
for _, s := range v {
|
||||
if s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInt64List(raw interface{}) []int64 {
|
||||
switch v := raw.(type) {
|
||||
case []interface{}:
|
||||
out := make([]int64, 0, len(v))
|
||||
for _, item := range v {
|
||||
switch n := item.(type) {
|
||||
case float64:
|
||||
out = append(out, int64(n))
|
||||
case int64:
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []int64:
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsString(haystack []string, needle string) bool {
|
||||
for _, s := range haystack {
|
||||
if s == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsInt64(haystack []int64, needle int64) bool {
|
||||
for _, n := range haystack {
|
||||
if n == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EnsureConfiguredWorkerNode creates or updates the bundled Docker Compose worker from environment variables.
|
||||
func EnsureConfiguredWorkerNode() {
|
||||
token := envFirst("WORKER_AUTH_TOKEN", "RSMON_AUTH_TOKEN")
|
||||
if token == "" {
|
||||
return
|
||||
}
|
||||
|
||||
deployEnv := envDefault("DEPLOY_ENV", "production")
|
||||
workerID := envDefault("RSMON_WORKER_ID", "worker-"+deployEnv+"-01")
|
||||
regionCode := envDefault("RSMON_REGION_CODE", deployEnv)
|
||||
version := envDefault("RSMON_WORKER_VERSION", envDefault("IMAGE_TAG", "latest"))
|
||||
concurrency := envInt("WORKER_CONCURRENCY", 20)
|
||||
workerURL := strings.TrimSpace(os.Getenv("WORKER_URL"))
|
||||
capabilities := NormalizeWorkerCapabilities(splitEnvList(envDefault(
|
||||
"RSMON_CAPABILITIES",
|
||||
"http,ssl,dns,ssh,ftp,whois,bssl,llm,llm-http,ping,tcp,udp",
|
||||
)))
|
||||
|
||||
region := Region{}
|
||||
if err := DB().Where("code = ?", regionCode).First(®ion).Error; err != nil {
|
||||
region = Region{Code: regionCode, Name: regionCode, Enabled: true}
|
||||
if err := DB().Create(®ion).Error; err != nil {
|
||||
log.Printf("worker: failed to create configured worker region %s: %v", regionCode, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
capJSON, err := json.Marshal(map[string]interface{}{
|
||||
"check_types": capabilities,
|
||||
"task_envelope": true,
|
||||
"notification_methods": parseStringList(notificationMethodsFromEnv()),
|
||||
"notification_accounts": parseInt64List(notificationAccountsFromEnv()),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("worker: failed to marshal configured worker capabilities: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
worker := WorkerNode{}
|
||||
DB().Where("worker_id = ? OR auth_token = ?", workerID, token).First(&worker)
|
||||
created := worker.ID == 0
|
||||
worker.WorkerID = workerID
|
||||
worker.RegionCode = regionCode
|
||||
worker.Status = "registered"
|
||||
worker.AuthToken = token
|
||||
worker.Version = version
|
||||
worker.URL = workerURL
|
||||
worker.Capabilities = datatypes.JSON(capJSON)
|
||||
worker.Concurrency = concurrency
|
||||
|
||||
if err := DB().Save(&worker).Error; err != nil {
|
||||
log.Printf("worker: failed to provision configured worker %s: %v", workerID, err)
|
||||
return
|
||||
}
|
||||
if created {
|
||||
log.Printf("worker: provisioned configured worker %s in region %s", workerID, regionCode)
|
||||
} else {
|
||||
log.Printf("worker: updated configured worker %s in region %s", workerID, regionCode)
|
||||
}
|
||||
}
|
||||
|
||||
func splitEnvList(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
items := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
items = append(items, part)
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func envFirst(keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func envDefault(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func envInt(key string, defaultValue int) int {
|
||||
value, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || value <= 0 {
|
||||
return defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// Notification method constants used across the package. Centralized here so
|
||||
// the literal does not appear three or more times (goconst).
|
||||
const (
|
||||
methodEmail = "email"
|
||||
methodTelegram = "telegram"
|
||||
methodWebhook = "webhook"
|
||||
methodMattermost = "mattermost"
|
||||
)
|
||||
|
||||
// defaultNotificationMethods is the operated-worker default notification method
|
||||
// list. Lives at package scope so goconst does not flag the literal across
|
||||
// the package (account.go and user.go already reference "email").
|
||||
var defaultNotificationMethods = []string{methodEmail, methodTelegram, methodWebhook, methodMattermost}
|
||||
|
||||
// notificationMethodsFromEnv reads the optional NOTIFICATION_METHODS env var.
|
||||
// Empty result yields the default "all four" list so the operated worker can
|
||||
// deliver email / telegram / webhook / mattermost out of the box.
|
||||
func notificationMethodsFromEnv() []string {
|
||||
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_METHODS"))
|
||||
if raw == "" {
|
||||
return append([]string{}, defaultNotificationMethods...)
|
||||
}
|
||||
out := make([]string, 0, 4)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notificationAccountsFromEnv reads the optional NOTIFICATION_ACCOUNTS env var.
|
||||
// Empty result means "all accounts allowed" (the RSMon-operated default).
|
||||
func notificationAccountsFromEnv() []int64 {
|
||||
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_ACCOUNTS"))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
out := make([]int64, 0, 4)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.ParseInt(part, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Ссылка в новой задаче
Block a user