Некоторые проверки не удались
CI / test (push) Successful in 2m5s
Docker / Build and publish worker image (push) Failing after 31s
389 строки
14 KiB
Go
389 строки
14 KiB
Go
package models
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"database/sql/driver"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/lib/pq"
|
|
"gorm.io/datatypes"
|
|
"gorm.io/gorm"
|
|
|
|
"rocketgit.ru/rsmon/worker/app/models/concerns"
|
|
)
|
|
|
|
// AccountMCPToken stores a one-way verifier. MCP tokens are bearer credentials
|
|
// and must remain valid even when the deployment has no encryption key.
|
|
type AccountMCPToken struct {
|
|
concerns.Model
|
|
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
|
Name string `gorm:"size:120;not null" json:"name"`
|
|
TokenEnc string `gorm:"column:token;type:char(64);not null;index" json:"-"`
|
|
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
|
concerns.Timestamped
|
|
}
|
|
|
|
func (AccountMCPToken) TableName() string { return "account_mcp_tokens" }
|
|
|
|
func (t *AccountMCPToken) SetToken(token string) error {
|
|
sum := sha256.Sum256([]byte(token))
|
|
t.TokenEnc = hex.EncodeToString(sum[:])
|
|
return nil
|
|
}
|
|
|
|
func (t *AccountMCPToken) TokenMatches(token string) bool {
|
|
sum := sha256.Sum256([]byte(token))
|
|
encoded := hex.EncodeToString(sum[:])
|
|
return subtle.ConstantTimeCompare([]byte(t.TokenEnc), []byte(encoded)) == 1
|
|
}
|
|
|
|
func GenerateMCPToken() string {
|
|
return "mcp_" + base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
|
|
}
|
|
|
|
// Server health states — worst-of-monitor-states rollup; see
|
|
// docs/plans/servers-and-hardware-metrics.md §5.1 for the data model.
|
|
const (
|
|
ServerHealthDown = "down"
|
|
ServerHealthWarn = "warn"
|
|
ServerHealthUp = "up"
|
|
ServerHealthPaused = "paused"
|
|
ServerHealthUnknown = "unknown"
|
|
)
|
|
|
|
// ServerEnvironments is the allow-list for Server.Environment.
|
|
// New environments require an explicit edit so they are visible in
|
|
// tests.
|
|
var ServerEnvironments = []string{"production", "staging", "dev", "test"}
|
|
|
|
// ServerKind is the rstuff-mirrored lifecycle label for a Server
|
|
// (production / staging / old). See
|
|
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable mapping.
|
|
type ServerKind string
|
|
|
|
// ServerKind values match rstuff's Server.kind enum exactly.
|
|
// New values require adding a Postgres enum value via
|
|
// app/models/migrate.go.
|
|
const (
|
|
ServerKindProduction ServerKind = "production"
|
|
ServerKindStaging ServerKind = "staging"
|
|
ServerKindOld ServerKind = "old"
|
|
)
|
|
|
|
// Scan implements sql.Scanner for ServerKind.
|
|
func (k *ServerKind) Scan(src any) error {
|
|
if src == nil {
|
|
*k = ""
|
|
return nil
|
|
}
|
|
switch v := src.(type) {
|
|
case string:
|
|
*k = ServerKind(v)
|
|
case []byte:
|
|
*k = ServerKind(string(v))
|
|
default:
|
|
return fmt.Errorf("server_kind: cannot scan %T", src)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Value implements driver.Valuer for ServerKind.
|
|
func (k ServerKind) Value() (driver.Value, error) {
|
|
if k == "" {
|
|
return nil, nil
|
|
}
|
|
return string(k), nil
|
|
}
|
|
|
|
// Server represents a customer-facing logical host (e.g. "prod-web-01").
|
|
//
|
|
// One Server can host many WorkerNodes (HA after a VM migration). Each
|
|
// WorkerNode carries a nullable ServerID so legacy "no server assigned"
|
|
// rows keep working. The 1:N relation is stored as a nullable FK on
|
|
// worker_nodes.server_id. The N:M relation to Monitor is the
|
|
// monitor_servers join table. See
|
|
// docs/plans/servers-and-hardware-metrics.md §3 for the layer model
|
|
// and §5.1 for the schema.
|
|
//
|
|
// The inventory fields (ExtID, Kind, Token, PriceCents, Comment,
|
|
// Meta) are added per docs/plans/inventory-management.md §6.2 so
|
|
// rstuff can push the same row in via Valkey Streams and
|
|
// deploymentd can authenticate via Token.
|
|
type Server struct {
|
|
concerns.Model
|
|
|
|
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
|
Account *Account `json:"-"`
|
|
Name string `gorm:"size:120;not null" json:"name"`
|
|
Slug string `gorm:"size:120;not null;index" json:"slug"`
|
|
Description *string `gorm:"type:text" json:"description"`
|
|
Region string `gorm:"size:64;not null;index" json:"region"`
|
|
Environment string `gorm:"size:32;not null;default:'production'" json:"environment"`
|
|
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
|
|
Icon *string `gorm:"size:16" json:"icon"`
|
|
Color *string `gorm:"size:16" json:"color"`
|
|
Paused bool `gorm:"not null;default:false" json:"paused"`
|
|
|
|
// Inventory fields (rstuff mirror; see inventory-management.md §6.2).
|
|
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
|
Kind ServerKind `gorm:"type:server_kind;not null;default:'production'" json:"kind"`
|
|
// Token is omitted from JSON because it is a bearer credential.
|
|
// Read it back only via /api/v1/servers/:id/token (operator-only)
|
|
// and never echoed in list/show responses.
|
|
Token *string `gorm:"size:64" json:"-"`
|
|
PriceCents int `gorm:"not null;default:0" json:"price_cents"`
|
|
Comment *string `gorm:"type:text" json:"comment,omitempty"`
|
|
// Meta is rstuff-style free-form jsonb; serialized via JSON
|
|
// encoding (gin renders it as a nested object).
|
|
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
|
|
|
HealthState string `gorm:"size:16;not null;default:'unknown';index" json:"health_state"`
|
|
LastCheckAt *time.Time `json:"last_check_at"`
|
|
Uptime24h float64 `gorm:"not null;default:1.0" json:"uptime_24h"`
|
|
Uptime30d float64 `gorm:"not null;default:1.0" json:"uptime_30d"`
|
|
|
|
Monitors []Monitor `gorm:"many2many:monitor_servers;joinForeignKey:server_id;joinReferences:monitor_id;" json:"monitors,omitempty"`
|
|
Workers []WorkerNode `gorm:"foreignKey:ServerID" json:"workers,omitempty"`
|
|
|
|
concerns.Timestamped
|
|
Audited
|
|
}
|
|
|
|
// TableName provides functionality.
|
|
func (Server) TableName() string { return "servers" }
|
|
|
|
// MonitorServer is the join row for the N:M relation between Monitor and
|
|
// Server. One Monitor can be hosted on many Servers (multi-region
|
|
// failover); one Server can host many Monitors.
|
|
type MonitorServer struct {
|
|
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id) ON DELETE CASCADE;primaryKey" json:"monitor_id"`
|
|
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;primaryKey" json:"server_id"`
|
|
Position int `gorm:"not null;default:0" json:"position"`
|
|
concerns.Timestamped
|
|
}
|
|
|
|
// TableName provides functionality.
|
|
func (MonitorServer) TableName() string { return "monitor_servers" }
|
|
|
|
// ServerMetric is the last-N-point cache written alongside VictoriaMetrics.
|
|
// The full time-series lives in TSDB; Postgres only keeps the most recent
|
|
// row per (server, source) for fast health badges and "last seen" cells.
|
|
// See docs/plans/servers-and-hardware-metrics.md §5.3.
|
|
type ServerMetric struct {
|
|
concerns.Model
|
|
|
|
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
|
WorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"worker_id"`
|
|
Source string `gorm:"size:32;not null;default:'worker'" json:"source"`
|
|
CPUPercent *float64 `json:"cpu_percent"`
|
|
MemUsed *int64 `json:"mem_used"`
|
|
MemTotal *int64 `json:"mem_total"`
|
|
DiskUsed *int64 `json:"disk_used"`
|
|
DiskTotal *int64 `json:"disk_total"`
|
|
NetRx *int64 `json:"net_rx"`
|
|
NetTx *int64 `json:"net_tx"`
|
|
HostUptimeSec *int64 `json:"host_uptime_sec"`
|
|
Load1 *float64 `json:"load1"`
|
|
Load5 *float64 `json:"load5"`
|
|
Load15 *float64 `json:"load15"`
|
|
ProcessCount *int `json:"process_count"`
|
|
Processes datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"processes"`
|
|
Networks datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"networks"`
|
|
|
|
concerns.Timestamped
|
|
}
|
|
|
|
// ServerAlertRule defines one account-owned threshold for a server metric.
|
|
// ClearThreshold implements hysteresis: a firing rule only recovers after the
|
|
// value drops below it, avoiding alert flapping around Threshold.
|
|
type ServerAlertRule struct {
|
|
concerns.Model
|
|
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
|
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
|
Metric string `gorm:"size:32;not null" json:"metric"`
|
|
Threshold float64 `gorm:"not null" json:"threshold"`
|
|
ClearThreshold float64 `gorm:"not null" json:"clear_threshold"`
|
|
DurationSec int `gorm:"not null;default:300" json:"duration_sec"`
|
|
NotificationID int64 `gorm:"type:bigint REFERENCES notifications(id) ON DELETE CASCADE;not null" json:"notification_id"`
|
|
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
|
State string `gorm:"size:16;not null;default:'ok'" json:"state"`
|
|
BreachSince *time.Time `json:"breach_since"`
|
|
LastValue *float64 `json:"last_value"`
|
|
LastMetricID *int64 `json:"last_metric_id"`
|
|
LastFiredAt *time.Time `json:"last_fired_at"`
|
|
concerns.Timestamped
|
|
}
|
|
|
|
func (ServerAlertRule) TableName() string { return "server_alert_rules" }
|
|
|
|
// ServerAlertEvent is the durable dedupe/audit record for threshold changes.
|
|
type ServerAlertEvent struct {
|
|
concerns.Model
|
|
RuleID int64 `gorm:"type:bigint REFERENCES server_alert_rules(id) ON DELETE CASCADE;not null;index" json:"rule_id"`
|
|
State string `gorm:"size:16;not null" json:"state"`
|
|
Value float64 `gorm:"not null" json:"value"`
|
|
concerns.Timestamped
|
|
}
|
|
|
|
func (ServerAlertEvent) TableName() string { return "server_alert_events" }
|
|
|
|
// TableName provides functionality.
|
|
func (ServerMetric) TableName() string { return "server_metrics" }
|
|
|
|
// ValidateEnvironment returns nil iff env is in the allow list.
|
|
func ValidateEnvironment(env string) error {
|
|
for _, e := range ServerEnvironments {
|
|
if env == e {
|
|
return nil
|
|
}
|
|
}
|
|
return errors.New("invalid environment")
|
|
}
|
|
|
|
// Slugify turns a server name into a URL-safe slug.
|
|
func Slugify(name string) string {
|
|
slug := strings.ToLower(strings.TrimSpace(name))
|
|
var b strings.Builder
|
|
for _, r := range slug {
|
|
switch {
|
|
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
|
b.WriteRune(r)
|
|
case r == ' ', r == '_', r == '-', r == '.':
|
|
b.WriteByte('-')
|
|
}
|
|
}
|
|
out := strings.Trim(b.String(), "-")
|
|
if out == "" {
|
|
out = "server"
|
|
}
|
|
return out
|
|
}
|
|
|
|
// AssignMonitors replaces the full set of monitors for a server. Saves
|
|
// the join table explicitly because gorm:association_autoupdate is
|
|
// disabled globally (CLAUDE.md GORM Conventions).
|
|
func (s *Server) AssignMonitors(tx *gorm.DB, monitorIDs []uint) error {
|
|
if tx == nil {
|
|
tx = DB()
|
|
}
|
|
if err := tx.Exec("DELETE FROM monitor_servers WHERE server_id = ?", s.ID).Error; err != nil {
|
|
return err
|
|
}
|
|
if len(monitorIDs) == 0 {
|
|
return nil
|
|
}
|
|
seen := make(map[uint]struct{}, len(monitorIDs))
|
|
for i, mid := range monitorIDs {
|
|
if _, ok := seen[mid]; ok {
|
|
continue
|
|
}
|
|
seen[mid] = struct{}{}
|
|
row := MonitorServer{ServerID: s.ID, MonitorID: int64(mid), Position: i}
|
|
if err := tx.Create(&row).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RollupHealthState computes the worst-of-monitor-states. Returns one of
|
|
// ServerHealth{Down,Warn,Up,Paused,Unknown}.
|
|
func (s *Server) RollupHealthState(monitors []Monitor) string {
|
|
if s.Paused {
|
|
return ServerHealthPaused
|
|
}
|
|
if len(monitors) == 0 {
|
|
return ServerHealthUnknown
|
|
}
|
|
allPaused := true
|
|
worst := ServerHealthUp
|
|
for i := range monitors {
|
|
m := &monitors[i]
|
|
if m.Enabled {
|
|
allPaused = false
|
|
}
|
|
switch m.State {
|
|
case stateERR, stateFail:
|
|
return ServerHealthDown
|
|
case stateWARN:
|
|
worst = ServerHealthWarn
|
|
}
|
|
}
|
|
if allPaused {
|
|
return ServerHealthPaused
|
|
}
|
|
return worst
|
|
}
|
|
|
|
// SortedTagList returns tags sorted ascending; helper for stable JSON.
|
|
func (s *Server) SortedTagList() []string {
|
|
out := make([]string, len(s.Tags))
|
|
copy(out, s.Tags)
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// HealthForServer recomputes and persists health_state + last_check_at
|
|
// for one server. Called by the distworker health ticker.
|
|
func HealthForServer(serverID int64) error {
|
|
server := Server{}
|
|
if err := DB().First(&server, serverID).Error; err != nil {
|
|
return err
|
|
}
|
|
var monitors []Monitor
|
|
if err := DB().Joins("JOIN monitor_servers ms ON ms.monitor_id = monitors.id").
|
|
Where("ms.server_id = ?", serverID).Find(&monitors).Error; err != nil {
|
|
return err
|
|
}
|
|
state := server.RollupHealthState(monitors)
|
|
now := time.Now()
|
|
updates := map[string]interface{}{"health_state": state}
|
|
if len(monitors) > 0 {
|
|
updates["last_check_at"] = &now
|
|
}
|
|
return DB().Model(&server).Updates(updates).Error
|
|
}
|
|
|
|
// LatestServerMetric returns the newest accepted worker snapshot for a server.
|
|
func LatestServerMetric(serverID int64) (*ServerMetric, error) {
|
|
metric := ServerMetric{}
|
|
err := DB().Where("server_id = ?", serverID).Order("id DESC").First(&metric).Error
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &metric, nil
|
|
}
|
|
|
|
// FindServerByToken returns the Server whose token column equals
|
|
// the given hex value, or nil with gorm.ErrRecordNotFound when no
|
|
// row matches. Used by the deploymentd receiver middleware.
|
|
func FindServerByToken(token string) (*Server, error) {
|
|
var s Server
|
|
err := DB().Where("token = ?", token).First(&s).Error
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &s, nil
|
|
}
|
|
|
|
// GenerateServerToken returns a 32-byte random hex string. Caller
|
|
// stores the plaintext exactly once (via /servers/:id/rotate-token)
|
|
// and updates Server.Token; the old value is no longer recoverable.
|
|
func GenerateServerToken() string {
|
|
b := make([]byte, 32)
|
|
if _, err := rand.Read(b); err != nil {
|
|
// crypto/rand should not fail on Linux; panic keeps the
|
|
// contract simple for callers in the rare fatal case.
|
|
panic(err)
|
|
}
|
|
return hex.EncodeToString(b)
|
|
}
|