feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
2026-07-13 17:55:14 +03:00
Коммит 2c7a0236da
309 изменённых файлов: 44004 добавлений и 0 удалений

8
.dockerignore Обычный файл
Просмотреть файл

@@ -0,0 +1,8 @@
.git
.github
.env
bin
dist
rsmon-worker
*.log
*.db*

26
.env.example Обычный файл
Просмотреть файл

@@ -0,0 +1,26 @@
# Required: create the worker at https://rsmon.ru/settings/workers first.
RSMON_URL=https://rsmon.ru
RSMON_TOKEN=replace-with-worker-token
# Local operator console. Both credentials are required while the web UI is on.
WORKER_HOST=0.0.0.0
WORKER_PORT=27401
WORKER_BIND_IP=127.0.0.1
WORKER_URL=
WORKER_LOGIN=admin
WORKER_PASSWORD=replace-with-a-long-random-password
# Persistent web application state.
RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp
# Optional Raft worker cluster.
WORKER_CLUSTER_ENABLED=false
WORKER_CLUSTER_ID=
WORKER_CLUSTER_HOST=0.0.0.0
WORKER_CLUSTER_PORT=37401
WORKER_CLUSTER_PEERS=
WORKER_CLUSTER_DATA_DIR=/var/lib/rsmon-worker/cluster
WORKER_CLUSTER_BOOTSTRAP=false
# Optional update page API endpoint.
WORKER_RELEASE_URL=

27
.github/workflows/ci.yml поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,27 @@
name: CI
on:
push:
branches:
- master
pull_request:
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.26.x'
cache: true
- name: Check module files
run: |
go mod tidy
git diff --exit-code -- go.mod go.sum
- name: Test worker packages
run: make test
- name: Build binary
run: make build

79
.github/workflows/docker.yml поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,79 @@
name: Docker
on:
push:
branches:
- master
tags:
- v*
jobs:
docker:
name: Build and publish worker image
runs-on: ubuntu-latest
env:
HARBOR_REGISTRY: ${{ secrets.HARBOR_REGISTRY }}
HARBOR_PROJECT: rsmon
IMAGE_NAME: rsmon-worker
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Harbor
uses: docker/login-action@v3
with:
registry: ${{ env.HARBOR_REGISTRY }}
username: ${{ secrets.HARBOR_USER }}
password: ${{ secrets.HARBOR_PASSWORD }}
- name: Compute image metadata
id: vars
env:
REF_NAME: ${{ gitea.ref_name }}
REF_TYPE: ${{ gitea.ref_type }}
SHA: ${{ gitea.sha }}
run: |
IMAGE="${HARBOR_REGISTRY}/${HARBOR_PROJECT}/${IMAGE_NAME}"
SHORT_SHA="$(printf '%s' "${SHA}" | cut -c1-12)"
TAGS="${IMAGE}:sha-${SHORT_SHA}"
if [ "${REF_TYPE}" = "branch" ] && [ "${REF_NAME}" = "master" ]; then
TAGS="${TAGS},${IMAGE}:latest"
fi
if [ "${REF_TYPE}" = "tag" ]; then
TAGS="${TAGS},${IMAGE}:${REF_NAME}"
fi
{
echo "image=${IMAGE}"
echo "tags=${TAGS}"
echo "build_date=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >> "${GITHUB_OUTPUT}"
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
pull: true
push: true
tags: ${{ steps.vars.outputs.tags }}
labels: |
org.opencontainers.image.source=${{ gitea.server_url }}/${{ gitea.repository }}
org.opencontainers.image.revision=${{ gitea.sha }}
org.opencontainers.image.version=${{ gitea.ref_name }}
org.opencontainers.image.licenses=LicenseRef-RSMon-Worker-Source-Available-1.0
build-args: |
VERSION=${{ gitea.ref_name }}
COMMIT=${{ gitea.sha }}
BUILD_DATE=${{ steps.vars.outputs.build_date }}
cache-from: type=registry,ref=${{ steps.vars.outputs.image }}:buildcache
cache-to: type=registry,ref=${{ steps.vars.outputs.image }}:buildcache,mode=max
sbom: true
provenance: mode=max

8
.gitignore поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,8 @@
/bin/
/dist/
/rsmon-worker
/.env
*.log
*.db
*.db-shm
*.db-wal

37
Dockerfile Обычный файл
Просмотреть файл

@@ -0,0 +1,37 @@
FROM reg.rsxx.ru/library/golang:1-trixie AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download && go mod verify
COPY . .
ARG VERSION=dev
ARG COMMIT=unknown
ARG BUILD_DATE=unknown
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.buildDate=${BUILD_DATE}" \
-o /out/rsmon-worker ./cmd/rsmon-worker
FROM reg.rsxx.ru/library/debian:13-slim
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates chromium tzdata \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --gid 10001 rsmon-worker \
&& useradd --uid 10001 --gid rsmon-worker --home-dir /var/lib/rsmon-worker --create-home --shell /usr/sbin/nologin rsmon-worker
COPY --from=builder /out/rsmon-worker /usr/local/bin/rsmon-worker
ENV HOME=/var/lib/rsmon-worker \
RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp
VOLUME ["/var/lib/rsmon-worker"]
EXPOSE 27401 37401
USER rsmon-worker
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
CMD ["/usr/local/bin/rsmon-worker", "health"]
ENTRYPOINT ["/usr/local/bin/rsmon-worker"]

62
LICENSE Обычный файл
Просмотреть файл

@@ -0,0 +1,62 @@
RSMon Worker Source-Available License 1.0
Copyright (c) 2026 RSMon. All rights reserved.
This software is not open source. No rights are granted except those expressly
stated in this license.
1. Permitted use
Subject to this license, RSMon grants you a limited, non-exclusive,
non-transferable, revocable license to:
* inspect and evaluate the source code;
* use the source code for private research and evaluation; and
* modify and build the software solely to operate an RSMon worker connected to
the monitoring service hosted at rsmon.ru.
2. Restrictions
You may not, without prior written permission from RSMon:
* use the software with any service other than rsmon.ru;
* use the software to provide a competing or commercial monitoring service;
* redistribute, publish, sell, sublicense, lease, or otherwise make the source
code, modified source code, binaries, or container images available to any
third party;
* remove or alter copyright, license, attribution, or proprietary notices;
* use RSMon names, logos, or trademarks except to identify compatibility with
rsmon.ru; or
* use the software or source code for any purpose not expressly permitted by
section 1.
3. Modifications
Modifications and derivative works are subject to this license. You must keep
this license and all copyright notices with every permitted copy. RSMon is not
obligated to support, accept, or maintain modifications.
4. Ownership
RSMon and its licensors retain all right, title, and interest in the software,
including all intellectual-property rights. No patent, trademark, or other
license is granted by implication or estoppel.
5. Termination
This license terminates automatically if you breach any term. Upon termination,
you must stop using the software and delete all copies in your possession or
control.
6. Disclaimer
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE, TITLE, AND NON-INFRINGEMENT. TO THE MAXIMUM EXTENT PERMITTED BY LAW,
RSMON WILL NOT BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL,
OR PUNITIVE DAMAGES, OR FOR LOSS OF DATA, PROFITS, OR BUSINESS, ARISING FROM OR
RELATED TO THE SOFTWARE OR THIS LICENSE.
7. Additional permission
For permissions beyond this license, contact RSMon through https://rsmon.ru.

43
Makefile Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
BINARY := bin/rsmon-worker
VERSION ?= dev
COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || printf unknown)
BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE)
.PHONY: build test check image clean
build:
mkdir -p bin
CGO_ENABLED=0 go build -trimpath -ldflags="$(LDFLAGS)" -o $(BINARY) ./cmd/rsmon-worker
test:
RSMON_ENV=test CWD=$(CURDIR) go test \
./internal/distworker \
./internal/webapp \
./internal/workercluster \
./internal/wire \
./internal/checkexec \
./checks/calls \
./checks/cbssl \
./checks/cdns \
./checks/cftp \
./checks/chttp \
./checks/cping \
./checks/cssh \
./checks/cssl \
./checks/ctcp \
./checks/cudp \
./checks/cwhois \
./checks/llmhttp
check:
go mod tidy
git diff --exit-code -- go.mod go.sum
$(MAKE) test
$(MAKE) build
image:
docker build --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) --build-arg BUILD_DATE=$(BUILD_DATE) -t rsmon-worker:local .
clean:
rm -rf bin dist

6
NOTICE Обычный файл
Просмотреть файл

@@ -0,0 +1,6 @@
RSMon Worker
Copyright (c) 2026 RSMon. All rights reserved.
This is source-available software, not open-source software. Use is restricted
to building and operating workers for rsmon.ru and to private research or
evaluation under the terms in LICENSE.

127
README.md Обычный файл
Просмотреть файл

@@ -0,0 +1,127 @@
# RSMon Worker
Standalone distributed monitoring worker for [rsmon.ru](https://rsmon.ru). It
connects to the RSMon control plane over WebSocket, executes checks locally,
delivers delegated notifications, and reports results back to the service.
This repository is **source available, not open source**. Building and running
the worker with rsmon.ru and private research/evaluation are permitted. See
[LICENSE](LICENSE) for the complete terms.
## Requirements
- A worker token created in the rsmon.ru worker settings.
- Outbound HTTPS/WebSocket access to rsmon.ru.
- Chromium for browser-backed HTTP checks when running the binary directly.
- `CAP_NET_RAW` or an unprivileged ICMP configuration for ping checks.
## Build
Go 1.26 or newer is required.
```bash
make build
./bin/rsmon-worker --version
```
The binary reads `.env` from its working directory when present. The minimum
configuration is `RSMON_URL`, `RSMON_TOKEN`, `WORKER_LOGIN`, and
`WORKER_PASSWORD`.
## Docker Compose
```bash
cp .env.example .env
# Edit .env and set the worker token and operator-console password.
docker compose up -d
docker compose logs -f worker
```
The operator console is bound to `127.0.0.1:27401` by default. Set
`WORKER_BIND_IP` only when a firewall or TLS reverse proxy protects the port.
Persistent web and cluster state is stored in the `worker-data` volume.
## Docker
```bash
docker build -t rsmon-worker:local .
docker run --rm \
--cap-add NET_RAW \
--env-file .env \
-p 127.0.0.1:27401:27401 \
-v rsmon-worker-data:/var/lib/rsmon-worker \
rsmon-worker:local
```
Published images use these tags:
- `sha-<12-character-commit>` for every push;
- `latest` for `master`;
- the exact `v*` tag for releases.
## systemd
Install host dependencies first. On Debian or Ubuntu:
```bash
sudo apt-get update
sudo apt-get install -y ca-certificates chromium libcap2-bin tzdata
```
Build and install:
```bash
make build
cp packaging/systemd/worker.env.example worker.env
# Edit worker.env.
sudo ./scripts/install-systemd.sh --env ./worker.env
```
To install an already downloaded release binary:
```bash
sudo ./scripts/install-systemd.sh --binary ./rsmon-worker --env ./worker.env
```
Operational commands:
```bash
systemctl status rsmon-worker
journalctl -u rsmon-worker -f
sudo systemctl restart rsmon-worker
```
The service runs as the dedicated `rsmon-worker` user, stores state below
`/var/lib/rsmon-worker`, reads secrets from `/etc/rsmon-worker/worker.env`, and
has only `CAP_NET_RAW` for ICMP checks.
## Configuration
| Variable | Required | Default | Purpose |
| --- | --- | --- | --- |
| `RSMON_URL` | yes | `https://rsmon.ru` for health only | Control-plane base URL. |
| `RSMON_TOKEN` | yes | none | Worker bearer token. |
| `WORKER_HOST` | no | `0.0.0.0` | Operator-console bind address. |
| `WORKER_PORT` | no | `27401` | Operator-console port. |
| `WORKER_URL` | no | none | Public URL advertised to the control plane. |
| `WORKER_LOGIN` | yes | none | Operator-console basic-auth login. |
| `WORKER_PASSWORD` | yes | none | Operator-console basic-auth password. |
| `RSMON_WEBAPP_DATA_DIR` | no | user data directory | SQLite and local UI state. |
| `WORKER_CLUSTER_ENABLED` | no | `false` | Enable the optional Raft cluster. |
| `WORKER_CLUSTER_ID` | with cluster | none | Unique Raft node ID. |
| `WORKER_CLUSTER_PORT` | no | `WORKER_PORT+10000` | Raft transport port. |
| `WORKER_CLUSTER_PEERS` | no | none | Comma-separated `node@host:port` peers. |
| `WORKER_CLUSTER_DATA_DIR` | with cluster | none | Persistent Raft state directory. |
The public liveness endpoint is `GET /healthz`. `rsmon-worker health` checks the
configured control plane's `/up` endpoint and is suitable for container health
checks.
## Security
- Do not commit `.env`, worker tokens, or operator-console credentials.
- Expose the operator console only on loopback or behind authenticated TLS.
- Each worker should have its own control-plane token.
- Keep `/etc/rsmon-worker/worker.env` mode `0600`.
Report security issues privately through the contact channel at rsmon.ru.

60
app/models/access.go Обычный файл
Просмотреть файл

@@ -0,0 +1,60 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Access represents membership of a User (or ApiKey) within a tenant
// Account, optionally scoped to a Group or Monitor.
//
// A User can hold many Access rows across many Accounts — the Access
// table is the source of truth for "who can see what". Each row answers:
//
// "Does user U have permission P on scope S of account A?"
//
// Where:
//
// - A = AccountID (tenant)
// - U = UserID (or ApiKeyID for service tokens)
// - P = Role ("owner" | "admin" | "manager" | "view" |
// "notify")
// - S = Kind + (GroupID | MonitorID) — defaults to account-wide when
// Kind = "account" and both ids are
// nil.
//
// One Access row may also reference the Invite that produced it via
// InviteID. The Invite is preserved after registration so the access
// history stays auditable — system-registered users and admin-added
// users have nil InviteID.
//
// See docs/plans/users-and-rbac.md for the full RBAC matrix.
type Access struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
// Kind access kind, account \ group \ monitor
Kind string `gorm:"not null;default:'account'" json:"kind"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
User *User `json:"-"`
ApiKeyID *int64 `gorm:"type:bigint REFERENCES api_keys(id)" json:"-"` //nolint:revive // accepted lint exception
ApiKey *ApiKey `json:"-"` //nolint:revive // accepted lint exception
InviteID *int64 `gorm:"type:bigint REFERENCES invites(id)" json:"-"`
Invite *Invite `json:"-"`
GroupID *int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty"`
MonitorID *int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Role string `json:"role"`
// SeatType is additive to Role: role remains the authorization decision,
// while seat type is the billing entitlement.
SeatType string `gorm:"size:16;not null;default:'login'" json:"seat_type"`
Status string `gorm:"size:16;not null;default:'active'" json:"status"`
NotifyOnly bool `gorm:"not null;default:false" json:"notify_only"`
SeatAddonID *int64 `json:"seat_addon_id,omitempty"`
concerns.Timestamped `json:"-"`
Audited
}

117
app/models/account.go Обычный файл
Просмотреть файл

@@ -0,0 +1,117 @@
package models
import (
"time"
"github.com/pkg/errors"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Account represents a tenant — an isolated billing/permissions boundary
// that groups users, contacts, groups, monitors and notifications.
//
// Access to an Account is granted via the Access join table (see Access).
// `Role` on Account is a gorm:"-":all" virtual column populated by the
// controllers for the current session — it is the role the calling user
// holds on THIS account, not a property of the account itself.
type Account struct {
concerns.Model
Name string `json:"name"`
Accesses []Access `json:"-"`
Contacts []Contact `json:"-"`
Groups []Group `json:"-"`
Notifications []Notification `json:"-"`
PlanID *int64 `gorm:"type:bigint REFERENCES plans(id)" json:"-"`
Plan *Plan `json:"plan"`
// Diagnostic overrides are available only to plans that include confirmations.
// Nil keeps the catalog value; bounds are enforced by DiagnosticSettings.
ConfirmTimeoutSec *int `json:"confirm_timeout_sec,omitempty"`
HealthWindowSec *int `json:"health_window_sec,omitempty"`
HealthRateThreshold *float64 `json:"health_rate_threshold,omitempty"`
HealthMinAttempts *int `json:"health_min_attempts,omitempty"`
Role string `gorm:"-:all" json:"role"`
Timezone string `json:"timezone"`
Language string `gorm:"default:'ru'" json:"language"`
Disabled bool `gorm:"not null;default:false" json:"disabled"`
Blocked bool `gorm:"not null;default:false" json:"blocked"`
PaidUntil *time.Time `json:"paid_until"`
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
Deleted bool `gorm:"not null;default:false"`
concerns.Timestamped
Audited
}
// Users provides functionality.
func (a Account) Users() []User { //nolint:gocritic // hugeParam: accepted for interface compatibility
users := make([]User, 0)
err := DB().Where("id IN (SELECT user_id FROM accesses WHERE account_id = ?)", a.ID).Find(&users).Error
if err != nil {
panic(err)
}
return users
}
// CreateAccountForUser provides functionality.
func CreateAccountForUser(name string, u *User) (*Account, error) {
trialPlan := Plan{}
if err := DB().Where("code = ? AND archived = FALSE", "team").First(&trialPlan).Error; err != nil {
return nil, errors.Wrap(err, "failed to find trial plan")
}
trialEndsAt := time.Now().UTC().AddDate(0, 0, 14)
account := Account{PlanID: &trialPlan.ID, TrialEndsAt: &trialEndsAt}
if name != "" {
account.Name = name
}
err := DB().Transaction(func(tx *gorm.DB) error {
if err := tx.Create(&account).Error; err != nil {
return errors.Wrap(err, "failed to create account")
}
if err := tx.Create(&Subscription{
AccountID: account.ID, PlanID: trialPlan.ID, Provider: "manual", Status: SubscriptionStatusTrialing,
BillingCycle: "monthly", Currency: trialPlan.Currency, AmountMinor: trialPlan.PriceMonthlyMinor, CurrentPeriodEnd: &trialEndsAt, TrialEndsAt: &trialEndsAt,
}).Error; err != nil {
return errors.Wrap(err, "failed to create subscription")
}
var subscription Subscription
if err := tx.Where("account_id = ?", account.ID).First(&subscription).Error; err != nil {
return errors.Wrap(err, "failed to load trial subscription")
}
if err := tx.Create(&SubscriptionEvent{SubscriptionID: subscription.ID, AccountID: account.ID, Provider: "manual", Kind: "trial_started", ToPlanID: &trialPlan.ID, ActorUserID: &u.ID, CreatedAt: time.Now().UTC()}).Error; err != nil {
return errors.Wrap(err, "failed to record trial")
}
access := Access{AccountID: account.ID, UserID: &u.ID, Role: "owner", SeatType: "admin"}
if err := tx.Create(&access).Error; err != nil {
return errors.Wrap(err, "failed to create access")
}
group := Group{AccountID: account.ID, Name: "Основные"}
if err := tx.Create(&group).Error; err != nil {
return errors.Wrap(err, "failed to create group")
}
notification := Notification{AccountID: account.ID, Name: "Основные", Enabled: true}
if err := tx.Create(&notification).Error; err != nil {
return errors.Wrap(err, "failed to create notification")
}
if u.Email != nil {
contact := Contact{AccountID: &account.ID, UserID: &u.ID, Kind: "email", Value: *u.Email}
if err := tx.Create(&contact).Error; err != nil {
return errors.Wrap(err, "failed to create contact")
}
if err := tx.Model(&notification).Association("Contacts").Append(&contact); err != nil {
return errors.Wrap(err, "failed to add contact to notification")
}
}
if err := tx.Model(&notification).Association("Groups").Append(&group); err != nil {
return errors.Wrap(err, "failed to add group to notification")
}
return nil
})
if err != nil {
return nil, err
}
return &account, nil
}

283
app/models/account_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,283 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestCreateAccountForUserStartsTeamTrial(t *testing.T) {
models.Drop()
models.Migrate()
email := "team-trial@example.test"
user := models.User{Name: "Trial User", Email: &email, Enabled: true, Confirmed: true}
require.NoError(t, models.DB().Create(&user).Error)
account, err := models.CreateAccountForUser("Trial Account", &user)
require.NoError(t, err)
require.NotNil(t, account.PlanID)
require.NotNil(t, account.TrialEndsAt)
var plan models.Plan
require.NoError(t, models.DB().First(&plan, *account.PlanID).Error)
assert.Equal(t, "team", plan.Code)
var subscription models.Subscription
require.NoError(t, models.DB().Where("account_id = ?", account.ID).First(&subscription).Error)
assert.Equal(t, models.SubscriptionStatusTrialing, subscription.Status)
assert.Equal(t, plan.ID, subscription.PlanID)
require.NotNil(t, subscription.TrialEndsAt)
assert.WithinDuration(t, *account.TrialEndsAt, *subscription.TrialEndsAt, time.Millisecond)
}
// TestAccountModel tests basic Account model functionality
func TestAccountModel(t *testing.T) {
// Test Account structure
account := models.Account{
Name: "Test Account",
Timezone: "UTC",
Language: "en",
Deleted: false,
}
assert.Equal(t, "Test Account", account.Name)
assert.Equal(t, "UTC", account.Timezone)
assert.Equal(t, "en", account.Language)
assert.False(t, account.Deleted)
}
// TestAccountDisplayName tests User.DisplayName method
func TestUserDisplayName(t *testing.T) {
tests := []struct {
name string
user models.User
expected string
}{
{
name: "User with email",
user: models.User{
Name: "John Doe",
Email: stringPtr("john@example.com"),
},
expected: "John Doe john@example.com",
},
{
name: "User without email",
user: models.User{
Name: "Jane Doe",
Email: nil,
},
expected: "Jane Doe",
},
{
name: "User with empty name and email",
user: models.User{
Name: "",
Email: stringPtr("test@example.com"),
},
expected: " test@example.com",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.user.DisplayName()
assert.Equal(t, tt.expected, result)
})
}
}
// TestUserGravatar tests User.Gravatar method
func TestUserGravatar(t *testing.T) {
tests := []struct {
name string
user models.User
size int
expected string
}{
{
name: "User with email",
user: models.User{
Email: stringPtr("test@example.com"),
},
size: 32,
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=32&d=blank",
},
{
name: "User without email",
user: models.User{Email: nil},
size: 32,
expected: "",
},
{
name: "Different size",
user: models.User{
Email: stringPtr("test@example.com"),
},
size: 64,
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=64&d=blank",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.user.Gravatar(tt.size)
assert.Equal(t, tt.expected, result)
})
}
}
// TestUserAsJSON tests User.AsJSON method
func TestUserAsJSON(t *testing.T) {
email := "test@example.com"
user := models.User{
ID: 123,
Email: &email,
Name: "Test User",
}
result := user.AsJSON()
assert.NotNil(t, result)
assert.Equal(t, int64(123), result["id"])
assert.Equal(t, &email, result["email"])
assert.Contains(t, result["avatar"], "gravatar.com")
}
// TestAccessModel tests Access model structure
func TestAccessModel(t *testing.T) {
access := models.Access{
AccountID: 1,
Kind: "account",
Role: "owner",
}
assert.Equal(t, int64(1), access.AccountID)
assert.Equal(t, "account", access.Kind)
assert.Equal(t, "owner", access.Role)
}
// TestGroupModel tests Group model structure
func TestGroupModel(t *testing.T) {
group := models.Group{
AccountID: 1,
Name: "Test Group",
MonitorsCount: 5,
}
assert.Equal(t, int64(1), group.AccountID)
assert.Equal(t, "Test Group", group.Name)
assert.Equal(t, 5, group.MonitorsCount)
}
// TestGroupIdsForAccountId tests GroupIdsForAccountId function
func TestGroupIdsForAccountId(t *testing.T) {
// This test would require a database connection
// For now, we test that it doesn't panic with invalid input
t.Run("handles zero account id", func(t *testing.T) {
// Note: This will panic without DB connection, which is expected behavior
// In a real test, we'd set up a test database
})
}
// TestAccountTableDrivenTests demonstrates table-driven testing pattern
func TestAccountValidationTableDriven(t *testing.T) {
tests := []struct {
name string
account models.Account
wantErr bool
}{
{
name: "Valid account",
account: models.Account{
Name: "Valid Account",
Timezone: "UTC",
Language: "en",
},
wantErr: false,
},
{
name: "Account with empty name",
account: models.Account{
Name: "",
Timezone: "UTC",
Language: "en",
},
wantErr: true, // Name should be required
},
{
name: "Account with invalid timezone",
account: models.Account{
Name: "Test Account",
Timezone: "Invalid/Timezone",
Language: "en",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Validation logic would go here
// For now, we just verify the test structure
assert.NotNil(t, tt.account)
})
}
}
// Helper function to create string pointer
func stringPtr(s string) *string {
return &s
}
// BenchmarkUserDisplayName benchmarks the DisplayName method
func BenchmarkUserDisplayName(b *testing.B) {
user := models.User{
Name: "Test User",
Email: stringPtr("test@example.com"),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = user.DisplayName()
}
}
// TestAccountConcurrentOperations tests concurrent access to account data
func TestAccountConcurrentOperations(t *testing.T) {
account := models.Account{
Name: "Concurrent Test",
Timezone: "UTC",
}
done := make(chan bool)
// Simulate concurrent reads
for i := 0; i < 10; i++ {
go func() {
_ = account.Name
done <- true
}()
}
// Wait for all goroutines
for i := 0; i < 10; i++ {
<-done
}
}
// ExampleAccountUsage provides an example of how to use Account model
func ExampleAccount() {
account := models.Account{
Name: "Example Account",
Timezone: "America/New_York",
Language: "en",
}
_ = account.Name
// Output:
}

38
app/models/api_key.go Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
package models
import (
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// ApiKey represents an API authentication key. //nolint:revive // accepted lint exception
type ApiKey struct { //nolint:revive // accepted lint exception
concerns.Model
Name string `json:"name" gorm:"not null"`
AccountID int64 `json:"account_id,omitempty"`
Account User `json:"-"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
User *User `json:"-"`
Accesses []Access `json:"accesses" gorm:"foreignkey:api_key_id"`
concerns.HasToken
concerns.Timestamped
Audited
}
func (n *ApiKey) BeforeCreate(tx *gorm.DB) error { //nolint:revive // accepted lint exception
n.SetToken()
return nil
}
// FillAccesses provides functionality.
func (n *ApiKey) FillAccesses() {
for k, a := range n.Accesses { //nolint:gocritic // range copy is acceptable here
if a.ID <= 0 {
n.Accesses[k].ID = 0
}
n.Accesses[k].AccountID = n.AccountID
}
}

63
app/models/audited.go Обычный файл
Просмотреть файл

@@ -0,0 +1,63 @@
// Package models provides GORM models and business logic.
// Audited models inspired by https://github.com/qor/audited
package models
import (
"gorm.io/gorm"
)
// AuditedCurrentUserKey is the GORM Set key for the current user.
const AuditedCurrentUserKey = "audited:current_user"
// Audited tracks creator and updater IDs.
type Audited struct {
CreatorID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
Creator *User `json:"-"`
UpdaterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
Updater *User `json:"-"`
}
func getCurrentUser(scope *gorm.DB) (int64, bool) {
var user interface{}
var hasUser bool
user, hasUser = scope.Get(AuditedCurrentUserKey)
// spew.Dump(user, hasUser)
if hasUser {
return user.(*User).ID, true
}
return 0, false
}
func assignCreatedBy(tx *gorm.DB) {
name := "CreatorID"
if field := tx.Statement.Schema.LookUpField(name); field != nil {
if user, ok := getCurrentUser(tx); ok {
tx.Statement.SetColumn(name, user)
}
}
}
func assignUpdatedBy(tx *gorm.DB) {
name := "UpdaterID"
if field := tx.Statement.Schema.LookUpField(name); field != nil {
if user, ok := getCurrentUser(tx); ok {
tx.Statement.SetColumn("UpdaterID", user, true)
}
}
}
// RegisterCallbacks register callback into GORM DB
func RegisterCallbacks(db *gorm.DB) {
callback := db.Callback()
if callback.Create().Get("audited:assign_created_by") == nil {
_ = callback.Create().After("gorm:before_create").Register("audited:assign_created_by", assignCreatedBy)
}
if callback.Update().Get("audited:assign_updated_by") == nil {
_ = callback.Update().After("gorm:before_update").Register("audited:assign_updated_by", assignUpdatedBy)
}
}

32
app/models/authidentity/auth_identity.go Обычный файл
Просмотреть файл

@@ -0,0 +1,32 @@
// Package authidentity provides the AuthIdentity and Basic types for QOR-style
// authentication identity management. Vendored from github.com/glebtv/auth/auth_identity
// to eliminate the rsgit.ru/rs/sessionmanager transitive dependency.
package authidentity
import "time"
// AuthIdentity combines Basic provider info with SignLogs for a full identity record.
type AuthIdentity struct {
Basic
SignLogs
}
// TableName returns the database table name for AuthIdentity.
func (AuthIdentity) TableName() string {
return "identities"
}
// Basic represents the core identity fields (provider, UID, encrypted password).
type Basic struct {
ID int64 `gorm:"primary_key" json:"id"`
Provider string
UID string `gorm:"column:uid"`
EncryptedPassword string
UserID *int64
ConfirmedAt *time.Time
}
// TableName returns the database table name for Basic.
func (Basic) TableName() string {
return "identities"
}

50
app/models/authidentity/sign_logs.go Обычный файл
Просмотреть файл

@@ -0,0 +1,50 @@
package authidentity
import (
"database/sql/driver"
"encoding/json"
"errors"
"time"
)
// SignLogs holds login history (log entries and sign-in count).
type SignLogs struct {
Log string `sql:"-"`
SignInCount uint
Logs []SignLog
}
// Scan implements sql.Scanner for deserializing SignLogs from JSON.
func (signLogs *SignLogs) Scan(data interface{}) (err error) {
switch values := data.(type) {
case []byte:
if len(values) != 0 {
return json.Unmarshal(values, signLogs)
}
case string:
return signLogs.Scan([]byte(values))
case []string:
for _, str := range values {
if err := signLogs.Scan(str); err != nil {
return err
}
}
default:
err = errors.New("unsupported driver -> Scan pair for SignLogs")
}
return
}
// Value implements driver.Valuer for serializing SignLogs to JSON.
func (signLogs SignLogs) Value() (driver.Value, error) {
results, err := json.Marshal(signLogs)
return string(results), err
}
// SignLog represents a single login event entry.
type SignLog struct {
UserAgent string
At *time.Time
IP string
}

31
app/models/bits.go Обычный файл
Просмотреть файл

@@ -0,0 +1,31 @@
package models
import "time"
// BeginningOfDay provides functionality.
func BeginningOfDay(t time.Time) time.Time {
year, month, day := t.Date()
return time.Date(year, month, day, 0, 0, 0, 0, t.Location())
}
// SetBit provides functionality.
// https://stackoverflow.com/questions/23192262/how-would-you-set-and-clear-a-single-bit-in-go
// Sets the bit at pos in the integer n.
func SetBit(n int, pos uint) int {
n |= (1 << pos)
return n
}
// ClearBit provides functionality.
// Clears the bit at pos in n.
func ClearBit(n int, pos uint) int {
mask := ^(1 << pos)
n &= mask
return n
}
// HasBit provides functionality.
func HasBit(n int, pos uint) bool {
val := n & (1 << pos)
return (val > 0)
}

171
app/models/check.go Обычный файл
Просмотреть файл

@@ -0,0 +1,171 @@
package models
import (
"encoding/json"
"strings"
"time"
"unicode"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
)
// Check provides functionality.
type Check struct {
ID int64 `gorm:"primarykey" json:"id"`
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
MonitorID int64 `gorm:"index;type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Monitor *Monitor `json:"-"`
Name *string `json:"name"`
Kind string `json:"kind"`
Interval int `json:"interval" validate:"required,gte=60"`
// URL to monitor
URL *string `json:"url,omitempty"`
// Other settings of the check
Settings datatypes.JSON `gorm:"not null;" json:"settings"`
State string `gorm:"not null;default:'UNK'" json:"state"`
LastStart *time.Time `json:"last_start"`
LastEnd *time.Time `json:"last_end"`
LastOk *time.Time `json:"last_ok"`
LastFail *time.Time `json:"last_fail"`
WasUp *time.Time `json:"was_up"`
Fails int `json:"fails"`
Expires *time.Time `json:"expires"`
Error *string `json:"error"`
Warnings pq.StringArray `gorm:"type:varchar(255)[]" json:"warnings"`
Infos pq.StringArray `gorm:"type:varchar(255)[]" json:"infos"`
// RequireQuorum enables multi-region result aggregation (Phase 3 of
// docs/todo.md): when >1 the check's State is NOT written directly by
// ApplyRemoteCheckResult — instead CheckRegionResult rows accumulate
// until app/models/check_aggregator.go decides OK/ERR/DEGRADED.
// Default 1 keeps the legacy single-region behavior unchanged.
RequireQuorum int `gorm:"not null;default:1" json:"require_quorum"`
// AggregationWindowSeconds is how long the aggregator waits for
// regional CheckRegionResult rows before deciding the check's State.
// Stored as int seconds (matching the existing GORM style — no
// time.Duration columns) and exposed via AggregationWindow(). Default
// 5s; ignored when RequireQuorum <= 1.
AggregationWindowSeconds int `gorm:"not null;default:5" json:"aggregation_window_seconds"`
IsNew bool `gorm:"-:all" sql:"-" json:"is_new,omitempty"`
Deleted bool `gorm:"-:all" sql:"-" json:"deleted,omitempty"`
Events []Event `json:"-" gorm:"many2many:event_checks;"`
Audited
}
// ExpScope provides functionality.
func ExpScope(q *gorm.DB) *gorm.DB {
return q.Where("kind IN ('whois', 'ssl')").
Preload("Monitor").
Preload("Monitor.Group").
Preload("Monitor.Group.Notifications").
Preload("Monitor.Group.Notifications.Contacts").
Where("expires < ?", time.Now().Add(time.Hour*7*24))
}
// IntervalOK provides functionality.
func (c *Check) IntervalOK() bool {
if c.Kind == kindRKN {
return true
}
if c.Kind == kindWhois {
return c.Interval >= 43200
}
return c.Interval >= 30
}
// GetLabel provides functionality.
func (c *Check) GetLabel() string {
if c.Name != nil {
return *c.Name
}
if c.URL != nil {
return *c.URL
}
return c.Kind
}
// KindLabel provides functionality.
func (c *Check) KindLabel() string {
if c.Kind == kindWhois {
return "регистрация домена"
}
if c.Kind == kindSSL {
return "SSL сертификат"
}
return c.Kind
}
// GetSettings provides functionality.
func (c *Check) GetSettings() CheckSettings {
d := CheckSettings{}
err := json.Unmarshal(c.Settings, &d)
if err != nil {
panic(err)
}
return d
}
// ValidateSettings provides functionality.
func (c *Check) ValidateSettings() error {
if len(c.Settings) == 0 {
c.Settings = []byte("{}")
}
return nil
}
// GetURL provides functionality.
func (c *Check) GetURL() (string, error) {
// return c.GetSettings()["url"].(string)
if c.URL != nil {
return *c.URL, nil
}
return "http://" + c.Monitor.Host, nil
}
// MetricName provides functionality.
func (c *Check) MetricName() string {
sanitized := strings.Map(func(r rune) rune {
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == ':' {
return r
}
return '_'
}, c.Kind)
return "c" + sanitized
}
// QuorumEnabled reports whether this check should be aggregated by
// app/models/check_aggregator.go. When false (RequireQuorum <= 1),
// ApplyRemoteCheckResult keeps the legacy direct State update path.
func (c *Check) QuorumEnabled() bool {
return c.RequireQuorum > 1
}
// AggregationWindow returns AggregationWindowSeconds as a time.Duration.
// Defaults to 5s when the underlying int is zero/negative, mirroring the
// GORM column default; callers can rely on a strictly positive value.
func (c *Check) AggregationWindow() time.Duration {
if c.AggregationWindowSeconds <= 0 {
return 5 * time.Second
}
return time.Duration(c.AggregationWindowSeconds) * time.Second
}

336
app/models/check_aggregator.go Обычный файл
Просмотреть файл

@@ -0,0 +1,336 @@
package models
import (
"context"
"log"
"time"
"gorm.io/gorm"
)
// Phase 3 of docs/todo.md — result aggregation for multi-region checks.
//
// When a check has RequireQuorum > 1, ApplyRemoteCheckResult does not
// write Check.State directly. Instead it appends a CheckRegionResult row
// and leaves AggregatedAt NULL. This file owns the background goroutine
// that walks those pending rows once their aggregation window has
// elapsed, decides OK/ERR/DEGRADED per the documented rule, writes the
// aggregate state onto Check, stamps AggregatedAt on the contributing
// rows, and triggers Monitor.UpdateStatusFromChecks so the monitor's own
// status follows.
//
// Aggregation rule (see docs/todo.md Phase 3 + checkSeverityRank in
// monitor.go for the corresponding severity order):
//
// - Aggregate only rows whose created_at is older than
// NOW() - Check.AggregationWindowSeconds. This is the "watermark"
// pattern: a row is eligible only when no fresher regional result
// could still arrive and tip the vote. The window is per-check so
// noisy checks can use a longer wait than fast ones.
// - If zero eligible rows exist for a check, leave Check.State
// untouched (the special case called out in the spec).
// - Otherwise count OK vs not-OK among the eligible rows:
// OK >= RequireQuorum → Check.State = OK
// not-OK >= RequireQuorum → Check.State = ERR
// neither side reaches quorum → Check.State = DEGRADED
// - Stamp AggregatedAt = NOW() on every contributing row so the next
// tick skips them. One transaction per check; per-row failures do
// not poison other checks.
// AggregatorTickInterval is the default cadence of StartCheckAggregator
// when the caller passes interval <= 0. Mirrors the 30s default used by
// the other reapers in this package so the three reapers all tick on
// the same wall clock cadence — easier to grep, easier to reason about
// in incident timelines.
const AggregatorTickInterval = 30 * time.Second
// EnsureCheckAggregatorIndexes adds the partial indexes the aggregator
// relies on. AutoMigrate creates AggregatedAt as a regular btree column,
// but the per-tick SELECT filters on `aggregated_at IS NULL` over what
// grows to be a busy table; a partial index keeps the working set
// tiny. Idempotent so it is safe to call from Migrate() and from tests.
func EnsureCheckAggregatorIndexes() error {
return DB().Exec(`
CREATE INDEX IF NOT EXISTS check_region_results_pending_idx
ON check_region_results (check_id, created_at)
WHERE aggregated_at IS NULL
`).Error
}
// aggregateCheckState holds the per-check aggregation inputs we need to
// keep the rule readable. Rows is the set of CheckRegionResult rows
// eligible for the current decision; quorum is Check.RequireQuorum.
type aggregateCheckState struct {
CheckID int64
Quorum int
Rows []CheckRegionResult
}
// decideAggregateState encodes the OK/ERR/DEGRADED rule described in
// the package doc. Pure function — no DB, no time — so it is trivially
// unit-testable from the test file.
func decideAggregateState(in aggregateCheckState) (string, bool) {
if len(in.Rows) == 0 || in.Quorum <= 1 {
// Zero eligible rows in the window OR a misconfigured check
// (QuorumEnabled false). Caller must leave Check.State alone
// in both cases.
return "", false
}
okCount := 0
badCount := 0
for i := range in.Rows {
if in.Rows[i].State == stateOK {
okCount++
} else {
badCount++
}
}
switch {
case okCount >= in.Quorum:
return stateOK, true
case badCount >= in.Quorum:
return stateERR, true
default:
return stateDegraded, true
}
}
// CheckAggregatorTick performs one pass of the aggregator. It is the
// per-tick body StartCheckAggregator calls. Exported so the test suite
// can call it directly without spinning up the goroutine; production
// always goes through StartCheckAggregator.
//
// The returned (aggregated, err) tuple lets the caller log a metric:
// aggregated counts how many Check rows had their State written this
// tick. The function is idempotent — a second call with no new
// unaggregated rows is a no-op that returns (0, nil).
func CheckAggregatorTick() (aggregated int, err error) {
// Step 1: collect candidate check IDs. The JOIN to checks is needed
// to read each check's window length and to filter on
// require_quorum > 1 (so we never aggregate the legacy path).
rows, err := DB().Raw(`
SELECT DISTINCT crr.check_id
FROM check_region_results crr
JOIN checks c ON c.id = crr.check_id
WHERE crr.aggregated_at IS NULL
AND c.require_quorum > 1
AND crr.created_at < NOW() - make_interval(secs => GREATEST(c.aggregation_window_seconds, 1))
ORDER BY crr.check_id
`).Rows()
if err != nil {
return 0, err
}
defer func() { _ = rows.Close() }()
var checkIDs []int64
for rows.Next() {
var id int64
if scanErr := rows.Scan(&id); scanErr != nil {
return 0, scanErr
}
checkIDs = append(checkIDs, id)
}
if scanErr := rows.Err(); scanErr != nil {
return 0, scanErr
}
if len(checkIDs) == 0 {
return 0, nil
}
for _, checkID := range checkIDs {
n, err := aggregateOneCheck(checkID)
if err != nil {
// Log and continue: one bad check must not stop the loop.
log.Printf("check_aggregator: check_id=%d error: %v", checkID, err)
continue
}
aggregated += n
}
return aggregated, nil
}
// aggregateOneCheck runs the aggregation logic for a single check inside
// a transaction. The transaction holds a FOR UPDATE row lock on the
// check so concurrent aggregator instances (multiple web processes) can
// not race on the same check — the second one waits for the first to
// commit, then sees AggregatedAt IS NOT NULL on every row and the
// candidate SELECT below returns an empty set.
func aggregateOneCheck(checkID int64) (int, error) {
tx := DB().Begin()
if tx.Error != nil {
return 0, tx.Error
}
defer func() {
if r := recover(); r != nil {
tx.Rollback()
panic(r)
}
}()
var check Check
if err := tx.Clauses(SkipLockedClause).First(&check, checkID).Error; err != nil {
tx.Rollback()
if err == gorm.ErrRecordNotFound {
// Check was deleted between candidate SELECT and lock; not
// an error, just nothing to do.
return 0, nil
}
return 0, err
}
// Defensive: only aggregate quorum-enabled checks. The candidate
// SELECT already filters on this, but a stale row that flipped off
// quorum between calls must not be re-aggregated here.
if !check.QuorumEnabled() {
tx.Rollback()
return 0, nil
}
var results []CheckRegionResult
if err := tx.
Where("check_id = ? AND aggregated_at IS NULL", checkID).
Order("created_at ASC").
Find(&results).Error; err != nil {
tx.Rollback()
return 0, err
}
decision, ok := decideAggregateState(aggregateCheckState{
CheckID: checkID,
Quorum: check.RequireQuorum,
Rows: results,
})
if !ok {
// Zero eligible rows — leave Check.State alone. There is also
// nothing to stamp, so just rollback and move on.
tx.Rollback()
return 0, nil
}
now := time.Now()
// Pull the latest error string from the contributing rows so the
// monitor event / notifier pipeline has something to show. Prefer
// the most recent ERR row's message; fall back to the most recent
// any-row message. UNK / empty stays NULL.
var lastError *string
for i := len(results) - 1; i >= 0; i-- {
if results[i].Error != nil && *results[i].Error != "" {
lastError = results[i].Error
break
}
}
upd := map[string]interface{}{
colState: decision,
colLastEnd: now,
}
if decision == stateOK {
// OK resets error — mirrors the legacy ApplyRemoteCheckResult
// path that sets `error = gorm.Expr("NULL")` when state==OK.
upd["error"] = gorm.Expr("NULL")
upd["last_ok"] = now
upd["fails"] = 0
upd["was_up"] = now
} else {
// Non-OK: bump the fail counter and only overwrite the error
// when one of the contributing rows actually carries a
// message. If none do, leave whatever was there before —
// mirrors the legacy `if report.Error != nil` branch.
upd["last_fail"] = now
upd["fails"] = gorm.Expr("fails + 1")
if lastError != nil {
upd["error"] = *lastError
}
}
if err := tx.Model(&Check{}).Where("id = ?", checkID).UpdateColumns(upd).Error; err != nil {
tx.Rollback()
return 0, err
}
if err := tx.Model(&CheckRegionResult{}).
Where("check_id = ? AND aggregated_at IS NULL", checkID).
UpdateColumns(map[string]interface{}{
"aggregated_at": now,
}).Error; err != nil {
tx.Rollback()
return 0, err
}
if err := tx.Commit().Error; err != nil {
return 0, err
}
// Mirror ApplyRemoteCheckResult: propagate the aggregate decision
// up to the monitor. We do this AFTER commit so a rollback does
// not leave the monitor in a state whose corresponding check is
// still pre-aggregate. The goroutine keeps the failure path of
// UpdateStatusFromChecks isolated from the aggregator's hot loop.
if check.MonitorID != 0 {
var mon Monitor
if err := DB().First(&mon, check.MonitorID).Error; err == nil {
go mon.UpdateStatusFromChecks()
} else {
log.Printf("check_aggregator: monitor lookup failed for check_id=%d: %v", checkID, err)
}
}
return 1, nil
}
// StartCheckAggregator launches a goroutine that calls
// CheckAggregatorTick on the given interval until ctx is canceled.
// Mirrors StartTaskReaper / StartDeadWorkerReaper in this package — same
// ticker shape, same default-interval fall-back, same per-tick recover
// so a malformed row cannot crash the web process.
//
// The default interval is AggregatorTickInterval (30s); values <= 0
// fall back to the default so the helper is safe to call from any call
// site without a guard. A nil context falls back to context.Background()
// the same way StartDeadWorkerReaper does, so main.init() and tests
// can both call it without ceremony.
//
// Wire from main.init() once per process. The aggregator is cheap in
// steady state (one indexed SELECT for candidates + a per-check
// transaction over a handful of unaggregated rows). Under load it
// scales horizontally — multiple web processes can each run their own
// StartCheckAggregator goroutine because FOR UPDATE SKIP LOCKED on the
// per-check transaction guarantees at-most-one winner per check.
func StartCheckAggregator(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = AggregatorTickInterval
}
if ctx == nil {
ctx = context.Background()
}
// Best-effort index bootstrap. AutoMigrate declares AggregatedAt as
// a regular btree column; the partial index speeds up the per-tick
// candidate SELECT. Idempotent — safe to call on every boot.
if err := EnsureCheckAggregatorIndexes(); err != nil {
log.Printf("check_aggregator: ensure index: %v", err)
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("check_aggregator: panic recovered: %v", r)
}
}()
n, err := CheckAggregatorTick()
if err != nil {
log.Printf("check_aggregator: error: %v", err)
return
}
if n > 0 {
log.Printf("check_aggregator: aggregated=%d", n)
}
}()
}
}
}()
}

448
app/models/check_aggregator_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,448 @@
package models_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
// seedAggregatorWorld creates the minimum fixture the aggregator tests
// need: a plan, account, group, monitor, and an http check whose
// RequireQuorum / AggregationWindowSeconds are set per call. The check
// is created with state=UNK so the test can observe the aggregator's
// effect on Check.State directly.
//
// Returns the freshly-created monitor + check; the check is what every
// test below mutates (RequireQuorum, AggregationWindowSeconds) and then
// asserts on. Cleanup is the caller's responsibility — most tests call
// models.Drop() at the top instead.
func seedAggregatorWorld(t *testing.T, quorum, windowSeconds int) (models.Monitor, models.Check) {
t.Helper()
plan := models.Plan{Name: "agg-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "agg-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
grp := &models.Group{AccountID: acc.ID, Name: "agg"}
require.NoError(t, models.DB().Create(grp).Error)
mon := models.Monitor{
Name: stringPtrAgg("agg.test"),
Host: "agg.test",
GroupID: grp.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&mon).Error)
enTrue := true
check := models.Check{
MonitorID: mon.ID,
Kind: "http",
Interval: 60,
Enabled: &enTrue,
State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
RequireQuorum: quorum,
AggregationWindowSeconds: windowSeconds,
}
require.NoError(t, models.DB().Create(&check).Error)
return mon, check
}
func stringPtrAgg(s string) *string { return &s }
// makeReport constructs a wire.CheckResultReport with sensible defaults
// for the OK or not-OK case. Tests use this to push results through
// ApplyRemoteCheckResult exactly the way a real worker would.
func makeReport(checkID, monitorID int64, state string) wire.CheckResultReport {
return wire.CheckResultReport{
JobID: "job-" + state,
CheckID: checkID,
MonitorID: monitorID,
State: state,
}
}
// regionResultWithErr is regionResultFor plus an error message. Used
// when the test wants to verify that the aggregator forwards the row's
// error string onto Check.Error (mimics a real worker reporting
// state=ERR with a diagnostic message).
func regionResultWithErr(t *testing.T, checkID int64, region, state, errMsg string, age time.Duration) models.CheckRegionResult {
t.Helper()
row := regionResultFor(t, checkID, region, state, age)
require.NoError(t, models.DB().Model(&row).UpdateColumn("error", errMsg).Error)
return row
}
// regionResultFor inserts a single CheckRegionResult row whose
// created_at and executed_at are both backdated by `age`, so the
// aggregator's window-based watermark picks it up immediately without
// needing a real time.Sleep. Returned row has its DB-assigned ID
// populated.
func regionResultFor(t *testing.T, checkID int64, region string, state string, age time.Duration) models.CheckRegionResult {
t.Helper()
// RegionCode has a FK to regions.code, so the region must exist
// before the result row is inserted. seedRegion is idempotent.
seedRegion(t, region)
row := models.CheckRegionResult{
CheckID: checkID,
RegionCode: region,
ExecutedAt: time.Now().Add(-age),
State: state,
}
require.NoError(t, models.DB().Create(&row).Error)
// Backdate CreatedAt too — the aggregator SQL keys on
// check_region_results.created_at (see CheckAggregatorTick). GORM
// auto-sets CreatedAt on insert, so we have to UPDATE it post-hoc.
require.NoError(t, models.DB().Model(&row).UpdateColumns(map[string]interface{}{
"created_at": time.Now().Add(-age),
"updated_at": time.Now().Add(-age),
}).Error)
return row
}
// loadCheck re-reads a Check row by ID — used after the aggregator
// runs so the test asserts against the post-tick state.
func loadCheck(t *testing.T, id int64) models.Check {
t.Helper()
var c models.Check
require.NoError(t, models.DB().First(&c, id).Error)
return c
}
// countPendingResults returns how many CheckRegionResult rows for
// checkID have aggregated_at IS NULL — the working set the next
// aggregator tick would consider.
func countPendingResults(t *testing.T, checkID int64) int64 {
t.Helper()
var n int64
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("check_id = ? AND aggregated_at IS NULL", checkID).
Count(&n).Error)
return n
}
// ---------------------------------------------------------------------------
// ApplyRemoteCheckResult: regression tests for the QuorumEnabled split.
// ---------------------------------------------------------------------------
// TestApplyRemoteCheckResult_DirectWhenQuorumOne pins the legacy path:
// when RequireQuorum==1 (the default), ApplyRemoteCheckResult still
// writes Check.State synchronously, exactly the way it did before Phase
// 3. This is the regression guard for the in-process RKN scheduler
// tests in internal/rknscheduler.
func TestApplyRemoteCheckResult_DirectWhenQuorumOne(t *testing.T) {
models.Drop()
models.Migrate()
mon, check := seedAggregatorWorld(t, 1, 5)
// Region must exist because CheckRegionResult has a FK to
// regions.code (seeded by Migrate, but the test region is custom).
seedRegion(t, "ru-msk")
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "OK"),
"ru-msk",
))
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "quorum=1 must keep the direct State update")
assert.NotNil(t, got.LastEnd, "legacy path must keep stamping last_end")
// One region result inserted with aggregated_at=NULL.
assert.EqualValues(t, 1, countPendingResults(t, check.ID),
"the region result row is always inserted even on the legacy path")
}
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
// Check.State — it only inserts the CheckRegionResult row. The check
// stays at its initial UNK and the unaggregated row count grows by
// exactly 1 per call.
func TestApplyRemoteCheckResult_BuffersWhenQuorumN(t *testing.T) {
models.Drop()
models.Migrate()
mon, check := seedAggregatorWorld(t, 3, 5)
seedRegion(t, "ru-msk")
seedRegion(t, "us-east")
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "OK"), "ru-msk",
))
require.NoError(t, models.ApplyRemoteCheckResult(
makeReport(check.ID, mon.ID, "ERR"), "us-east",
))
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State,
"quorum>1 must NOT touch Check.State — the aggregator owns it")
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
"two results buffered, both with aggregated_at=NULL")
}
// ---------------------------------------------------------------------------
// CheckAggregatorTick: rule tests.
// ---------------------------------------------------------------------------
// TestAggregator_QuorumOK: with RequireQuorum=2 and two OK results
// buffered, the aggregator must decide OK and stamp AggregatedAt on
// both contributing rows.
func TestAggregator_QuorumOK(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n, "one check aggregated this tick")
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "2 OK results >= quorum=2 → state=OK")
assert.NotNil(t, got.LastOk, "OK decision must stamp last_ok")
assert.EqualValues(t, 0, got.Fails, "fails must reset on OK")
assert.EqualValues(t, 0, countPendingResults(t, check.ID),
"both contributing rows must be stamped aggregated_at")
}
// TestAggregator_QuorumFail: with RequireQuorum=2 and two ERR results
// buffered, the aggregator must decide ERR and surface the most recent
// row's error message on Check.Error.
func TestAggregator_QuorumFail(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultWithErr(t, check.ID, "ru-msk", "ERR", "connection refused", 5*time.Second)
regionResultWithErr(t, check.ID, "us-east", "ERR", "timeout", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "ERR", got.State, "2 ERR >= quorum=2 → state=ERR")
assert.NotNil(t, got.LastFail)
assert.NotNil(t, got.Error, "ERR decision must carry an error message from the rows")
assert.Contains(t, *got.Error, "timeout",
"aggregator should surface the latest row's error message")
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
}
// TestAggregator_DegradedWhenPartial: with RequireQuorum=3 and 1 OK +
// 2 ERR (mixed within window), neither side reaches the quorum of 3 so
// the aggregator must decide DEGRADED. The state must NOT silently
// become OK or ERR.
func TestAggregator_DegradedWhenPartial(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 3, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "ERR", 4*time.Second)
regionResultFor(t, check.ID, "eu-west", "ERR", 3*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "DEGRADED", got.State,
"neither OK nor ERR reaches quorum=3 → state=DEGRADED")
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
}
// TestAggregator_NotEnoughRegionsAlsoDegraded covers the single-region-
// only-delivered case: with RequireQuorum=3 and only 1 result buffered
// (and it aged past the window), the rule still says "neither side
// reached quorum" → DEGRADED. This is the documented behavior for slow
// regions that never report in time.
func TestAggregator_NotEnoughRegionsAlsoDegraded(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 3, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 1, n)
got := loadCheck(t, check.ID)
assert.Equal(t, "DEGRADED", got.State,
"single OK row vs quorum=3 → DEGRADED (below quorum on both sides)")
}
// TestAggregator_NoResultsLeavesStateAlone is the "special case" from
// the spec: when the aggregator tick finds no eligible rows for a
// check, Check.State must NOT change. Pre-set the check to OK and
// verify it stays OK.
func TestAggregator_NoResultsLeavesStateAlone(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
// Pre-set state and a previous LastEnd so we can detect any
// accidental overwrite.
prevEnd := time.Now().Add(-time.Hour)
require.NoError(t, models.DB().Model(&models.Check{}).
Where("id = ?", check.ID).
Updates(map[string]interface{}{
"state": "OK",
"last_end": prevEnd,
"last_ok": prevEnd,
}).Error)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "no eligible rows → nothing aggregated")
got := loadCheck(t, check.ID)
assert.Equal(t, "OK", got.State, "state must not change with zero eligible rows")
assert.WithinDuration(t, prevEnd, *got.LastEnd, time.Second,
"last_end must not be touched when there are no eligible rows")
}
// TestAggregator_MultipleChecksIndependent verifies that a single tick
// processes every check with pending results, not just the first one.
// Two checks, each with 2 regions, each should flip to OK after the
// tick.
func TestAggregator_MultipleChecksIndependent(t *testing.T) {
models.Drop()
models.Migrate()
_, c1 := seedAggregatorWorld(t, 2, 1)
_, c2 := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, c1.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, c1.ID, "us-east", "OK", 4*time.Second)
regionResultFor(t, c2.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, c2.ID, "eu-west", "OK", 4*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 2, n, "both checks aggregated in the same tick")
assert.Equal(t, "OK", loadCheck(t, c1.ID).State)
assert.Equal(t, "OK", loadCheck(t, c2.ID).State)
assert.EqualValues(t, 0, countPendingResults(t, c1.ID))
assert.EqualValues(t, 0, countPendingResults(t, c2.ID))
}
// TestAggregator_AlreadyAggregatedRowsSkipped pins the idempotency
// story: a second tick with no new rows must be a no-op. We pre-mark
// the rows aggregated_at=NOW() and verify the tick returns (0, nil)
// without touching Check.State.
func TestAggregator_AlreadyAggregatedRowsSkipped(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
r1 := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
r2 := regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
// Pretend a previous aggregator pass already stamped them.
now := time.Now()
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("id IN ?", []int64{r1.ID, r2.ID}).
UpdateColumns(map[string]interface{}{"aggregated_at": now}).Error)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "no candidate checks → 0 aggregated")
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State, "already-aggregated rows must not cause a re-decision")
}
// TestAggregator_IgnoresRowsInsideWindow verifies the watermark: rows
// whose CreatedAt is NEWER than (NOW() - window) are NOT eligible and
// must NOT be stamped. With AggregationWindowSeconds=10 and rows aged
// only 2s, the aggregator finds nothing to do.
func TestAggregator_IgnoresRowsInsideWindow(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 10)
regionResultFor(t, check.ID, "ru-msk", "OK", 2*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 1*time.Second)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "rows still inside window → no aggregation")
got := loadCheck(t, check.ID)
assert.Equal(t, "UNK", got.State)
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
"rows inside window stay unaggregated for the next tick")
}
// TestAggregator_SkipsChecksWithQuorumOne guards the candidate SELECT
// filter: even though CheckRegionResult rows are written for
// RequireQuorum=1 checks (via StoreCheckRegionResult), the aggregator
// must not re-decide their state because ApplyRemoteCheckResult
// already did. We simulate by inserting a region row with aggregated_at
// NULL for a quorum=1 check and verifying the tick ignores it.
func TestAggregator_SkipsChecksWithQuorumOne(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 1, 1)
r := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
require.Nil(t, r.AggregatedAt)
n, err := models.CheckAggregatorTick()
require.NoError(t, err)
assert.Equal(t, 0, n, "quorum=1 checks must be filtered out by the candidate SELECT")
// The row must stay unaggregated too — the aggregator has no
// business stamping it.
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
}
// ---------------------------------------------------------------------------
// StartCheckAggregator: ticker smoke test.
// ---------------------------------------------------------------------------
// TestStartCheckAggregator_TickerFiresOnce is the smoke test for the
// background helper: spin up the aggregator with a tight 10ms ticker
// and a cancellable context, wait for one tick to flip a seeded
// check's state, then cancel so the goroutine exits cleanly. Mirrors
// TestStartDeadWorkerReaper_TickerFiresOnce in shape.
func TestStartCheckAggregator_TickerFiresOnce(t *testing.T) {
models.Drop()
models.Migrate()
_, check := seedAggregatorWorld(t, 2, 1)
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
models.StartCheckAggregator(ctx, 10*time.Millisecond)
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
got := loadCheck(t, check.ID)
if got.State == "OK" {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("aggregator goroutine did not flip check to OK within 2s; state=%q", loadCheck(t, check.ID).State)
}
// silence unused import warnings when individual helpers are inlined by
// editors — the package-level references below keep the imports live.
var (
_ = gorm.ErrRecordNotFound
)

14
app/models/check_data.go Обычный файл
Просмотреть файл

@@ -0,0 +1,14 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/internal/influx"
)
// CheckData provides functionality.
type CheckData struct {
Uptime int `json:"uptime"`
Data []influx.InfluxData `json:"data"`
LastCheck *time.Time `json:"last_check"`
}

415
app/models/check_jobs.go Обычный файл
Просмотреть файл

@@ -0,0 +1,415 @@
package models
import (
"encoding/json"
"fmt"
"log"
"time"
"github.com/google/uuid"
"github.com/lib/pq"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/internal/influx"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
// defaultRegionCode is the historical default region seeded by Migrate()
// (see app/models/migrate.go) and used as a catch-all bucket for results
// reported without a region code. Aliased to the exported Region
// constant (DefaultRegionCode) so admin endpoints and the in-process
// job router share one source of truth.
const defaultRegionCode = DefaultRegionCode
// ChecksForWorker returns checks that need to be executed by a distributed worker.
// It uses FOR UPDATE SKIP LOCKED to prevent race conditions between concurrent workers.
// The worker specifies which check kinds it can handle via the kinds parameter.
//
// Phase 2 of docs/plans/worker-notifier-mvp.md adds regional job routing: when
// worker is non-nil, the candidate monitor set is filtered by
// applyRegionRouting so a worker only sees checks that explicitly allow its
// region. Pass nil for the legacy "no region scoping" path used by
// diagnostics/dashboard tooling.
func ChecksForWorker(worker *WorkerNode, kinds []string, limit int) []*Check {
tx := DB().Begin()
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
Where("monitors.enabled").
Where("checks.enabled AND checks.kind IN (?)", kinds)
if worker != nil {
q = applyRegionRouting(q, worker)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
}
// Flagged workers remain visible for audit/history but never receive new work.
if worker.NetworkProblemActive(time.Now()) {
tx.Rollback()
return nil
}
}
// Allow faster retry for failed http/dns checks
notOk := ""
hasHTTPOrDNS := false
for _, k := range kinds {
if k == kindHTTP || k == kindDNS {
hasHTTPOrDNS = true
break
}
}
if hasHTTPOrDNS {
notOk = `OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
}
whereClause := `
(checks.last_start IS NULL) OR
(checks.last_start + (checks.interval || ' second')::interval < now())
`
if notOk != "" {
whereClause += notOk
}
rq := q.Where(whereClause)
// Use SKIP LOCKED to avoid contention between workers. The same
// FOR UPDATE SKIP LOCKED clause also gives us implicit load balancing
// across workers in the same region: each concurrent worker call
// grabs a disjoint slice of the pending checks and a row leased by
// worker A is invisible to worker B until A's transaction commits
// (or rolls back / lease expires).
var checks []*Check
rq.Clauses(SkipLockedClause).
Limit(limit).
Preload("Monitor").
Find(&checks)
for _, c := range checks {
log.Println("worker: assigned remote check:", c.ID, c.Kind)
tx.Model(&c).Where("id = ?", c.ID).Update(colLastStart, time.Now())
}
tx.Commit()
return checks
}
// EnqueueDueCheckTasks atomically turns due normal checks into durable generic
// task envelopes. ChecksForWorker remains for the HTTP polling compatibility
// endpoint, while websocket scheduling uses this task-producing path.
func EnqueueDueCheckTasks(worker *WorkerNode, kinds []string, limit int) error {
if worker == nil || len(kinds) == 0 || limit <= 0 || worker.NetworkProblemActive(time.Now()) {
return nil
}
return DB().Transaction(func(tx *gorm.DB) error {
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
Where("monitors.enabled").Where("checks.enabled AND checks.kind IN (?)", kinds)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
}
q = applyRegionRouting(q, worker)
where := `(checks.last_start IS NULL) OR (checks.last_start + (checks.interval || ' second')::interval < now())`
for _, kind := range kinds {
if kind == kindHTTP || kind == kindDNS {
where += ` OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
break
}
}
var checks []*Check
if err := q.Where(where).Clauses(SkipLockedClause).Limit(limit).Preload("Monitor.Group").Find(&checks).Error; err != nil {
return err
}
now := time.Now()
for _, check := range checks {
if check.Monitor == nil || check.Monitor.Group == nil {
continue
}
job := JobForCheck(check)
payload, err := json.Marshal(job)
if err != nil {
return err
}
checkID, monitorID := check.ID, check.MonitorID
bucket := now.UTC().Unix() / int64(check.Interval)
task := Task{
JobID: job.JobID, Kind: TaskKindCheck, State: TaskStateQueued,
AccountID: check.Monitor.Group.AccountID, CheckID: &checkID, MonitorID: &monitorID,
Payload: payload, NotBefore: now, MaxAttempts: DefaultTaskMaxAttempts,
IdempotencyKey: fmt.Sprintf("check:%d:%d", check.ID, bucket),
}
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&task).Error; err != nil {
return err
}
if err := tx.Model(check).Update(colLastStart, now).Error; err != nil {
return err
}
}
return nil
})
}
// applyRegionRouting narrows the monitor JOIN in ChecksForWorker to the
// subset whose routing rule matches the calling worker's region. The filter
// is applied at the SQL layer so the SKIP LOCKED page only scans/leases
// rows that this worker is allowed to run, instead of leasing and then
// discarding forbidden checks.
//
// The function intentionally mirrors Monitor.WantsRegion so the helper can
// be reused from non-SQL callers (UI preview, plan validation, etc.).
//
// SQL form:
//
// - monitors.region_mode IN ('any', 'all', ”)
// → unconditional match; legacy / Phase 3 placeholder behavior.
// - monitors.region_mode = 'specific' AND
// (monitors.preferred_regions IS NULL OR
// cardinality(monitors.preferred_regions) = 0 OR
// ? = ANY(monitors.preferred_regions))
// → empty array falls back to "any"; otherwise the worker code must
// be in the whitelist.
func applyRegionRouting(q *gorm.DB, worker *WorkerNode) *gorm.DB {
if worker == nil || worker.RegionCode == "" {
return q
}
if worker.RegionCode == defaultRegionCode {
// The default "local" region is the historical catch-all; the
// in-process scheduler (not ChecksForWorker) handles those
// monitors. Skip regional filtering entirely so we don't leak
// Phase 1 in-process workers through the new router.
return q
}
// TODO(phase3): split RegionMode="all" into N assignments, one per
// preferred region, so the result aggregator can build a quorum.
// Today it is treated as "any" so existing checks keep flowing.
return q.Where(
`(monitors.region_mode IN ('any', 'all', '') OR `+
`(monitors.region_mode = 'specific' AND `+
`(monitors.preferred_regions IS NULL OR `+
`coalesce(array_length(monitors.preferred_regions, 1), 0) = 0 OR `+
`? = ANY(monitors.preferred_regions))))`,
worker.RegionCode,
)
}
// JobForCheck creates a CheckJob from a Check model for sending to a worker
func JobForCheck(c *Check) wire.CheckJob {
jobID := uuid.New().String()
var urlStr *string
if c.URL != nil {
urlStr = c.URL
}
return wire.CheckJob{
JobID: jobID,
CheckID: c.ID,
MonitorID: c.MonitorID,
Kind: c.Kind,
Host: c.Monitor.Host,
URL: urlStr,
Interval: c.Interval,
Settings: json.RawMessage(c.Settings),
}
}
// QueueMonitorChecks makes enabled checks for a monitor immediately eligible for remote assignment.
func QueueMonitorChecks(monitorID int64) error {
return DB().Model(&Check{}).
Where("monitor_id = ? AND enabled", monitorID).
Updates(map[string]interface{}{
colLastStart: nil,
colLastEnd: nil,
}).Error
}
// QueueMonitorChecksKind makes enabled checks of one kind immediately eligible for remote assignment.
func QueueMonitorChecksKind(monitorID int64, kind string) error {
return DB().Model(&Check{}).
Where("monitor_id = ? AND kind = ? AND enabled", monitorID, kind).
Updates(map[string]interface{}{
colLastStart: nil,
colLastEnd: nil,
}).Error
}
// ApplyRemoteCheckResult applies a check result reported by a distributed worker.
// It updates the check state in the database and triggers monitor status aggregation.
//
// Phase 3 of docs/todo.md (multi-region quorum aggregation): when the
// check has RequireQuorum > 1, the per-region result is recorded in
// check_region_results but Check.State is NOT touched here — that is
// the job of app/models/check_aggregator.go, which decides OK/ERR/
// DEGRADED once enough regional results have arrived or the aggregation
// window has elapsed. QuorumEnabled() == false preserves the legacy
// direct-update path so single-region / non-aggregated monitors keep
// the same behavior.
func ApplyRemoteCheckResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
return ApplyRemoteCheckResultFromWorker(report, regionCode, nil)
}
// ApplyRemoteCheckResultFromWorker persists worker attribution before changing
// legacy check state. A confirmation result is consumed exactly once and never
// overwrites the original check result.
func ApplyRemoteCheckResultFromWorker(report wire.CheckResultReport, regionCode string, worker *WorkerNode) error { //nolint:gocritic,lll // hugeParam: wire compatibility
var monitor *Monitor
err := DB().Transaction(func(tx *gorm.DB) error {
var err error
monitor, err = ApplyRemoteCheckResultFromWorkerTx(tx, report, regionCode, worker)
return err
})
if err != nil {
return err
}
if monitor != nil {
monitor.UpdateStatusFromChecks()
}
// VictoriaMetrics is outside PostgreSQL and is deliberately post-commit.
// A caller that retries after this error will not duplicate relational state;
// metric points are external at-least-once observations and need TSDB repair
// if the write remains unavailable.
return StoreRemoteCheckMetrics(report.Metrics)
}
// ApplyRemoteCheckResultFromWorkerTx applies all relational result effects using
// the caller's transaction. It intentionally does not write VictoriaMetrics or
// aggregate monitor state: both must happen only after the transaction commits.
// A nil monitor means the report was a consumed diagnostic attempt.
func ApplyRemoteCheckResultFromWorkerTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, worker *WorkerNode) (*Monitor, error) { //nolint:gocritic,lll // hugeParam: wire compatibility
if tx == nil {
return nil, fmt.Errorf("apply check result: nil transaction")
}
now := time.Now()
if worker != nil {
handled := false
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
return nil, err
}
if handled {
return nil, nil
}
}
check := Check{}
if err := tx.Preload("Monitor").First(&check, report.CheckID).Error; err != nil {
log.Println("worker: check not found:", report.CheckID, err)
return nil, err
}
// Always persist the per-region result first so the aggregator can
// pick it up regardless of which path we take next. We rely on
// StoreCheckRegionResult to default AggregatedAt=NULL (the column
// type is *time.Time, so a zero value writes SQL NULL).
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
log.Println("worker: error storing region result:", report.CheckID, err)
return nil, err
}
// Quorum-enabled checks: write nothing to Check.State here. The
// aggregator will compute the aggregate state once the window has
// elapsed (or enough regions have reported) and stamp AggregatedAt on
// the contributing CheckRegionResult rows.
if check.QuorumEnabled() {
return nil, nil
}
update := map[string]interface{}{
colState: report.State,
colLastEnd: now,
colWarnings: pq.StringArray(report.Warnings),
colInfos: pq.StringArray(report.Infos),
}
if report.State == "OK" {
update["was_up"] = now
update["last_ok"] = now
update["fails"] = 0
update["error"] = gorm.Expr("NULL")
} else {
update["last_fail"] = now
update["fails"] = gorm.Expr("fails + 1")
if report.Error != nil {
update["error"] = *report.Error
}
}
if report.ExpiresAt != nil {
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
if err == nil {
update["expires"] = t
}
}
if err := tx.Model(&check).UpdateColumns(update).Error; err != nil {
log.Println("worker: error updating check:", report.CheckID, err)
return nil, err
}
if worker != nil {
payload, _ := json.Marshal(report)
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
if attempt.JobID == "" {
attempt.JobID = uuid.New().String()
}
// A duplicate websocket/HTTP delivery must not create another attempt.
if err := tx.Where("job_id = ?", attempt.JobID).FirstOrCreate(&attempt).Error; err != nil {
return nil, err
}
switch report.State {
case stateERR, stateFail:
if err := StartConfirmationTx(tx, check.ID, worker.ID, now); err != nil {
return nil, err
}
case stateOK:
if err := RecoverDiagnosticTx(tx, check.ID, now); err != nil {
return nil, err
}
}
}
return check.Monitor, nil
}
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
func StoreRemoteCheckMetrics(metrics []wire.MetricPoint) error {
for _, metric := range metrics {
if metric.Metric == "" || len(metric.Fields) == 0 {
continue
}
if err := influx.WriteOne(metric.Metric, metric.Tags, metric.Fields); err != nil {
return err
}
}
return nil
}
// StoreCheckRegionResult stores a per-region check result for distributed monitoring analytics
func StoreCheckRegionResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
return StoreCheckRegionResultTx(DB(), report, regionCode, time.Now())
}
// StoreCheckRegionResultTx stores a regional result in the caller's transaction.
func StoreCheckRegionResultTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, executedAt time.Time) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
if tx == nil {
return fmt.Errorf("store region result: nil transaction")
}
if regionCode == "" {
regionCode = defaultRegionCode
}
result := CheckRegionResult{
CheckID: report.CheckID,
RegionCode: regionCode,
ExecutedAt: executedAt,
State: report.State,
DurationMs: report.DurationMs,
Error: report.Error,
}
return tx.Create(&result).Error
}
// StaleWorkers marks workers as inactive or dead based on last_seen time
func StaleWorkers() {
// Mark workers with no heartbeat for 2 minutes as inactive
DB().Model(&WorkerNode{}).
Where("status = ? AND last_seen < ?", "active", time.Now().Add(-2*time.Minute)).
Update("status", "inactive")
// Mark workers with no heartbeat for 5 minutes as dead
DB().Model(&WorkerNode{}).
Where("status IN (?, ?) AND last_seen < ?", "active", "inactive", time.Now().Add(-5*time.Minute)).
Update("status", "dead")
}

451
app/models/check_jobs_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,451 @@
package models_test
import (
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// seedRegionRoutingWorld builds two regions and three groups (one per
// monitor) plus three workers. The monitors and their PreferredRegions /
// RegionMode are configured by the caller via a callback so each test can
// express the exact routing scenario it wants to assert on.
//
// Returns a routerTestWorld that knows how to look up each fixture by name
// for readable assertions.
type routerTestWorld struct {
groupAny *models.Group
groupSpec *models.Group
groupAll *models.Group
workerMsk *models.WorkerNode
workerUSEast *models.WorkerNode
workerLocal *models.WorkerNode
}
// seedWorker creates a worker bound to regionCode. AuthToken is unique per
// worker so the FOR UPDATE SKIP LOCKED path can exercise two distinct
// concurrent callers.
func seedWorker(t *testing.T, id, regionCode string) *models.WorkerNode {
t.Helper()
seedRegion(t, regionCode)
w := &models.WorkerNode{
WorkerID: id,
RegionCode: regionCode,
Status: "active",
AuthToken: "tok-" + id,
Concurrency: 4,
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
}
require.NoError(t, models.DB().Create(w).Error)
return w
}
// seedRouterMonitor creates a Monitor with the given region routing
// attributes and one ready-to-run http Check. The check has Interval=60
// (minimum allowed) and LastStart=nil so ChecksForWorker picks it up
// immediately on the next poll.
func seedRouterMonitor(t *testing.T, group *models.Group, host, regionMode string, preferred []string) (models.Monitor, models.Check) {
t.Helper()
enTrue := true
mon := models.Monitor{
Name: stringPtrRouter(host),
Host: host,
GroupID: group.ID,
Enabled: true,
}
if regionMode != "" {
mon.RegionMode = regionMode
}
if preferred != nil {
mon.PreferredRegions = models.RegionCodesFromSlice(preferred)
}
require.NoError(t, models.DB().Create(&mon).Error)
check := models.Check{
MonitorID: mon.ID,
Kind: "http",
Interval: 60,
Enabled: &enTrue,
State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&check).Error)
return mon, check
}
func stringPtrRouter(s string) *string { return &s }
// seedRouterWorld is the common fixture for the TestRegionRouting_* table.
// It provisions two regions (ru-msk, us-east) and three monitors pinned to
// different routing modes; the workers are created lazily by the caller.
func seedRouterWorld(t *testing.T) routerTestWorld {
t.Helper()
models.Drop()
models.Migrate()
seedRegion(t, "ru-msk")
seedRegion(t, "us-east")
plan := models.Plan{Name: "router", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "router-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
groupAny := &models.Group{AccountID: acc.ID, Name: "any"}
groupSpec := &models.Group{AccountID: acc.ID, Name: "spec"}
groupAll := &models.Group{AccountID: acc.ID, Name: "all"}
require.NoError(t, models.DB().Create(groupAny).Error)
require.NoError(t, models.DB().Create(groupSpec).Error)
require.NoError(t, models.DB().Create(groupAll).Error)
seedRouterMonitor(t, groupAny, "any.test", models.RegionModeAny, nil)
seedRouterMonitor(t, groupSpec, "spec-msk.test", models.RegionModeSpecific, []string{"ru-msk"})
seedRouterMonitor(t, groupSpec, "spec-us.test", models.RegionModeSpecific, []string{"us-east"})
seedRouterMonitor(t, groupAll, "all.test", models.RegionModeAll, []string{"ru-msk", "us-east"})
return routerTestWorld{
groupAny: groupAny,
groupSpec: groupSpec,
groupAll: groupAll,
workerMsk: seedWorker(t, "w-msk", "ru-msk"),
workerUSEast: seedWorker(t, "w-us", "us-east"),
workerLocal: seedWorker(t, "w-local", "local"),
}
}
// idsOf returns a sorted list of monitor IDs assigned to the worker for
// easier assertions across checks.
func idsOf(checks []*models.Check) []int64 {
out := make([]int64, 0, len(checks))
for _, c := range checks {
if c.Monitor == nil {
continue
}
out = append(out, c.Monitor.ID)
}
return out
}
// TestRegionRouting_AnyReturnsAll confirms the SQL filter preserves the
// legacy behavior for region_mode='any' monitors and the Phase 3
// placeholder 'all' monitors: a worker in a region nobody explicitly
// whitelisted must still see them, while monitors pinned to specific
// regions stay filtered out.
//
// seedRouterWorld configures four monitors:
// - any.test → region_mode='any', no PreferredRegions
// - spec-msk.test → region_mode='specific', preferred=[ru-msk]
// - spec-us.test → region_mode='specific', preferred=[us-east]
// - all.test → region_mode='all', preferred=[ru-msk, us-east]
//
// A worker in region "remote" (whitelisted by nobody) must see exactly
// {any.test, all.test} — the two monitors whose region_mode bypasses the
// whitelist — and nothing else.
func TestRegionRouting_AnyReturnsAll(t *testing.T) {
seedRouterWorld(t)
seedRegion(t, "remote")
w := seedWorker(t, "w-remote", "remote")
checks := models.ChecksForWorker(w, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{"any.test", "all.test"}, hosts,
"region_mode='any' and the Phase 3 'all' placeholder must bypass the whitelist")
assert.NotContains(t, hosts, "spec-msk.test",
"specific-mode monitor with whitelisted ru-msk must NOT reach a remote worker")
assert.NotContains(t, hosts, "spec-us.test",
"specific-mode monitor with whitelisted us-east must NOT reach a remote worker")
}
// TestRegionRouting_SpecificFiltersByRegion proves the core Phase 2 promise:
// workers in different regions never receive a monitor whose PreferredRegions
// does not include their region code. The check is run with parallel
// goroutines because ChecksForWorker stamps `last_start` on every row it
// leases — a sequential second poll would always see an already-claimed
// queue and the filter would have nothing to test against.
//
// Under the FOR UPDATE SKIP LOCKED race, whichever SELECT fires first grabs
// every matching row, so the *exact* per-worker host list is non-deterministic.
// The deterministic invariant the test asserts is the no-leak property: a
// worker in ru-msk must never see spec-us.test, and vice versa.
func TestRegionRouting_SpecificFiltersByRegion(t *testing.T) {
world := seedRouterWorld(t)
var (
wg sync.WaitGroup
mskChecks []*models.Check
usChecks []*models.Check
)
wg.Add(2)
go func() {
defer wg.Done()
mskChecks = models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
}()
go func() {
defer wg.Done()
usChecks = models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
}()
wg.Wait()
mskHosts := hostsOf(mskChecks)
usHosts := hostsOf(usChecks)
// Aggregate coverage: together the two workers must see every check
// the routing layer would ever allow them — the four seeded monitors.
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
},
append(append([]string{}, mskHosts...), usHosts...),
"union of both workers' slices must cover every seeded monitor (any/specific/all × region)")
// Core Phase 2 invariant: regional filtering never leaks across
// PreferredRegions boundaries. This is the only assertion a
// concurrent SKIP LOCKED race lets us pin deterministically.
assert.NotContains(t, mskHosts, "spec-us.test",
"ru-msk worker must never see a monitor whitelisted for us-east only")
assert.NotContains(t, usHosts, "spec-msk.test",
"us-east worker must never see a monitor whitelisted for ru-msk only")
}
// TestRegionRouting_SpecificEmptyPreferredFallsBackToAny confirms the
// documented fall-back: a monitor in RegionModeSpecific with no
// PreferredRegions behaves like RegionModeAny so the field is safe to
// leave blank. We poll from the us-east worker — without the fall-back it
// would only see any.test + all.test + spec-us.test.
func TestRegionRouting_SpecificEmptyPreferredFallsBackToAny(t *testing.T) {
world := seedRouterWorld(t)
// Reset the spec-msk monitor to have an empty PreferredRegions list
// (the seed above gave it one). The Monitor row's RegionMode stays
// 'specific'.
require.NoError(t, models.DB().Model(&models.Monitor{}).
Where("host = ?", "spec-msk.test").
Update("preferred_regions", models.RegionCodesFromSlice(nil)).Error)
checks := models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.Contains(t, hosts, "spec-msk.test",
"empty PreferredRegions with region_mode=specific must fall back to 'any'")
}
// TestRegionRouting_AllDeferredToAny pins the Phase 3 placeholder behavior:
// region_mode='all' is logged and treated as 'any' today. The test asserts
// the monitor flows to a worker in any region (the TODO log marker is
// emitted from applyRegionRouting — pinned here as a code-grep contract).
func TestRegionRouting_AllDeferredToAny(t *testing.T) {
world := seedRouterWorld(t)
checks := models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.Contains(t, hosts, "all.test",
"region_mode='all' must currently behave like 'any' so existing checks keep flowing")
}
// TestRegionRouting_LocalWorkerBypass ensures the historic "local" region
// still routes everything: the in-process scheduler handles those monitors
// and we don't want the Phase 2 filter to leak platform workers through it.
func TestRegionRouting_LocalWorkerBypass(t *testing.T) {
world := seedRouterWorld(t)
checks := models.ChecksForWorker(world.workerLocal, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
}, hosts, "worker in region 'local' must receive every check (bypass)")
}
// TestRegionRouting_NilWorkerReturnsAll asserts the diagnostic-friendly
// escape hatch: passing nil for the worker skips the routing filter and
// returns every check the kinds/limit envelope allows.
func TestRegionRouting_NilWorkerReturnsAll(t *testing.T) {
seedRouterWorld(t)
checks := models.ChecksForWorker(nil, []string{"http"}, 50)
hosts := hostsOf(checks)
assert.ElementsMatch(t, []string{
"any.test",
"spec-msk.test",
"spec-us.test",
"all.test",
}, hosts, "nil worker must bypass the routing filter")
}
// TestRegionRouting_LoadBalanceImplicit confirms the SKIP LOCKED implicit
// load-balancing story: when two workers in the same region race for a pool
// of pending checks, each of them receives a non-empty disjoint slice. The
// two polls run in parallel goroutines so the FOR UPDATE SKIP LOCKED race
// window is actually exercised.
//
// IMPORTANT: SKIP LOCKED with a large LIMIT is unfair — whichever
// transaction's SELECT fires first grabs everything. The test therefore
// uses LIMIT=4 with 10 pending rows so each worker is forced to leave some
// rows unlocked for the other worker to pick up. Together they must cover
// at most 8 rows (LIMIT × workers) without overlap; the remaining rows are
// intentionally left for a future poll cycle, which mirrors production
// behavior where workers continually drain a backlog.
func TestRegionRouting_LoadBalanceImplicit(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "shared")
plan := models.Plan{Name: "lb-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "lb", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := &models.Group{AccountID: acc.ID, Name: "lb-g"}
require.NoError(t, models.DB().Create(group).Error)
enTrue := true
for i := 0; i < 10; i++ {
host := "lb-" + string(rune('a'+i)) + ".test"
mon := models.Monitor{
Name: stringPtrRouter(host),
Host: host,
GroupID: group.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&mon).Error)
ck := models.Check{
MonitorID: mon.ID, Kind: "http", Interval: 60,
Enabled: &enTrue, State: "UNK",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&ck).Error)
}
w1 := seedWorker(t, "lb-w1", "shared")
w2 := seedWorker(t, "lb-w2", "shared")
const limitPerWorker = 4
var (
wg sync.WaitGroup
aChecks, bChecks []*models.Check
)
wg.Add(2)
go func() {
defer wg.Done()
aChecks = models.ChecksForWorker(w1, []string{"http"}, limitPerWorker)
}()
go func() {
defer wg.Done()
bChecks = models.ChecksForWorker(w2, []string{"http"}, limitPerWorker)
}()
wg.Wait()
assert.Greater(t, len(aChecks), 0, "worker 1 must receive at least one check")
assert.Greater(t, len(bChecks), 0, "worker 2 must receive at least one check")
assert.LessOrEqual(t, len(aChecks)+len(bChecks), 2*limitPerWorker,
"two concurrent workers with LIMIT each can lease at most LIMIT*2 rows per cycle")
assert.Empty(t, intersectHosts(aChecks, bChecks),
"the two slices must be disjoint (FOR UPDATE SKIP LOCKED must not double-lease)")
}
// TestMonitorValidateRegionMode exercises the documented enum on the
// Monitor type so the validator surface does not regress.
func TestMonitorValidateRegionMode(t *testing.T) {
cases := []struct {
mode string
wantErr bool
}{
{"", false},
{"any", false},
{"specific", false},
{"all", false},
{"round-robin", true},
{"RANDOM", true},
}
for _, c := range cases {
t.Run("mode="+c.mode, func(t *testing.T) {
m := models.Monitor{RegionMode: c.mode}
err := m.ValidateRegionMode()
if c.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// TestMonitorWantsRegion pins the public WantsRegion helper that powers the
// non-SQL callers (UI preview, plan validation). Phase 3 will swap the
// 'all' branch's behavior, so the table covers all three modes today.
func TestMonitorWantsRegion(t *testing.T) {
cases := []struct {
name string
mode string
regions []string
workerRC string
want bool
}{
{"any_always_true", "any", []string{"ru-msk"}, "us-east", true},
{"any_empty_pref_still_true", "any", nil, "us-east", true},
{"specific_match", "specific", []string{"ru-msk", "eu-west"}, "ru-msk", true},
{"specific_no_match", "specific", []string{"ru-msk", "eu-west"}, "us-east", false},
{"specific_empty_pref_fallback", "specific", nil, "us-east", true},
{"all_placeholder_true", "all", []string{"ru-msk", "us-east"}, "ru-msk", true},
{"all_placeholder_foreign_region", "all", []string{"ru-msk", "us-east"}, "eu-west", true},
{"empty_mode_defaults_to_any", "", nil, "us-east", true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
m := models.Monitor{
RegionMode: c.mode,
PreferredRegions: models.RegionCodesFromSlice(c.regions),
}
assert.Equal(t, c.want, m.WantsRegion(c.workerRC))
})
}
}
// hostsOf extracts the hostnames from the assigned checks for readable
// assertions in table-driven tests.
func hostsOf(checks []*models.Check) []string {
out := make([]string, 0, len(checks))
for _, c := range checks {
if c.Monitor == nil {
continue
}
out = append(out, c.Monitor.Host)
}
return out
}
// intersectHosts returns the hostnames present in both slices — used to
// prove two concurrent workers did not lease the same check twice.
func intersectHosts(a, b []*models.Check) []string {
set := make(map[string]struct{}, len(a))
for _, c := range a {
if c.Monitor != nil {
set[c.Monitor.Host] = struct{}{}
}
}
var out []string
for _, c := range b {
if c.Monitor == nil {
continue
}
if _, ok := set[c.Monitor.Host]; ok {
out = append(out, c.Monitor.Host)
}
}
return out
}

42
app/models/check_metric_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,42 @@
package models
import (
"testing"
"unicode"
"github.com/stretchr/testify/assert"
)
func TestCheckMetricName(t *testing.T) {
tests := []struct {
kind string
want string
}{
{"http", "chttp"},
{"ssl", "cssl"},
{"bssl", "cbssl"},
{"ssh", "cssh"},
{"ftp", "cftp"},
{"dns", "cdns"},
{"whois", "cwhois"},
{"rkn", "crkn"},
{"llm", "cllm"},
{"llm-http", "cllm_http"},
{"weird-kind", "cweird_kind"},
}
for _, tt := range tests {
t.Run(tt.kind, func(t *testing.T) {
c := &Check{Kind: tt.kind, ID: 1}
got := c.MetricName()
assert.Equal(t, tt.want, got, "MetricName() for kind %q", tt.kind)
metric := got
for _, ch := range metric {
if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' && ch != ':' {
t.Errorf("metric name %q contains invalid character %q", metric, ch)
}
}
})
}
}

33
app/models/check_region_result.go Обычный файл
Просмотреть файл

@@ -0,0 +1,33 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// CheckRegionResult stores check results from distributed workers by region
type CheckRegionResult struct {
concerns.Model
CheckID int64 `gorm:"index;not null" json:"check_id"`
Check *Check `gorm:"foreignKey:CheckID" json:"check,omitempty"`
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id"`
WorkerNode *WorkerNode `gorm:"foreignKey:WorkerNodeID" json:"worker_node,omitempty"`
ExecutedAt time.Time `gorm:"not null" json:"executed_at"`
State string `gorm:"size:10;not null" json:"state"`
DurationMs int64 `json:"duration_ms"`
Error *string `json:"error"`
// AggregatedAt is set by app/models/check_aggregator.go once a row has
// been folded into a Check.State decision. NULL means "still waiting
// for the aggregator"; non-NULL means "this row has already been
// counted in a quorum decision and must not be re-counted". The
// column is indexed (see check_aggregator.go index helper) so the
// per-tick SELECT that finds unaggregated rows is O(matching rows)
// rather than scanning the whole table.
AggregatedAt *time.Time `gorm:"index" json:"aggregated_at,omitempty"`
concerns.Timestamped
}

155
app/models/check_settings.go Обычный файл
Просмотреть файл

@@ -0,0 +1,155 @@
package models
import (
"errors"
"fmt"
"net/http"
"strings"
)
// CheckHeader provides functionality.
type CheckHeader struct {
Key string `json:"key"`
Value string `json:"value"`
}
// CheckSettings provides functionality.
type CheckSettings struct {
// HTTP Basic Auth Username
HTTPUsername string `json:"http_username,omitempty"`
// HTTP Basic Auth Password
HTTPPassword string `json:"http_password,omitempty"`
// ExpectedAnswer (default, redirect, custom)
ExpectedAnswer string `json:"expected_answer,omitempty"`
// Expected redirect location
ExpectedLocation string `json:"expected_location,omitempty"`
// Expected HTTP status codes for custom
ExpectedHTTPCode int `json:"expected_http_code,omitempty"`
// Keyword search type (off, present or absent)
KeywordType string `json:"keyword_type,omitempty"`
// Keyword to search for
KeywordValue string `json:"keyword_value,omitempty"`
SlowTime int `json:"slow_time"`
Timeout int `json:"timeout"`
// Request Settings
CheckIp bool `json:"checkip"` //nolint:revive // accepted lint exception
CheckIPv6 bool `json:"checkipv6"`
RequestHeader []CheckHeader `json:"request_headers"`
RequestMethod string `json:"request_method"`
RequestType string `json:"request_type"`
RequestContent string `json:"request_content"`
// SSH Settings
Port string `json:"port"`
// Host optionally overrides the monitor host for checks that do
// not naturally target the Monitor.Host (e.g. an ICMP/TCP/UDP
// probe to a separate machine, or a different IP family). Used by
// cping/ctcp/cudp.
Host string `json:"host,omitempty"`
// Count is the per-check packet count for ping. Defaults to 1 in
// cping when zero or negative.
Count int `json:"count,omitempty"`
// PacketSize is the ICMP payload size for ping (bytes). Defaults
// to 56 in cping when zero or negative.
PacketSize int `json:"packet_size,omitempty"`
// Distributed marks this check as eligible for execution on the
// distributed worker pool (multi-region, multi-worker) instead of
// only the in-process scheduler. Reserved for paid plans; the
// controller layer enforces the plan check.
Distributed bool `json:"distributed"`
}
// redirectCodes contains all HTTP redirect status codes.
var redirectCodes = []int{300, 301, 302, 303, 307, 308}
// CheckKeyword provides functionality.
func (s *CheckSettings) CheckKeyword(body []byte, warns []string) ([]string, error) {
var err error
switch s.KeywordType {
case "", "off":
return warns, nil
case "present":
if !strings.Contains(string(body), s.KeywordValue) {
err = errors.New("expected keyword " + s.KeywordValue + " not found")
}
case "absent":
if strings.Contains(string(body), s.KeywordValue) {
err = errors.New("unexpected keyword " + s.KeywordValue + " found")
}
}
return warns, err
}
// CheckAnswer provides functionality.
func (s *CheckSettings) CheckAnswer(resp *http.Response, body []byte) ([]string, error) {
// log.Println("check answer, settings:")
// spew.Dump(s)
isRedirect := false
for _, rc := range redirectCodes {
if resp.StatusCode == rc {
isRedirect = true
}
}
location := resp.Header.Get("location")
// log.Println("redirect?", isRedirect, location)
switch s.ExpectedAnswer {
case "", "default":
if isRedirect {
return s.CheckKeyword(body, []string{"redirect: " + location})
} else if resp.StatusCode != 200 {
return []string{}, fmt.Errorf("bad status code: %d", resp.StatusCode)
}
case "redirect":
if !isRedirect {
return []string{}, fmt.Errorf("bad status code: %d (expected redirect)", resp.StatusCode)
}
if s.ExpectedLocation != "" {
if location != s.ExpectedLocation {
return []string{}, fmt.Errorf(
"bad location: %s (expected %s)",
location,
s.ExpectedLocation,
)
}
}
case "custom":
if s.ExpectedHTTPCode == 0 {
s.ExpectedHTTPCode = 200
}
if resp.StatusCode != s.ExpectedHTTPCode {
return []string{}, fmt.Errorf("bad status code: %d (expected %d)", resp.StatusCode, s.ExpectedHTTPCode)
}
if s.ExpectedLocation != "" {
if location != s.ExpectedLocation {
return []string{}, fmt.Errorf(
"bad location: %s (expected %s)",
location,
s.ExpectedLocation,
)
}
}
default:
panic("bad expectedAnswer")
}
// return []string{}, nil
return s.CheckKeyword(body, []string{})
}

60
app/models/check_settings_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,60 @@
package models
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCheckSettingsDistributed_Default(t *testing.T) {
s := CheckSettings{}
assert.False(t, s.Distributed, "new CheckSettings should default to Distributed=false")
}
func TestCheckSettingsDistributed_JSON(t *testing.T) {
original := CheckSettings{
HTTPUsername: "user",
HTTPPassword: "pass",
ExpectedAnswer: "default",
Timeout: 30,
Distributed: true,
}
data, err := json.Marshal(original)
assert.NoError(t, err)
assert.Contains(t, string(data), `"distributed":true`)
var decoded CheckSettings
err = json.Unmarshal(data, &decoded)
assert.NoError(t, err)
assert.Equal(t, original, decoded)
assert.True(t, decoded.Distributed)
}
func TestCheckSettingsDistributed_OmitFalse(t *testing.T) {
s := CheckSettings{HTTPUsername: "user"}
data, err := json.Marshal(s)
assert.NoError(t, err)
assert.Contains(t, string(data), `"distributed":false`)
}
func TestCheckSettingsDistributed_CheckUnmarshal(t *testing.T) {
c := &Check{
Settings: []byte(`{"distributed": true, "timeout": 60}`),
}
got := c.GetSettings()
assert.True(t, got.Distributed)
assert.Equal(t, 60, got.Timeout)
}
func TestPlanAllowsDistributed(t *testing.T) {
var nilPlan *Plan
assert.False(t, nilPlan.AllowsDistributed(), "nil plan must not allow distributed")
free := &Plan{Price: 0}
assert.False(t, free.AllowsDistributed(), "free plan must not allow distributed")
paid := &Plan{Price: 100}
assert.True(t, paid.AllowsDistributed(), "paid plan must allow distributed")
}

181
app/models/cleanup_stale_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,181 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// makeStaleUser inserts a user whose LastActiveAt is older than the
// 3-month cutoff used by the cleanup filter. The returned user is what
// the candidates query should pick up.
func makeStaleUser(t *testing.T, email string, lastActive *time.Time) *models.User {
t.Helper()
u := &models.User{
Email: &email,
Name: "stale " + email,
Enabled: true,
Confirmed: true,
LastActiveAt: lastActive,
}
require.NoError(t, models.DB().Create(u).Error)
return u
}
// TestFindStaleAccounts_EmptyWhenNoCandidates checks the obvious
// negative case: a fresh account with an active owner is not eligible.
func TestFindStaleAccounts_EmptyWhenNoCandidates(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "fresh", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
// Group + owner access + active user
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
recent := time.Now().Add(-1 * time.Hour)
user := makeStaleUser(t, "active@example.com", &recent)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got, "an account with an active owner is not stale")
}
// TestFindStaleAccounts_PicksStaleEmptyAccount checks the happy path:
// account with no monitors + single user + last login > 3 months ago.
func TestFindStaleAccounts_PicksStaleEmptyAccount(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "ghost", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
neverLoggedIn := (*time.Time)(nil)
user := makeStaleUser(t, "ghost@example.com", neverLoggedIn)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
require.Len(t, got, 1, "the empty stale account should be picked up")
assert.Equal(t, acc.ID, got[0].AccountID)
assert.Equal(t, user.ID, got[0].UserID)
}
// TestFindStaleAccounts_SkipsAccountWithMonitors makes sure the
// "zero monitors" gate is enforced.
func TestFindStaleAccounts_SkipsAccountWithMonitors(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "active", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "default"}
require.NoError(t, models.DB().Create(&group).Error)
// One monitor → account is NOT eligible even if the user is stale.
monitor := models.Monitor{GroupID: group.ID, Host: "example.com"}
require.NoError(t, models.DB().Create(&monitor).Error)
stale := time.Now().Add(-365 * 24 * time.Hour)
user := makeStaleUser(t, "owner@example.com", &stale)
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
require.NoError(t, models.DB().Create(&access).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got)
}
// TestFindStaleAccounts_SkipsUserWithMultipleAccounts verifies that a
// user holding two accounts disqualifies BOTH accounts.
func TestFindStaleAccounts_SkipsUserWithMultipleAccounts(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc1 := models.Account{Name: "acc1", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc1).Error)
acc2 := models.Account{Name: "acc2", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc2).Error)
models.DB().Create(&models.Group{AccountID: acc1.ID, Name: "g1"})
models.DB().Create(&models.Group{AccountID: acc2.ID, Name: "g2"})
stale := time.Now().Add(-365 * 24 * time.Hour)
user := makeStaleUser(t, "shared@example.com", &stale)
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc1.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc2.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
got, err := models.FindStaleAccounts()
require.NoError(t, err)
assert.Empty(t, got, "user with two accounts disqualifies both accounts")
}
// TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans verifies the
// end-to-end cleanup: matching account + user are removed, and
// recently-active accounts survive.
func TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
// Stale account with a stale user that has only this one account.
staleAcc := models.Account{Name: "ghost", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&staleAcc).Error)
staleGroup := models.Group{AccountID: staleAcc.ID, Name: "g"}
require.NoError(t, models.DB().Create(&staleGroup).Error)
staleUser := makeStaleUser(t, "ghost@example.com", nil)
require.NoError(t, models.DB().Create(&models.Access{AccountID: staleAcc.ID, UserID: &staleUser.ID, Kind: "account", Role: "owner"}).Error)
// Active account with a recent user — must NOT be touched.
freshAcc := models.Account{Name: "fresh", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&freshAcc).Error)
require.NoError(t, models.DB().Create(&models.Group{AccountID: freshAcc.ID, Name: "g"}).Error)
recent := time.Now().Add(-1 * time.Hour)
freshUser := makeStaleUser(t, "fresh@example.com", &recent)
require.NoError(t, models.DB().Create(&models.Access{AccountID: freshAcc.ID, UserID: &freshUser.ID, Kind: "account", Role: "owner"}).Error)
deleted, err := models.CleanupStaleAccounts()
require.NoError(t, err)
assert.Equal(t, 1, deleted, "only the stale empty account should be deleted")
// Stale account is gone.
var count int64
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", staleAcc.ID).Count(&count).Error)
assert.Equal(t, int64(0), count)
// Stale user is orphaned → also hard-deleted by the cleanup pass.
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", staleUser.ID).Count(&count).Error)
assert.Equal(t, int64(0), count)
// Fresh account and user survive.
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", freshAcc.ID).Count(&count).Error)
assert.Equal(t, int64(1), count)
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", freshUser.ID).Count(&count).Error)
assert.Equal(t, int64(1), count)
}

41
app/models/concerns/has_token.go Обычный файл
Просмотреть файл

@@ -0,0 +1,41 @@
// Package concerns provides functionality.
package concerns
import (
"bytes"
"crypto/rand"
"encoding/base64"
)
// HasToken provides functionality.
type HasToken struct {
Token string `json:"-" gorm:"unique_index"`
}
// SetToken provides functionality.
func (m *HasToken) SetToken() {
tk := RandomToken(32)
m.Token = base64.RawURLEncoding.EncodeToString(tk)
if m.Token == "" {
panic("RandomToken failed: token not set")
}
if len(m.Token) < 32 {
panic("RandomToken failed: token too short")
}
}
// RandomToken provides functionality.
func RandomToken(tokenLen int) []byte {
b := make([]byte, tokenLen)
n, err := rand.Read(b)
if err != nil {
panic(err)
}
if n != tokenLen {
panic("RandomToken failed: bad len")
}
if bytes.Equal(b, make([]byte, tokenLen)) {
panic("RandomToken failed: generated empty token")
}
return b
}

14
app/models/concerns/model.go Обычный файл
Просмотреть файл

@@ -0,0 +1,14 @@
// Source: https://gorm.io/gorm/blob/master/model.go
// Use 64 bit keys
package concerns
// Model base model definition, including fields `ID`, `CreatedAt`, `UpdatedAt`, `DeletedAt`, which could be embedded in your models
// type User struct {
// gorm.Model
// }
// Model provides functionality.
type Model struct {
ID int64 `gorm:"primarykey" json:"id"`
}

7
app/models/concerns/renderable.go Обычный файл
Просмотреть файл

@@ -0,0 +1,7 @@
package concerns
// Renderable provides functionality.
type Renderable struct {
Raw string
Rendered string
}

12
app/models/concerns/soft_delete.go Обычный файл
Просмотреть файл

@@ -0,0 +1,12 @@
package concerns
import (
"time"
)
// SoftDelete provides functionality.
type SoftDelete struct {
DeletedAt *time.Time `gorm:"index" json:"-"`
DeleterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
// Deleter *User `json:"-"`
}

11
app/models/concerns/timestamped.go Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
package concerns
import (
"time"
)
// Timestamped provides functionality.
type Timestamped struct {
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}

86
app/models/contact.go Обычный файл
Просмотреть файл

@@ -0,0 +1,86 @@
package models
import (
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Contact represents a notification contact.
//
// Ownership / tenancy:
//
// - A Contact belongs to exactly one Account (tenant). AccountID may be
// nil for system-level admin contacts (see SystemContacts).
// - A Contact may belong to at most one User. UserID is set when the
// contact was created on behalf of a specific user (the typical
// case for self-service "My email" / "My Telegram" contacts) and is
// nil for account-wide contacts.
//
// `User` is omitempty because most list payloads don't preload it; the
// `/settings/users` page loads it server-side via the AccountUserRow
// payload.
type Contact struct {
concerns.Model
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id)"`
Account *Account `json:"-"`
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
User *User `json:"user,omitempty"`
Name string `json:"name" gorm:"not null"`
Kind string `json:"kind" gorm:"not null;index:contact_value"`
Value string `json:"value" gorm:"index:contact_value"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
Data datatypes.JSON `json:"data"`
IsSystem *bool `json:"is_system" gorm:"default:false"`
Notifications []Notification `json:"-" gorm:"many2many:notification_contacts;"`
NotificationsCount int `gorm:"-:all" json:"notifications_count"`
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
Messages []Message `json:"-"`
concerns.Timestamped
concerns.HasToken
Audited
}
// SystemContacts returns all contacts marked as system/admin (is_system=true).
// These are the contacts distributed workers notify directly when the main
// API is unreachable (see docs/worker-protocol.md "System Selfcheck").
func SystemContacts() ([]Contact, error) {
var contacts []Contact
err := DB().Where("is_system = ? AND enabled = ?", true, true).Find(&contacts).Error
return contacts, err
}
// ContactsCounts fills NotificationsCount and MonitorsCount for each contact.
func ContactsCounts(contacts *[]Contact) {
groupIDs := make(map[int64]bool, 0)
groupCount := make(map[int64]int, 0)
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
(*contacts)[i].NotificationsCount = len(c.Notifications)
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
groupIDs[g.ID] = true
groupCount[g.ID] = 0
}
}
}
gids := make([]int64, 0, len(groupIDs))
for g := range groupIDs {
gids = append(gids, g)
}
CountGroups(gids, &groupCount)
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
cnt := 0
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
cnt += groupCount[g.ID]
}
}
(*contacts)[i].MonitorsCount = cnt
}
}

65
app/models/contact_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,65 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestSystemContacts verifies that SystemContacts returns only contacts with
// is_system=true and ignores contacts with is_system=false or nil.
func TestSystemContacts(t *testing.T) {
models.Drop()
models.Migrate()
account := &models.Account{Name: "test-account"}
require.NoError(t, models.DB().Create(account).Error)
accountID := account.ID
trueVal, falseVal := true, false
systemContact := &models.Contact{
AccountID: &accountID,
Name: "system-admin",
Kind: "email",
Value: "ops@example.com",
IsSystem: &trueVal,
}
regularContact := &models.Contact{
AccountID: &accountID,
Name: "regular",
Kind: "email",
Value: "user@example.com",
IsSystem: &falseVal,
}
nilSystemContact := &models.Contact{
AccountID: &accountID,
Name: "nil-system",
Kind: "email",
Value: "nil@example.com",
}
require.NoError(t, models.DB().Create(systemContact).Error)
require.NoError(t, models.DB().Create(regularContact).Error)
require.NoError(t, models.DB().Create(nilSystemContact).Error)
got, err := models.SystemContacts()
require.NoError(t, err)
var ids []int64
var names []string
for _, c := range got {
ids = append(ids, c.ID)
names = append(names, c.Name)
}
assert.Contains(t, names, "system-admin")
assert.NotContains(t, names, "regular")
assert.NotContains(t, names, "nil-system")
assert.Contains(t, ids, systemContact.ID)
assert.NotContains(t, ids, regularContact.ID)
assert.NotContains(t, ids, nilSystemContact.ID)
}

97
app/models/credential_crypto.go Обычный файл
Просмотреть файл

@@ -0,0 +1,97 @@
package models
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"io"
"os"
"strings"
"rsgit.ru/rsmon/rsmon/config/secrets"
)
const credentialKeyEnv = "RSMON_CRED_KEY"
// encryptSecret encrypts plaintext with AES-GCM. If no key is configured
// (dev/test), stores the value with a "plain:" prefix. Production deployments
// must set RSMON_CRED_KEY or config/secrets.yml crypto.pepper.
func encryptSecret(plaintext string) (string, error) {
key := credentialKey()
if key == "" {
return "plain:" + plaintext, nil
}
block, err := aes.NewCipher(deriveKey(key))
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
sealed := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return "enc:" + base64.StdEncoding.EncodeToString(sealed), nil
}
// decryptSecret decrypts a value produced by encryptSecret. Values without a
// known prefix are returned as-is for backward compatibility with legacy
// plaintext records.
func decryptSecret(stored string) (string, error) {
if stored == "" {
return "", nil
}
switch {
case strings.HasPrefix(stored, "plain:"):
return strings.TrimPrefix(stored, "plain:"), nil
case strings.HasPrefix(stored, "enc:"):
key := credentialKey()
if key == "" {
return "", errors.New("credential key not configured but secret is encrypted")
}
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(stored, "enc:"))
if err != nil {
return "", err
}
block, err := aes.NewCipher(deriveKey(key))
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
if len(raw) < gcm.NonceSize() {
return "", errors.New("ciphertext too short")
}
nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", err
}
return string(plaintext), nil
default:
return stored, nil
}
}
func deriveKey(s string) []byte {
h := sha256.Sum256([]byte(s))
return h[:32]
}
func credentialKey() string {
if k := os.Getenv(credentialKeyEnv); k != "" {
return k
}
if secrets.Crypto != nil && secrets.Crypto.Pepper != "" {
return secrets.Crypto.Pepper
}
return ""
}

120
app/models/dead_worker_reaper.go Обычный файл
Просмотреть файл

@@ -0,0 +1,120 @@
package models
import (
"context"
"log"
"time"
)
// DeadWorkerHeartbeatTimeout is the threshold for ReapDeadWorkers — a worker
// whose last_seen is older than this is considered dead and any leased tasks
// it owns are reassigned to the pool. Five minutes mirrors the StaleWorkers()
// check in check_jobs.go so the two reapers cannot disagree about who is
// dead. See docs/todo.md Phase 4 §5.
const DeadWorkerHeartbeatTimeout = 5 * time.Minute
// ReapDeadWorkers marks any non-dead worker whose last_seen is older than
// DeadWorkerHeartbeatTimeout as "dead", then reassigns its leased tasks
// back to the queued pool so other workers (or freshly registered ones)
// can pick them up. It mirrors the structure of ReapExpiredTasks — two
// short UPDATE statements, cheap enough to run from the web process
// every 30s.
//
// The returned tuple is (reaped, reassigned, err): reaped counts the
// workers that flipped to dead during this call; reassigned counts the
// leased tasks that were given back to the pool. A zero count on either
// is normal — the reaper is idempotent and the call is silent when
// nothing is due.
func ReapDeadWorkers() (reaped int, reassigned int, err error) {
now := time.Now()
cutoff := now.Add(-DeadWorkerHeartbeatTimeout)
// First flip the workers to dead so the second UPDATE can match the
// freshly-stamped ids without having to re-derive them in Go.
r := DB().Exec(`
UPDATE worker_nodes
SET status = ?, updated_at = ?
WHERE status <> ? AND last_seen IS NOT NULL AND last_seen < ?`,
"dead", now, "dead", cutoff,
)
if r.Error != nil {
return 0, 0, r.Error
}
reaped = int(r.RowsAffected)
// Nothing flipped → no tasks to return. Cheaper than running an
// UPDATE that touches 0 rows on every tick when the fleet is healthy.
if reaped == 0 {
return 0, 0, nil
}
// Second: clear any leased tasks owned by the now-dead workers.
// The selector stashes the worker's WorkerID string in lease_owner;
// matching against the (now-stale) worker row's WorkerID is the
// same identifier the selector uses, so we don't need an extra
// join. Tasks in other states (queued, succeeded, dead, …) are
// unaffected — only leased work the dead worker still owned has
// to go back to the queue.
r2 := DB().Exec(`
UPDATE tasks
SET state = ?, lease_owner = '', lease_expires_at = NULL, updated_at = ?
WHERE state = ? AND lease_owner IN (
SELECT worker_id FROM worker_nodes WHERE status = ?
)`,
TaskStateQueued, now, TaskStateLeased, "dead",
)
if r2.Error != nil {
return reaped, 0, r2.Error
}
reassigned = int(r2.RowsAffected)
return reaped, reassigned, nil
}
// StartDeadWorkerReaper launches a goroutine that runs ReapDeadWorkers
// on the given interval until ctx is canceled. Mirrors StartTaskReaper
// in this package: same ticker pattern, same logging style, same
// recover() safety net so a malformed row cannot crash the web process.
//
// The default interval is 30s; values <= 0 fall back to the default so
// the helper is safe to call from any call site without a guard. Wire
// from main.init() once per process — the reaper uses short row-level
// locks and is cheap under load (two indexed UPDATEs of <= a few
// hundred rows in steady state).
//
// Passing a nil context falls back to context.Background() so callers
// can write `models.StartDeadWorkerReaper(nil, ...)` in one-liners
// (main, tests) without having to import "context" first.
func StartDeadWorkerReaper(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 30 * time.Second
}
if ctx == nil {
ctx = context.Background()
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("dead_worker_reaper: panic recovered: %v", r)
}
}()
reaped, reassigned, err := ReapDeadWorkers()
if err != nil {
log.Printf("dead_worker_reaper: error: %v", err)
return
}
if reaped > 0 || reassigned > 0 {
log.Printf("dead_worker_reaper: reaped=%d reassigned=%d", reaped, reassigned)
}
}()
}
}
}()
}

274
app/models/dead_worker_reaper_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,274 @@
package models_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestReapDeadWorkers_SkipsHealthyAndKillsSilent mirrors the
// reaper-shape test for ReapExpiredTasks in task_test.go: seed three
// workers (heartbeat fresh / heartbeat stale / already dead) plus two
// leased tasks owned by the stale worker and one leased task on a
// healthy worker (which must NOT be touched). Then assert:
//
// - the fresh worker is left active
// - the silent worker flips to "dead"
// - the already-dead worker is left dead (idempotent)
// - the silent worker's leased tasks are reset to queued
// - the healthy worker's leased task is untouched
func TestReapDeadWorkers_SkipsHealthyAndKillsSilent(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
healthy := &models.WorkerNode{
WorkerID: "w-healthy",
RegionCode: "test",
Status: "active",
AuthToken: "tok-healthy",
LastSeen: timePtr(time.Now()),
}
stale := &models.WorkerNode{
WorkerID: "w-stale",
RegionCode: "test",
Status: "active",
AuthToken: "tok-stale",
LastSeen: timePtr(time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)),
}
alreadyDead := &models.WorkerNode{
WorkerID: "w-dead",
RegionCode: "test",
Status: "dead",
AuthToken: "tok-dead",
LastSeen: timePtr(time.Now().Add(-time.Hour)),
}
require.NoError(t, models.DB().Create(healthy).Error)
require.NoError(t, models.DB().Create(stale).Error)
require.NoError(t, models.DB().Create(alreadyDead).Error)
// Two leased tasks on the stale worker — both must come back.
task1 := mustLeaseTask(t, stale.WorkerID, "test-acct")
task2 := mustLeaseTask(t, stale.WorkerID, "test-acct")
// One leased task on the healthy worker — must stay leased.
healthyTask := mustLeaseTask(t, healthy.WorkerID, "test-acct")
// Already-dead worker with a leased task — not part of THIS reaper's
// reaping set (it would only be touched by a fresh reaper pass), so
// leave it leased to demonstrate that we don't accidentally reassign
// from prior-dead leases too.
deadPriorTask := mustLeaseTask(t, alreadyDead.WorkerID, "test-acct")
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 1, reaped, "only the stale worker should flip (already-dead is left untouched)")
// 3 tasks come back: the 2 leased by the stale worker (just flipped
// to dead) + the 1 leased by the prior-dead worker, which had never
// been cleaned up because no previous reaper ran. The reaper matches
// dead workers by status, so any leased task on a dead worker is a
// stranded lease and must be returned to the pool regardless of when
// the worker flipped.
assert.Equal(t, 3, reassigned)
var healthyRow, staleRow, deadRow models.WorkerNode
require.NoError(t, models.DB().First(&healthyRow, healthy.ID).Error)
assert.Equal(t, "active", healthyRow.Status, "healthy worker must stay active")
require.NoError(t, models.DB().First(&staleRow, stale.ID).Error)
assert.Equal(t, "dead", staleRow.Status, "stale worker should flip to dead")
require.NoError(t, models.DB().First(&deadRow, alreadyDead.ID).Error)
assert.Equal(t, "dead", deadRow.Status, "already-dead worker should not be touched")
got := func(id int64) models.Task {
var row models.Task
require.NoError(t, models.DB().First(&row, id).Error)
return row
}
t1 := got(task1.ID)
assert.Equal(t, models.TaskStateQueued, t1.State, "stale worker task must come back to queued")
assert.Empty(t, t1.LeaseOwner)
assert.Nil(t, t1.LeaseExpiresAt)
t2 := got(task2.ID)
assert.Equal(t, models.TaskStateQueued, t2.State)
assert.Empty(t, t2.LeaseOwner)
assert.Nil(t, t2.LeaseExpiresAt)
ht := got(healthyTask.ID)
assert.Equal(t, models.TaskStateLeased, ht.State, "healthy worker's lease must be untouched")
assert.Equal(t, healthy.WorkerID, ht.LeaseOwner)
dt := got(deadPriorTask.ID)
assert.Equal(t, models.TaskStateQueued, dt.State,
"prior-dead task must also be returned to the pool — any leased task on a dead worker is a stranded lease")
assert.Empty(t, dt.LeaseOwner)
}
// TestReapDeadWorkers_NoOpOnHealthyFleet verifies the cheap path: when
// no workers are stale the reaper returns (0, 0, nil) without doing any
// work. Mirrors the "reaped = int(r.RowsAffected)" branch in
// ReapExpiredTasks.
func TestReapDeadWorkers_NoOpOnHealthyFleet(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "w-only-healthy",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: timePtr(time.Now()),
}
require.NoError(t, models.DB().Create(w).Error)
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 0, reaped)
assert.Equal(t, 0, reassigned)
var row models.WorkerNode
require.NoError(t, models.DB().First(&row, w.ID).Error)
assert.Equal(t, "active", row.Status)
}
// TestReapDeadWorkers_OnlyReassignsLeasedNotOthers ensures that the
// reaper does not touch queued/succeeded/failed_retry tasks on the dead
// worker — only leased ones need to be returned to the queue. Tasks in
// other states either belong to no one (queued) or are terminal/semi-
// terminal and have their own audit trail.
func TestReapDeadWorkers_OnlyReassignsLeasedNotOthers(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
now := time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)
w := &models.WorkerNode{
WorkerID: "w-mix",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: &now,
}
require.NoError(t, models.DB().Create(w).Error)
leased := mustLeaseTask(t, w.WorkerID, "acct-mix")
succeeded := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateSucceeded)
failedRetry := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedRetry)
failedPerm := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedPerm)
otherWorker := mustLeaseTask(t, "w-other", "acct-mix")
reaped, reassigned, err := models.ReapDeadWorkers()
require.NoError(t, err)
assert.Equal(t, 1, reaped)
assert.Equal(t, 1, reassigned, "exactly the one leased task on the dead worker")
got := func(id int64) string {
var row models.Task
require.NoError(t, models.DB().First(&row, id).Error)
return row.State
}
assert.Equal(t, models.TaskStateQueued, got(leased.ID))
assert.Equal(t, models.TaskStateSucceeded, got(succeeded.ID), "succeeded must not move")
assert.Equal(t, models.TaskStateFailedRetry, got(failedRetry.ID), "failed_retry must not move")
assert.Equal(t, models.TaskStateFailedPerm, got(failedPerm.ID), "failed_perm must not move")
assert.Equal(t, models.TaskStateLeased, got(otherWorker.ID),
"tasks leased by another worker must not move")
}
// TestStartDeadWorkerReaper_TickerFiresOnce is a smoke test for the
// background helper: spin up the reaper with a tight 10ms ticker and
// a cancellable context, wait for the first tick, then cancel the
// context so the goroutine exits cleanly without leaking. Mirrors the
// shape of how main.init() uses StartTaskReaper (it can't be torn
// down, but for tests we always pass a cancellable context).
func TestStartDeadWorkerReaper_TickerFiresOnce(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "w-ticker",
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
LastSeen: timePtr(time.Now().Add(-2 * models.DeadWorkerHeartbeatTimeout)),
}
require.NoError(t, models.DB().Create(w).Error)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
models.StartDeadWorkerReaper(ctx, 10*time.Millisecond)
deadline := time.Now().Add(2 * time.Second)
var got models.WorkerNode
for time.Now().Before(deadline) {
require.NoError(t, models.DB().First(&got, w.ID).Error)
if got.Status == "dead" {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("reaper goroutine did not flip worker to dead within 2s; last status=%q", got.Status)
}
// ---------------------------------------------------------------------------
// test helpers
// ---------------------------------------------------------------------------
func timePtr(t time.Time) *time.Time { return &t }
// mustLeaseTask inserts a leased Task row owned by workerID. The state
// is the only field that matters for reaper tests; payload/idempotency
// are stubs.
func mustLeaseTask(t *testing.T, workerID, accountLabel string) models.Task {
t.Helper()
return mustInsertTask(t, workerID, accountLabel, models.TaskStateLeased)
}
func mustInsertTask(t *testing.T, workerID, accountLabel string, state string) models.Task {
t.Helper()
leaseUntil := time.Now().Add(time.Hour)
stamp := time.Now().UnixNano()
jobID := fmt.Sprintf("reap-%s-%s-%d", state, workerID, stamp)
// Cap at 64 chars (tasks.job_id VARCHAR(64)). The components above
// already stay under the limit because mustLeaseTask keeps workerID
// short ("w-mix", "w-stale", …) and state is bounded.
if len(jobID) > 64 {
jobID = jobID[:64]
}
idemp := fmt.Sprintf("reap:%s:%d", workerID, stamp)
if len(idemp) > 255 {
idemp = idemp[:255]
}
task := models.Task{
JobID: jobID,
Kind: models.TaskKindNotification,
State: state,
AccountID: 1,
Payload: datatypes.JSON([]byte(`{"method":"email"}`)),
NotBefore: time.Now().Add(-time.Minute),
LeaseOwner: workerID,
LeaseExpiresAt: &leaseUntil,
Attempts: 1,
MaxAttempts: 5,
IdempotencyKey: idemp,
}
if state != models.TaskStateLeased {
task.LeaseOwner = ""
task.LeaseExpiresAt = nil
}
require.NoError(t, models.DB().Create(&task).Error)
require.NotZero(t, task.ID)
return task
}

582
app/models/deletion.go Обычный файл
Просмотреть файл

@@ -0,0 +1,582 @@
package models
import (
"errors"
"log"
"time"
"gorm.io/gorm"
)
// OnUserCacheInvalidate is called when a user is mutated in a way that
// invalidates the cached representation in auth/cache.go. It is wired up
// from the auth package during init() to avoid an import cycle.
var OnUserCacheInvalidate func(userID int64)
// DeletionGracePeriod is the time between a user requesting account deletion
// and the scheduled hard-delete. During this period the user can cancel the
// request and monitoring is paused for all of the user's accounts.
const DeletionGracePeriod = 7 * 24 * time.Hour
// RequestUserDeletion sets the user's DeletionRequestedAt to now,
// starting the 7-day grace period.
func RequestUserDeletion(user *User) error {
if user == nil {
return errors.New("nil user")
}
if user.DeletionPending() {
return nil
}
now := time.Now()
user.DeletionRequestedAt = &now
if err := DB().Save(user).Error; err != nil {
return err
}
if OnUserCacheInvalidate != nil {
OnUserCacheInvalidate(user.ID)
}
log.Printf("user %d requested account deletion (grace until %s)", user.ID, now.Add(DeletionGracePeriod))
return nil
}
// CancelUserDeletion clears the user's DeletionRequestedAt field,
// canceling the pending account deletion.
func CancelUserDeletion(user *User) error {
if user == nil {
return errors.New("nil user")
}
if !user.DeletionPending() {
return nil
}
user.DeletionRequestedAt = nil
if err := DB().Save(user).Error; err != nil {
return err
}
if OnUserCacheInvalidate != nil {
OnUserCacheInvalidate(user.ID)
}
log.Printf("user %d canceled account deletion", user.ID)
return nil
}
// HardDeleteAccount removes an account and all of its data (monitors, checks,
// events, messages, contacts, groups, notifications, etc.) in a single
// transaction. Accesses and invites pointing at the account are also cleaned
// up. Users themselves are kept (they may belong to other accounts). This is
// the admin "purge" path used to remove spam/error accounts immediately.
func HardDeleteAccount(accountID int64) error {
return DB().Transaction(func(tx *gorm.DB) error {
// Remove accesses pointing at this account first (FK constraint).
if err := tx.Where("account_id = ?", accountID).Delete(&Access{}).Error; err != nil {
return err
}
// Remove invites for this account.
if err := tx.Where("account_id = ?", accountID).Delete(&Invite{}).Error; err != nil {
return err
}
// Delete all account data and the account row itself.
return deleteAccountData(tx, accountID)
})
}
// HardDeleteUser removes the user and all of their owned data: accesses,
// contacts, accounts, monitors, checks, events and messages. The deletion
// is wrapped in a transaction to make sure partial failures don't leave
// the database in a broken state.
func HardDeleteUser(userID int64) error {
return DB().Transaction(func(tx *gorm.DB) error {
user := User{}
if err := tx.First(&user, userID).Error; err != nil {
return err
}
// 1) Find all accounts the user has any access to.
accesses := make([]Access, 0)
if err := tx.Where("user_id = ?", userID).Find(&accesses).Error; err != nil {
return err
}
accountIDs := make([]int64, 0, len(accesses))
for i := range accesses {
accountIDs = append(accountIDs, accesses[i].AccountID)
}
// 2) Find all contacts that belong to those accounts or directly to the user.
if err := tx.Where("user_id = ?", userID).Delete(&Contact{}).Error; err != nil {
return err
}
// 3) For each account: delete the related data, then the account itself.
for _, accountID := range accountIDs {
if err := deleteAccountData(tx, accountID); err != nil {
return err
}
}
// 4) Remove accesses.
if err := tx.Where("user_id = ?", userID).Delete(&Access{}).Error; err != nil {
return err
}
// 5) Remove invites issued by or for this user.
if err := tx.Exec("DELETE FROM invites WHERE invitee_id = ? OR account_id IN (?)",
userID, accountIDs).Error; err != nil {
return err
}
// 6) Remove auth identities (password, social).
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
return err
}
// 7) Remove api keys owned by the user.
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
return err
}
// 8) Remove sessions.
if err := tx.Exec("DELETE FROM remember_tokens WHERE user_id = ?", userID).Error; err != nil {
return err
}
// 9) Finally remove the user row.
if err := tx.Delete(&user).Error; err != nil {
return err
}
log.Printf("user %d hard-deleted (cascade accounts=%v)", userID, accountIDs)
return nil
})
}
// deleteAccountData removes everything associated with a single account:
// contacts, monitors, checks, events, messages, notifications, groups and
// the account itself. The user accesses are removed separately.
func deleteAccountData(tx *gorm.DB, accountID int64) error {
// All contacts that reference this account (both account-scoped ones
// with user_id IS NULL and per-user ones created in
// CreateAccountForUser that set both account_id and user_id).
// User-only contacts (account_id IS NULL) are independent of the
// account and survive the deletion. contacts.account_id has a FK
// to accounts(id) without ON DELETE CASCADE, so all referencing
// rows must be removed before the account row goes away.
contactIDs := make([]int64, 0)
if err := tx.Model(&Contact{}).Where("account_id = ?", accountID).Pluck("id", &contactIDs).Error; err != nil {
return err
}
notificationIDs := make([]int64, 0)
if err := tx.Model(&Notification{}).Where("account_id = ?", accountID).Pluck("id", &notificationIDs).Error; err != nil {
return err
}
if len(contactIDs) > 0 {
if err := tx.Exec("DELETE FROM notification_contacts WHERE contact_id IN (?)", contactIDs).Error; err != nil {
return err
}
}
if len(notificationIDs) > 0 {
if err := tx.Exec("DELETE FROM notification_contacts WHERE notification_id IN (?)", notificationIDs).Error; err != nil {
return err
}
}
messageQuery := tx.Model(&Message{})
switch {
case len(contactIDs) > 0 && len(notificationIDs) > 0:
messageQuery = messageQuery.Where("contact_id IN (?) OR notification_id IN (?)", contactIDs, notificationIDs)
case len(contactIDs) > 0:
messageQuery = messageQuery.Where("contact_id IN (?)", contactIDs)
case len(notificationIDs) > 0:
messageQuery = messageQuery.Where("notification_id IN (?)", notificationIDs)
default:
messageQuery = nil
}
if messageQuery != nil {
messageIDs := make([]int64, 0)
if err := messageQuery.Pluck("id", &messageIDs).Error; err != nil {
return err
}
if len(messageIDs) > 0 {
if err := tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error; err != nil {
return err
}
if err := tx.Where("id IN (?)", messageIDs).Delete(&Message{}).Error; err != nil {
return err
}
}
}
if err := tx.Where("account_id = ?", accountID).Delete(&Contact{}).Error; err != nil {
return err
}
// Monitors (and their checks/events/messages via cascade below).
monitors := make([]Monitor, 0)
if err := tx.Joins("JOIN groups ON monitors.group_id = groups.id").
Where("groups.account_id = ?", accountID).Find(&monitors).Error; err != nil {
return err
}
monitorIDs := make([]int64, 0, len(monitors))
for i := range monitors {
monitorIDs = append(monitorIDs, monitors[i].ID)
}
if len(monitorIDs) > 0 {
// Checks
checks := make([]Check, 0)
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&checks).Error; err != nil {
return err
}
checkIDs := make([]int64, 0, len(checks))
for i := range checks {
checkIDs = append(checkIDs, checks[i].ID)
}
// Events
events := make([]Event, 0)
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&events).Error; err != nil {
return err
}
eventIDs := make([]int64, 0, len(events))
for i := range events {
eventIDs = append(eventIDs, events[i].ID)
}
// Join tables first to avoid FK violations
if len(checkIDs) > 0 {
if err := tx.Exec("DELETE FROM event_checks WHERE check_id IN (?)", checkIDs).Error; err != nil {
return err
}
// check_region_results.check_id has a FK to checks(id) without
// ON DELETE CASCADE, so it must be purged before checks go away.
if err := tx.Exec("DELETE FROM check_region_results WHERE check_id IN (?)", checkIDs).Error; err != nil {
return err
}
}
if len(eventIDs) > 0 {
if err := tx.Exec("DELETE FROM event_messages WHERE event_id IN (?)", eventIDs).Error; err != nil {
return err
}
}
if len(checkIDs) > 0 {
if err := tx.Where("id IN (?)", checkIDs).Delete(&Check{}).Error; err != nil {
return err
}
}
if len(eventIDs) > 0 {
if err := tx.Where("id IN (?)", eventIDs).Delete(&Event{}).Error; err != nil {
return err
}
}
// DNS records
if err := tx.Where("monitor_id IN (?)", monitorIDs).Delete(&DNSRecord{}).Error; err != nil {
return err
}
// Monitors themselves
if err := tx.Where("id IN (?)", monitorIDs).Delete(&Monitor{}).Error; err != nil {
return err
}
}
// Notification <-> group links
if err := tx.Exec(
"DELETE FROM notification_groups WHERE group_id IN (SELECT id FROM groups WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
// Notifications
if err := tx.Where("account_id = ?", accountID).Delete(&Notification{}).Error; err != nil {
return err
}
// LLMs scoped to this account. worker_llms.llm_id has a FK to llms(id)
// without ON DELETE CASCADE, so the join rows must go first.
if err := tx.Exec(
"DELETE FROM worker_llms WHERE llm_id IN (SELECT id FROM llms WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&LLM{}).Error; err != nil {
return err
}
// Groups
if err := tx.Where("account_id = ?", accountID).Delete(&Group{}).Error; err != nil {
return err
}
// Inventory entities (docs/plans/inventory-management.md §6). All
// four tables hold account_id FKs to accounts(id) without ON
// DELETE CASCADE, so we wipe them in dependency order: deployments
// and domains first (both FK into sites and servers), then
// server_ips, then sites, then servers. Anything the account does
// not own is left alone (e.g. shared infra servers are filtered
// by account_id and survive).
if err := tx.Where("account_id = ?", accountID).Delete(&Deployment{}).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Domain{}).Error; err != nil {
return err
}
if err := tx.Exec(
"DELETE FROM server_ips WHERE server_id IN (SELECT id FROM servers WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Site{}).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&Server{}).Error; err != nil {
return err
}
// Private workers scoped to this account (NULL account_id workers
// are platform-operated and survive account deletion). The
// worker_llms and check_region_results joins both FK to
// worker_nodes(id) without ON DELETE CASCADE, so the join rows
// must be cleared before the worker rows go away.
if err := tx.Exec(
"DELETE FROM worker_llms WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Exec(
"DELETE FROM check_region_results WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
accountID,
).Error; err != nil {
return err
}
if err := tx.Where("account_id = ?", accountID).Delete(&WorkerNode{}).Error; err != nil {
return err
}
// The account itself
return tx.Delete(&Account{}, accountID).Error
}
// ProcessPendingDeletions hard-deletes users whose 7-day grace period has
// elapsed. Designed to be called from a periodic scheduler.
func ProcessPendingDeletions() (int, error) {
cutoff := time.Now().Add(-DeletionGracePeriod)
users := make([]User, 0)
if err := DB().Where("deletion_requested_at IS NOT NULL AND deletion_requested_at < ?", cutoff).
Find(&users).Error; err != nil {
return 0, err
}
deleted := 0
for i := range users {
if err := HardDeleteUser(users[i].ID); err != nil {
log.Printf("ProcessPendingDeletions: failed to delete user %d: %v", users[i].ID, err)
continue
}
deleted++
}
if deleted > 0 {
log.Printf("ProcessPendingDeletions: hard-deleted %d user(s)", deleted)
}
return deleted, nil
}
// StaleAccountInactivity is the minimum inactivity window before a stale
// account becomes eligible for admin cleanup. Picked at 3 months per the
// /admin/accounts "Удалить старые" button.
const StaleAccountInactivity = 90 * 24 * time.Hour
// StaleAccountCandidate describes an account that matched the
// admin-cleanup eligibility filter but has not yet been deleted. The
// snapshot is what the frontend shows in the confirmation dialog.
type StaleAccountCandidate struct {
AccountID int64 `json:"account_id"`
AccountName string `json:"account_name"`
UserID int64 `json:"user_id"`
UserEmail *string `json:"user_email"`
LastActiveAt *time.Time `json:"last_active_at"`
}
// FindStaleAccounts returns the accounts that are eligible for the
// admin "Удалить старые" cleanup:
//
// - the account has zero monitors configured (via group.account_id)
// - every user with access to the account has exactly one account
// membership total (so removing the account also orphans them)
// - every such user's last_active_at is older than
// StaleAccountInactivity (3 months). Users who have never logged in
// (last_active_at IS NULL) are also eligible.
func FindStaleAccounts() ([]StaleAccountCandidate, error) {
cutoff := time.Now().Add(-StaleAccountInactivity)
// Step 1: account IDs that have zero monitors.
accountsWithMonitors := make([]int64, 0)
if err := DB().
Table("monitors").
Select("DISTINCT groups.account_id").
Joins("JOIN groups ON groups.id = monitors.group_id").
Scan(&accountsWithMonitors).Error; err != nil {
return nil, err
}
accounts := make([]Account, 0)
q := DB().Order("id ASC")
if len(accountsWithMonitors) > 0 {
q = q.Where("id NOT IN (?)", accountsWithMonitors)
}
if err := q.Find(&accounts).Error; err != nil {
return nil, err
}
out := make([]StaleAccountCandidate, 0, len(accounts))
for i := range accounts {
acc := accounts[i]
// Step 2: every user with access to this account must have
// exactly one account membership total.
type userAccessCount struct {
UserID int64
Cnt int
}
counts := make([]userAccessCount, 0)
err := DB().
Table("accesses AS a1").
Select("a1.user_id AS user_id, (SELECT COUNT(*) FROM accesses AS a2 WHERE a2.user_id = a1.user_id) AS cnt").
Where("a1.account_id = ? AND a1.user_id IS NOT NULL", acc.ID).
Group("a1.user_id").
Scan(&counts).Error
if err != nil {
return nil, err
}
if len(counts) == 0 {
// An account with no user accesses is a config bug
// (the owner access should always exist). Skip.
continue
}
allSingle := true
for _, c := range counts {
if c.Cnt != 1 {
allSingle = false
break
}
}
if !allSingle {
continue
}
// Step 3: every such user must be inactive beyond the cutoff.
userIDs := make([]int64, 0, len(counts))
for _, c := range counts {
userIDs = append(userIDs, c.UserID)
}
users := make([]User, 0)
if err := DB().Where("id IN (?)", userIDs).Find(&users).Error; err != nil {
return nil, err
}
allStale := true
for j := range users {
u := users[j]
if u.LastActiveAt != nil && u.LastActiveAt.After(cutoff) {
allStale = false
break
}
}
if !allStale {
continue
}
// All gates passed — emit one candidate per user so the UI
// can list which specific accounts would be removed.
for j := range users {
out = append(out, StaleAccountCandidate{
AccountID: acc.ID,
AccountName: acc.Name,
UserID: users[j].ID,
UserEmail: users[j].Email,
LastActiveAt: users[j].LastActiveAt,
})
}
}
return out, nil
}
// CleanupStaleAccounts hard-deletes every account eligible for the
// admin "Удалить старые" sweep. Returns the number of accounts that
// were deleted. The matching users (each of whom only belonged to one
// account) are deleted by HardDeleteAccount's cascading access cleanup
// only if they no longer have any other account — that final teardown
// is done here.
func CleanupStaleAccounts() (int, error) {
candidates, err := FindStaleAccounts()
if err != nil {
return 0, err
}
accountIDs := make([]int64, 0, len(candidates))
seen := make(map[int64]bool, len(candidates))
for _, c := range candidates {
if !seen[c.AccountID] {
seen[c.AccountID] = true
accountIDs = append(accountIDs, c.AccountID)
}
}
deleted := 0
for _, accountID := range accountIDs {
if err := HardDeleteAccount(accountID); err != nil {
log.Printf("CleanupStaleAccounts: failed to delete account %d: %v", accountID, err)
continue
}
deleted++
}
// Users that no longer have any accesses after the cascade are
// clearly orphaned — purge them so the auth layer doesn't keep
// dangling rows around. We bypass HardDeleteUser here because the
// account-level data (monitors, checks, groups, etc.) was already
// removed by the HardDeleteAccount loop above, so only the user
// row and the dangling identities / contacts need cleanup.
if deleted > 0 {
orphans := make([]int64, 0)
err := DB().
Table("users").
Select("users.id").
Joins("LEFT JOIN accesses ON accesses.user_id = users.id").
Where("accesses.id IS NULL").
Pluck("users.id", &orphans).Error
if err != nil {
log.Printf("CleanupStaleAccounts: orphan user scan failed: %v", err)
return deleted, nil
}
for _, userID := range orphans {
tx := DB().Begin()
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: identities delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Where("user_id = ? AND account_id IS NULL", userID).Delete(&Contact{}).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: contacts delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: api_keys delete failed for user %d: %v", userID, err)
continue
}
if err := tx.Delete(&User{}, userID).Error; err != nil {
tx.Rollback()
log.Printf("CleanupStaleAccounts: user delete failed for %d: %v", userID, err)
continue
}
if err := tx.Commit().Error; err != nil {
log.Printf("CleanupStaleAccounts: commit failed for user %d: %v", userID, err)
}
}
}
if deleted > 0 {
log.Printf("CleanupStaleAccounts: hard-deleted %d stale account(s)", deleted)
}
return deleted, nil
}

352
app/models/deletion_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,352 @@
package models_test
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/icrowley/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
var gormErrRecordNotFound = gorm.ErrRecordNotFound
func deletionStringPtr(s string) *string { return &s }
func deletionBoolPtr(b bool) *bool { return &b }
func init() {
database.Init()
}
// TestUserDeletionPending tests the DeletionPending method
func TestUserDeletionPending(t *testing.T) {
t.Run("returns false when no deletion requested", func(t *testing.T) {
user := models.User{}
assert.False(t, user.DeletionPending())
})
t.Run("returns true when deletion requested", func(t *testing.T) {
now := time.Now()
user := models.User{DeletionRequestedAt: &now}
assert.True(t, user.DeletionPending())
})
}
// TestUserAsJSONIncludesDeletionField verifies that the AsJSON output
// contains the deletion_requested_at field so the frontend can react to it.
func TestUserAsJSONIncludesDeletionField(t *testing.T) {
email := "test@example.com"
now := time.Now()
user := models.User{
ID: 42,
Email: &email,
Name: "Test User",
DeletionRequestedAt: &now,
}
result := user.AsJSON()
assert.NotNil(t, result)
assert.Contains(t, result, "deletion_requested_at")
assert.Equal(t, &now, result["deletion_requested_at"])
}
// TestRequestUserDeletionNilUser ensures nil-safety
func TestRequestUserDeletionNilUser(t *testing.T) {
err := models.RequestUserDeletion(nil)
assert.Error(t, err)
}
// TestCancelUserDeletionNilUser ensures nil-safety
func TestCancelUserDeletionNilUser(t *testing.T) {
err := models.CancelUserDeletion(nil)
assert.Error(t, err)
}
// TestCacheInvalidationHook_CalledByRequestDeletion verifies that
// RequestUserDeletion calls OnUserCacheInvalidate with the correct user ID.
func TestCacheInvalidationHook_CalledByRequestDeletion(t *testing.T) {
models.Drop()
models.Migrate()
email := fake.EmailAddress()
user := models.User{Email: &email, Name: "CacheHookTest"}
assert.NoError(t, models.DB().Create(&user).Error)
var calledID int64
originalHook := models.OnUserCacheInvalidate
models.OnUserCacheInvalidate = func(userID int64) {
calledID = userID
}
defer func() { models.OnUserCacheInvalidate = originalHook }()
assert.NoError(t, models.RequestUserDeletion(&user))
assert.Equal(t, user.ID, calledID)
models.DB().Unscoped().Delete(&user)
}
// TestCacheInvalidationHook_CalledByCancelUserDeletion verifies that
// CancelUserDeletion calls OnUserCacheInvalidate with the correct user ID.
func TestCacheInvalidationHook_CalledByCancelUserDeletion(t *testing.T) {
models.Drop()
models.Migrate()
email := fake.EmailAddress()
now := time.Now()
user := models.User{Email: &email, Name: "CacheHookCancel", DeletionRequestedAt: &now}
assert.NoError(t, models.DB().Create(&user).Error)
var calledID int64
originalHook := models.OnUserCacheInvalidate
models.OnUserCacheInvalidate = func(userID int64) {
calledID = userID
}
defer func() { models.OnUserCacheInvalidate = originalHook }()
assert.NoError(t, models.CancelUserDeletion(&user))
assert.Equal(t, user.ID, calledID)
models.DB().Unscoped().Delete(&user)
}
// TestHardDeleteAccount_FKCascadeRegression covers the FK regressions
// that surfaced as a series of distinct SQL errors when an operator
// purged an account that owned monitors with distributed-worker
// activity or per-user contacts:
//
// 1. check_region_results.check_id → checks(id) had no ON DELETE
// CASCADE, so deleting a check while it still had region-result
// rows raised 23503.
// 2. worker_nodes did not have an account_id column, so the
// "Workers scoped to this account" delete raised 42703.
// 3. contacts.account_id → accounts(id) had no ON DELETE CASCADE,
// and the original delete filter required user_id IS NULL, so
// per-user contacts created in CreateAccountForUser (both
// account_id and user_id set) survived and blocked the account
// delete with 23503.
//
// The test seeds an account with a monitor, a check with a region
// result row, an account-scoped LLM, a per-user contact, and both a
// private worker (with account_id) and an operated worker (NULL
// account_id), then runs HardDeleteAccount and asserts that the
// account, the private worker, the LLM, the monitor/check/region
// result, and the per-user contact all disappear, while the operated
// worker and an unrelated user-only contact survive.
func TestHardDeleteAccount_FKCascadeRegression(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "hard-delete-regression"}
require.NoError(t, models.DB().Create(&plan).Error)
account := models.Account{Name: "victim", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&account).Error)
otherAccount := models.Account{Name: "survivor", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&otherAccount).Error)
owner := models.User{Name: "owner", Email: deletionStringPtr("owner@example.com"), Timezone: "UTC"}
require.NoError(t, models.DB().Create(&owner).Error)
stranger := models.User{Name: "stranger", Email: deletionStringPtr("stranger@example.com"), Timezone: "UTC"}
require.NoError(t, models.DB().Create(&stranger).Error)
region := models.Region{}
if err := models.DB().Where("code = ?", "test").First(&region).Error; err != nil {
require.NoError(t, models.DB().Create(&models.Region{Code: "test", Name: "test", Enabled: true}).Error)
}
group := models.Group{Name: "g", AccountID: account.ID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{
Name: deletionStringPtr("m"),
Host: "example.com",
GroupID: group.ID,
Enabled: true,
}
require.NoError(t, models.DB().Create(&monitor).Error)
check := models.Check{
MonitorID: monitor.ID,
Kind: "http",
Interval: 60,
Settings: datatypes.JSON([]byte(`{}`)),
Enabled: deletionBoolPtr(true),
State: "UNK",
}
require.NoError(t, models.DB().Create(&check).Error)
// Per-user contact that references both the victim account and a
// user. CreateAccountForUser writes a contact in this shape, and
// the old "user_id IS NULL" filter would let it survive and block
// the account delete with contacts_account_id_fkey 23503.
ownerContact := models.Contact{
AccountID: &account.ID,
UserID: &owner.ID,
Name: "owner-email",
Kind: "email",
Value: "owner@example.com",
}
require.NoError(t, models.DB().Create(&ownerContact).Error)
// Account-only contact (user_id IS NULL) — also tied to the account
// via FK and must be removed.
accountContact := models.Contact{
AccountID: &account.ID,
Name: "ops",
Kind: "email",
Value: "ops@example.com",
}
require.NoError(t, models.DB().Create(&accountContact).Error)
notification := models.Notification{AccountID: account.ID, Name: "alerts", Enabled: true}
require.NoError(t, models.DB().Create(&notification).Error)
require.NoError(t, models.DB().Model(&notification).Association("Contacts").Append(&accountContact))
message := models.Message{
NotificationID: notification.ID,
ContactID: accountContact.ID,
Kind: "test",
State: "OK",
CreatedAt: time.Now(),
SentAt: time.Now(),
}
require.NoError(t, models.DB().Create(&message).Error)
// User-only contact on a stranger — must survive account deletion.
userOnlyContact := models.Contact{
UserID: &stranger.ID,
Name: "stranger",
Kind: "email",
Value: "stranger@example.com",
}
require.NoError(t, models.DB().Create(&userOnlyContact).Error)
// Region result row: this is the row that used to trigger the
// fk_check_region_results_check FK violation when Check was
// deleted. Without the fix the entire HardDeleteAccount would
// fail here.
privateWorker := &models.WorkerNode{
WorkerID: "private-" + uuid.New().String(),
RegionCode: "test",
Status: "active",
AuthToken: "priv-tok-" + uuid.New().String(),
AccountID: &account.ID,
}
require.NoError(t, models.DB().Create(privateWorker).Error)
operatedWorker := &models.WorkerNode{
WorkerID: "operated-" + uuid.New().String(),
RegionCode: "test",
Status: "active",
AuthToken: "op-tok-" + uuid.New().String(),
AccountID: nil,
}
require.NoError(t, models.DB().Create(operatedWorker).Error)
require.NoError(t, models.DB().Create(&models.CheckRegionResult{
CheckID: check.ID,
RegionCode: "test",
WorkerNodeID: &privateWorker.ID,
ExecutedAt: time.Now(),
State: "OK",
}).Error)
// LLM scoped to the victim account + linked to the private worker.
// worker_llms.llm_id has a FK to llms(id) without ON DELETE
// CASCADE, so the join row used to block LLM deletion too.
llm := models.LLM{
AccountID: &account.ID,
Name: "private-llm",
URL: "https://llm.example.com",
ModelName: "gpt-test",
APIKey: "secret",
Kind: "openai",
}
require.NoError(t, models.DB().Create(&llm).Error)
require.NoError(t, models.DB().Exec(
"INSERT INTO worker_llms (worker_node_id, llm_id) VALUES (?, ?)",
privateWorker.ID, llm.ID,
).Error)
// Inventory entities scoped to the victim account. Each has an
// account_id FK to accounts(id) without ON DELETE CASCADE so they
// must be removed before the account row goes away. The
// shared-infra server belongs to another account and must
// survive.
victimServer := models.Server{Name: "victim-srv", AccountID: account.ID}
require.NoError(t, models.DB().Create(&victimServer).Error)
victimServerIP := models.ServerIp{ServerID: victimServer.ID, Address: "10.0.0.1"}
require.NoError(t, models.DB().Create(&victimServerIP).Error)
victimSite := models.Site{
AccountID: account.ID, ServerID: &victimServer.ID,
Slug: "victim-site", Name: "victim-site", Kind: "production", IsActive: true,
}
require.NoError(t, models.DB().Create(&victimSite).Error)
victimDeployment := models.Deployment{
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
Kind: "production", Mode: "compose",
}
require.NoError(t, models.DB().Create(&victimDeployment).Error)
victimDomain := models.Domain{
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
Name: "victim.example.com",
}
require.NoError(t, models.DB().Create(&victimDomain).Error)
otherServer := models.Server{Name: "shared-srv", AccountID: otherAccount.ID}
require.NoError(t, models.DB().Create(&otherServer).Error)
require.NoError(t, models.HardDeleteAccount(account.ID))
// Account and all account-scoped rows must be gone.
assert.ErrorIs(t, models.DB().First(&models.Account{}, account.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Check{}, check.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Monitor{}, monitor.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Group{}, group.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Notification{}, notification.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Message{}, message.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.LLM{}, llm.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Contact{}, ownerContact.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Contact{}, accountContact.ID).Error, gormErrRecordNotFound)
var notificationContactCount int64
require.NoError(t, models.DB().Table("notification_contacts").
Where("notification_id = ? OR contact_id = ?", notification.ID, accountContact.ID).
Count(&notificationContactCount).Error)
assert.Zero(t, notificationContactCount)
// Private worker must be gone; operated worker must survive.
assert.ErrorIs(t, models.DB().First(&models.WorkerNode{}, privateWorker.ID).Error, gormErrRecordNotFound)
var stillOperated models.WorkerNode
require.NoError(t, models.DB().First(&stillOperated, operatedWorker.ID).Error)
assert.Nil(t, stillOperated.AccountID, "operated worker account_id must remain NULL")
// User-only contact (no account_id) must survive.
var stillUserOnly models.Contact
require.NoError(t, models.DB().First(&stillUserOnly, userOnlyContact.ID).Error)
// Inventory entities scoped to the account must be gone.
assert.ErrorIs(t, models.DB().First(&models.Server{}, victimServer.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Site{}, victimSite.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Deployment{}, victimDeployment.ID).Error, gormErrRecordNotFound)
assert.ErrorIs(t, models.DB().First(&models.Domain{}, victimDomain.ID).Error, gormErrRecordNotFound)
var leftover int64
require.NoError(t, models.DB().Model(&models.ServerIp{}).
Where("server_id = ?", victimServer.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "server_ips for the deleted server must be cleaned up")
// Other-account inventory must survive.
var stillOtherServer models.Server
require.NoError(t, models.DB().First(&stillOtherServer, otherServer.ID).Error)
// Region-result rows for the deleted check must be gone.
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("check_id = ?", check.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "check_region_results must be cleaned up before checks")
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
Where("worker_node_id = ?", privateWorker.ID).Count(&leftover).Error)
assert.Zero(t, leftover, "check_region_results referencing a private worker must be cleaned up")
require.NoError(t, models.DB().Raw(
"SELECT COUNT(*) FROM worker_llms WHERE llm_id = ? OR worker_node_id = ?",
llm.ID, privateWorker.ID,
).Scan(&leftover).Error)
assert.Zero(t, leftover, "worker_llms rows for the deleted LLM and private worker must be gone")
// Sanity: the other account and its data are untouched.
var stillOther models.Account
require.NoError(t, models.DB().First(&stillOther, otherAccount.ID).Error)
}

240
app/models/deployment.go Обычный файл
Просмотреть файл

@@ -0,0 +1,240 @@
package models
import (
"database/sql/driver"
"fmt"
"time"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DeploymentKind is the rstuff-mirrored lifecycle label for a
// deployment (production / production_prev / production_next /
// internal / staging / old). See
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable numeric
// mapping (which Postgres stores alphabetically, not numerically).
type DeploymentKind string
// Lifecycle labels for a Deployment row. See DeploymentKind for
// the matching rstuff enum. The label set is closed; new values
// require adding a Postgres enum value via app/models/migrate.go.
const (
// DeploymentKindProduction is the customer-facing "live" deployment.
DeploymentKindProduction DeploymentKind = "production"
DeploymentKindProductionPrev DeploymentKind = "production_prev"
DeploymentKindProductionNext DeploymentKind = "production_next"
// DeploymentKindInternal is for ops/admin tooling (not customer-facing).
DeploymentKindInternal DeploymentKind = "internal"
DeploymentKindStaging DeploymentKind = "staging"
DeploymentKindOld DeploymentKind = "old"
)
// DeploymentMode is the host-side lifecycle label (kubernetes /
// compose / dedicated / vds / user). `dedicated` covers a single
// nginx vhost; `compose` covers a Docker Compose project.
type DeploymentMode string
// DeploymentMode values map onto rstuff's Deployment.mode enum.
// `dedicated` covers a single nginx vhost; `compose` covers a
// Docker Compose project; the others are reserved for future
// v2 surfaces (Kubernetes, VDS, user-owned).
const (
DeploymentModeKubernetes DeploymentMode = "kubernetes"
DeploymentModeCompose DeploymentMode = "compose"
DeploymentModeDedicated DeploymentMode = "dedicated"
DeploymentModeVDS DeploymentMode = "vds"
DeploymentModeUser DeploymentMode = "user"
)
// DeploymentAction is the reconciliation state. Updated by the
// deploymentd receiver on every POST and by the 90s reconcile sweep
// (see app/models/deployment.go ReconcileMissing).
type DeploymentAction string
// DeploymentAction values. Pending/PendingMove/PendingDrop are
// transient (operator or receiver-initiated); Deleted/Missing are
// sticky until the deployment shows up again on a future POST.
const (
DeploymentActionOk DeploymentAction = "ok"
DeploymentActionPending DeploymentAction = "pending"
DeploymentActionPendingMove DeploymentAction = "pending_move"
DeploymentActionPendingDrop DeploymentAction = "pending_drop"
DeploymentActionDeleted DeploymentAction = "deleted"
DeploymentActionMissing DeploymentAction = "missing"
)
// Deployment represents a single host-side binding: one nginx
// vhost, one Docker Compose service, or one Kubernetes service. The
// shape mirrors rstuff's `deployments` table. See
// docs/plans/inventory-management.md §4 / §6.1.
type Deployment struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
ServiceName *string `gorm:"size:120" json:"service_name,omitempty"`
Kind DeploymentKind `gorm:"type:deployment_kind;not null;default:'production'" json:"kind"`
Mode DeploymentMode `gorm:"type:deployment_mode;not null;default:'dedicated'" json:"mode"`
Action DeploymentAction `gorm:"type:deployment_action;not null;default:'ok'" json:"action"`
URL *string `gorm:"type:text" json:"url,omitempty"`
SSHUser *string `gorm:"size:64" json:"ssh_user,omitempty"`
RootPath *string `gorm:"type:text" json:"root_path,omitempty"`
ConfigPath *string `gorm:"type:text" json:"config_path,omitempty"`
IP *string `gorm:"type:inet" json:"ip,omitempty"`
Listen pq.StringArray `gorm:"type:varchar(64)[];not null;default:'{}'" json:"listen"`
ServerName pq.StringArray `gorm:"type:varchar(255)[];not null;default:'{}'" json:"server_name"`
Auth bool `gorm:"not null;default:false" json:"auth"`
IsProxied bool `gorm:"not null;default:false" json:"is_proxied"`
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
concerns.Timestamped
Audited
}
// TableName provides functionality.
func (Deployment) TableName() string { return "deployments" }
// Scan implements sql.Scanner so a Postgres enum value can land in
// our typed string alias without a code-generation step.
func (d *DeploymentKind) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentKind(v)
case []byte:
*d = DeploymentKind(string(v))
default:
return fmt.Errorf("deployment_kind: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for the inverse direction.
func (d DeploymentKind) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// Scan implements sql.Scanner for DeploymentMode.
func (d *DeploymentMode) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentMode(v)
case []byte:
*d = DeploymentMode(string(v))
default:
return fmt.Errorf("deployment_mode: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for DeploymentMode.
func (d DeploymentMode) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// Scan implements sql.Scanner for DeploymentAction.
func (d *DeploymentAction) Scan(src any) error {
if src == nil {
*d = ""
return nil
}
switch v := src.(type) {
case string:
*d = DeploymentAction(v)
case []byte:
*d = DeploymentAction(string(v))
default:
return fmt.Errorf("deployment_action: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for DeploymentAction.
func (d DeploymentAction) Value() (driver.Value, error) {
if d == "" {
return nil, nil
}
return string(d), nil
}
// ReconcileMissingDeployments flips action='missing' on every Deployment for the
// given server whose last_seen_at is older than cutoff. Called by
// the deploymentd receiver after every successful upsert so the
// "missing" badge appears within one POST cycle.
//
// Idempotent: re-running with the same cutoff is a no-op.
func ReconcileMissingDeployments(serverID int64, mode DeploymentMode, cutoff time.Time) (int64, error) {
res := DB().Model(&Deployment{}).
Where("server_id = ? AND mode = ? AND action NOT IN ?", serverID, mode,
[]DeploymentAction{DeploymentActionDeleted, DeploymentActionMissing, DeploymentActionPendingDrop}).
Where("last_seen_at IS NULL OR last_seen_at < ?", cutoff).
Update("action", DeploymentActionMissing)
return res.RowsAffected, res.Error
}
// UpsertNginxDeployment finds or creates a Deployment by
// (server_id, config_path) for an nginx vhost. The caller fills in
// the lifecycle fields (listen, server_name, etc.) after the upsert
// returns. The return value is the row to mutate; the caller MUST
// also touch last_seen_at and save.
func UpsertNginxDeployment(tx *gorm.DB, accountID int64, serverID int64, configPath string) (*Deployment, error) {
if tx == nil {
tx = DB()
}
var d Deployment
err := tx.Where("server_id = ? AND config_path = ?", serverID, configPath).First(&d).Error
if err == nil {
return &d, nil
}
if err != gorm.ErrRecordNotFound {
return nil, err
}
d = Deployment{
AccountID: accountID,
ServerID: &serverID,
Kind: DeploymentKindProduction,
Mode: DeploymentModeDedicated,
Action: DeploymentActionOk,
ConfigPath: &configPath,
}
if err := tx.Create(&d).Error; err != nil {
return nil, err
}
return &d, nil
}
// RotateServerToken sets a new random token for a server and returns
// the plaintext. Called by the operator-only
// POST /api/v1/servers/:id/rotate-token endpoint. The plaintext is
// returned exactly once — it is not stored anywhere recoverable.
func RotateServerToken(tx *gorm.DB, serverID int64, newToken string) error {
if tx == nil {
tx = DB()
}
return tx.Model(&Server{}).Where("id = ?", serverID).Update("token", newToken).Error
}

80
app/models/dns_record.go Обычный файл
Просмотреть файл

@@ -0,0 +1,80 @@
package models
import (
"log"
"rsgit.ru/rsmon/rsmon/internal/netaddr"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DNSRecord provides functionality.
type DNSRecord struct {
concerns.Model
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
Monitor *Monitor `json:"-"`
Name string `json:"name"`
Kind string `json:"kind"`
Value netaddr.Inet `json:"value" gorm:"type:bytea;"`
}
// SaveIps provides functionality.
func SaveIps(m *Monitor, _ Check, ips []NSRecord) { //nolint:gocritic // hugeParam: accepted for interface compatibility
// log.Println("save ips for monitor")
// spew.Dump(m)
var err error
tx := DB().Begin()
currentRecords := make([]DNSRecord, 0)
_ = tx.Model(&m).Association("DNSRecords").Find(&currentRecords)
recordHash := make(map[string]DNSRecord, 0)
for _, record := range currentRecords {
recordHash[record.Name] = record
}
nextRecords := make(map[string]bool, 0)
for _, ip := range ips {
if record, ok := recordHash[ip.Name]; ok {
// log.Println("old value:", record)
record.MonitorID = m.ID
record.Value = ip.Value
record.Kind = ip.Kind
err = tx.Model(&m).Association("DNSRecords").Replace(&record)
if err != nil {
log.Println("fatal error in saveips", err)
return
}
} else {
record = DNSRecord{
MonitorID: m.ID,
Name: ip.Name,
Kind: ip.Kind,
Value: ip.Value,
}
if _, ok := nextRecords[ip.Name]; !ok {
nextRecords[ip.Name] = true
err = tx.Model(&m).Association("DNSRecords").Append(&record)
if err != nil {
log.Println("fatal error in saveips", err)
return
}
}
}
}
// spew.Dump(ips_hash)
err = tx.Commit().Error
if err != nil {
log.Println("fatal error in saveips", err)
return
}
}
// NSRecord provides functionality.
type NSRecord struct {
Name string
Kind string
Value netaddr.Inet
}

32
app/models/domain.go Обычный файл
Просмотреть файл

@@ -0,0 +1,32 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Domain represents a customer-facing DNS name. The shape mirrors
// rstuff's `domains` table — see docs/parity/rstuff-inventory.md §2
// and docs/plans/inventory-management.md §4.
//
// Distinct from RknDomain (the RKN blocklist cache, see
// app/models/rkn_domain.go): RknDomain is read-only data about
// blocked domains; Domain is the customer-side name→site/server
// pointer that monitoring reasons about.
type Domain struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
Name string `gorm:"size:255;not null" json:"name"`
Env string `gorm:"size:32;not null;default:'production'" json:"env"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
concerns.Timestamped
}
// TableName provides functionality.
func (Domain) TableName() string { return "domains" }

96
app/models/drop.go Обычный файл
Просмотреть файл

@@ -0,0 +1,96 @@
package models
import (
"fmt"
"strings"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
)
// Drop removes all test data from the database.
// It uses TRUNCATE ... CASCADE for join tables and deletes leaf-to-root for main tables.
// Safe to call from multiple goroutines within a single test binary; concurrent calls
// from separate test binaries are serialized by the advisory lock in Migrate().
func Drop() {
var dbname string
if err := DB().Raw("SELECT current_database()").Scan(&dbname).Error; err != nil {
panic(fmt.Sprintf("Drop: cannot read database name: %v", err))
}
if !strings.Contains(dbname, "test") {
panic(fmt.Sprintf(
"Drop() refused: database '%s' is not a test database. "+
"Set DATABASE_NAME=rsmon_test to run tests safely.", dbname,
))
}
// Truncate many2many join tables first to avoid FK violations.
DB().Exec("TRUNCATE event_checks, event_messages, notification_contacts, notification_groups RESTART IDENTITY CASCADE")
// Delete main tables in dependency order (leaf tables first).
DB().Where("1=1").Delete(&NotificationDelivery{})
DB().Where("1=1").Delete(&TaskReplay{})
DB().Where("1=1").Delete(&TelegramBotMessage{})
DB().Where("1=1").Delete(&TelegramBotStatus{})
DB().Where("1=1").Delete(&Task{})
DB().Where("1=1").Delete(&Access{})
DB().Where("1=1").Delete(&Invite{})
DB().Where("1=1").Delete(&Message{})
DB().Where("1=1").Delete(&Event{})
// Durable network-diagnostic rows reference checks, monitors, and workers.
DB().Where("1=1").Delete(&DiagnosticAuditEvent{})
DB().Where("1=1").Delete(&CheckAttempt{})
// CheckRegionResult FKs check_id; it must be cleared before Check.
DB().Where("1=1").Delete(&CheckRegionResult{})
DB().Where("1=1").Delete(&Check{})
DB().Where("1=1").Delete(&Notification{})
// DNSRecord FKs monitor_id; it must be cleared before Monitor.
DB().Where("1=1").Delete(&DNSRecord{})
DB().Where("1=1").Delete(&Monitor{})
DB().Where("1=1").Delete(&Group{})
DB().Where("1=1").Delete(&Contact{})
DB().Where("1=1").Delete(&NotificationCredential{})
DB().Where("1=1").Delete(&WorkerLogEvent{})
DB().Where("1=1").Delete(&WorkerNode{})
DB().Where("1=1").Delete(&LLM{})
DB().Where("1=1").Delete(&Region{})
// Inventory (docs/plans/inventory-management.md §6): leaf tables
// (sites, deployments, server_ips) reference accounts/servers, so
// they must be cleared before Server is deleted.
DB().Where("1=1").Delete(&Deployment{})
DB().Where("1=1").Delete(&SiteRepo{})
DB().Where("1=1").Delete(&Site{})
DB().Where("1=1").Delete(&Repo{})
DB().Where("1=1").Delete(&ServerIp{})
DB().Where("1=1").Delete(&Domain{})
DB().Where("1=1").Delete(&Server{})
// Account-scoped tag metadata (account_id FK to accounts(id)).
// Cleared before Account so a future cascade change cannot orphan
// rows mid-truncate.
DB().Where("1=1").Delete(&Tag{})
// Status pages (docs/plans/status-pages.md §3.1–3.5). Children
// reference status_pages(id) with ON DELETE CASCADE so GORM
// ordering would already wipe them, but we delete them
// explicitly so the test DB stays clean even if a future model
// change drops the cascade.
DB().Where("1=1").Delete(&StatusPageDomain{})
DB().Unscoped().Where("1=1").Delete(&Maintenance{})
DB().Where("1=1").Delete(&StatusPageMaintenance{})
DB().Where("1=1").Delete(&StatusPageIncident{})
DB().Where("1=1").Delete(&StatusPageDelivery{})
DB().Where("1=1").Delete(&StatusPageDigestSchedule{})
DB().Where("1=1").Delete(&StatusPageSubscriber{})
DB().Unscoped().Where("1=1").Delete(&StatusPage{})
DB().Where("1=1").Delete(&SubscriptionEvent{})
DB().Where("1=1").Delete(&Subscription{})
DB().Where("1=1").Delete(&Account{})
DB().Unscoped().Where("1=1").Delete(&authidentity.AuthIdentity{})
// User has a unique email index, so test fixtures must be physically
// removed rather than soft-deleted between tests.
DB().Unscoped().Where("1=1").Delete(&User{})
// A few audited tables added by feature migrations can retain a user FK that
// is intentionally not modeled as an association. CASCADE keeps fixture
// cleanup deterministic instead of silently leaving unique emails behind.
DB().Exec("TRUNCATE users CASCADE")
DB().Exec("TRUNCATE regions CASCADE")
}

83
app/models/event.go Обычный файл
Просмотреть файл

@@ -0,0 +1,83 @@
package models
import (
"fmt"
"time"
"github.com/lib/pq"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/util"
)
// Event provides functionality.
type Event struct {
concerns.Model
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id"`
Monitor *Monitor `json:"monitor,omitempty"`
StartTime *time.Time `json:"start_time"`
EndTime *time.Time `json:"end_time"`
Duration int `json:"duration"`
Errors int `json:"errors"`
Oks int `json:"oks"`
State string `gorm:"index" json:"state"`
Reason string `json:"reason"`
Messages []Message `json:"messages" gorm:"many2many:event_messages;"`
ChecksDown pq.StringArray `gorm:"type:varchar(255)[]" json:"checks_down"`
Checks []Check `json:"-" gorm:"many2many:event_checks;"`
ExpiresAt *time.Time `json:"-"`
Audited
}
// EventScope provides functionality.
func EventScope(q *gorm.DB) *gorm.DB {
return q.Where("state IN ('current', 'ended')").
Preload("Checks").
Preload("Monitor").
Preload("Monitor.Group").
Preload("Monitor.Group.Notifications").
Preload("Monitor.Group.Notifications.Contacts")
}
// GetDuration provides functionality.
func (e *Event) GetDuration(tn time.Time) int64 {
endTime := e.EndTime
if endTime == nil {
endTime = &tn
}
return int64(endTime.Sub(*e.StartTime) / time.Second)
}
// FormatDuration provides functionality.
func (e *Event) FormatDuration() string {
d := e.GetDuration(time.Now())
return util.FormatDuration(d)
}
// Inspect provides functionality.
func (e *Event) Inspect() string {
var st, et string
if e.StartTime != nil {
st = e.StartTime.Format("2006-01-02 15:04:05")
}
if e.EndTime != nil {
et = e.EndTime.Format("2006-01-02 15:04:05")
}
return fmt.Sprintf(
"Event<id: %d, monitor_id: %d, start_time: %s, end_time: %s, reason: %s, duration: %d>",
e.ID,
e.MonitorID,
st,
et,
e.Reason,
e.Duration,
)
}

76
app/models/group.go Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// Group represents a monitor group.
type Group struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
Name string `json:"name" gorm:"not null"`
IsSystem *bool `json:"is_system" gorm:"default:false"`
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
Monitors []Monitor `json:"-"`
Notifications []Notification `gorm:"many2many:notification_groups;" json:"-"`
concerns.Timestamped `json:"-"`
Audited
}
// SystemGroups returns all groups marked as system/internal (is_system=true).
// These groups are converted to distributed monitors (system checks running
// on the distributed worker pool).
func SystemGroups() ([]Group, error) {
var groups []Group
err := DB().Where("is_system = ?", true).Find(&groups).Error
return groups, err
}
// GroupIdsForAccountId returns all group IDs for the given account.
func GroupIdsForAccountId(accountID int64) []int64 { //nolint:revive // accepted lint exception
rows, err := DB().Raw("SELECT id FROM groups WHERE account_id = ?", accountID).Rows()
if err != nil {
panic(err)
}
defer rows.Close() //nolint:errcheck // accepted lint exception
var cid int64
groupIDs := make([]int64, 0)
for rows.Next() {
rows.Scan(&cid) //nolint:errcheck // accepted lint exception
groupIDs = append(groupIDs, cid)
}
return groupIDs
}
// CountGroups counts monitors per group.
func CountGroups(groupIDs []int64, groupCount *map[int64]int) { //nolint:gocritic // ptrToRefParam: accepted pattern
rows, err := DB().Raw("select group_id, count(id) from monitors where group_id IN (?) group by group_id ", groupIDs).Rows()
if err != nil {
panic(err)
}
defer rows.Close() //nolint:errcheck // accepted lint exception
var gid int64
var count int
for rows.Next() {
rows.Scan(&gid, &count) //nolint:errcheck // accepted lint exception
(*groupCount)[gid] = count
}
}
// GroupsCounts fills MonitorsCount for each group.
func GroupsCounts(groups *[]Group) {
groupIDs := make([]int64, len(*groups))
groupCount := make(map[int64]int, len(*groups))
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
groupIDs[i] = g.ID
groupCount[g.ID] = 0
}
CountGroups(groupIDs, &groupCount)
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
(*groups)[i].MonitorsCount = groupCount[g.ID]
}
}

58
app/models/group_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,58 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestSystemGroups verifies that SystemGroups returns only groups with
// is_system=true and ignores groups with is_system=false or nil.
func TestSystemGroups(t *testing.T) {
models.Drop()
models.Migrate()
account := &models.Account{Name: "test-account"}
require.NoError(t, models.DB().Create(account).Error)
trueVal, falseVal := true, false
systemGroup := &models.Group{
AccountID: account.ID,
Name: "system-internal",
IsSystem: &trueVal,
}
regularGroup := &models.Group{
AccountID: account.ID,
Name: "regular",
IsSystem: &falseVal,
}
nilSystemGroup := &models.Group{
AccountID: account.ID,
Name: "nil-system",
}
require.NoError(t, models.DB().Create(systemGroup).Error)
require.NoError(t, models.DB().Create(regularGroup).Error)
require.NoError(t, models.DB().Create(nilSystemGroup).Error)
got, err := models.SystemGroups()
require.NoError(t, err)
var ids []int64
var names []string
for _, g := range got {
ids = append(ids, g.ID)
names = append(names, g.Name)
}
assert.Contains(t, names, "system-internal")
assert.NotContains(t, names, "regular")
assert.NotContains(t, names, "nil-system")
assert.Contains(t, ids, systemGroup.ID)
assert.NotContains(t, ids, regularGroup.ID)
assert.NotContains(t, ids, nilSystemGroup.ID)
}

30
app/models/init.go Обычный файл
Просмотреть файл

@@ -0,0 +1,30 @@
package models
import (
"context"
"github.com/fatih/structs"
"gorm.io/gorm"
)
func init() {
structs.DefaultTagName = "json"
}
// db Gorm DB
var db *gorm.DB
// DB provides functionality.
func DB() *gorm.DB {
return db.WithContext(context.TODO())
}
// SetDB provides functionality.
func SetDB(newDb *gorm.DB) {
db = newDb
}
// IsDBAvailable returns true if the database has been initialized
func IsDBAvailable() bool {
return db != nil
}

222
app/models/inventory_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,222 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestServer_RotateToken(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "rotate-token-account"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "rotate-target",
Slug: "rotate-target",
Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
t1 := models.GenerateServerToken()
require.NoError(t, models.RotateServerToken(nil, srv.ID, t1))
got, err := models.FindServerByToken(t1)
require.NoError(t, err)
assert.Equal(t, srv.ID, got.ID)
t2 := models.GenerateServerToken()
require.NoError(t, models.RotateServerToken(nil, srv.ID, t2))
// Old token no longer matches.
_, err = models.FindServerByToken(t1)
assert.ErrorIs(t, err, gorm.ErrRecordNotFound)
got2, err := models.FindServerByToken(t2)
require.NoError(t, err)
assert.Equal(t, srv.ID, got2.ID)
// Generated tokens are hex-encoded 32 bytes (64 chars).
assert.Len(t, t1, 64)
assert.NotEqual(t, t1, t2)
}
func TestServer_InventoryFields_Default(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "inv-defaults"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "fresh-server",
Slug: "fresh-server",
Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
got := models.Server{}
require.NoError(t, models.DB().First(&got, srv.ID).Error)
assert.Equal(t, models.ServerKindProduction, got.Kind)
assert.Equal(t, 0, got.PriceCents)
assert.False(t, got.Paused)
}
func TestServer_KindEnum_OnlyAllowsValidValues(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "inv-enum"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID,
Name: "kinder",
Slug: "kinder",
Region: "local",
Kind: models.ServerKindStaging,
}
require.NoError(t, models.DB().Create(srv).Error)
got := models.Server{}
require.NoError(t, models.DB().First(&got, srv.ID).Error)
assert.Equal(t, models.ServerKindStaging, got.Kind)
// Inserting an invalid value via raw SQL fails the enum check.
err := models.DB().Exec(
"INSERT INTO servers (account_id, name, slug, region, kind) VALUES (?, ?, ?, ?, ?)",
acc.ID, "bad-kinder", "bad-kinder", "local", "scrapped",
).Error
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid input value for enum")
}
func TestSite_Slugify(t *testing.T) {
cases := []struct {
in, want string
}{
{"Cafe", "cafe"}, // cyrillic stripped (latin-only rule, see Slugify)
{" Spaces Everywhere ", "spaces-everywhere"},
{"dots.and-dashes_and spaces", "dots-and-dashes-and-spaces"},
{"", "site"},
{"-leading-and-trailing-", "leading-and-trailing"},
{"mix_of.dots-dashes spaces", "mix-of-dots-dashes-spaces"},
}
for _, c := range cases {
t.Run(c.in, func(t *testing.T) {
assert.Equal(t, c.want, models.SiteSlugify(c.in))
})
}
}
func TestSite_FindOrCreateBySlug(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "site-foc"}
require.NoError(t, models.DB().Create(acc).Error)
got, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
require.NoError(t, err)
require.NotZero(t, got.ID, "row should be persisted")
assert.Equal(t, "my-site", got.Slug)
assert.Equal(t, "production", got.Kind)
assert.True(t, got.IsActive)
// Second call returns the same row (idempotent).
got2, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
require.NoError(t, err)
assert.Equal(t, got.ID, got2.ID)
}
func TestDeployment_UpsertNginx_MatchesByConfigPath(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "dep-upsert"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID, Name: "host1", Slug: "host1", Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
d1, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
require.NoError(t, err)
require.NoError(t, models.DB().Save(d1).Error)
// Second call with the same config_path returns the existing row.
d2, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
require.NoError(t, err)
assert.Equal(t, d1.ID, d2.ID, "should reuse the same row on identical config_path")
// Different config_path creates a new row.
d3, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/b.conf")
require.NoError(t, err)
assert.NotEqual(t, d1.ID, d3.ID)
}
func TestDeployment_ReconcileMissing_FlipsAction(t *testing.T) {
models.Drop()
models.Migrate()
acc := &models.Account{Name: "reconcile"}
require.NoError(t, models.DB().Create(acc).Error)
srv := &models.Server{
AccountID: acc.ID, Name: "rec", Slug: "rec", Region: "local",
}
require.NoError(t, models.DB().Create(srv).Error)
old := time.Now().Add(-2 * time.Hour)
fresh := models.Deployment{
AccountID: acc.ID,
ServerID: &srv.ID,
Kind: models.DeploymentKindProduction,
Mode: models.DeploymentModeDedicated,
Action: models.DeploymentActionOk,
ConfigPath: ptr("/etc/nginx/old.conf"),
LastSeenAt: &old,
}
require.NoError(t, models.DB().Create(&fresh).Error)
recent := models.Deployment{
AccountID: acc.ID,
ServerID: &srv.ID,
Kind: models.DeploymentKindProduction,
Mode: models.DeploymentModeDedicated,
Action: models.DeploymentActionOk,
ConfigPath: ptr("/etc/nginx/recent.conf"),
LastSeenAt: ptrTime(time.Now()),
}
require.NoError(t, models.DB().Create(&recent).Error)
cutoff := time.Now().Add(-90 * time.Second)
marked, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
require.NoError(t, err)
assert.EqualValues(t, 1, marked, "only the old row should flip")
var oldAfter models.Deployment
require.NoError(t, models.DB().First(&oldAfter, fresh.ID).Error)
assert.Equal(t, models.DeploymentActionMissing, oldAfter.Action)
var recentAfter models.Deployment
require.NoError(t, models.DB().First(&recentAfter, recent.ID).Error)
assert.Equal(t, models.DeploymentActionOk, recentAfter.Action, "fresh row stays ok")
// Re-running with the same cutoff is a no-op.
marked2, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
require.NoError(t, err)
assert.EqualValues(t, 0, marked2)
}
func ptr(s string) *string { return &s }
func ptrTime(t time.Time) *time.Time { return &t }

52
app/models/invite.go Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
package models
import (
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Invite provides functionality.
type Invite struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
InviterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"inviter_id"`
Inviter *User `json:"inviter"`
InviteeID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"invitee_id"`
Invitee *User `json:"invitee"`
Name string `json:"name"`
Email string `json:"email"`
// invite state, FAIL - failed to send, SENT - not regestired, OK - registered
State string `gorm:"not null;default:'UNK'" json:"state"`
CreatedAt time.Time `json:"created_at"`
RegisteredAt time.Time `json:"registered_at"`
SentAt time.Time `json:"sent_at"`
Accesses []Access `json:"accesses" gorm:"foreignkey:invite_id"`
concerns.HasToken
Audited
}
// BeforeCreate runs before creating an Invite record.
func (i *Invite) BeforeCreate(_ *gorm.DB) error {
i.SetToken()
return nil
}
// FillAccesses provides functionality.
func (i *Invite) FillAccesses() {
for k, a := range i.Accesses { //nolint:gocritic // range copy is acceptable here
if a.ID <= 0 {
i.Accesses[k].ID = 0
}
i.Accesses[k].AccountID = i.AccountID
}
}

17
app/models/llm.go Обычный файл
Просмотреть файл

@@ -0,0 +1,17 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// LLM stores an OpenAI-compatible LLM endpoint available to checks.
type LLM struct {
concerns.Model
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id);index"`
Account *Account `json:"-"`
Name string `json:"name" gorm:"not null"`
URL string `json:"url" gorm:"not null"`
ModelName string `json:"model" gorm:"column:model;not null"`
APIKey string `json:"-" gorm:"not null"`
Kind string `json:"kind" gorm:"not null;default:'openai'"`
Workers []WorkerNode `json:"-" gorm:"many2many:worker_llms;"`
concerns.Timestamped
}

324
app/models/maintenance.go Обычный файл
Просмотреть файл

@@ -0,0 +1,324 @@
package models
import (
"fmt"
"strings"
"time"
"github.com/lib/pq"
"github.com/robfig/cron/v3"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
MaintenanceManual = "manual"
MaintenanceSingle = "single"
MaintenanceCron = "cron"
MaintenanceRecurringInterval = "recurring-interval"
MaintenanceRecurringWeekday = "recurring-weekday"
MaintenanceRecurringDayOfMonth = "recurring-day-of-month"
)
// Maintenance is account-owned planned downtime. Times are stored as UTC;
// Timezone only defines how recurring wall-clock fields are interpreted.
type Maintenance struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
Account *Account `json:"-"`
Title string `gorm:"size:200;not null" json:"title"`
Description string `gorm:"type:text;not null;default:''" json:"description"`
Strategy string `gorm:"size:32;not null" json:"strategy"`
Cron string `gorm:"type:text;not null;default:''" json:"cron"`
DurationSec int `gorm:"not null;default:0" json:"duration_sec"`
StartDate *time.Time `json:"start_date,omitempty"`
EndDate *time.Time `json:"end_date,omitempty"`
StartTime string `gorm:"size:5;not null;default:''" json:"start_time"`
EndTime string `gorm:"size:5;not null;default:''" json:"end_time"`
Weekdays pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"weekdays"`
DaysOfMonth pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"days_of_month"`
IntervalDay int `gorm:"not null;default:1" json:"interval_day"`
Timezone string `gorm:"size:64;not null;default:'UTC'" json:"timezone"`
Active bool `gorm:"not null;default:true" json:"active"`
LastStartDate *time.Time `json:"last_start_date,omitempty"`
LegacyStatusPageMaintenanceID *int64 `gorm:"uniqueIndex" json:"-"`
ShowOnAllStatusPages bool `gorm:"not null;default:true" json:"show_on_all_status_pages"`
Monitors []Monitor `gorm:"many2many:maintenance_monitors;constraint:OnDelete:CASCADE" json:"monitors,omitempty"`
StatusPages []StatusPage `gorm:"many2many:maintenance_status_pages;constraint:OnDelete:CASCADE" json:"status_pages,omitempty"`
concerns.Timestamped
Audited
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
}
func (Maintenance) TableName() string { return "maintenances" }
func (m *Maintenance) location() (*time.Location, error) {
if m.Timezone == "" || m.Timezone == "SAME_AS_SERVER" {
return time.UTC, nil
}
return time.LoadLocation(m.Timezone)
}
func (m *Maintenance) generatedCron() (string, error) {
if m.Strategy == MaintenanceCron {
return m.Cron, nil
}
if m.Strategy == MaintenanceManual || m.Strategy == MaintenanceSingle {
return "", nil
}
parts := strings.Split(m.StartTime, ":")
if len(parts) != 2 {
return "", fmt.Errorf("start_time must be HH:MM")
}
base := parts[1] + " " + parts[0]
switch m.Strategy {
case MaintenanceRecurringInterval:
return "", nil
case MaintenanceRecurringWeekday:
if len(m.Weekdays) == 0 {
return "", fmt.Errorf("at least one weekday is required")
}
values := make([]string, len(m.Weekdays))
for i, day := range m.Weekdays {
if day < 0 || day > 6 {
return "", fmt.Errorf("weekday must be 0 through 6")
}
values[i] = fmt.Sprint(day)
}
return base + " * * " + strings.Join(values, ","), nil
case MaintenanceRecurringDayOfMonth:
if len(m.DaysOfMonth) == 0 {
return "", fmt.Errorf("at least one day of month is required")
}
values := make([]string, 0, len(m.DaysOfMonth))
for _, day := range m.DaysOfMonth {
if day == "lastDay1" {
values = append(values, "28-31")
} else {
values = append(values, day)
}
}
return base + " " + strings.Join(values, ",") + " * *", nil
default:
return "", fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
}
}
// Validate normalizes generated schedules and rejects ambiguous or invalid
// input before it can reach the scheduler.
func (m *Maintenance) Validate() error {
m.Title = strings.TrimSpace(m.Title)
if m.Title == "" || len(m.Title) > 200 {
return fmt.Errorf("title is required and must be at most 200 characters")
}
if _, err := m.location(); err != nil {
return fmt.Errorf("invalid timezone: %w", err)
}
switch m.Strategy {
case MaintenanceManual:
return nil
case MaintenanceSingle:
if m.StartDate == nil || m.EndDate == nil || !m.EndDate.After(*m.StartDate) {
return fmt.Errorf("single maintenance requires end_date after start_date")
}
m.DurationSec = int(m.EndDate.Sub(*m.StartDate).Seconds())
return nil
case MaintenanceRecurringInterval:
if m.DurationSec <= 0 || m.IntervalDay <= 0 || (m.IntervalDay > 1 && m.StartDate == nil) {
return fmt.Errorf("recurring interval requires positive duration_sec and interval_day; intervals over one day require start_date")
}
if _, _, err := parseMaintenanceTime(m.StartTime); err != nil {
return err
}
m.Cron = ""
return nil
case MaintenanceCron, MaintenanceRecurringWeekday, MaintenanceRecurringDayOfMonth:
if m.DurationSec <= 0 {
return fmt.Errorf("duration_sec must be positive")
}
cronText, err := m.generatedCron()
if err != nil {
return err
}
if _, err := cron.ParseStandard(cronText); err != nil {
return fmt.Errorf("invalid cron: %w", err)
}
m.Cron = cronText
return nil
default:
return fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
}
}
func (m *Maintenance) BeforeSave(_ *gorm.DB) error { return m.Validate() }
// IsUnderMaintenance evaluates durable data only. This intentionally avoids
// scheduler-owned state so a process restart and multiple web pods agree.
func (m *Maintenance) IsUnderMaintenance(now time.Time) bool {
if !m.Active {
return false
}
if m.Strategy == MaintenanceManual {
return true
}
if m.Strategy == MaintenanceSingle {
return m.StartDate != nil && m.EndDate != nil && !now.Before(*m.StartDate) && now.Before(*m.EndDate)
}
if m.Strategy == MaintenanceRecurringInterval {
return m.isUnderInterval(now)
}
if m.DurationSec <= 0 {
return false
}
loc, err := m.location()
if err != nil {
return false
}
schedule, err := cron.ParseStandard(m.Cron)
if err != nil {
return false
}
// Ask cron for each candidate since the earliest possible active start.
// Cron is minute-granular, hence the extra minute catches exact boundaries.
from := now.In(loc).Add(-time.Duration(m.DurationSec)*time.Second - time.Minute)
to := now.In(loc)
for candidate := schedule.Next(from); !candidate.After(to); candidate = schedule.Next(candidate) {
if !m.allowsRecurringCandidate(candidate.In(loc)) {
continue
}
start := candidate.UTC()
if !now.Before(start) && now.Before(start.Add(time.Duration(m.DurationSec)*time.Second)) {
return true
}
}
return false
}
func parseMaintenanceTime(value string) (int, int, error) {
parsed, err := time.Parse("15:04", value)
if err != nil {
return 0, 0, fmt.Errorf("start_time must be HH:MM")
}
return parsed.Hour(), parsed.Minute(), nil
}
func (m *Maintenance) intervalStartOn(date time.Time, loc *time.Location) (time.Time, bool) {
if m.IntervalDay <= 0 {
return time.Time{}, false
}
hour, minute, err := parseMaintenanceTime(m.StartTime)
if err != nil {
return time.Time{}, false
}
if m.IntervalDay == 1 && m.StartDate == nil {
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
}
if m.StartDate == nil {
return time.Time{}, false
}
anchor := m.StartDate.In(loc)
anchorDay := time.Date(anchor.Year(), anchor.Month(), anchor.Day(), 0, 0, 0, 0, loc)
candidateDay := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc)
// Compare civil dates rather than elapsed hours: a local day can be 23 or
// 25 hours when the maintenance timezone crosses a DST boundary.
days := civilDaysBetween(anchorDay, candidateDay)
if days < 0 || days%m.IntervalDay != 0 {
return time.Time{}, false
}
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
}
func civilDaysBetween(from, to time.Time) int {
fromDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
toDay := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC)
return int(toDay.Sub(fromDay) / (24 * time.Hour))
}
func (m *Maintenance) isUnderInterval(now time.Time) bool {
loc, err := m.location()
if err != nil || m.DurationSec <= 0 {
return false
}
localNow := now.In(loc)
for day := 0; day <= int(time.Duration(m.DurationSec)/24/time.Hour)+1; day++ {
start, ok := m.intervalStartOn(localNow.AddDate(0, 0, -day), loc)
if ok && !now.Before(start.UTC()) && now.Before(start.UTC().Add(time.Duration(m.DurationSec)*time.Second)) {
return true
}
}
return false
}
// robfig/cron cannot express "last day". The generated 28-31 range is only
// a candidate generator; this final predicate makes lastDay1 exact.
func (m *Maintenance) allowsRecurringCandidate(candidate time.Time) bool {
if m.Strategy != MaintenanceRecurringDayOfMonth {
return true
}
lastDay := candidate.AddDate(0, 0, 1).Month() != candidate.Month()
for _, value := range m.DaysOfMonth {
if value == "lastDay1" && lastDay {
return true
}
if value == fmt.Sprint(candidate.Day()) {
return true
}
}
return false
}
func (m *Maintenance) NextRun(now time.Time) *time.Time {
if !m.Active || m.Strategy == MaintenanceManual {
return nil
}
if m.Strategy == MaintenanceSingle {
if m.StartDate != nil && m.StartDate.After(now) {
return m.StartDate
}
return nil
}
if m.Strategy == MaintenanceRecurringInterval {
loc, err := m.location()
if err != nil {
return nil
}
localNow := now.In(loc)
for day := 0; day <= m.IntervalDay; day++ {
if next, ok := m.intervalStartOn(localNow.AddDate(0, 0, day), loc); ok && next.After(localNow) {
result := next.UTC()
return &result
}
}
return nil
}
loc, err := m.location()
if err != nil {
return nil
}
s, err := cron.ParseStandard(m.Cron)
if err != nil {
return nil
}
for candidate := s.Next(now.In(loc)); ; candidate = s.Next(candidate) {
if m.allowsRecurringCandidate(candidate.In(loc)) {
next := candidate.UTC()
return &next
}
}
}
// MonitorUnderMaintenance is the notifier/public-page lookup.
func MonitorUnderMaintenance(monitorID int64, now time.Time) (bool, error) {
var rows []Maintenance
err := DB().Joins("JOIN maintenance_monitors mm ON mm.maintenance_id = maintenances.id").Where("mm.monitor_id = ? AND maintenances.active = TRUE", monitorID).Find(&rows).Error
if err != nil {
return false, err
}
for i := range rows {
if rows[i].IsUnderMaintenance(now) {
return true, nil
}
}
return false, nil
}

33
app/models/maintenance_migration_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,33 @@
package models
import (
"testing"
"time"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
)
func TestMigratePreservesEveryLegacyStatusPageMaintenance(t *testing.T) {
Drop()
Migrate()
plan := Plan{Name: "legacy migration plan"}
require.NoError(t, DB().Create(&plan).Error)
account := Account{Name: "legacy migration account", PlanID: &plan.ID}
require.NoError(t, DB().Create(&account).Error)
page := StatusPage{AccountID: account.ID, Slug: "legacy-maintenance-migration", Name: "Legacy"}
require.NoError(t, DB().Create(&page).Error)
start := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
for i := 0; i < 2; i++ {
row := StatusPageMaintenance{StatusPageID: page.ID, Title: "same", StartsAt: start, EndsAt: start.Add(time.Hour), MonitorIDs: pq.Int64Array{}}
require.NoError(t, DB().Create(&row).Error)
}
Migrate()
var migrated []Maintenance
require.NoError(t, DB().Where("legacy_status_page_maintenance_id IS NOT NULL").Find(&migrated).Error)
require.Len(t, migrated, 2)
var joins int64
require.NoError(t, DB().Table("maintenance_status_pages").Where("status_page_id = ?", page.ID).Count(&joins).Error)
require.EqualValues(t, 2, joins)
}

44
app/models/maintenance_notifications.go Обычный файл
Просмотреть файл

@@ -0,0 +1,44 @@
package models
import (
"fmt"
"time"
"gorm.io/gorm"
)
const maintenanceStartNotificationPrefix = "maintenance:%d:start:"
// MaintenanceStartNotificationKey uniquely identifies one contact's warning
// for one schedule revision and occurrence. Updated windows get a new revision
// while concurrent scheduler replicas share the same key.
func MaintenanceStartNotificationKey(maintenanceID int64, revision, startsAt time.Time, timezone string, notificationID, contactID int64) string {
loc, err := time.LoadLocation(timezone)
if err != nil || timezone == "SAME_AS_SERVER" || timezone == "" {
loc = time.UTC
}
// A fall-back hour can have two UTC instants for one wall-clock occurrence.
// Warnings are once per civil occurrence, matching the recurrence editor.
civilOccurrence := startsAt.In(loc).Format("200601021504")
return fmt.Sprintf("maintenance:%d:start:%d:%s:notification:%d:contact:%d", maintenanceID, revision.UnixNano(), civilOccurrence, notificationID, contactID)
}
// CancelMaintenanceStartNotificationsTx prevents queued warnings from being
// delivered after an operator pauses, changes, or deletes the maintenance.
// Leased work may already be executing and cannot be recalled from a worker.
func CancelMaintenanceStartNotificationsTx(tx *gorm.DB, maintenanceID int64, reason string) error {
prefix := fmt.Sprintf(maintenanceStartNotificationPrefix, maintenanceID) + "%"
var tasks []Task
if err := tx.Clauses(SkipLockedClause).Where("idempotency_key LIKE ? AND state IN ?", prefix, []string{TaskStateQueued, TaskStateFailedRetry}).Find(&tasks).Error; err != nil {
return err
}
for i := range tasks {
if err := tx.Model(&Task{}).Where("id = ? AND state IN ?", tasks[i].ID, []string{TaskStateQueued, TaskStateFailedRetry}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "canceled", "canceled: "+reason); err != nil {
return err
}
}
return nil
}

105
app/models/maintenance_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,105 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestMaintenanceValidateGeneratesRecurringCron(t *testing.T) {
tests := []struct {
name string
m models.Maintenance
want string
}{
{"interval", models.Maintenance{Title: "interval", Strategy: models.MaintenanceRecurringInterval, StartDate: maintenanceTimePtr(time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)), StartTime: "02:30", IntervalDay: 3, DurationSec: 60, Timezone: "UTC"}, ""},
{"weekday", models.Maintenance{Title: "weekdays", Strategy: models.MaintenanceRecurringWeekday, StartTime: "02:30", Weekdays: []int64{1, 5}, DurationSec: 60, Timezone: "UTC"}, "30 02 * * 1,5"},
{"month", models.Maintenance{Title: "month", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "02:30", DaysOfMonth: []string{"1", "lastDay1"}, DurationSec: 60, Timezone: "UTC"}, "30 02 1,28-31 * *"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) { require.NoError(t, test.m.Validate()); assert.Equal(t, test.want, test.m.Cron) })
}
}
func TestMaintenanceIsUnderMaintenanceBoundariesAndTimezone(t *testing.T) {
start := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
single := models.Maintenance{Title: "single", Strategy: models.MaintenanceSingle, StartDate: &start, EndDate: &end, Active: true, Timezone: "UTC"}
assert.True(t, single.IsUnderMaintenance(start))
assert.True(t, single.IsUnderMaintenance(end.Add(-time.Nanosecond)))
assert.False(t, single.IsUnderMaintenance(end))
cron := models.Maintenance{Title: "moscow", Strategy: models.MaintenanceCron, Cron: "0 12 * * *", DurationSec: 3600, Active: true, Timezone: "Europe/Moscow"}
require.NoError(t, cron.Validate())
assert.True(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 9, 30, 0, 0, time.UTC)), "12:30 Moscow is 09:30 UTC in July")
assert.False(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)))
}
func TestMaintenanceValidationRejectsInvalidSchedules(t *testing.T) {
badCron := models.Maintenance{Title: "bad", Strategy: models.MaintenanceCron, Cron: "not cron", DurationSec: 1, Timezone: "UTC"}
badSingle := models.Maintenance{Title: "bad", Strategy: models.MaintenanceSingle, Timezone: "UTC"}
assert.Error(t, badCron.Validate())
assert.Error(t, badSingle.Validate())
}
func TestMaintenanceLastDayIsNotEveryDayInExpandedCronRange(t *testing.T) {
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 3600, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 3, 28, 12, 30, 0, 0, time.UTC)))
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 31, 12, 30, 0, 0, time.UTC)))
}
func TestMaintenanceRecurringIntervalUsesAnchorAndIntervalDay(t *testing.T) {
anchor := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
m := models.Maintenance{Title: "every three days", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "12:00", IntervalDay: 3, DurationSec: 3600, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 7, 4, 12, 30, 0, 0, time.UTC)))
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 7, 5, 12, 30, 0, 0, time.UTC)))
next := m.NextRun(time.Date(2026, 7, 2, 13, 0, 0, 0, time.UTC))
require.NotNil(t, next)
assert.Equal(t, time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC), *next)
}
func TestMaintenanceRecurringIntervalRequiresAnchorForMultiDayAndPositiveInterval(t *testing.T) {
m := models.Maintenance{Title: "invalid", Strategy: models.MaintenanceRecurringInterval, StartTime: "12:00", IntervalDay: 2, DurationSec: 60, Timezone: "UTC"}
assert.Error(t, m.Validate())
}
func TestMaintenanceRecurringIntervalKeepsCivilDayAcrossDST(t *testing.T) {
loc, err := time.LoadLocation("Europe/Berlin")
require.NoError(t, err)
// March 29, 2026 is the spring-forward day in Berlin. The second run is
// still two civil days after the anchor, not one because a day was 23h.
anchor := time.Date(2026, 3, 27, 0, 0, 0, 0, loc)
m := models.Maintenance{Title: "DST", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "03:30", IntervalDay: 2, DurationSec: 3600, Active: true, Timezone: "Europe/Berlin"}
require.NoError(t, m.Validate())
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 29, 4, 0, 0, 0, loc).UTC()))
}
func TestMaintenanceStartNotificationKeyIncludesRevisionAndOccurrence(t *testing.T) {
revision := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
start := revision.Add(5 * time.Minute)
key := models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13)
assert.Equal(t, key, models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13))
assert.NotEqual(t, key, models.MaintenanceStartNotificationKey(7, revision.Add(time.Second), start, "Europe/Berlin", 11, 13))
}
func TestMaintenanceStartNotificationKeyDeduplicatesDSTFallbackCivilOccurrence(t *testing.T) {
revision := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
first := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC)
second := first.Add(time.Hour)
assert.Equal(t, models.MaintenanceStartNotificationKey(7, revision, first, "Europe/Berlin", 11, 13), models.MaintenanceStartNotificationKey(7, revision, second, "Europe/Berlin", 11, 13))
}
func TestMaintenanceNextRunSkipsNonFinalLastDayCandidates(t *testing.T) {
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 60, Active: true, Timezone: "UTC"}
require.NoError(t, m.Validate())
next := m.NextRun(time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC))
require.NotNil(t, next)
assert.Equal(t, time.Date(2026, 4, 30, 12, 0, 0, 0, time.UTC), *next)
}
func maintenanceTimePtr(value time.Time) *time.Time { return &value }

48
app/models/message.go Обычный файл
Просмотреть файл

@@ -0,0 +1,48 @@
package models
import (
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Message info about performed notification
type Message struct {
concerns.Model
NotificationID int64 `json:"notification_id"`
Notification *Notification `json:"notification,omitempty"`
ContactID int64 `gorm:"index;type:bigint REFERENCES contacts(id)" json:"contact_id"`
Contact *Contact `json:"contact,omitempty"`
// Events are for up/down messages
Events []Event `json:"events" gorm:"many2many:event_messages;"`
// Checks are for expires messages
CheckID *int64 `gorm:"index;type:bigint REFERENCES checks(id)" json:"check_id"`
Check *Check `json:"check,omitempty"`
Kind string `json:"kind"`
State string `json:"state"`
Error *string `json:"error"`
Response *string `json:"response"`
Tries int `json:"-"`
CreatedAt time.Time `json:"created_at"`
SentAt time.Time `json:"sent_at"`
}
// MessageScope provides functionality.
func MessageScope(q *gorm.DB) *gorm.DB {
return q.
Preload("Events").
Preload("Events.Checks").
Preload("Events.Monitor").
Preload("Notification").
Preload("Contact").
Preload("Check").
Preload("Check.Monitor")
}

750
app/models/migrate.go Обычный файл
Просмотреть файл

@@ -0,0 +1,750 @@
package models
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
)
// isTypeExistsError returns true if the error is a Postgres "type already exists" error.
// This happens when AutoMigrate is called concurrently from multiple test processes.
func isTypeExistsError(err error) bool {
if err == nil {
return false
}
s := err.Error()
// SQLSTATE 42710 = duplicate_object (type already exists)
return strings.Contains(s, "42710") ||
strings.Contains(s, "already exists") ||
strings.Contains(s, "pg_type_typname_nsp_index")
}
// migrateOrIgnoreTypeExists runs AutoMigrate and ignores "type already exists" errors
// that can occur when parallel test processes both try to create the same Postgres types.
func migrateOrIgnoreTypeExists(models ...interface{}) {
err := DB().AutoMigrate(models...)
if err != nil && !isTypeExistsError(err) {
panic(err)
}
if err != nil {
log.Printf("migrate: ignoring type-exists error (expected during parallel test runs): %v", err)
}
}
// ensureSingleCurrentEventInvariant pins cleanup and index creation to one
// transaction/connection. The global migration lock is session-scoped through a
// pool, so it is not sufficient for this multi-statement invariant by itself.
func ensureSingleCurrentEventInvariant() error {
return DB().Transaction(func(tx *gorm.DB) error {
if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", int64(1234567892)).Error; err != nil {
return err
}
if err := tx.Exec(`WITH ranked AS (
SELECT id, row_number() OVER (PARTITION BY monitor_id ORDER BY start_time DESC NULLS LAST, id DESC) AS n
FROM events WHERE state = 'current'
) UPDATE events SET state = 'ended', end_time = COALESCE(end_time, now())
FROM ranked WHERE events.id = ranked.id AND ranked.n > 1`).Error; err != nil {
return err
}
return tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS events_one_current_per_monitor
ON events (monitor_id) WHERE state = 'current'`).Error
})
}
// Migrate run db migration
var migrateMu sync.Mutex
func Migrate() {
migrateMu.Lock()
defer migrateMu.Unlock()
withMigrationAdvisoryLock(migrateLocked)
}
// withMigrationAdvisoryLock holds the session lock on a dedicated connection
// while migration work uses GORM's normal pool. Reusing the lock connection
// for GORM transactions can leave its *sql.Conn closed after commit.
func withMigrationAdvisoryLock(migrate func()) {
const migrateAdvisoryLock = int64(1234567890)
sqlDB, err := DB().DB()
if err != nil {
panic(fmt.Sprintf("migrate: database handle: %v", err))
}
ctx := context.Background()
conn, err := sqlDB.Conn(ctx)
if err != nil {
panic(fmt.Sprintf("migrate: lock connection: %v", err))
}
defer conn.Close() //nolint:errcheck // closing releases the session lock after a migration panic
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", migrateAdvisoryLock); err != nil {
panic(fmt.Sprintf("migrate: advisory lock: %v", err))
}
unlocked := false
defer func() {
if unlocked {
return
}
if _, unlockErr := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); unlockErr != nil {
log.Printf("migrate: unlock after failure: %v", unlockErr)
}
}()
migrate()
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); err != nil {
panic(fmt.Sprintf("migrate: unlock: %v", err))
}
unlocked = true
}
func migrateLocked() {
// M0 replaces the old flat plans table without rewriting historical rows.
// Rename before AutoMigrate so GORM creates the canonical table rather than
// adding columns to the incompatible legacy schema.
if err := prepareCanonicalPlansTable(); err != nil {
panic(fmt.Sprintf("migrate: prepare canonical plans: %v", err))
}
// Step 0: create inventory Postgres enum types FIRST. The DO/EXCEPTION
// blocks are idempotent so concurrent migrateOrIgnoreTypeExists
// reruns from parallel test binaries are safe (the type already
// exists → duplicate_object is swallowed). The enum types MUST
// exist before any AutoMigrate below because GORM emits
// `kind server_kind` literals in CREATE TABLE for the Server
// struct (referenced transitively from Monitor.Site → Site → Server).
for _, ddl := range []string{
`DO $$ BEGIN
CREATE TYPE server_kind AS ENUM ('production','staging','old');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_kind AS ENUM
('production','production_prev','production_next','internal','staging','old');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_mode AS ENUM
('kubernetes','compose','dedicated','vds','user');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
CREATE TYPE deployment_action AS ENUM
('ok','pending','pending_move','pending_drop','deleted','missing');
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
} {
if err := DB().Exec(ddl).Error; err != nil {
panic(fmt.Sprintf("migrate: enum creation: %v", err))
}
}
var err error
// Step 1: Migrate core models (User, Plan, Account, ApiKey, AuthIdentity, Invite)
// Note: Access is moved to Step 2 because it has FKs to groups/monitors
// Seed and remap plans before Account migration recreates its plan FK.
migrateOrIgnoreTypeExists(&Plan{})
if err := seedCanonicalPlansAndBackfill(true); err != nil {
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
}
migrateOrIgnoreTypeExists(
&User{},
&Account{},
&ApiKey{},
&authidentity.AuthIdentity{}, // After User (has FK to users)
&Invite{}, // After Account/User (has FKs to accounts/users)
&Subscription{}, // After Account/Plan
&SubscriptionEvent{}, // After Subscription
)
// Monitor and metric rows reference worker_nodes, while worker_nodes itself
// references servers. Create the two roots without their associations before
// migrating Monitor/ServerMetric on a fresh database.
migrateOrIgnoreTypeExists(&Region{}, &LLM{})
if err = DB().Omit("Monitors", "Workers").AutoMigrate(&Server{}); err != nil {
panic(err)
}
migrateOrIgnoreTypeExists(&WorkerNode{})
// Step 2: Migrate Group, Monitor, and Access (which has FKs to groups/monitors)
// This ensures the groups table exists when GORM creates foreign keys
migrateOrIgnoreTypeExists(
&Group{},
&Monitor{},
&Check{},
// Server is a customer-facing logical host, distinct from the
// WorkerNode executor. Keep the join/cache models here so a fresh
// database gets the complete server metrics schema in one migration.
&Server{},
&AccountMCPToken{},
&MonitorServer{},
&ServerMetric{},
&ServerAlertRule{},
&ServerAlertEvent{},
&RknIP{},
&RknDomain{},
&DNSRecord{},
&Contact{},
&Whois{},
&Payment{},
&Message{},
&TelegramBotMessage{},
&TelegramBotStatus{},
&Event{},
&SelfCheck{},
&Notification{}, // After Group/Monitor so notification_groups FK works
&Access{}, // After Group/Monitor so access FKs work
&NotificationCredential{}, // No FKs to other domain tables; safe here.
)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS servers_account_slug_unique ON servers (account_id, slug)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS monitor_servers_position_idx ON monitor_servers (server_id, position)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_server_recent_idx ON server_metrics (server_id, id DESC)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_worker_idx ON server_metrics (worker_id)`)
// Step 3: Clean up orphaned references (now that all tables exist)
// Fix typo in old table name (only present on DBs migrated from older versions)
err = DB().Exec("ALTER TABLE IF EXISTS envent_checks RENAME TO event_checks;").Error
if err != nil {
log.Println(err)
}
err = DB().Exec("DROP TABLE IF EXISTS envent_messages;").Error
if err != nil {
log.Println(err)
}
// Defense-in-depth: make sure the columns that the in-process notifier
// scheduler eagerly queries at startup exist, even if AutoMigrate above
// was skipped or the column was dropped by a manual operation. Without
// these, a fresh restore from a pre-soft-delete production dump will
// panic the first time RunExp preloads Contacts or
// ProcessPendingDeletions queries Users (see internal/notifier for the
// defensive recover() that catches the resulting query errors).
err = DB().Exec(
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT TRUE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS disabled BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS blocked BOOLEAN NOT NULL DEFAULT FALSE",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE users ADD COLUMN IF NOT EXISTS deletion_requested_at TIMESTAMPTZ",
).Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS webhook_token VARCHAR(128)",
).Error
if err != nil {
log.Println(err)
}
var telegramCreds []NotificationCredential
if err = DB().Where("kind = ? AND (webhook_token IS NULL OR webhook_token = '')", CredentialKindTelegram).Find(&telegramCreds).Error; err != nil {
log.Println(err)
}
for i := range telegramCreds {
telegramCreds[i].EnsureWebhookToken()
if err = DB().Save(&telegramCreds[i]).Error; err != nil {
log.Println(err)
}
}
err = DB().Exec("DROP INDEX IF EXISTS idx_notification_credentials_webhook_token").Error
if err != nil {
log.Println(err)
}
err = DB().Exec(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_credentials_webhook_token ON notification_credentials (webhook_token) WHERE webhook_token IS NOT NULL AND webhook_token <> ''",
).Error
if err != nil {
log.Println(err)
}
// Replace the early M0 per-event-id index with provider-scoped webhook
// idempotency: PSP event IDs are only unique inside a provider.
if err = DB().Exec("DROP INDEX IF EXISTS idx_subscription_events_provider_event_id").Error; err != nil {
log.Println(err)
}
if err = DB().Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_subscription_events_provider_event ON subscription_events (provider, provider_event_id) WHERE provider_event_id IS NOT NULL").Error; err != nil {
log.Println(err)
}
// Clean up orphaned event_checks
err = DB().Exec("DELETE FROM event_checks where event_id NOT IN (select id from events)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM event_checks where check_id NOT IN (select id from checks)").Error
if err != nil {
panic(err)
}
// Clean up orphaned event_messages
err = DB().Exec("DELETE FROM event_messages where event_id NOT IN (select id from events)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM event_messages where message_id NOT IN (select id from messages)").Error
if err != nil {
panic(err)
}
// Clean up orphaned notification_contacts
err = DB().Exec("DELETE FROM notification_contacts where notification_id NOT IN (select id from notifications)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM notification_contacts where contact_id NOT IN (select id from contacts)").Error
if err != nil {
panic(err)
}
// Clean up orphaned notification_groups (now safe - groups table exists)
err = DB().Exec("DELETE FROM notification_groups where notification_id NOT IN (select id from notifications)").Error
if err != nil {
panic(err)
}
err = DB().Exec("DELETE FROM notification_groups where group_id NOT IN (select id from groups)").Error
if err != nil {
panic(err)
}
DB().Raw("CREATE INDEX IF NOT EXISTS not_old_events on events (monitor_id, id) where state != 'old'")
DB().Raw("CREATE INDEX IF NOT EXISTS current_events ON event (monitor_id, start_time) WHERE state = 'current'")
DB().Raw("CREATE INDEX IF NOT EXISTS ended_events ON event (monitor_id, start_time) WHERE state = 'ended'")
DB().Raw("CREATE INDEX IF NOT EXISTS queued_messages ON message (id) WHERE state = 'queued'")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, account_id)")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, group_id)")
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS invite_email ON invites (account_id, email)")
// RKN indexes — see app/models/rkn_ip.go EnsureRknIndexes. GORM
// AutoMigrate above declared the uniqueIndex on RknDomain.Domain
// and the cidr column type on RknIP, but GiST on rkn_ips.network
// is not expressible via the GORM tag language; we add it here so
// the (>>=) containment operator used by IsRknIPBlocked has an
// index to back it.
if err := EnsureRknIndexes(); err != nil {
log.Printf("migrate: EnsureRknIndexes failed: %v", err)
}
// Distributed worker models
migrateOrIgnoreTypeExists(
&WorkerLogEvent{},
&CheckAttempt{},
&DiagnosticAuditEvent{},
&CheckRegionResult{},
&Task{},
&TaskReplay{},
&NotificationDelivery{},
&Tag{},
)
// Tags — (account_id, name) is the unique key so a single account
// cannot register two metadata rows for the same tag string. The
// tag name itself is also the join key against monitors.tags, so
// uniqueness is enforced at the table level (not just on the
// metadata row).
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tags_account_name_unique ON tags (account_id, name)`)
// Status pages use globally unique public slugs. A public URL has no
// account component, so account-scoped uniqueness would make /status/:slug
// ambiguous. Soft-deleted rows do not reserve their slug.
// subscriber email index uses lower(email) for case-insensitive
// matching (the codebase does not adopt citext). All five tables
// are created together so M0 ships a consistent schema baseline
// regardless of which milestone first writes rows.
migrateOrIgnoreTypeExists(
&StatusPage{},
&StatusPageSubscriber{},
&StatusPageIncident{},
&StatusPageMaintenance{},
&StatusPageDomain{},
&StatusPageDelivery{},
&StatusPageDigestSchedule{},
&Maintenance{},
)
DB().Exec(`DROP INDEX IF EXISTS status_pages_account_slug_unique`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_pages_slug_unique
ON status_pages (slug) WHERE deleted_at IS NULL`)
if err := ensureSingleCurrentEventInvariant(); err != nil {
panic(fmt.Sprintf("migrate: current event invariant: %v", err))
}
DB().Exec(
`CREATE UNIQUE INDEX IF NOT EXISTS status_page_subscribers_active_email
ON status_page_subscribers (status_page_id, lower(email))
WHERE unsubscribed_at IS NULL`,
)
// Existing installations can already have subscriber rows. Keep the legacy
// token column during the nullable transition: outstanding links remain
// valid, while each resend/confirmation rotates it into a hash.
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS confirm_token_hash varchar(64)`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS token_expires_at timestamptz`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS unsubscribe_token_hash varchar(64)`)
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS contact_id bigint REFERENCES contacts(id) ON DELETE SET NULL`)
DB().Exec(`UPDATE status_page_subscribers SET token_expires_at = created_at + interval '24 hours' WHERE token_expires_at IS NULL`)
for {
var subscribers []StatusPageSubscriber
if err := DB().Where("confirm_token_hash IS NULL AND confirm_token IS NOT NULL").Limit(500).Find(&subscribers).Error; err != nil || len(subscribers) == 0 {
break
}
for i := range subscribers {
subscribers[i].ConfirmTokenHash = HashStatusPageToken(*subscribers[i].LegacyConfirmToken)
_ = DB().Model(&subscribers[i]).Update("confirm_token_hash", subscribers[i].ConfirmTokenHash).Error
}
}
DB().Exec(`UPDATE status_page_subscribers SET confirm_token_hash = '' WHERE confirm_token_hash IS NULL`)
DB().Exec(`UPDATE status_page_subscribers SET unsubscribe_token_hash = '' WHERE unsubscribe_token_hash IS NULL`)
// A page may expose more than one verified hostname; older schema reserved
// only one domain per page.
DB().Exec(`DROP INDEX IF EXISTS idx_status_page_domains_status_page_id`)
DB().Exec(
`CREATE INDEX IF NOT EXISTS status_page_incidents_started
ON status_page_incidents (status_page_id, started_at DESC)`,
)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_page_incidents_event_unique
ON status_page_incidents (status_page_id, event_id) WHERE event_id IS NOT NULL`)
DB().Exec(
`CREATE INDEX IF NOT EXISTS status_page_maintenance_starts
ON status_page_maintenance (status_page_id, starts_at DESC)`,
)
DB().Exec(`CREATE INDEX IF NOT EXISTS maintenances_account_active ON maintenances (account_id) WHERE deleted_at IS NULL`)
// Legacy status-page rows predate account-scoped maintenance. Keep their
// source IDs so equal title/start rows remain distinct and reruns can safely
// preserve every window and each of its joins.
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS maintenances_legacy_status_page_maintenance_unique
ON maintenances (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL`)
DB().Exec(`INSERT INTO maintenances (account_id, title, description, strategy, duration_sec, start_date, end_date, timezone, active, show_on_all_status_pages, legacy_status_page_maintenance_id, created_at, updated_at)
SELECT sp.account_id, old.title, old.description, 'single', EXTRACT(EPOCH FROM (old.ends_at - old.starts_at))::int, old.starts_at, old.ends_at, 'UTC', TRUE, FALSE, old.id, old.created_at, old.updated_at
FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id
ON CONFLICT (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL DO NOTHING`)
DB().Exec(`INSERT INTO maintenance_status_pages (maintenance_id, status_page_id)
SELECT m.id, old.status_page_id FROM status_page_maintenance old JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id
ON CONFLICT DO NOTHING`)
DB().Exec(`INSERT INTO maintenance_monitors (maintenance_id, monitor_id)
SELECT m.id, monitor_id FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id,
LATERAL unnest(CASE WHEN cardinality(old.monitor_ids) > 0 THEN old.monitor_ids ELSE sp.monitor_ids END) AS monitor_id
ON CONFLICT DO NOTHING`)
// Worker-driven task queue indexes.
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_queued_due ON tasks (kind, not_before) WHERE state = 'queued'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_leased_expires ON tasks (lease_expires_at) WHERE state = 'leased'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_failed_retry_due ON tasks (not_before) WHERE state = 'failed_retry'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_dead_kind ON tasks (kind, account_id) WHERE state = 'dead'`)
DB().Exec(`CREATE INDEX IF NOT EXISTS check_attempts_worker_finished ON check_attempts (worker_node_id, finished_at DESC)`)
err = DB().Exec("ALTER TABLE worker_nodes ALTER COLUMN concurrency SET DEFAULT 10").Error
if err != nil {
log.Println(err)
}
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET DEFAULT '{}'::jsonb").Error
if err != nil {
panic(err)
}
err = DB().Exec("UPDATE checks SET settings = '{}'::jsonb WHERE settings IS NULL").Error
if err != nil {
panic(err)
}
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET NOT NULL").Error
if err != nil {
panic(err)
}
// Inventory models (docs/plans/inventory-management.md §6, §10 M0).
// Enum types are created at Step 0 above so they exist before any
// AutoMigrate. The numeric labels of each enum value are taken
// verbatim from rstuff (`/data/int/rstuff/app/models/*.rb`) so a
// future sync layer does not need a value-mapping table — see
// docs/parity/rstuff-inventory.md §6.1.
// Servers — extend with the rstuff-shaped inventory fields. All
// statements are IF NOT EXISTS so existing rows keep working
// untouched (ext_id/token/price_cents default sensibly; meta gets
// an empty jsonb).
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS ext_id VARCHAR(64)`)
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_ext_id_unique ON servers (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS kind server_kind NOT NULL DEFAULT 'production'`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS token VARCHAR(64)`)
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_token_unique ON servers (token) WHERE token IS NOT NULL`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS price_cents INTEGER NOT NULL DEFAULT 0`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS comment TEXT`)
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS meta JSONB NOT NULL DEFAULT '{}'::jsonb`)
// WorkerNodes — optional server_id join for inventory correlation.
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS server_id BIGINT REFERENCES servers(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_server_idx ON worker_nodes (server_id)`)
// WorkerNodes — optional account_id FK marking a private (customer-
// operated) worker per docs/distributed/private-workers.md. NULL
// rows are platform-operated workers eligible to serve any account;
// non-NULL rows are pinned to a single account and are removed by
// HardDeleteAccount before the account row itself is dropped.
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_account_idx ON worker_nodes (account_id)`)
// Notification credentials are either platform-managed (account_id IS NULL)
// or owned by one account. Replace the legacy global name constraint with
// scope-aware unique indexes.
DB().Exec(`ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE CASCADE`)
DB().Exec(`CREATE INDEX IF NOT EXISTS notification_credentials_account_idx ON notification_credentials (account_id)`)
DB().Exec(`DROP INDEX IF EXISTS cred_kind_name`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_system_kind_name_unique ON notification_credentials (kind, name) WHERE account_id IS NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_account_kind_name_unique ON notification_credentials (account_id, kind, name) WHERE account_id IS NOT NULL`)
// Inventory entities. Order matters: server_ips before sites
// (FK), sites before deployments (FK), deployments before
// domains (FK). Audit columns (creator_id/updater_id) use the
// Audited mixin via concerns.Timestamped + Audited.
migrateOrIgnoreTypeExists(
&ServerIp{},
&Repo{},
&Site{},
&Deployment{},
&SiteRepo{},
&Domain{},
)
// Monitors is migrated earlier for historical FK ordering. Add the optional
// site reference only after sites exists on fresh databases.
DB().Exec(`ALTER TABLE monitors ADD COLUMN IF NOT EXISTS site_id BIGINT REFERENCES sites(id) ON DELETE SET NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS monitors_site_idx ON monitors (site_id)`)
// Indexes — kept here so the AutoMigrate path stays the single
// source of truth. IF NOT EXISTS guards the rerun case.
DB().Exec(`CREATE INDEX IF NOT EXISTS server_ips_server_idx ON server_ips (server_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS server_ips_address_unique ON server_ips (server_id, address)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_account_idx ON sites (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_server_idx ON sites (server_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_ext_id_unique ON sites (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_account_slug_unique ON sites (account_id, slug)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_ext_id_unique ON repos (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_gitlab_id_unique ON repos (gitlab_id) WHERE gitlab_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_account_idx ON deployments (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_server_idx ON deployments (server_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_site_idx ON deployments (site_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_ext_id_unique ON deployments (ext_id) WHERE ext_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_account_idx ON domains (account_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_server_idx ON domains (server_id)`)
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_site_idx ON domains (site_id)`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS domains_name_unique ON domains (name)`)
// Dedupe by (server_id, config_path) for nginx sites and by
// (server_id, site_id, service_name) for compose services — matches
// the upsert keys in app/controllers/api/inventory_deploymentd.go.
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_nginx
ON deployments (server_id, config_path) WHERE mode = 'dedicated' AND config_path IS NOT NULL`)
DB().Exec(`DROP INDEX IF EXISTS deployments_unique_compose`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_compose
ON deployments (server_id, site_id, service_name) WHERE mode = 'compose' AND site_id IS NOT NULL AND service_name IS NOT NULL`)
if err := seedCanonicalPlansAndBackfill(false); err != nil {
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
}
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS plans_active_code_unique ON plans (code) WHERE archived = FALSE`)
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS subscriptions_provider_external_unique ON subscriptions (provider, provider_subscription_id) WHERE provider_subscription_id IS NOT NULL`)
DB().Exec(`CREATE INDEX IF NOT EXISTS subscriptions_account_status_idx ON subscriptions (account_id, status)`)
DB().Exec(`DO $$ BEGIN
ALTER TABLE plans ADD CONSTRAINT plans_price_nonnegative CHECK (price_monthly_minor >= 0 AND price_annual_minor >= 0);
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
DB().Exec(`DO $$ BEGIN
ALTER TABLE plans ADD CONSTRAINT plans_limits_valid CHECK (monitor_cap >= 0 AND interval_min_seconds >= 30);
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
// Seed default region. Use defaultRegionCode (declared in check_jobs.go)
// so the literal does not appear three times in the package.
localRegion := Region{}
DB().Where("code = ?", defaultRegionCode).First(&localRegion)
if localRegion.ID == 0 {
DB().Create(&Region{Code: defaultRegionCode, Name: "Local (default)", Enabled: true, Priority: 100})
}
DB().Exec(`
WITH t AS (
select u.id as user_id, i.encrypted_password as encrypted_password
from users as u
join identities as i on u.id = i.user_id
where u.encrypted_password is NULL
)
UPDATE users
SET encrypted_password = t.encrypted_password
from t
where users.id = t.user_id
`)
log.Println("migrated DB.")
}
func prepareCanonicalPlansTable() error {
if !DB().Migrator().HasTable("plans") || DB().Migrator().HasColumn("plans", "code") {
return nil
}
return DB().Transaction(func(tx *gorm.DB) error {
return tx.Exec(`ALTER TABLE plans RENAME TO plans_legacy`).Error
})
}
func planForeignKeyReferences(tx *gorm.DB, table, target string) (bool, error) {
var references bool
err := tx.Raw(`SELECT EXISTS (
SELECT 1 FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.contype = 'f' AND c.conrelid = to_regclass(?)
AND c.confrelid = to_regclass(?) AND a.attname = 'plan_id'
)`, table, target).Scan(&references).Error
return references, err
}
func seedCanonicalPlansAndBackfill(remapAccounts bool) error {
plans := []Plan{
{Code: "free", NameRU: "Бесплатный", NameEN: "Free", Currency: "RUB", MonitorCap: 50, IntervalMinSeconds: 300, StatusPagesCap: 1, MaintenanceCap: -1, LoginSeatsIncluded: 3, NotifySeatsIncluded: 0, Integrations: []string{"email", "telegram"}, CheckKinds: []string{"http", "ssl", "dns", "whois", "ping"}, DataRetentionMonths: 3, IsDefault: true},
{Code: "solo", NameRU: "Соло", NameEN: "Solo", PriceMonthlyMinor: 74900, PriceAnnualMinor: 64900, Currency: "RUB", MonitorCap: 10, IntervalMinSeconds: 60, StatusPagesCap: 3, MaintenanceCap: 5, LoginSeatsIncluded: 5, NotifySeatsIncluded: 3, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping"}, DataRetentionMonths: 12, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 60},
{Code: "team", NameRU: "Команда", NameEN: "Team", PriceMonthlyMinor: 299000, PriceAnnualMinor: 254900, Currency: "RUB", MonitorCap: 100, IntervalMinSeconds: 60, StatusPagesCap: 100, MaintenanceCap: 50, LoginSeatsIncluded: 5, NotifySeatsIncluded: 5, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 24, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 45},
{Code: "enterprise", NameRU: "Предприятие", NameEN: "Enterprise", PriceMonthlyMinor: 549000, PriceAnnualMinor: 464900, Currency: "RUB", MonitorCap: 200, IntervalMinSeconds: 30, StatusPagesCap: 0, MaintenanceCap: 0, LoginSeatsIncluded: 0, NotifySeatsIncluded: 0, UnlimitedSeats: true, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook", "sso_saml"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 36, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 30, SOC2: true, GDPRDPA: true},
}
return DB().Transaction(func(tx *gorm.DB) error {
for i := range plans {
var existing Plan
err := tx.Where("code = ?", plans[i].Code).First(&existing).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
if err := tx.Create(&plans[i]).Error; err != nil {
return err
}
} else if err != nil {
return err
}
}
// Catalog rows already existed before maintenance_cap was introduced.
// Update this new entitlement only; do not overwrite customer-adjusted
// prices or other existing plan attributes during a normal migration.
if err := tx.Exec(`UPDATE plans SET maintenance_cap = CASE code
WHEN 'free' THEN -1 WHEN 'solo' THEN 5 WHEN 'team' THEN 50 WHEN 'enterprise' THEN 0 ELSE maintenance_cap END
WHERE code IN ('free', 'solo', 'team', 'enterprise')`).Error; err != nil {
return err
}
hasLegacy := tx.Migrator().HasTable("plans_legacy")
hasMigratedLegacy := tx.Migrator().HasTable("plans_legacy_migrated")
if !hasLegacy && !hasMigratedLegacy {
return nil
}
hasAccounts := tx.Migrator().HasTable("accounts")
accountsNeedRemap := false
if remapAccounts && hasLegacy && hasAccounts {
referencesCanonical, err := planForeignKeyReferences(tx, "accounts", "plans")
if err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
return err
}
accountsNeedRemap = !referencesCanonical
if accountsNeedRemap {
if err := tx.Exec(`UPDATE accounts SET plan_id = CASE
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id)`).Error; err != nil {
return err
}
}
}
hasSubscriptions := tx.Migrator().HasTable("subscriptions")
subscriptionsNeedRemap := false
if remapAccounts && hasLegacy && hasSubscriptions {
referencesCanonical, err := planForeignKeyReferences(tx, "subscriptions", "plans")
if err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
return err
}
subscriptionsNeedRemap = !referencesCanonical
if subscriptionsNeedRemap {
if err := tx.Exec(`UPDATE subscriptions SET plan_id = CASE
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id)`).Error; err != nil {
return err
}
}
}
if remapAccounts && hasLegacy && (accountsNeedRemap || subscriptionsNeedRemap) && tx.Migrator().HasTable("subscription_events") {
if err := tx.Exec(`UPDATE subscription_events e SET from_plan_id = CASE
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
FROM plans_legacy l WHERE e.from_plan_id = l.id`).Error; err != nil {
return err
}
if err := tx.Exec(`UPDATE subscription_events e SET to_plan_id = CASE
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
ELSE (SELECT id FROM plans WHERE code = 'solo') END
FROM plans_legacy l WHERE e.to_plan_id = l.id`).Error; err != nil {
return err
}
}
if !remapAccounts && hasAccounts && hasSubscriptions {
now := time.Now().UTC()
if err := tx.Exec(`INSERT INTO subscriptions (account_id, plan_id, provider, status, billing_cycle, current_period_start, current_period_end, currency, amount_minor, metadata_json, created_at, updated_at)
SELECT a.id, a.plan_id, 'manual', 'active', 'monthly', ?, ?, p.currency, p.price_monthly_minor, '{}'::jsonb, ?, ?
FROM accounts a JOIN plans p ON p.id = a.plan_id
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.account_id = a.id)`, now, now.AddDate(0, 0, 30), now, now).Error; err != nil {
return err
}
}
if !remapAccounts || !hasLegacy {
return nil
}
if hasAccounts {
// AutoMigrate names this association fk_accounts_plan, while an older
// migration used accounts_plan_id_fkey. Either name may still point at
// plans_legacy after the table rename, so replace both deterministically.
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE accounts ADD CONSTRAINT fk_accounts_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
return err
}
}
if hasSubscriptions {
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
return err
}
if err := tx.Exec(`ALTER TABLE subscriptions ADD CONSTRAINT fk_subscriptions_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
return err
}
}
return tx.Exec("ALTER TABLE plans_legacy RENAME TO plans_legacy_migrated").Error
})
}

162
app/models/migrate_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,162 @@
package models
import (
"context"
"fmt"
"os"
"strings"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
func TestMigrateUpgradesLegacyPlansWithoutClosingLockConnection(t *testing.T) {
var databaseName string
require.NoError(t, DB().Raw("SELECT current_database()").Scan(&databaseName).Error)
require.Contains(t, databaseName, "test")
original := db
schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
require.NoError(t, original.Exec("CREATE SCHEMA "+schema).Error)
var isolatedSQLDB interface{ Close() error }
t.Cleanup(func() {
SetDB(original)
if isolatedSQLDB != nil {
_ = isolatedSQLDB.Close()
}
original.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE")
})
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable search_path=%s",
testDatabaseEnv("DATABASE_HOST", "POSTGRES_HOST", "localhost"),
testDatabaseEnv("DATABASE_PORT", "POSTGRES_PORT", "5432"),
testDatabaseEnv("DATABASE_USER", "POSTGRES_USER", "rsmon"),
testDatabaseEnv("DATABASE_PASSWORD", "POSTGRES_PASSWORD", "rsmon"),
databaseName,
schema,
)
isolated, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
require.NoError(t, err)
sqlDB, err := isolated.DB()
require.NoError(t, err)
isolatedSQLDB = sqlDB
RegisterCallbacks(isolated)
SetDB(isolated.Set("gorm:association_autoupdate", false))
require.NoError(t, DB().Exec(`CREATE TABLE plans (
id BIGSERIAL PRIMARY KEY,
name TEXT NOT NULL,
price BIGINT NOT NULL DEFAULT 0,
total_monitors BIGINT NOT NULL DEFAULT 0,
"default" BOOLEAN NOT NULL DEFAULT FALSE
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE accounts (
id BIGSERIAL PRIMARY KEY,
name TEXT NOT NULL,
plan_id BIGINT REFERENCES plans(id)
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE subscriptions (
id BIGSERIAL PRIMARY KEY,
account_id BIGINT NOT NULL REFERENCES accounts(id),
plan_id BIGINT NOT NULL REFERENCES plans(id),
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
status VARCHAR(24) NOT NULL DEFAULT 'active',
billing_cycle VARCHAR(8) NOT NULL DEFAULT 'monthly',
currency VARCHAR(3) NOT NULL DEFAULT 'RUB',
amount_minor BIGINT NOT NULL DEFAULT 0,
metadata_json JSONB NOT NULL DEFAULT '{}'
)`).Error)
require.NoError(t, DB().Exec(`CREATE TABLE subscription_events (
id BIGSERIAL PRIMARY KEY,
subscription_id BIGINT NOT NULL,
account_id BIGINT NOT NULL,
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
kind VARCHAR(32) NOT NULL,
from_plan_id BIGINT,
to_plan_id BIGINT,
payload_json JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)`).Error)
require.NoError(t, DB().Exec(`INSERT INTO plans (id, name, price, total_monitors, "default")
VALUES (42, 'Legacy free', 0, 10, TRUE)`).Error)
require.NoError(t, DB().Exec("INSERT INTO accounts (id, name, plan_id) VALUES (7, 'Legacy account', 42)").Error)
require.NoError(t, DB().Exec("INSERT INTO subscriptions (id, account_id, plan_id) VALUES (9, 7, 42)").Error)
require.NoError(t, DB().Exec(`INSERT INTO subscription_events
(id, subscription_id, account_id, kind, from_plan_id, to_plan_id)
VALUES (11, 9, 7, 'legacy_change', 42, 42)`).Error)
Migrate()
require.True(t, DB().Migrator().HasColumn("plans", "code"))
require.False(t, DB().Migrator().HasTable("plans_legacy"))
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
var codes []string
require.NoError(t, DB().Table("plans").Order("code").Pluck("code", &codes).Error)
require.ElementsMatch(t, CanonicalPlanCodes(), codes)
var accountPlanCode string
require.NoError(t, DB().Table("accounts").Select("plans.code").
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
Scan(&accountPlanCode).Error)
require.Equal(t, "free", accountPlanCode)
var subscriptionPlanCode string
require.NoError(t, DB().Table("subscriptions").Select("plans.code").
Joins("JOIN plans ON plans.id = subscriptions.plan_id").Where("subscriptions.id = 9").
Scan(&subscriptionPlanCode).Error)
require.Equal(t, "free", subscriptionPlanCode)
var eventPlanCodes struct {
FromCode string
ToCode string
}
require.NoError(t, DB().Table("subscription_events e").
Select("fp.code AS from_code, tp.code AS to_code").
Joins("JOIN plans fp ON fp.id = e.from_plan_id").
Joins("JOIN plans tp ON tp.id = e.to_plan_id").
Where("e.id = 11").Scan(&eventPlanCodes).Error)
require.Equal(t, "free", eventPlanCodes.FromCode)
require.Equal(t, "free", eventPlanCodes.ToCode)
require.NoError(t, DB().Exec("SELECT 1").Error)
// The previous implementation retained this name after remapping. Its
// canonical FKs must prevent a retry from remapping the same rows again.
require.NoError(t, DB().Exec("ALTER TABLE plans_legacy_migrated RENAME TO plans_legacy").Error)
require.NotPanics(t, Migrate)
accountPlanCode = ""
require.NoError(t, DB().Table("accounts").Select("plans.code").
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
Scan(&accountPlanCode).Error)
require.Equal(t, "free", accountPlanCode)
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
}
func TestMigrationAdvisoryLockIsReleasedAfterPanic(t *testing.T) {
sqlDB, err := DB().DB()
require.NoError(t, err)
otherConn, err := sqlDB.Conn(context.Background())
require.NoError(t, err)
t.Cleanup(func() { _ = otherConn.Close() })
require.PanicsWithValue(t, "migration failed", func() {
withMigrationAdvisoryLock(func() { panic("migration failed") })
})
const migrateAdvisoryLock = int64(1234567890)
var acquired bool
require.NoError(t, otherConn.QueryRowContext(context.Background(),
"SELECT pg_try_advisory_lock($1)", migrateAdvisoryLock).Scan(&acquired))
require.True(t, acquired)
_, err = otherConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock)
require.NoError(t, err)
}
func testDatabaseEnv(primary, fallback, defaultValue string) string {
if value := os.Getenv(primary); value != "" {
return value
}
if value := os.Getenv(fallback); value != "" {
return value
}
return defaultValue
}

534
app/models/monitor.go Обычный файл
Просмотреть файл

@@ -0,0 +1,534 @@
package models
import (
"log"
"sync"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/lib/pq"
"github.com/pkg/errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
stateOK = "OK"
stateERR = "ERR"
stateWARN = "WARN"
stateFail = "FAIL"
stateDegraded = "DEGRADED"
// Event states.
stateEnded = "ended"
// Check column names used in map[string]interface{} GORM updates. Defining
// them as constants keeps GORM column references in sync with model fields.
colLastStart = "last_start"
colLastEnd = "last_end"
colState = "state"
colWarnings = "warnings"
colInfos = "infos"
// Check kinds. Used to avoid sprinkling magic strings across the codebase.
kindHTTP = "http"
kindSSL = "ssl"
kindSSH = "ssh"
kindFTP = "ftp"
kindDNS = "dns"
kindWhois = "whois"
kindRKN = "rkn"
kindBSSL = "bssl"
kindLLM = "llm"
kindLLMHTTP = "llm-http"
kindPing = "ping"
kindTCP = "tcp"
kindUDP = "udp"
)
// Monitor monitor
type Monitor struct {
concerns.Model
// activity status
Enabled bool `gorm:"not null;default:true" json:"enabled"`
// check state, OK - all green, ERR - some checks have failed, UNK - new or not run, FAIL - unable to check
State string `gorm:"not null;default:'UNK'" json:"state"`
ConfirmState string `gorm:"size:32;not null;default:'none';index" json:"confirm_state"`
ConfirmAt *time.Time `json:"confirm_at,omitempty"`
ConfirmedByWorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"confirmed_by_worker_id,omitempty"`
// Group ID
GroupID int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty" validate:"required"`
Group *Group `json:"group,omitempty"`
// Optional inventory Site join (see docs/plans/inventory-management.md §6.3).
// Lets the operator navigate monitor → site → deployments → server in one query.
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
Site *Site `json:"site,omitempty"`
// Tags for monitor grouping/searching
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
// PreferredRegions is the optional whitelist of region codes a distributed
// worker must be in to receive this monitor's checks. Empty/NULL means
// "no preference" — any worker can pick it up. Used by Phase 2 of
// docs/plans/worker-notifier-mvp.md (regional job routing); see
// app/models/check_jobs.go applyRegionRouting.
PreferredRegions pq.StringArray `gorm:"type:varchar(64)[]" json:"preferred_regions,omitempty"`
// RegionMode controls how PreferredRegions is interpreted by the
// distributed-worker job router. Defaults to "any" so monitors without
// explicit routing still match every worker — backwards-compatible with
// Phase 1 deployments.
// "any" — no region filter; legacy behavior (default)
// "specific" — only workers whose region_code is in PreferredRegions
// "all" — Phase 3 placeholder; today behaves like "any". The
// multi-region quorum aggregation is not implemented yet,
// see docs/todo.md Phase 3.
RegionMode string `gorm:"size:16;not null;default:'any'" json:"region_mode" validate:"omitempty,oneof=any specific all"`
// Monitor name
Name *string `json:"name,omitempty"`
// Host to monitor
Host string `json:"host" validate:"required"`
// UserID specify user for this monitor (info field)
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
User *User `json:"user,omitempty"`
// Comment (info field)
Comment *string `json:"comment"`
Checks []Check `json:"checks,omitempty"`
DNSRecords []DNSRecord `json:"-"`
StatsData `gorm:"-:all" sql:"-" json:"stats"`
concerns.Timestamped
Audited
}
// KINDS Check kinds
var KINDS = []string{kindHTTP, kindSSL, kindSSH, kindFTP, kindDNS, kindWhois, kindRKN, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
// ValidCheckKind is the single canonical allow-list for user supplied check
// kinds. Keep it beside the kind constants so every transport validates the
// same set before a check reaches a worker.
func ValidCheckKind(kind string) bool {
for _, candidate := range KINDS {
if kind == candidate {
return true
}
}
return false
}
// Region routing mode constants used by Monitor.RegionMode and
// app/models/check_jobs.go applyRegionRouting. Centralized so the literal
// values are not sprinkled through the codebase.
const (
// RegionModeAny keeps the legacy behavior: every worker is eligible,
// PreferredRegions is ignored. Default for newly-created monitors.
RegionModeAny = "any"
// RegionModeSpecific limits eligible workers to those whose RegionCode
// is contained in PreferredRegions. Empty PreferredRegions falls back to
// RegionModeAny so the field is safe to leave blank in the UI.
RegionModeSpecific = "specific"
// RegionModeAll is the Phase 3 placeholder: a monitor pinned to all of
// its preferred regions for quorum aggregation. Phase 2 treats it as
// RegionModeAny and logs a TODO marker so it is easy to grep for.
RegionModeAll = "all"
)
// RegionCodesFromSlice is a convenience wrapper so callers (mainly tests
// and HTTP handlers) can pass a plain []string and get the pq.StringArray
// type the model expects. nil/empty input is preserved as a nil slice so
// the GORM column writes a SQL NULL instead of an empty array, matching
// the column default.
func RegionCodesFromSlice(in []string) pq.StringArray {
if len(in) == 0 {
return nil
}
out := make(pq.StringArray, len(in))
copy(out, in)
return out
}
// Int64ArrayFromSlice mirrors RegionCodesFromSlice for bigint[] columns
// such as status_pages.monitor_ids and status_page_maintenance.monitor_ids.
// The GORM pq.Int64Array driver expects a non-nil slice for ordered
// inserts; callers that always have a non-empty list (the dashboard list
// filter, the maintenance form) can rely on this to write a stable shape.
func Int64ArrayFromSlice(in []int64) pq.Int64Array {
if len(in) == 0 {
return pq.Int64Array{}
}
out := make(pq.Int64Array, len(in))
copy(out, in)
return out
}
// ValidateRegionMode returns an error when RegionMode is not one of the
// documented values ("any", "specific", "all"). Empty strings are treated as
// "any" for backwards compatibility with monitors persisted before the field
// existed; the DB column also defaults to "any".
func (m *Monitor) ValidateRegionMode() error {
switch m.RegionMode {
case "", RegionModeAny, RegionModeSpecific, RegionModeAll:
return nil
default:
return errors.Errorf("invalid region_mode %q (expected any|specific|all)", m.RegionMode)
}
}
// WantsRegion returns true when the monitor should be routed to a worker
// operating in the given region code. Callers use this in
// app/models/check_jobs.go to filter the eligible worker pool per check.
//
// - RegionModeAny: always true (no preference).
// - RegionModeAll (Phase 3 placeholder): behaves like Any today; returns
// true unconditionally so every region sees the check.
// - RegionModeSpecific: true when code is contained in PreferredRegions,
// or when PreferredRegions is empty (fall-back to Any).
func (m *Monitor) WantsRegion(code string) bool {
switch m.RegionMode {
case RegionModeSpecific:
if len(m.PreferredRegions) == 0 {
return true
}
for _, r := range m.PreferredRegions {
if r == code {
return true
}
}
return false
case RegionModeAll:
// TODO(phase3): enumerate PreferredRegions and emit one assignment
// per region so the result aggregator can do quorum. Today we
// behave like Any so existing workers keep getting checks.
return true
default:
return true
}
}
// GetLabel provides functionality.
func (m *Monitor) GetLabel() string {
if m.Name != nil {
return *m.Name
}
return m.Host
}
// ProcessChecks provides functionality.
func (m *Monitor) ProcessChecks(tx *gorm.DB) error {
log.Println("process checks")
checks := make([]Check, 0)
for _, c := range m.Checks { //nolint:gocritic // range copy is acceptable here
log.Println("maybe delete check", c.ID, c.Deleted, c.IsNew)
if c.Deleted {
if !c.IsNew {
log.Println("delete check", c.ID)
err := tx.Exec("delete from event_checks where check_id = ?", c.ID).Error
if err != nil {
return err
}
// First, find all message IDs for this check
var messageIDs []int64
err = tx.Model(&Message{}).Where("check_id = ?", c.ID).Pluck("id", &messageIDs).Error
if err != nil {
return err
}
// Delete event_messages (join table) first to avoid FK constraint violation
if len(messageIDs) > 0 {
err = tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error
if err != nil {
return err
}
}
// Now delete the messages
err = tx.Where("check_id = ?", c.ID).Delete(Message{}).Error
if err != nil {
return err
}
err = tx.Where("id = ? AND monitor_id = ?", c.ID, m.ID).Delete(Check{}).Error
if err != nil {
return err
}
}
continue
}
if c.IsNew {
c.ID = 0
}
err := c.ValidateSettings()
if err != nil {
return errors.Wrap(err, "check validation error")
}
checks = append(checks, c)
}
m.Checks = checks
return nil
}
// ActiveEvent provides functionality.
func (m *Monitor) ActiveEvent() Event {
evt := Event{}
DB().Where("monitor_id = ? AND state != 'old'", m.ID).First(&evt)
if evt.ID != 0 {
evt.MonitorID = m.ID
t := time.Now()
evt.StartTime = &t
}
return evt
}
var mutex sync.Mutex
// checkSeverityRank assigns an ordinal to each check state so the monitor
// aggregator can pick the highest-severity child deterministically.
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
// real check state takes precedence over them.
func checkSeverityRank(state string) int {
switch state {
case stateFail:
return 5
case stateERR:
return 4
case stateDegraded:
return 3
case stateWARN:
return 2
case stateOK:
return 1
default:
return 0
}
}
// UpdateStatusFromChecks updates the monitor status based on its checks.
func (m *Monitor) UpdateStatusFromChecks() {
mutex.Lock()
checks := make([]Check, 0)
tx := DB().Begin()
var locked Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&locked, m.ID).Error; err != nil {
tx.Rollback()
mutex.Unlock()
log.Println("UpdateStatusFromChecks lock monitor", err)
return
}
m.State = locked.State
_ = tx.Model(m).Association("Checks").Find(&checks)
prevState := m.State
m.State = stateOK
// Pick the highest-severity enabled check. The previous implementation
// inlined three if-statements with non-obvious precedence (a WARN that
// appeared AFTER an ERR in the iteration would never downgrade back,
// but a FAIL after ERR would silently get clobbered). Using a single
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
// independent of slice ordering — the same rule Phase 3 introduces
// for DEGRADED, applied uniformly to the existing states too.
bestRank := checkSeverityRank(stateOK)
bestState := stateOK
hasChecks := false
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
if check.Enabled == nil || !*check.Enabled {
continue
}
hasChecks = true
if r := checkSeverityRank(check.State); r > bestRank {
bestRank = r
bestState = check.State
}
}
m.State = bestState
if !hasChecks {
m.State = stateWARN
}
if m.State != prevState {
err := tx.Model(&m).UpdateColumn("state", m.State).Error
if err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks fail update state", err)
mutex.Unlock()
return
}
}
evt := Event{}
if err := tx.Where("monitor_id = ? AND state = ?", m.ID, "current").First(&evt).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
tx.Rollback()
mutex.Unlock()
log.Println("UpdateStatusFromChecks active event", err)
return
}
// log.Println("state", m.State, "active event:", evt.ID)
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
if check.State == stateERR || check.State == stateFail {
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
evt.Checks = append(evt.Checks, check)
if check.Error != nil {
evt.Reason = *check.Error
} else {
evt.Reason = "unknown error"
}
}
}
switch m.State {
case stateOK, stateWARN, stateDegraded:
// DEGRADED is treated like OK/WARN for the event lifecycle: we do
// NOT open a new "current" outage event for a partial regional
// failure. Operators see DEGRADED on the check detail page and
// the monitor list, but the existing notifier pipeline (down /
// restore events) only fires for full ERR/FAIL. A future
// improvement can add a separate "degraded" message kind.
if evt.ID != 0 {
upd := map[string]interface{}{
"duration": time.Since(*evt.StartTime).Seconds(),
"state": stateEnded,
"oks": evt.Oks + 1,
}
if evt.EndTime == nil {
upd["end_time"] = time.Now()
}
err := tx.Model(&evt).UpdateColumns(upd).Error
if err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks fail update to ended", err)
mutex.Unlock()
return
}
}
case stateERR, stateFail:
if evt.ID == 0 {
tn := time.Now()
evt.StartTime = &tn
evt.Duration = 0
evt.State = "current"
evt.MonitorID = m.ID
err := tx.Save(&evt).Error
if err != nil {
tx.Rollback()
spew.Dump(evt)
log.Println("UpdateStatusFromChecks fail create", err)
mutex.Unlock()
return
}
} else {
upd := map[string]interface{}{
"end_time": nil,
"state": "current",
"errors": evt.Errors + 1,
}
if evt.StartTime == nil {
upd["start_time"] = time.Now()
upd["duration"] = 0
} else {
upd["duration"] = time.Since(*evt.StartTime).Seconds()
}
err := tx.Model(&evt).UpdateColumns(upd).Error
if err != nil {
tx.Rollback()
spew.Dump(evt)
spew.Dump(upd)
log.Println("UpdateStatusFromChecks fail update to current", err)
mutex.Unlock()
return
}
}
}
if err := m.syncStatusPageIncidentsTx(tx, &evt); err != nil {
tx.Rollback()
log.Println("UpdateStatusFromChecks status page incident", err)
mutex.Unlock()
return
}
err := tx.Commit().Error
mutex.Unlock()
if err != nil {
log.Println("UpdateStatusFromChecks commit fail", err)
return
}
if m.State != prevState {
m.invalidateStatusPages()
}
}
func (m *Monitor) invalidateStatusPages() {
var ids []int64
if err := DB().Model(&StatusPage{}).Where("? = ANY(monitor_ids)", m.ID).Pluck("id", &ids).Error; err != nil {
return
}
for _, id := range ids {
InvalidateStatusPageCache(id)
}
}
func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
if event == nil || event.ID == 0 {
return nil
}
var pages []StatusPage
if err := tx.Where("auto_open_incidents = TRUE AND ? = ANY(monitor_ids)", m.ID).Find(&pages).Error; err != nil {
return err
}
for i := range pages {
page := &pages[i]
var incident StatusPageIncident
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
if m.State == stateERR || m.State == "FAIL" {
if err != nil {
// The database uniqueness constraint makes concurrent state updates idempotent.
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
if err := tx.Create(&incident).Error; err != nil {
return err
}
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "opened"); err != nil {
return err
}
} else if incident.BodyMD != event.Reason {
if err := tx.Model(&incident).Update("body_md", event.Reason).Error; err != nil {
return err
}
incident.BodyMD, incident.UpdatedAt = event.Reason, time.Now()
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "updated"); err != nil {
return err
}
}
} else if err == nil && incident.ResolvedAt == nil {
now := time.Now()
if err := tx.Model(&incident).Update("resolved_at", now).Error; err != nil {
return err
}
incident.ResolvedAt, incident.UpdatedAt = &now, now
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "resolved"); err != nil {
return err
}
}
}
return nil
}

215
app/models/monitor_state_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,215 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// monitorStateTestWorld bundles the account/group/monitor fixture the
// monitor-state tests need. It deliberately re-creates the row in each
// test rather than sharing, because Monitor.UpdateStatusFromChecks
// mutates the row in place and the per-test assertions need a clean
// baseline.
type monitorStateTestWorld struct {
plan models.Plan
account models.Account
group models.Group
monitor models.Monitor
}
// seedMonitorStateWorld provisions one plan/account/group/monitor with
// the requested initial state. The monitor is enabled so
// UpdateStatusFromChecks treats its checks as live.
func seedMonitorStateWorld(t *testing.T, initialState string) monitorStateTestWorld {
t.Helper()
models.Drop()
models.Migrate()
plan := models.Plan{Name: "ms-plan", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
account := models.Account{Name: "ms-acc", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&account).Error)
group := models.Group{AccountID: account.ID, Name: "ms"}
require.NoError(t, models.DB().Create(&group).Error)
mon := models.Monitor{
Name: stringPtrAgg("mon.test"),
Host: "mon.test",
GroupID: group.ID,
Enabled: true,
State: initialState,
}
require.NoError(t, models.DB().Create(&mon).Error)
return monitorStateTestWorld{
plan: plan,
account: account,
group: group,
monitor: mon,
}
}
// attachCheck creates an enabled check on the monitor with the given
// state. Returns the persisted check so the test can re-load it.
func attachCheck(t *testing.T, monitorID int64, kind string, state string) models.Check {
t.Helper()
enTrue := true
c := models.Check{
MonitorID: monitorID,
Kind: kind,
Interval: 60,
Enabled: &enTrue,
State: state,
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&c).Error)
return c
}
// reloadMonitor pulls the latest monitor state from the DB so the test
// can compare against the post-UpdateStatusFromChecks row.
func reloadMonitor(t *testing.T, id int64) models.Monitor {
t.Helper()
var m models.Monitor
require.NoError(t, models.DB().First(&m, id).Error)
return m
}
// ---------------------------------------------------------------------------
// Phase 3 severity rules: FAIL > ERR > DEGRADED > WARN > OK.
// ---------------------------------------------------------------------------
// TestMonitorStateFromChecks_Degraded: monitor with 1 OK + 1 DEGRADED
// child → monitor.State = DEGRADED. The new severity rank must
// correctly promote DEGRADED above OK.
func TestMonitorStateFromChecks_Degraded(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "DEGRADED", got.State,
"DEGRADED child must promote monitor above OK")
}
// TestMonitorStateFromChecks_DegradedWithError: monitor with 1 ERR +
// 1 DEGRADED → monitor.State = ERR. ERR beats DEGRADED in the
// severity order.
func TestMonitorStateFromChecks_DegradedWithError(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "ERR")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "ERR", got.State,
"ERR must beat DEGRADED — the order is FAIL > ERR > DEGRADED > WARN > OK")
}
// TestMonitorStateFromChecks_DegradedOnlyOK covers the single-DEGRADED
// case explicitly so a regression that treats DEGRADED as "WARN-ish"
// would flip this assertion.
func TestMonitorStateFromChecks_DegradedOnlyOK(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "DEGRADED", got.State,
"mixed OK+DEGRADED monitor must be DEGRADED, not OK")
}
// TestMonitorStateFromChecks_SeverityOrderingTable is a table-driven
// sweep of the FAIL > ERR > DEGRADED > WARN > OK ladder. The case set
// is intentionally small — every pair that could reveal a wrong
// winner under the new severity rank. Keeping it table-driven makes
// it trivial to add more cases if a future state is introduced.
func TestMonitorStateFromChecks_SeverityOrderingTable(t *testing.T) {
cases := []struct {
name string
checks []string
wantMon string
}{
{"all_ok", []string{"OK", "OK"}, "OK"},
{"all_warn", []string{"WARN", "WARN"}, "WARN"},
{"ok_with_warn", []string{"OK", "WARN"}, "WARN"},
{"warn_with_ok", []string{"WARN", "OK"}, "WARN"}, // order independence
{"ok_with_degraded", []string{"OK", "DEGRADED"}, "DEGRADED"},
{"warn_with_degraded", []string{"WARN", "DEGRADED"}, "DEGRADED"},
{"degraded_with_warn", []string{"DEGRADED", "WARN"}, "DEGRADED"},
{"err_with_degraded", []string{"ERR", "DEGRADED"}, "ERR"},
{"degraded_with_err", []string{"DEGRADED", "ERR"}, "ERR"},
{"fail_with_err", []string{"FAIL", "ERR"}, "FAIL"},
{"err_with_fail", []string{"ERR", "FAIL"}, "FAIL"}, // order independence
{"fail_alone", []string{"FAIL"}, "FAIL"},
{"all_degraded", []string{"DEGRADED", "DEGRADED"}, "DEGRADED"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
for i, st := range c.checks {
attachCheck(t, world.monitor.ID,
"http-"+string(rune('a'+i)), st)
}
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, c.wantMon, got.State,
"severity winner for child states %v must be %s", c.checks, c.wantMon)
})
}
}
// TestMonitorStateFromChecks_NoChecksIsWarn pins the existing "no
// enabled checks → WARN" behavior — the new severity rank must not
// accidentally produce OK for an empty monitor.
func TestMonitorStateFromChecks_NoChecksIsWarn(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "WARN", got.State,
"an enabled monitor with zero checks must remain WARN")
}
// TestMonitorStateFromChecks_DisabledCheckIgnored verifies that a
// disabled check is not folded into the severity decision. Otherwise
// a stuck-in-ERR check that has been disabled would keep tripping the
// monitor forever.
func TestMonitorStateFromChecks_DisabledCheckIgnored(t *testing.T) {
world := seedMonitorStateWorld(t, "UNK")
attachCheck(t, world.monitor.ID, "http", "OK")
enFalse := false
disabled := models.Check{
MonitorID: world.monitor.ID,
Kind: "http-disabled",
Interval: 60,
Enabled: &enFalse,
State: "ERR",
Settings: datatypes.JSON([]byte(`{}`)),
}
require.NoError(t, models.DB().Create(&disabled).Error)
world.monitor.UpdateStatusFromChecks()
got := reloadMonitor(t, world.monitor.ID)
assert.Equal(t, "OK", got.State,
"disabled check must be ignored — only the enabled OK check counts")
}
// guard against time import being pruned by an editor when individual
// test bodies stop referencing it directly.
var _ = time.Second

94
app/models/monitor_transfer_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,94 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models"
)
// TestMonitorTransferGroupSwap verifies that swapping a monitor's
// group_id between two accounts correctly re-homes the monitor without
// touching any other monitor data. This is the database primitive that
// POST /api/v1/monitors/:id/transfer relies on.
func TestMonitorTransferGroupSwap(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
// Two accounts, each with their own default group.
accA := models.Account{Name: "A", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&accA).Error)
accB := models.Account{Name: "B", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&accB).Error)
groupA := models.Group{AccountID: accA.ID, Name: "A-default"}
require.NoError(t, models.DB().Create(&groupA).Error)
groupB := models.Group{AccountID: accB.ID, Name: "B-default"}
require.NoError(t, models.DB().Create(&groupB).Error)
// Monitor lives in account A with one HTTP check.
monitor := models.Monitor{
GroupID: groupA.ID,
Host: "example-a.test",
}
require.NoError(t, models.DB().Create(&monitor).Error)
check := models.Check{
MonitorID: monitor.ID,
Kind: "http",
URL: ptrString("https://example-a.test/"),
Interval: 300,
Settings: datatypes.JSON([]byte("{}")),
}
require.NoError(t, models.DB().Create(&check).Error)
// Simulate the controller-side update.
require.NoError(t, models.DB().
Model(&models.Monitor{}).
Where("id = ?", monitor.ID).
Update("group_id", groupB.ID).Error)
// Monitor now lives in B; check follows by FK on monitor_id.
var reloaded models.Monitor
require.NoError(t, models.DB().Preload("Group").First(&reloaded, monitor.ID).Error)
assert.Equal(t, groupB.ID, reloaded.GroupID, "monitor group_id should now point at account B's group")
assert.Equal(t, accB.ID, reloaded.Group.AccountID, "preloaded group should belong to account B")
var checkCount int64
require.NoError(t, models.DB().Model(&models.Check{}).
Where("monitor_id = ?", monitor.ID).Count(&checkCount).Error)
assert.Equal(t, int64(1), checkCount, "check rows must follow the monitor across the move")
}
// TestMonitorTransferSameAccountGuard documents the early-return path: the
// controller must not silently no-op when the caller picks the current
// account, and the test exercises that the DB stays untouched.
func TestMonitorTransferSameAccountGuard(t *testing.T) {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "only", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "only"}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: "x.test"}
require.NoError(t, models.DB().Create(&monitor).Error)
// No update is issued because the controller rejects same-account moves
// before the SQL UPDATE. Verify the row is unchanged.
var reloaded models.Monitor
require.NoError(t, models.DB().First(&reloaded, monitor.ID).Error)
assert.Equal(t, group.ID, reloaded.GroupID)
}
func ptrString(s string) *string { return &s }

528
app/models/network_diagnostics.go Обычный файл
Просмотреть файл

@@ -0,0 +1,528 @@
package models
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/influx"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
const (
ConfirmStateNone = "none"
ConfirmStatePending = "pending"
ConfirmStateConfirmed = "confirmed" // A different worker reproduced the failure.
ConfirmStateTimeout = "confirmed_by_timeout"
AttemptKindRegular = "regular"
AttemptKindConfirm = "confirmation"
AttemptStateQueued = "queued"
AttemptStateLeased = "leased"
AttemptStateFinished = "finished"
diagnosticSoft = "soft"
diagnosticHard = "hard"
diagnosticRecovery = "recovery"
)
type diagnosticSettings struct {
confirmTimeout time.Duration
healthWindow time.Duration
healthRate float64
healthMin int64
}
func settingsForAccount(account *Account) diagnosticSettings {
settings := diagnosticSettings{confirmTimeout: 90 * time.Second, healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
if account == nil || account.Plan == nil {
return settings
}
plan := account.Plan
if plan.ConfirmTimeoutSec > 0 {
settings.confirmTimeout = time.Duration(plan.ConfirmTimeoutSec) * time.Second
}
if plan.HealthWindowSec > 0 {
settings.healthWindow = time.Duration(plan.HealthWindowSec) * time.Second
}
if plan.HealthRateThreshold > 0 && plan.HealthRateThreshold <= 1 {
settings.healthRate = plan.HealthRateThreshold
}
if plan.HealthMinAttempts > 0 {
settings.healthMin = int64(plan.HealthMinAttempts)
}
if !plan.Confirmations {
return settings
}
if account.ConfirmTimeoutSec != nil && *account.ConfirmTimeoutSec >= 15 {
settings.confirmTimeout = time.Duration(*account.ConfirmTimeoutSec) * time.Second
}
if account.HealthWindowSec != nil && *account.HealthWindowSec >= 60 {
settings.healthWindow = time.Duration(*account.HealthWindowSec) * time.Second
}
if account.HealthRateThreshold != nil && *account.HealthRateThreshold > 0 && *account.HealthRateThreshold <= 1 {
settings.healthRate = *account.HealthRateThreshold
}
if account.HealthMinAttempts != nil && *account.HealthMinAttempts > 0 {
settings.healthMin = int64(*account.HealthMinAttempts)
}
return settings
}
// CheckAttempt is the durable worker-attribution record. Unlike check state,
// it is append-only and therefore remains useful after a worker is deweighted.
type CheckAttempt struct {
concerns.Model
JobID string `gorm:"uniqueIndex;size:64;not null" json:"job_id"`
CheckID int64 `gorm:"index;not null" json:"check_id"`
MonitorID int64 `gorm:"index;not null" json:"monitor_id"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
WorkerNode *WorkerNode `json:"worker_node,omitempty"`
SourceWorkerNodeID *int64 `gorm:"index" json:"source_worker_node_id,omitempty"`
Kind string `gorm:"size:32;not null" json:"kind"`
State string `gorm:"size:32;not null" json:"state"`
ResultState string `gorm:"size:16" json:"result_state"`
Result datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"result"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
LeaseToken string `gorm:"size:64" json:"-"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
Deweighted bool `gorm:"not null;default:false" json:"deweighted"`
concerns.Timestamped
}
// DiagnosticAuditEvent is a compact, queryable control-plane audit record.
type DiagnosticAuditEvent struct {
concerns.Model
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
Kind string `gorm:"size:64;index;not null" json:"kind"`
Metadata datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"metadata"`
concerns.Timestamped
}
func auditDiagnostic(tx *gorm.DB, kind string, monitorID, workerID *int64, metadata map[string]interface{}) {
b, _ := json.Marshal(metadata)
_ = tx.Create(&DiagnosticAuditEvent{MonitorID: monitorID, WorkerNodeID: workerID, Kind: kind, Metadata: b}).Error
}
// AuditNetworkRecovery is used by the admin incident-response endpoint.
func AuditNetworkRecovery(tx *gorm.DB, workerID int64) {
auditDiagnostic(tx, "worker.network_problem_force_recover", nil, &workerID, nil)
}
// enqueueDiagnosticDelivery creates the message and its durable delivery task in
// the transition transaction. A transaction advisory lock prevents concurrent
// result frames from leaving duplicate messages when the task dedupe wins.
func enqueueDiagnosticDelivery(tx *gorm.DB, monitor *Monitor, account *Account, tier string, now time.Time) error {
if account == nil || account.Plan == nil {
return nil
}
if tier == diagnosticHard && !account.Plan.AllowHardAlerts {
return nil
}
var notifications []Notification
if err := tx.Joins("JOIN notification_groups ON notification_groups.notification_id = notifications.id").
Where("notifications.account_id = ? AND notifications.enabled AND notification_groups.group_id = ?", account.ID, monitor.GroupID).
Preload("Contacts", "enabled = ?", true).Find(&notifications).Error; err != nil {
return err
}
for i := range notifications {
for j := range notifications[i].Contacts {
contact := notifications[i].Contacts[j]
method := diagnosticContactMethod(contact.Kind)
if method == "" || (tier == diagnosticSoft && method != "email" && method != "telegram") {
continue
}
key := fmt.Sprintf("diagnostic:%d:%s:%d:%d", monitor.ID, tier, notifications[i].ID, contact.ID)
if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", key).Error; err != nil {
return err
}
var existing Task
if err := tx.Where("idempotency_key = ?", key).First(&existing).Error; err == nil {
continue
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
message := Message{NotificationID: notifications[i].ID, ContactID: contact.ID, Kind: "diagnostic_" + tier, State: TaskStateQueued}
if err := tx.Create(&message).Error; err != nil {
return err
}
monitorID, messageID := monitor.ID, message.ID
payload, err := json.Marshal(wire.NotificationTask{AccountID: account.ID, MessageID: messageID, NotificationID: notifications[i].ID, MonitorID: &monitorID, Method: method, Contact: wire.NotificationContact{ID: contact.ID, Kind: contact.Kind, Value: contact.Value, Name: contact.Name}, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind})
if err != nil {
return err
}
if _, err = EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notifications[i].ID, ContactID: contact.ID, MessageID: &messageID, MonitorID: &monitorID, Method: method, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind, NotBefore: now, Payload: payload, IdempotencyKey: key}); err != nil {
return err
}
}
}
return nil
}
func diagnosticContactMethod(kind string) string {
switch kind {
case "email":
return "email"
case "telegram_private", "telegram_group":
return "telegram"
case "webhook", "mattermost", "sms", "voice":
return kind
}
return ""
}
func diagnosticSubject(monitor *Monitor, tier string) string {
return fmt.Sprintf("Monitor %s: %s", monitor.Host, tier)
}
func diagnosticBody(monitor *Monitor, tier string) string {
return fmt.Sprintf("Network diagnostic %s for monitor %s.", tier, monitor.Host)
}
// ConfirmationJobsForWorker atomically leases confirmation jobs assigned to this
// worker or left unassigned by an expired lease. Unassigned attempts still retain
// SourceWorkerNodeID, so the original failing worker can never claim them.
func ConfirmationJobsForWorker(worker *WorkerNode, kinds []string, limit int) ([]wire.CheckJob, error) {
if worker == nil || worker.AccountID != nil || !worker.SupportsTaskEnvelope() || worker.NetworkProblemActive(time.Now()) || limit < 1 {
return nil, nil
}
var jobs []wire.CheckJob
err := DB().Transaction(func(tx *gorm.DB) error {
var attempts []CheckAttempt
if err := tx.Clauses(SkipLockedClause).Where("(worker_node_id = ? OR worker_node_id IS NULL) AND kind = ? AND state = ?", worker.ID, AttemptKindConfirm, AttemptStateQueued).Order("id").Limit(limit).Find(&attempts).Error; err != nil {
return err
}
for i := range attempts {
var check Check
if err := tx.Preload("Monitor").First(&check, attempts[i].CheckID).Error; err != nil {
continue
}
if check.Monitor == nil || !check.Monitor.Enabled {
continue
}
if attempts[i].SourceWorkerNodeID != nil && *attempts[i].SourceWorkerNodeID == worker.ID {
continue
}
if !containsString(kinds, check.Kind) || !containsString(worker.CheckTypes(), check.Kind) {
continue
}
now := time.Now()
leaseToken := uuid.NewString()
leaseUntil := now.Add(DefaultTaskLeaseTTL)
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateLeased, "started_at": now, "lease_token": leaseToken, "lease_expires_at": leaseUntil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
return err
}
jobs = append(jobs, wire.CheckJob{JobID: attempts[i].JobID, LeaseToken: leaseToken, CheckID: check.ID, MonitorID: check.MonitorID, Kind: check.Kind, Host: check.Monitor.Host, URL: check.URL, Interval: check.Interval, Settings: json.RawMessage(check.Settings)})
}
return nil
})
return jobs, err
}
// StartConfirmation creates exactly one targeted confirmation for a new outage.
func StartConfirmation(checkID, sourceWorkerID int64, now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
return StartConfirmationTx(tx, checkID, sourceWorkerID, now)
})
}
// StartConfirmationTx is StartConfirmation's transaction-aware form.
func StartConfirmationTx(tx *gorm.DB, checkID, sourceWorkerID int64, now time.Time) error {
if tx == nil {
return errors.New("start confirmation: nil transaction")
}
{
var check Check
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
return err
}
// Confirmations are a paid distributed-check entitlement. Free accounts
// retain the legacy direct soft alert path and do not consume worker budget.
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil || check.Monitor.Group.Account.Plan == nil || !check.Monitor.Group.Account.Plan.Confirmations {
return nil
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
return err
}
if monitor.ConfirmState == ConfirmStatePending || monitor.ConfirmState == ConfirmStateConfirmed {
return nil
}
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, 0, now)
if err != nil {
monitor.ConfirmState, monitor.ConfirmAt = ConfirmStateTimeout, &now
auditDiagnostic(tx, "check.confirm_unavailable", &monitor.ID, &sourceWorkerID, nil)
if err := tx.Save(&monitor).Error; err != nil {
return err
}
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticHard, now)
}
monitor.ConfirmState, monitor.ConfirmAt, monitor.ConfirmedByWorkerID = ConfirmStatePending, &now, &worker.ID
if err := tx.Save(&monitor).Error; err != nil {
return err
}
if err := enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticSoft, now); err != nil {
return err
}
attempt := CheckAttempt{JobID: uuid.NewString(), CheckID: check.ID, MonitorID: monitor.ID, WorkerNodeID: &worker.ID, SourceWorkerNodeID: &sourceWorkerID, Kind: AttemptKindConfirm, State: AttemptStateQueued}
if err := tx.Create(&attempt).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_assign", &monitor.ID, &worker.ID, map[string]interface{}{"exclude_worker_id": sourceWorkerID, "job_id": attempt.JobID})
return nil
}
}
// confirmationWorkerTx selects an independent active platform worker that can
// execute this exact kind. Capability filtering is deliberately performed in
// Go because the JSON capability format also supports legacy rows safely.
func confirmationWorkerTx(tx *gorm.DB, checkKind string, sourceWorkerID, excludeWorkerID int64, now time.Time) (*WorkerNode, error) {
var workers []WorkerNode
if err := tx.Where("id <> ? AND id <> ? AND account_id IS NULL AND status = 'active' AND (network_problems = FALSE OR network_problems_until <= ? OR network_problems_until IS NULL)", sourceWorkerID, excludeWorkerID, now).Order("id").Find(&workers).Error; err != nil {
return nil, err
}
for i := range workers {
if workers[i].SupportsTaskEnvelope() && containsString(workers[i].CheckTypes(), checkKind) {
return &workers[i], nil
}
}
return nil, gorm.ErrRecordNotFound
}
// ApplyDiagnosticResult resolves a targeted attempt once. Duplicate reports are ignored.
func ApplyDiagnosticResult(report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
if report.JobID == "" || worker == nil {
return false, nil
}
handled := true
err := DB().Transaction(func(tx *gorm.DB) error {
return ApplyDiagnosticResultTx(tx, report, worker, now, &handled)
})
return handled, err
}
// ApplyDiagnosticResultTx resolves a diagnostic attempt within the caller's
// transaction. handled distinguishes a normal check result from a diagnostic.
func ApplyDiagnosticResultTx(tx *gorm.DB, report wire.CheckResultReport, worker *WorkerNode, now time.Time, handled *bool) error {
if tx == nil {
return errors.New("apply diagnostic: nil transaction")
}
if handled == nil {
return errors.New("apply diagnostic: nil handled result")
}
*handled = true
{
var attempt CheckAttempt
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("job_id = ?", report.JobID).First(&attempt).Error; err != nil {
if err == gorm.ErrRecordNotFound {
*handled = false
return nil
}
return err
}
if attempt.Kind != AttemptKindConfirm {
*handled = false
return nil
}
if attempt.State == AttemptStateFinished {
*handled = false
return nil
}
if attempt.State != AttemptStateLeased || attempt.WorkerNodeID == nil || *attempt.WorkerNodeID != worker.ID {
return gorm.ErrRecordNotFound
}
if attempt.LeaseToken == "" || report.LeaseToken == "" || report.LeaseToken != attempt.LeaseToken || attempt.LeaseExpiresAt == nil || !attempt.LeaseExpiresAt.After(now) {
return errors.New("apply diagnostic: lease token is invalid or expired")
}
payload, _ := json.Marshal(report)
deweighted := worker.NetworkProblemActive(now)
if err := tx.Model(&attempt).Where("state = ? AND lease_token = ? AND lease_expires_at > ?", AttemptStateLeased, report.LeaseToken, now).Updates(map[string]interface{}{"state": AttemptStateFinished, "result_state": report.State, "result": payload, "finished_at": now, "lease_token": "", "lease_expires_at": nil, "deweighted": deweighted}).Error; err != nil {
return err
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, attempt.MonitorID).Error; err != nil {
return err
}
if attempt.Kind == AttemptKindConfirm && monitor.ConfirmState == ConfirmStatePending {
if report.State == stateERR || report.State == stateFail {
monitor.ConfirmState = ConfirmStateConfirmed
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticHard, now); err != nil {
return err
}
} else {
monitor.ConfirmState = ConfirmStateNone
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticRecovery, now); err != nil {
return err
}
}
if err := tx.Save(&monitor).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_result", &monitor.ID, &worker.ID, map[string]interface{}{"state": report.State, "deweighted": deweighted})
}
return nil
}
}
func monitorAccount(tx *gorm.DB, monitor *Monitor) *Account {
var group Group
if err := tx.Preload("Account.Plan").First(&group, monitor.GroupID).Error; err != nil {
return nil
}
return group.Account
}
// RecoverDiagnostic clears a completed hard escalation only once and queues the
// corresponding recovery tasks in the same transaction.
func RecoverDiagnostic(checkID int64, now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
return RecoverDiagnosticTx(tx, checkID, now)
})
}
// RecoverDiagnosticTx is RecoverDiagnostic's transaction-aware form.
func RecoverDiagnosticTx(tx *gorm.DB, checkID int64, now time.Time) error {
if tx == nil {
return errors.New("recover diagnostic: nil transaction")
}
{
var check Check
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
return err
}
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil {
return nil
}
var monitor Monitor
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
return err
}
if monitor.ConfirmState != ConfirmStateConfirmed && monitor.ConfirmState != ConfirmStateTimeout {
return nil
}
monitor.ConfirmState = ConfirmStateNone
if err := tx.Save(&monitor).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_recovery", &monitor.ID, nil, nil)
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticRecovery, now)
}
}
// NetworkDiagnosticsTick expires confirmations and derives worker health from durable attempts.
func NetworkDiagnosticsTick(now time.Time) error {
return DB().Transaction(func(tx *gorm.DB) error {
if err := reapExpiredConfirmationAttemptsTx(tx, now); err != nil {
return err
}
var monitors []Monitor
if err := tx.Clauses(SkipLockedClause).Preload("Group.Account.Plan").Where("confirm_state = ?", ConfirmStatePending).Find(&monitors).Error; err != nil {
return err
}
for i := range monitors {
settings := settingsForAccount(monitors[i].Group.Account)
if monitors[i].ConfirmAt == nil || monitors[i].ConfirmAt.After(now.Add(-settings.confirmTimeout)) {
continue
}
if err := tx.Model(&monitors[i]).Update("confirm_state", ConfirmStateTimeout).Error; err != nil {
return err
}
auditDiagnostic(tx, "check.confirm_timeout", &monitors[i].ID, nil, nil)
if err := enqueueDiagnosticDelivery(tx, &monitors[i], monitors[i].Group.Account, diagnosticHard, now); err != nil {
return err
}
}
var workers []WorkerNode
if err := tx.Find(&workers).Error; err != nil {
return err
}
for i := range workers {
var total, failed int64
// Operated workers serve accounts on different plans. Use the most
// sensitive entitled setting among their recent attempts.
settings := diagnosticSettings{healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
q := tx.Model(&CheckAttempt{}).Where("worker_node_id = ? AND finished_at >= ?", workers[i].ID, now.Add(-settings.healthWindow))
q.Count(&total)
q.Where("result_state IN ?", []string{stateERR, stateFail}).Count(&failed)
flagged := total >= settings.healthMin && float64(failed)/float64(total) >= settings.healthRate
updates := map[string]interface{}{"last_total_count": total, "last_failure_count": failed}
if flagged {
updates["network_problems"] = true
updates["network_problems_until"] = now.Add(10 * time.Minute)
}
if workers[i].NetworkProblems && workers[i].NetworkProblemsUntil != nil && workers[i].NetworkProblemsUntil.Before(now) && !flagged {
updates["network_problems"] = false
updates["network_problems_until"] = nil
auditDiagnostic(tx, "worker.network_problem_unflag", nil, &workers[i].ID, nil)
}
if flagged && !workers[i].NetworkProblems {
auditDiagnostic(tx, "worker.network_problem_flag", nil, &workers[i].ID, map[string]interface{}{"failures": failed, "total": total})
}
if err := tx.Model(&workers[i]).Updates(updates).Error; err != nil {
return err
}
_ = influx.WriteOne("worker_health", map[string]string{"worker_id": workers[i].WorkerID}, map[string]interface{}{"failures": failed, "total": total, "failure_rate": float64(failed) / float64(maxInt64(total, 1))})
}
return nil
})
}
func reapExpiredConfirmationAttemptsTx(tx *gorm.DB, now time.Time) error {
var attempts []CheckAttempt
if err := tx.Clauses(SkipLockedClause).Where("kind = ? AND state = ? AND lease_expires_at <= ?", AttemptKindConfirm, AttemptStateLeased, now).Find(&attempts).Error; err != nil {
return err
}
for i := range attempts {
var check Check
if err := tx.First(&check, attempts[i].CheckID).Error; err != nil {
return err
}
oldWorkerID := int64(0)
if attempts[i].WorkerNodeID != nil {
oldWorkerID = *attempts[i].WorkerNodeID
}
sourceWorkerID := int64(0)
if attempts[i].SourceWorkerNodeID != nil {
sourceWorkerID = *attempts[i].SourceWorkerNodeID
}
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, oldWorkerID, now)
if errors.Is(err, gorm.ErrRecordNotFound) {
// No replacement is available now. Remove the stale assignment so a
// later capable independent worker can claim this queued attempt.
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": nil, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", nil).Error; err != nil {
return err
}
continue
}
if err != nil {
return err
}
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
return err
}
}
return nil
}
func maxInt64(a, b int64) int64 {
if a > b {
return a
}
return b
}

193
app/models/notification.go Обычный файл
Просмотреть файл

@@ -0,0 +1,193 @@
package models
import (
"log"
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/workdays"
)
// Notification provides functionality.
type Notification struct {
concerns.Model
Name string `json:"name" gorm:"not null"`
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
Account *Account `json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
ContactIDs []int64 `gorm:"-:all" json:"contact_ids"`
Contacts []Contact `gorm:"many2many:notification_contacts;" json:"contacts,omitempty"`
GroupIDs []int64 `gorm:"-:all" json:"group_ids"`
Groups []Group `gorm:"many2many:notification_groups;" json:"-"`
AlertDelay *int64 `json:"alert_delay,omitempty"`
// RepeatAlert *int64 `json:"repeat_alert,omitempty"`
BeforeExpiration *int64 `json:"before_expiration,omitempty"`
NotifyDown bool `gorm:"default:true" json:"notify_down"`
NotifyRestore bool `gorm:"default:true" json:"notify_restore"`
NotifyWHOIS bool `gorm:"default:true" json:"notify_whois"`
NotifySSL bool `gorm:"default:true" json:"notify_ssl"`
NotifyDays *int `json:"notify_days"`
NotifyDayStart *int `json:"notify_day_start"`
NotifyDayEnd *int `json:"notify_day_end"`
NotifyHolidays bool `gorm:"default:true" json:"notify_holidays"`
Messages []Message `json:"-"`
concerns.Timestamped
Audited
}
const notificationDebug = false
// EnabledNow checks if the notification is enabled at the given time.
func (n *Notification) EnabledNow(tn *time.Time) bool {
weekday := int(tn.Weekday())
// делаем из 0-воскр 1-пн 6-сб вариант 0-пн 6-воскр
if weekday == 0 {
weekday = 7
}
weekday--
if notificationDebug {
log.Println("notification", n.ID, "check enabled now at", tn, "for day", weekday)
}
if !n.NotifyHolidays {
c := workdays.GetCalendar()
if !c.IsWorkday(*tn) {
if notificationDebug {
log.Println("notification", n.ID, "is not enabled on holiday", tn)
}
return false
}
}
minusOneDay := false
notifyFromDay := true
if n.NotifyDayStart != nil && n.NotifyDayEnd != nil {
bod := BeginningOfDay(*tn)
secondsToday := int(tn.Sub(bod) / time.Second)
ds := *n.NotifyDayStart
de := *n.NotifyDayEnd
// вариант 9 утра - 2 часа ночи
if ds == de { //nolint:gocritic // complex condition chain
notifyFromDay = true
} else if ds > de {
// с 0 до DayEnd
if secondsToday < de { //nolint:gocritic // complex condition chain
minusOneDay = true
notifyFromDay = true
} else if secondsToday < ds {
// с DayEnd до DayStart
notifyFromDay = false
} else {
notifyFromDay = true
}
} else {
if secondsToday < ds { //nolint:gocritic // complex condition chain
// с 0 до DayStart
notifyFromDay = false
} else if secondsToday > de {
notifyFromDay = false
} else {
notifyFromDay = true
}
}
// log.Println(secondsToday)
}
if !notifyFromDay {
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled as notifyFromDay", tn)
}
return false
}
// Если время 0-dayStart считаем что это прошлый день
if minusOneDay {
weekday--
if weekday < 0 {
weekday = 6
}
}
if n.NotifyDays != nil {
if !HasBit(*n.NotifyDays, uint(weekday)) {
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled on weekday", tn.Weekday(), weekday, tn)
}
return false
}
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is enabled on weekday", tn.Weekday(), weekday, tn)
}
}
if notificationDebug {
log.Println("notification", n.ID, *n.NotifyDays, "is enabled", tn)
}
return true
}
// NotificationLoadIDs provides functionality.
func NotificationLoadIDs(notifications *[]Notification) {
for i, n := range *notifications { //nolint:gocritic // range copy is acceptable here
cids := make([]int64, len(n.Contacts))
for i, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
cids[i] = c.ID
}
(*notifications)[i].ContactIDs = cids
gids := make([]int64, len(n.Groups))
for j, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
gids[j] = g.ID
}
(*notifications)[i].GroupIDs = gids
}
}
// PersistRelations provides functionality.
func (n *Notification) PersistRelations() error {
cts := make([]Contact, len(n.ContactIDs))
for i, c := range n.ContactIDs {
ct := Contact{}
ct.ID = c
cts[i] = ct
}
err := DB().Model(&n).Association("Contacts").Replace(cts)
if err != nil {
return err
}
grp := make([]Group, len(n.GroupIDs))
for i, g := range n.GroupIDs {
gr := Group{}
gr.ID = g
grp[i] = gr
}
err = DB().Model(&n).Association("Groups").Replace(grp)
if err != nil {
return err
}
return nil
}
// GetContacts returns the contacts associated with this notification.
// Errors are logged and an empty slice is returned instead of panicking so
// that a single misconfigured notification cannot kill the scheduler goroutine
// that processes expiry alerts (see internal/notifier.RunExp).
func (n *Notification) GetContacts() []Contact {
contacts := make([]Contact, 0)
err := DB().Model(*n).Where("enabled = ?", true).Association("Contacts").Find(&contacts)
if err != nil {
log.Printf("notification %d: GetContacts failed: %v", n.ID, err)
return contacts
}
return contacts
}

167
app/models/notification_credential.go Обычный файл
Просмотреть файл

@@ -0,0 +1,167 @@
package models
import (
"encoding/base64"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// NotificationCredential kinds stored in the database.
const (
CredentialKindSMTP = "smtp"
CredentialKindTelegram = "telegram"
)
// NotificationCredential stores per-method delivery credentials (SMTP login,
// Telegram bot token, etc.) encrypted at rest. Credentials are pushed to
// workers through the init/config websocket refresh (see docs/worker-protocol.md
// "Credentials Push").
type NotificationCredential struct {
concerns.Model
// AccountID is nil for platform-managed credentials and set for credentials
// owned by one customer account.
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;index" json:"account_id,omitempty"`
Kind string `gorm:"not null" json:"kind"`
Name string `gorm:"not null" json:"name"`
// SMTP-specific
Server *string `json:"server,omitempty"`
Port *int `json:"port,omitempty"`
Login *string `json:"login,omitempty"`
FromName *string `json:"from_name,omitempty"`
FromAddr *string `json:"from_address,omitempty"`
InsecureSkipVerify bool `gorm:"default:false" json:"insecure_skip_verify"`
// Telegram-specific
BotName *string `json:"bot_name,omitempty"`
APIURL *string `json:"api_url,omitempty"`
WebhookToken string `gorm:"size:128" json:"webhook_token,omitempty"`
// SecretEnc holds the encrypted (or "plain:"-prefixed fallback) secret
// value — SMTP password or Telegram bot token. Decrypt via GetSecret.
SecretEnc string `gorm:"column:secret;type:text" json:"-"`
SecretMasked string `gorm:"-" json:"secret_masked,omitempty"`
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
Meta datatypes.JSON `gorm:"type:jsonb;default:'{}'" json:"meta,omitempty"`
concerns.Timestamped
Audited
}
// SetSecret encrypts and stores the plaintext secret value.
func (c *NotificationCredential) SetSecret(plaintext string) error {
enc, err := encryptSecret(plaintext)
if err != nil {
return err
}
c.SecretEnc = enc
return nil
}
// GetSecret decrypts and returns the secret value.
func (c *NotificationCredential) GetSecret() (string, error) {
return decryptSecret(c.SecretEnc)
}
// FillSecretMasked populates SecretMasked with a display-safe version of the credential secret.
func (c *NotificationCredential) FillSecretMasked() {
secret, err := c.GetSecret()
if err != nil {
return
}
c.SecretMasked = maskSecret(secret)
}
// EnsureWebhookToken creates the per-bot webhook URL token if it is missing.
func (c *NotificationCredential) EnsureWebhookToken() {
if c.WebhookToken != "" {
return
}
c.WebhookToken = base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
}
func maskSecret(secret string) string {
if secret == "" {
return ""
}
if len(secret) == 1 {
return secret
}
return secret[:1] + "***" + secret[len(secret)-1:]
}
// TableName overrides the default table name.
func (NotificationCredential) TableName() string {
return "notification_credentials"
}
// AllNotificationCredentials returns all credentials ordered by kind and name.
func AllNotificationCredentials() ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id IS NULL").Order("kind ASC, name ASC").Find(&creds).Error
for i := range creds {
creds[i].FillSecretMasked()
}
return creds, err
}
// AccountNotificationCredentials returns credentials owned by accountID.
func AccountNotificationCredentials(accountID int64) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id = ?", accountID).Order("kind ASC, name ASC").Find(&creds).Error
for i := range creds {
creds[i].FillSecretMasked()
}
return creds, err
}
// EnabledCredentialsByKind returns enabled credentials of the given kind.
func EnabledCredentialsByKind(kind string) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id IS NULL AND kind = ? AND enabled = ?", kind, true).Order("name ASC").Find(&creds).Error
return creds, err
}
// EnabledCredentialsByAccountAndKind returns only enabled credentials owned by
// accountID. It never falls back to platform credentials.
func EnabledCredentialsByAccountAndKind(accountID int64, kind string) ([]NotificationCredential, error) {
var creds []NotificationCredential
err := DB().Where("account_id = ? AND kind = ? AND enabled = ?", accountID, kind, true).Order("name ASC").Find(&creds).Error
return creds, err
}
// FindCredential returns a credential by id.
func FindCredential(id int64) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().First(&c, id).Error; err != nil {
return nil, err
}
c.FillSecretMasked()
return &c, nil
}
// FindCredentialByName returns a credential by kind and name.
func FindCredentialByName(kind, name string) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().Where("kind = ? AND name = ?", kind, name).First(&c).Error; err != nil {
return nil, err
}
return &c, nil
}
// FindTelegramCredentialByWebhookToken returns an enabled Telegram credential by webhook token.
func FindTelegramCredentialByWebhookToken(token string) (*NotificationCredential, error) {
var c NotificationCredential
if err := DB().Where("kind = ? AND webhook_token = ? AND enabled = ?", CredentialKindTelegram, token, true).First(&c).Error; err != nil {
return nil, err
}
return &c, nil
}
// DeleteCredential removes a credential by id.
func DeleteCredential(id int64) error {
return DB().Delete(&NotificationCredential{}, id).Error
}

186
app/models/notification_credential_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,186 @@
package models_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// TestNotificationCredential_EncryptDecryptRoundTrip verifies that SetSecret
// followed by GetSecret returns the original plaintext regardless of which
// encryption mode (AES-GCM or plain: fallback) is active.
func TestNotificationCredential_EncryptDecryptRoundTrip(t *testing.T) {
c := &models.NotificationCredential{}
plaintext := "super-secret-smtp-password"
require.NoError(t, c.SetSecret(plaintext))
got, err := c.GetSecret()
require.NoError(t, err)
assert.Equal(t, plaintext, got)
}
// TestNotificationCredential_EncryptDecryptWithKey verifies that when
// RSMON_CRED_KEY is configured the stored value is prefixed with "enc:" and
// can still be decrypted back to the original plaintext.
func TestNotificationCredential_EncryptDecryptWithKey(t *testing.T) {
t.Setenv("RSMON_CRED_KEY", "test-key-rotate-me-1234567890")
c := &models.NotificationCredential{}
plaintext := "bot-token-9876543210:ABCDEFG"
require.NoError(t, c.SetSecret(plaintext))
assert.True(t, strings.HasPrefix(c.SecretEnc, "enc:"),
"expected SecretEnc to start with 'enc:' prefix, got %q", c.SecretEnc)
assert.NotEqual(t, plaintext, c.SecretEnc, "encrypted value must not equal plaintext")
got, err := c.GetSecret()
require.NoError(t, err)
assert.Equal(t, plaintext, got)
}
// TestNotificationCredential_CRUD exercises create / find-by-id / find-by-name
// / delete against the test database.
func TestNotificationCredential_CRUD(t *testing.T) {
models.Drop()
models.Migrate()
server := "smtp.example.com"
port := 587
login := "alerts@example.com"
enabled := true
c := &models.NotificationCredential{
Kind: models.CredentialKindSMTP,
Name: "primary",
Server: &server,
Port: &port,
Login: &login,
Enabled: &enabled,
}
require.NoError(t, c.SetSecret("smtp-password-xyz"))
require.NoError(t, models.DB().Create(c).Error)
require.NotZero(t, c.ID, "expected ID to be assigned after Create")
found, err := models.FindCredential(c.ID)
require.NoError(t, err)
assert.Equal(t, "primary", found.Name)
assert.Equal(t, models.CredentialKindSMTP, found.Kind)
require.NotNil(t, found.Server)
assert.Equal(t, "smtp.example.com", *found.Server)
gotSecret, err := found.GetSecret()
require.NoError(t, err)
assert.Equal(t, "smtp-password-xyz", gotSecret)
assert.Equal(t, "s***z", found.SecretMasked)
byName, err := models.FindCredentialByName(models.CredentialKindSMTP, "primary")
require.NoError(t, err)
assert.Equal(t, c.ID, byName.ID)
require.NoError(t, models.DeleteCredential(c.ID))
_, err = models.FindCredential(c.ID)
assert.Error(t, err, "FindCredential should fail after delete")
}
// TestNotificationCredential_UniqueKindName verifies that two credentials with
// the same (kind, name) pair violate the unique index.
func TestNotificationCredential_UniqueKindName(t *testing.T) {
models.Drop()
models.Migrate()
enTrue := true
first := &models.NotificationCredential{
Kind: models.CredentialKindTelegram,
Name: "main-bot",
Enabled: &enTrue,
}
require.NoError(t, first.SetSecret("token-a"))
require.NoError(t, models.DB().Create(first).Error)
second := &models.NotificationCredential{
Kind: models.CredentialKindTelegram,
Name: "main-bot",
Enabled: &enTrue,
}
require.NoError(t, second.SetSecret("token-b"))
err := models.DB().Create(second).Error
require.Error(t, err, "expected unique constraint violation for duplicate (kind, name)")
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"expected error mentioning unique/duplicate, got: %v", err)
}
// TestEnabledCredentialsByKind verifies the kind+enabled filter.
func TestEnabledCredentialsByKind(t *testing.T) {
models.Drop()
models.Migrate()
enTrue := true
enFalse := false
enabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "enabled-1", Enabled: &enTrue}
disabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "disabled-1", Enabled: &enFalse}
require.NoError(t, models.DB().Create(enabled).Error)
require.NoError(t, models.DB().Create(disabled).Error)
got, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
require.NoError(t, err)
var names []string
for _, c := range got {
names = append(names, c.Name)
}
assert.Contains(t, names, "enabled-1")
assert.NotContains(t, names, "disabled-1")
}
func TestNotificationCredentialsAreScopedToSystemOrAccount(t *testing.T) {
models.Drop()
models.Migrate()
accountA := &models.Account{Name: "credential-a"}
accountB := &models.Account{Name: "credential-b"}
require.NoError(t, models.DB().Create(accountA).Error)
require.NoError(t, models.DB().Create(accountB).Error)
enabled := true
system := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "system", Enabled: &enabled}
ownedA := &models.NotificationCredential{AccountID: &accountA.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
ownedB := &models.NotificationCredential{AccountID: &accountB.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
require.NoError(t, models.DB().Create(system).Error)
require.NoError(t, models.DB().Create(ownedA).Error)
require.NoError(t, models.DB().Create(ownedB).Error)
systemCreds, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
require.NoError(t, err)
require.Len(t, systemCreds, 1)
assert.Equal(t, system.ID, systemCreds[0].ID)
accountCreds, err := models.EnabledCredentialsByAccountAndKind(accountA.ID, models.CredentialKindSMTP)
require.NoError(t, err)
require.Len(t, accountCreds, 1)
assert.Equal(t, ownedA.ID, accountCreds[0].ID)
}
func TestNotificationCredentialSecretMasked(t *testing.T) {
models.Drop()
models.Migrate()
cred := models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "smtp"}
require.NoError(t, cred.SetSecret("password"))
require.NoError(t, models.DB().Create(&cred).Error)
loaded, err := models.FindCredential(cred.ID)
require.NoError(t, err)
assert.Equal(t, "p***d", loaded.SecretMasked)
}

146
app/models/notification_get_contacts_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,146 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/spec/factories"
)
// TestNotificationGetContactsIncludesSystemContact exercises the regression
// reported when the production dump was restored into dev: the
// contacts.is_system column was missing and the GORM preload panicked on
// GetContacts. The fix has two layers:
//
// 1. AutoMigrate must add is_system (and deletion_requested_at) before the
// notifier scheduler starts running.
// 2. GetContacts itself must not panic on a query error so a single bad row
// cannot tear down the scheduler goroutine.
//
// This test verifies both layers by:
// - asserting that the schema post-Migrate includes is_system, so the
// production-like scenario no longer panics; and
// - building a notification that contains a contact flagged is_system=true
// and checking GetContacts returns it.
func TestNotificationGetContactsIncludesSystemContact(t *testing.T) {
models.Drop()
models.Migrate()
// Column must exist after Migrate; otherwise GetContacts would fail
// with the same panic we saw in production.
assertColumnExists(t, "contacts", "is_system")
account := &models.Account{Name: "acct-get-contacts"}
require.NoError(t, models.DB().Create(account).Error)
accountID := account.ID
trueVal := true
contact := &models.Contact{
AccountID: &accountID,
Name: "system-admin",
Kind: "email",
Value: "ops@example.com",
IsSystem: &trueVal,
}
require.NoError(t, models.DB().Create(contact).Error)
notification := factories.PersistedNotification(
account, []int64{contact.ID}, nil, 300, false,
)
// Reload so the model has its persisted ID; the factory's PersistRelations
// may have left ContactIDs empty on the returned value depending on the
// GORM version, so fetch fresh.
require.NoError(t, models.DB().
Preload("Contacts").
First(&notification, notification.ID).Error)
got := notification.GetContacts()
ids := make([]int64, 0, len(got))
for _, c := range got {
ids = append(ids, c.ID)
}
assert.Contains(t, ids, contact.ID, "GetContacts must include the is_system contact")
}
// TestUserDeletionRequestedAtColumnAndRoundTrip verifies the second missing
// column reported by the panic: users.deletion_requested_at. It asserts that
// AutoMigrate creates the column and that the field round-trips through the
// DB correctly. ProcessPendingDeletions (the consumer of this column) relies
// on it being present and queryable.
func TestUserDeletionRequestedAtColumnAndRoundTrip(t *testing.T) {
models.Drop()
models.Migrate()
assertColumnExists(t, "users", "deletion_requested_at")
user := factories.PersistedUser("deletion-roundtrip@test.ru", "secret")
now := time.Now().UTC().Truncate(time.Microsecond)
user.DeletionRequestedAt = &now
require.NoError(t, models.DB().Save(&user).Error)
reloaded := models.User{}
require.NoError(t, models.DB().First(&reloaded, user.ID).Error)
require.NotNil(t, reloaded.DeletionRequestedAt, "deletion_requested_at must round-trip via Save/First")
assert.True(t, reloaded.DeletionRequestedAt.Equal(now),
"deletion_requested_at must preserve the timestamp value (got %v, want %v)",
reloaded.DeletionRequestedAt, now)
// ProcessPendingDeletions should not panic on the populated schema and
// must respect the cutoff: a recently-set deletion_requested_at is
// still inside the 7-day grace period, so no hard-delete must occur.
deleted, err := models.ProcessPendingDeletions()
require.NoError(t, err)
assert.Equal(t, 0, deleted, "users within the 7-day grace period must not be hard-deleted")
}
// TestProcessPendingDeletionsQueriesMissingColumnGracefully asserts that even
// if the deletion_requested_at column were missing, ProcessPendingDeletions
// would not panic (the panic-on-error pattern was historically present in
// other notifier helpers). We force the failure by renaming the column back,
// calling ProcessPendingDeletions, then restoring the column.
func TestProcessPendingDeletionsQueriesMissingColumnGracefully(t *testing.T) {
models.Drop()
models.Migrate()
assertColumnExists(t, "users", "deletion_requested_at")
// Simulate the production-missing-column scenario in a contained way:
// rename the column so the SELECT against deletion_requested_at fails.
require.NoError(t, models.DB().
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at TO deletion_requested_at_hidden").Error)
t.Cleanup(func() {
// Restore so subsequent tests in this package keep working.
_ = models.DB().
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at_hidden TO deletion_requested_at").Error
})
// Must not panic; must return an error.
assert.NotPanics(t, func() {
_, err := models.ProcessPendingDeletions()
assert.Error(t, err, "missing column must surface as an error, not a panic")
})
}
// assertColumnExists checks that the given table has the given column by
// querying information_schema. It is the canary for the AutoMigrate step
// ordering bug: if the column is missing, every test that touches it will
// panic with SQLSTATE 42703.
func assertColumnExists(t *testing.T, table, column string) {
t.Helper()
var n int
err := models.DB().Raw(
`SELECT COUNT(*) FROM information_schema.columns
WHERE table_schema = current_schema()
AND table_name = ? AND column_name = ?`,
table, column,
).Scan(&n).Error
require.NoError(t, err, "information_schema query must succeed")
assert.Equal(t, 1, n, "table %q must have column %q after Migrate()", table, column)
}

128
app/models/notification_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,128 @@
package models
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var (
mondayFriday int
allDays int
)
var time1, time3, time8, time9, time10, time23 time.Time
func init() {
// database.ConfigFile = "." + database.ConfigFile
// database.Init()
// Migrate()
// Drop()
mondayFriday = 0
mondayFriday = SetBit(mondayFriday, 0)
mondayFriday = SetBit(mondayFriday, 1)
mondayFriday = SetBit(mondayFriday, 2)
mondayFriday = SetBit(mondayFriday, 3)
mondayFriday = SetBit(mondayFriday, 4)
allDays = 0
allDays = SetBit(allDays, 0)
allDays = SetBit(allDays, 1)
allDays = SetBit(allDays, 2)
allDays = SetBit(allDays, 3)
allDays = SetBit(allDays, 4)
allDays = SetBit(allDays, 5)
allDays = SetBit(allDays, 6)
time1 = time.Date(2019, time.May, 26, 1, 0, 0, 0, time.Local)
time3 = time.Date(2019, time.May, 26, 3, 0, 0, 0, time.Local)
time8 = time.Date(2019, time.May, 26, 8, 0, 0, 0, time.Local)
time9 = time.Date(2019, time.May, 26, 9, 0, 0, 0, time.Local)
time10 = time.Date(2019, time.May, 26, 10, 0, 0, 0, time.Local)
time23 = time.Date(2019, time.May, 26, 23, 0, 0, 0, time.Local)
}
func TestNotificationEnabledNowHolidays(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.ID = 1
n.NotifyHolidays = true
holiday := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.Local)
assert.Equal(t, true, n.EnabledNow(&holiday), "notification by default should be enabled on holidays")
n.ID = 2
n.NotifyHolidays = false
assert.Equal(t, false, n.EnabledNow(&holiday), "notification with NotifyHolidays=fasle should not be enabled on holiday")
}
func TestNotificationEnabledNowMondayFriday(t *testing.T) {
n := &Notification{}
n.NotifyHolidays = true
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
n.ID = 3
n.NotifyDays = &mondayFriday
assert.Equal(t, false, n.EnabledNow(&weekend), "notification mon-fri should not be enabled on sunday")
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-fri should be enabled on monday")
}
func TestNotificationEnabledNowAllDays(t *testing.T) {
n := &Notification{}
n.NotifyHolidays = true
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
n.ID = 4
n.NotifyDays = &allDays
assert.Equal(t, true, n.EnabledNow(&weekend), "notification mon-sat should be enabled on sunday")
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-sat should be enabled on monday")
}
func TestNotificationEnabledNowNormal(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.NotifyHolidays = true
// 9am - 18pm
n.ID = 5
start := 9 * 3600
end := 18 * 3600
n.NotifyDayStart = &start
n.NotifyDayEnd = &end
assert.Equal(t, false, n.EnabledNow(&time1), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
assert.Equal(t, false, n.EnabledNow(&time23), "notification should not be enabled outside day")
}
// TestNotificationEnabledNowRollover tests time range that crosses midnight
//
//nolint:dupl // Test structure similar to TestNotificationEnabledNowNormal but tests different behavior (rollover vs normal time range)
func TestNotificationEnabledNowRollover(t *testing.T) {
n := &Notification{}
n.NotifyDays = &allDays
n.NotifyHolidays = true
// 9am - 2am
n.ID = 6
start := 9 * 3600
end := 2 * 3600
n.NotifyDayStart = &start
n.NotifyDayEnd = &end
assert.Equal(t, true, n.EnabledNow(&time1), "notification should be enabled inside day")
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
assert.Equal(t, true, n.EnabledNow(&time23), "notification should be enabled inside day")
}

18
app/models/payment.go Обычный файл
Просмотреть файл

@@ -0,0 +1,18 @@
package models
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
// Payment provides functionality.
type Payment struct {
concerns.Model
AccountID int64 `json:"account_id"`
Account User `json:"-"`
Kind string
ExtID string
Amount int
concerns.Timestamped
Audited
}

108
app/models/plan.go Обычный файл
Просмотреть файл

@@ -0,0 +1,108 @@
package models
import (
"log"
"github.com/google/uuid"
"github.com/lib/pq"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
var canonicalPlanCodes = []string{"free", "solo", "team", "enterprise"}
// CanonicalPlanCodes returns the supported public catalog codes.
func CanonicalPlanCodes() []string {
return append([]string(nil), canonicalPlanCodes...)
}
// Plan is a versioned-by-code catalog entry. A zero cap means unlimited.
type Plan struct {
concerns.Model
Code string `gorm:"size:32;not null" json:"code"`
NameRU string `gorm:"size:64;not null" json:"name_ru"`
NameEN string `gorm:"size:64;not null" json:"name_en"`
PriceMonthlyMinor int64 `gorm:"not null;default:0" json:"price_monthly_minor"`
PriceAnnualMinor int64 `gorm:"not null;default:0" json:"price_annual_minor"`
Currency string `gorm:"size:3;not null;default:'RUB'" json:"currency"`
MonitorCap int64 `gorm:"not null;default:0" json:"monitor_cap"`
IntervalMinSeconds int `gorm:"not null;default:30" json:"interval_min_seconds"`
StatusPagesCap int64 `gorm:"not null;default:0" json:"status_pages_cap"`
MaintenanceCap int64 `gorm:"not null;default:0" json:"maintenance_cap"`
LoginSeatsIncluded int64 `gorm:"not null;default:0" json:"login_seats_included"`
NotifySeatsIncluded int64 `gorm:"not null;default:0" json:"notify_seats_included"`
UnlimitedSeats bool `gorm:"not null;default:false" json:"unlimited_seats"`
Integrations pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"integrations"`
CheckKinds pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"check_kinds"`
DataRetentionMonths int `gorm:"not null;default:3" json:"data_retention_months"`
DistributedWorkers bool `gorm:"not null;default:false" json:"distributed_workers"`
Confirmations bool `gorm:"not null;default:false" json:"confirmations"`
AllowHardAlerts bool `gorm:"not null;default:false" json:"allow_hard_alerts"`
ConfirmTimeoutSec int `gorm:"not null;default:90" json:"confirm_timeout_sec"`
HealthWindowSec int `gorm:"not null;default:300" json:"health_window_sec"`
HealthRateThreshold float64 `gorm:"not null;default:0.5" json:"health_rate_threshold"`
HealthMinAttempts int `gorm:"not null;default:10" json:"health_min_attempts"`
SOC2 bool `gorm:"not null;default:false" json:"soc2"`
GDPRDPA bool `gorm:"not null;default:false" json:"gdpr_dpa"`
IsDefault bool `gorm:"not null;default:false" json:"is_default"`
Archived bool `gorm:"not null;default:false" json:"archived"`
// Deprecated source-compatibility fields. The legacy plans table is retained
// as plans_legacy; these values are never written to the canonical catalog.
Default bool `gorm:"-" json:"Default,omitempty"`
Name string `gorm:"-" json:"Name,omitempty"`
HTTPMonitors *int64 `gorm:"-" json:"HTTPMonitors,omitempty"`
DNSMonitors *int64 `gorm:"-" json:"DNSMonitors,omitempty"`
WHOISMonitors *int64 `gorm:"-" json:"WHOISMonitors,omitempty"`
TotalMonitors *int64 `gorm:"-" json:"TotalMonitors,omitempty"`
Price int `gorm:"-" json:"Price,omitempty"`
TrialPeriod int `gorm:"-" json:"TrialPeriod,omitempty"`
concerns.Timestamped
Audited
}
func (p *Plan) BeforeCreate(_ *gorm.DB) error {
if p.Code == "" {
p.Code = "legacy-" + uuid.NewString()[:24]
}
if p.NameRU == "" {
p.NameRU = p.Name
}
if p.NameEN == "" {
p.NameEN = p.NameRU
}
if p.Currency == "" {
p.Currency = "RUB"
}
if p.IntervalMinSeconds == 0 {
p.IntervalMinSeconds = 30
}
return nil
}
func (p *Plan) AfterFind(_ *gorm.DB) error {
p.Name = p.NameRU
p.Price = int(p.PriceMonthlyMinor / 100)
p.Default = p.IsDefault
p.TotalMonitors = &p.MonitorCap
return nil
}
// DefaultPlan returns the canonical free plan.
func DefaultPlan() Plan {
pl := Plan{}
if err := DB().Where("code = ? AND archived = FALSE", "free").First(&pl).Error; err != nil {
log.Println("unable to find default plan")
panic(err)
}
return pl
}
// AllowsDistributed keeps legacy callers working while using the canonical
// entitlement flag for catalog plans.
func (p *Plan) AllowsDistributed() bool {
return p != nil && (p.DistributedWorkers || p.Price > 0)
}

20
app/models/region.go Обычный файл
Просмотреть файл

@@ -0,0 +1,20 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// DefaultRegionCode is the historical default region code seeded by
// Migrate() (see app/models/migrate.go). Centralized here so admin
// endpoints and check_jobs.go agree on the literal.
const DefaultRegionCode = "local"
// Region represents a geographic region where distributed workers can run
type Region struct {
concerns.Model
Code string `gorm:"uniqueIndex;size:20;not null" json:"code"` // e.g. "ru-msk", "us-east", "eu-west"
Name string `gorm:"not null" json:"name"` // "Moscow, Russia"
Enabled bool `gorm:"not null;default:true" json:"enabled"`
Priority int `gorm:"not null;default:0" json:"priority"`
concerns.Timestamped
}

38
app/models/repo.go Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
package models
import (
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Repo is a source repository shared by one or more account-scoped sites.
// Repositories themselves are global rstuff mirrors; SiteRepo supplies the
// account boundary through its Site.
type Repo struct {
concerns.Model
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
GitlabID *int64 `json:"gitlab_id,omitempty"`
Name string `gorm:"size:120;not null" json:"name"`
Namespace *string `gorm:"size:120" json:"namespace,omitempty"`
Path *string `gorm:"size:255" json:"path,omitempty"`
Description *string `gorm:"type:text" json:"description,omitempty"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
concerns.Timestamped
}
// TableName returns the repository table name.
func (Repo) TableName() string { return "repos" }
// SiteRepo is the explicit site/repository join. Role is intentionally data,
// rather than an enum, to preserve the rstuff contract as it evolves.
type SiteRepo struct {
SiteID int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE CASCADE;primaryKey" json:"site_id"`
RepoID int64 `gorm:"type:bigint REFERENCES repos(id) ON DELETE CASCADE;primaryKey" json:"repo_id"`
Role string `gorm:"size:32;not null;default:'primary'" json:"role"`
concerns.Timestamped
}
// TableName returns the repository assignment table name.
func (SiteRepo) TableName() string { return "site_repos" }

135
app/models/rkn_domain.go Обычный файл
Просмотреть файл

@@ -0,0 +1,135 @@
package models
import (
"strings"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// RknDomain is one row of the locally cached ru-blocked domains list. The
// `Domain` field carries a lowercased domain string and is the unique key.
type RknDomain struct {
concerns.Model
Domain string `gorm:"size:255;uniqueIndex;not null" json:"domain"`
}
// TableName pins the underlying table name so GORM migrations and raw SQL
// (used by IsRknDomainBlocked and the rkn updater) agree.
func (RknDomain) TableName() string { return "rkn_domains" }
// ReplaceRknDomains deletes every existing row and inserts the given domains
// in batches. Caller-supplied domains are lower-cased and de-duplicated, and
// blank entries are dropped. The whole operation runs in a single
// transaction so readers see either the old set or the new set — there is no
// in-between state where the table is half-flushed.
//
// Designed to be called once per parser-updater tick (default every 6h); the
// uniqueIndex on Domain guarantees idempotent re-runs even when the caller
// forgets to filter.
func ReplaceRknDomains(domains []string) error {
if domains == nil {
domains = []string{}
}
deduped := make([]string, 0, len(domains))
seen := make(map[string]struct{}, len(domains))
for _, d := range domains {
d = strings.ToLower(strings.TrimSpace(d))
if d == "" {
continue
}
if _, ok := seen[d]; ok {
continue
}
seen[d] = struct{}{}
deduped = append(deduped, d)
}
tx := DB().Begin()
if tx.Error != nil {
return tx.Error
}
// Step 1 — wipe the existing cache. Using a scoped Where("1 = 1") Delete
// instead of TRUNCATE so the advisory lock taken by Migrate() does not
// become a bottleneck and so any in-flight foreign-key checks against
// rkn_domains remain satisfied (the table has no FKs today, but this
// matches the convention used by Drop()).
if err := tx.Where("1 = 1").Delete(&RknDomain{}).Error; err != nil {
_ = tx.Rollback().Error
return err
}
// Step 2 — bulk-insert in chunks of 1000 rows. CreateInBatches runs N
// multi-row INSERT statements, which for the typical ~30k ru-blocked
// entries is ~3ms per batch — significantly cheaper than per-row
// Create() in tight loops (the previous AddRknDomain implementation).
const batchSize = 1000
for start := 0; start < len(deduped); start += batchSize {
end := start + batchSize
if end > len(deduped) {
end = len(deduped)
}
rows := make([]RknDomain, 0, end-start)
for _, d := range deduped[start:end] {
rows = append(rows, RknDomain{Domain: d})
}
if err := tx.CreateInBatches(rows, batchSize).Error; err != nil {
_ = tx.Rollback().Error
return err
}
}
return tx.Commit().Error
}
// IsRknDomainBlocked returns true iff `domain` (or its root label, or any
// parent suffix already recorded as `*.parent.tld`) is present in the
// rkn_domains table.
//
// Matching rules — see checks/crkn/rkn_init.go for the original logic we
// consolidate here:
// 1. Exact match against the stored domain string.
// 2. Root-domain match (last two labels of the input) — covers the case
// where the user passed a subdomain but the upstream only lists the
// apex.
// 3. Suffix match (`stored LIKE '%' || input || ?`) — covers the case
// where the user passed the apex (or a higher-level label) but the
// upstream lists a child subdomain.
//
// All three checks are combined into a single SQL statement via OR so the
// table is scanned at most once and the SQL planner can pick a single
// index access path.
func IsRknDomainBlocked(domain string) (bool, error) {
domain = strings.ToLower(strings.TrimSpace(domain))
if domain == "" {
return false, nil
}
rootDomain := rootDomainOf(domain)
// Build the suffix patterns once. Note: every ".X" entry in the table
// (i.e. a domain that begins with a dot) matches any subdomain whose
// suffix is domain.
suffixPattern := "%." + domain
var count int64
err := DB().Raw(
"SELECT COUNT(*) FROM rkn_domains WHERE domain = ? OR domain = ? OR domain LIKE ?",
domain, rootDomain, suffixPattern,
).Scan(&count).Error
if err != nil {
return false, err
}
return count > 0, nil
}
// rootDomainOf returns the last two labels of `domain` (e.g. "a.b.c" → "b.c").
// Returns `domain` unchanged when it has fewer than three labels, because a
// one- or two-label input IS already the apex/root domain.
func rootDomainOf(domain string) string {
i := strings.LastIndex(domain, ".")
if i < 0 {
return domain
}
j := strings.LastIndex(domain[:i], ".")
if j < 0 {
return domain
}
return domain[j+1:]
}

90
app/models/rkn_domain_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,90 @@
package models_test
import (
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestReplaceRknDomains_BulkAndDedup(t *testing.T) {
models.Drop()
models.Migrate()
input := []string{
"Foo.example",
"foo.example", // dup after lowercasing
" bar.example ",
"",
"baz.example",
"qux.example",
"qux.example", // dup within input
}
require.NoError(t, models.ReplaceRknDomains(input))
got := []string{}
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &got).Error)
want := []string{"bar.example", "baz.example", "foo.example", "qux.example"}
sort.Strings(want)
assert.Equal(t, want, got)
}
func TestReplaceRknDomains_ReplacesExisting(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknDomains([]string{"old1.example", "old2.example"}))
var n int64
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
assert.Equal(t, int64(2), n)
// Second call wipes and replaces — no overlap with old set.
require.NoError(t, models.ReplaceRknDomains([]string{"new1.example", "new2.example", "new3.example"}))
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
assert.Equal(t, int64(3), n)
var domains []string
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &domains).Error)
assert.Equal(t, []string{"new1.example", "new2.example", "new3.example"}, domains)
}
func TestIsRknDomainBlocked(t *testing.T) {
models.Drop()
models.Migrate()
input := []string{
"example.com",
"foo.bar",
"sub.test",
}
require.NoError(t, models.ReplaceRknDomains(input))
cases := []struct {
host string
want bool
}{
{"example.com", true}, // exact
{"EXAMPLE.com", true}, // case insensitive (caller lowercases)
{"www.example.com", true}, // stored has apex; query apex → root-domain match
{"deep.nested.example.com", true}, // suffix match via LIKE '%.X'
{"foo.bar", true},
{"sub.test", true},
{"a.sub.test", true},
{"unrelated.org", false},
{"two.labels", false}, // 2-label input not in list → must not collapse to "labels"
{"", false},
}
for _, c := range cases {
t.Run(c.host, func(t *testing.T) {
got, err := models.IsRknDomainBlocked(c.host)
require.NoError(t, err)
assert.Equal(t, c.want, got)
})
}
}

193
app/models/rkn_ip.go Обычный файл
Просмотреть файл

@@ -0,0 +1,193 @@
package models
import (
"fmt"
"log"
"net"
"strings"
"github.com/davecgh/go-spew/spew"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/internal/netaddr"
)
// RknIP stores a single CIDR from the ru-blocked IP list. The cidr column
// uses Postgres's native cidr type so the (>>) containment operator works
// directly inside IsRknIPBlocked queries — GiST index recommended for any
// table large enough to amortize the planner cost; see EnsureRknIndexes.
type RknIP struct {
concerns.Model
Network *netaddr.Cidr `json:"network" gorm:"type:cidr;"`
}
// TableName pins the underlying table name.
func (RknIP) TableName() string { return "rkn_ips" }
// PanicOnErr panics with a spew-formatted error dump if err is non-nil.
// Kept for callers that still use the old add-by-row InsertInBatches path
// (none after this commit, but kept in case external scripts reference it).
func PanicOnErr(err error) {
if err != nil {
spew.Dump(err)
panic(err)
}
}
// FindRknIP returns true iff the IP inside `ip` (treated as a /32 or /128
// host) falls inside any CIDR already stored in rkn_ips. Kept for callers
// that already construct an internal/netaddr.Inet.
func FindRknIP(ip netaddr.Inet) (bool, error) {
ipstr := ip.Inet.String()
if !strings.Contains(ipstr, "/") {
if ip.Inet.To4() != nil {
ipstr += "/32"
} else {
ipstr += "/128"
}
}
_, network, err := net.ParseCIDR(ipstr)
if err != nil {
return false, err
}
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
var matched []RknIP
if err := DB().Raw("SELECT id FROM rkn_ips WHERE network >>= ?", &cidr).Scan(&matched).Error; err != nil {
return false, err
}
return len(matched) > 0, nil
}
// AddRknIP is the legacy per-row insert path. Deprecated: callers should
// invoke ReplaceRknIPs from the rkn updater. Kept around so existing cron
// scripts that import the symbol keep compiling.
func AddRknIP(data []string, count int) {
tx := DB().Begin()
for k, dataIP := range data {
if count > 0 && k > count-1 {
break
}
if !strings.Contains(dataIP, "/") {
dataIP += "/32"
}
_, network, err := net.ParseCIDR(dataIP)
PanicOnErr(err)
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
var rknIPs []RknIP
err = tx.Raw("SELECT id FROM rkn_ips where network >>= ?", &cidr).Scan(&rknIPs).Error
PanicOnErr(err)
if len(rknIPs) == 0 {
rknIPs = []RknIP{}
err = tx.Raw("SELECT id FROM rkn_ips where network << ?", &cidr).Scan(&rknIPs).Error
PanicOnErr(err)
if len(rknIPs) > 0 {
for _, r := range rknIPs {
PanicOnErr(tx.Delete(&r).Error)
}
}
PanicOnErr(tx.Create(&RknIP{Network: &cidr}).Error)
}
log.Println("created:", dataIP)
}
PanicOnErr(tx.Commit().Error)
}
// ReplaceRknIPs deletes every existing row and bulk-inserts the given CIDRs
// in batches of 1000. Whole operation runs in a single transaction so a
// partially-applied update can never leave the table in a hybrid state.
//
// CIDR.parseCIDR-loop uses net.ParseCIDR to canonicalise the address —
// upstream .dat files occasionally contain range/mask pairs that aren't
// already reduced (e.g. 192.168.0.0/16 spelled as 192.168.5.0/16); the
// Postgres cidr type normalises on insert.
func ReplaceRknIPs(nets []*net.IPNet) error {
tx := DB().Begin()
if tx.Error != nil {
return tx.Error
}
rows := make([]RknIP, 0, len(nets))
seen := make(map[string]struct{}, len(nets))
for _, n := range nets {
if n == nil || n.IP == nil {
continue
}
// Canonicalise by routing through net.ParseCIDR. This drops the
// host bits (a common bug in upstream dumps where a /24 range
// is written with the .5 host bit set) and stamps the right
// address family flag for Postgres.
canonical := n.String()
if _, parsed, err := net.ParseCIDR(canonical); err == nil {
n = parsed
canonical = parsed.String()
}
if _, ok := seen[canonical]; ok {
continue
}
seen[canonical] = struct{}{}
cidr := netaddr.Cidr{Cidr: *n, Valid: true}
rows = append(rows, RknIP{Network: &cidr})
}
// Wipe + re-insert in one transaction.
if err := tx.Where("1 = 1").Delete(&RknIP{}).Error; err != nil {
_ = tx.Rollback().Error
return err
}
const batchSize = 1000
for start := 0; start < len(rows); start += batchSize {
end := start + batchSize
if end > len(rows) {
end = len(rows)
}
if err := tx.CreateInBatches(rows[start:end], batchSize).Error; err != nil {
_ = tx.Rollback().Error
return err
}
}
return tx.Commit().Error
}
// IsRknIPBlocked returns true iff `ip` (any textual form ParseCIDR accepts)
// falls inside any CIDR stored in the rkn_ips table. The query uses the
// cidr >>= inet containment operator — see EnsureRknIndexes for the GiST
// index that makes this fast at scale.
func IsRknIPBlocked(ip string) (bool, error) {
ip = strings.TrimSpace(ip)
if ip == "" {
return false, nil
}
var count int64
if err := DB().Raw("SELECT COUNT(*) FROM rkn_ips WHERE network >>= ?::inet", ip).Scan(&count).Error; err != nil {
return false, err
}
return count > 0, nil
}
// EnsureRknIndexes creates the indexes that AutoMigrate cannot express —
// the GiST index on rkn_ips.network uses cidr >>= cidr containment (the
// expression index `network` already covers equality and prefix ranges,
// but the planner benefits from a GiST for `network >>= <other cidr>`
// queries against ~30k rows). The unique index on rkn_domains.domain is
// also declared in the GORM tag, this function only adds what GORM can't
// (GiST) and is idempotent so it's safe to call repeatedly during boot
// or migration.
func EnsureRknIndexes() error {
// GiST on cidr requires the btree_gist contrib — its `cidr_ops`
// opclass exposes cidr to GiST. CREATE EXTENSION IF NOT EXISTS is
// idempotent.
if err := DB().Exec("CREATE EXTENSION IF NOT EXISTS btree_gist").Error; err != nil {
return fmt.Errorf("ensure btree_gist: %w", err)
}
// rkn_ips GiST index on the cidr column supports the >>= containment
// operator that IsRknIPBlocked uses. Without it a 30k-row table makes
// every IP check a sequential scan; with it each check is an index
// probe.
if err := DB().Exec(
"CREATE INDEX IF NOT EXISTS idx_rkn_ips_network ON rkn_ips USING gist (network)",
).Error; err != nil {
return fmt.Errorf("ensure idx_rkn_ips_network: %w", err)
}
return nil
}

114
app/models/rkn_ip_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,114 @@
package models_test
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func mustIPNet(t *testing.T, cidr string) *net.IPNet {
t.Helper()
_, n, err := net.ParseCIDR(cidr)
require.NoError(t, err)
return n
}
func TestReplaceRknIPs_BulkAndDedup(t *testing.T) {
models.Drop()
models.Migrate()
input := []*net.IPNet{
mustIPNet(t, "10.0.0.0/8"),
mustIPNet(t, "10.5.0.0/8"), // same canonical /8 — dedup
mustIPNet(t, "192.168.1.0/24"),
mustIPNet(t, "2001:db8::/32"),
}
require.NoError(t, models.ReplaceRknIPs(input))
// The uniqueIndex on rkn_ips.network ensures the dedup actually drops
// duplicates; ReplaceRknIPs does an in-memory dedup, but the DB-level
// constraint is the guarantee.
var count int64
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&count).Error)
assert.Equal(t, int64(3), count, "expected dedup to 3 unique CIDRs")
}
func TestReplaceRknIPs_ReplacesExisting(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "8.8.8.0/24"),
}))
var n int64
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
assert.Equal(t, int64(1), n)
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "1.0.0.0/8"),
mustIPNet(t, "2.0.0.0/8"),
}))
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
assert.Equal(t, int64(2), n)
}
func TestIsRknIPBlocked(t *testing.T) {
models.Drop()
models.Migrate()
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
mustIPNet(t, "10.0.0.0/8"),
mustIPNet(t, "192.168.1.0/24"),
mustIPNet(t, "2001:db8::/32"),
}))
cases := []struct {
ip string
want bool
}{
{"10.5.7.9", true},
{"10.255.255.255", true},
{"192.168.1.42", true},
{"11.0.0.1", false},
{"8.8.8.8", false},
{"2001:db8::1", true},
{"2001:db9::1", false},
{"", false},
}
for _, c := range cases {
t.Run(c.ip, func(t *testing.T) {
got, err := models.IsRknIPBlocked(c.ip)
require.NoError(t, err)
assert.Equal(t, c.want, got, "ip=%s", c.ip)
})
}
}
func TestEnsureRknIndexes_Idempotent(t *testing.T) {
// Calling EnsureRknIndexes twice must not error — it's used both by
// Migrate() and could be called from boot scripts.
models.Drop()
models.Migrate()
require.NoError(t, models.EnsureRknIndexes())
require.NoError(t, models.EnsureRknIndexes())
// GiST index must exist on rkn_ips.network.
var exists bool
err := models.DB().Raw(`
SELECT EXISTS (
SELECT 1 FROM pg_indexes
WHERE schemaname='public' AND tablename='rkn_ips'
AND indexname='idx_rkn_ips_network'
)
`).Scan(&exists).Error
require.NoError(t, err)
assert.True(t, exists, "idx_rkn_ips_network must exist")
}

55
app/models/selfcheck.go Обычный файл
Просмотреть файл

@@ -0,0 +1,55 @@
package models
import (
"time"
"github.com/pkg/errors"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// SelfCheck provides functionality.
type SelfCheck struct {
concerns.Model
Kind string `gorm:"not null;uniqueIndex:selfchecks" json:"kind"`
Server *string `gorm:"uniqueIndex:selfchecks" json:"server"`
Info string `json:"info"`
LastCheck time.Time `json:"created_at"`
}
// LogCheck provides functionality.
func LogCheck(kind string) error {
m := SelfCheck{
Kind: kind,
Server: nil,
}
DB().FirstOrInit(&m, m)
m.LastCheck = time.Now()
return DB().Save(&m).Error
}
// IsOk checks if the selfcheck for the given kind ran recently.
func IsOk(kind string) (bool, string, error) {
m := SelfCheck{
Kind: kind,
Server: nil,
}
DB().First(&m, m)
if m.ID == 0 {
return false, "", errors.New("not run")
}
var ago time.Time
if kind == "exp" {
ago = time.Now().Add(-3 * time.Hour)
} else {
ago = time.Now().Add(-15 * time.Minute)
}
isOk := m.LastCheck.After(ago)
return isOk, m.LastCheck.Format(time.RFC3339), nil
}

388
app/models/server.go Обычный файл
Просмотреть файл

@@ -0,0 +1,388 @@
package models
import (
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"database/sql/driver"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/lib/pq"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// AccountMCPToken stores a one-way verifier. MCP tokens are bearer credentials
// and must remain valid even when the deployment has no encryption key.
type AccountMCPToken struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
Name string `gorm:"size:120;not null" json:"name"`
TokenEnc string `gorm:"column:token;type:char(64);not null;index" json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
concerns.Timestamped
}
func (AccountMCPToken) TableName() string { return "account_mcp_tokens" }
func (t *AccountMCPToken) SetToken(token string) error {
sum := sha256.Sum256([]byte(token))
t.TokenEnc = hex.EncodeToString(sum[:])
return nil
}
func (t *AccountMCPToken) TokenMatches(token string) bool {
sum := sha256.Sum256([]byte(token))
encoded := hex.EncodeToString(sum[:])
return subtle.ConstantTimeCompare([]byte(t.TokenEnc), []byte(encoded)) == 1
}
func GenerateMCPToken() string {
return "mcp_" + base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
}
// Server health states — worst-of-monitor-states rollup; see
// docs/plans/servers-and-hardware-metrics.md §5.1 for the data model.
const (
ServerHealthDown = "down"
ServerHealthWarn = "warn"
ServerHealthUp = "up"
ServerHealthPaused = "paused"
ServerHealthUnknown = "unknown"
)
// ServerEnvironments is the allow-list for Server.Environment.
// New environments require an explicit edit so they are visible in
// tests.
var ServerEnvironments = []string{"production", "staging", "dev", "test"}
// ServerKind is the rstuff-mirrored lifecycle label for a Server
// (production / staging / old). See
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable mapping.
type ServerKind string
// ServerKind values match rstuff's Server.kind enum exactly.
// New values require adding a Postgres enum value via
// app/models/migrate.go.
const (
ServerKindProduction ServerKind = "production"
ServerKindStaging ServerKind = "staging"
ServerKindOld ServerKind = "old"
)
// Scan implements sql.Scanner for ServerKind.
func (k *ServerKind) Scan(src any) error {
if src == nil {
*k = ""
return nil
}
switch v := src.(type) {
case string:
*k = ServerKind(v)
case []byte:
*k = ServerKind(string(v))
default:
return fmt.Errorf("server_kind: cannot scan %T", src)
}
return nil
}
// Value implements driver.Valuer for ServerKind.
func (k ServerKind) Value() (driver.Value, error) {
if k == "" {
return nil, nil
}
return string(k), nil
}
// Server represents a customer-facing logical host (e.g. "prod-web-01").
//
// One Server can host many WorkerNodes (HA after a VM migration). Each
// WorkerNode carries a nullable ServerID so legacy "no server assigned"
// rows keep working. The 1:N relation is stored as a nullable FK on
// worker_nodes.server_id. The N:M relation to Monitor is the
// monitor_servers join table. See
// docs/plans/servers-and-hardware-metrics.md §3 for the layer model
// and §5.1 for the schema.
//
// The inventory fields (ExtID, Kind, Token, PriceCents, Comment,
// Meta) are added per docs/plans/inventory-management.md §6.2 so
// rstuff can push the same row in via Valkey Streams and
// deploymentd can authenticate via Token.
type Server struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
Name string `gorm:"size:120;not null" json:"name"`
Slug string `gorm:"size:120;not null;index" json:"slug"`
Description *string `gorm:"type:text" json:"description"`
Region string `gorm:"size:64;not null;index" json:"region"`
Environment string `gorm:"size:32;not null;default:'production'" json:"environment"`
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
Icon *string `gorm:"size:16" json:"icon"`
Color *string `gorm:"size:16" json:"color"`
Paused bool `gorm:"not null;default:false" json:"paused"`
// Inventory fields (rstuff mirror; see inventory-management.md §6.2).
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
Kind ServerKind `gorm:"type:server_kind;not null;default:'production'" json:"kind"`
// Token is omitted from JSON because it is a bearer credential.
// Read it back only via /api/v1/servers/:id/token (operator-only)
// and never echoed in list/show responses.
Token *string `gorm:"size:64" json:"-"`
PriceCents int `gorm:"not null;default:0" json:"price_cents"`
Comment *string `gorm:"type:text" json:"comment,omitempty"`
// Meta is rstuff-style free-form jsonb; serialized via JSON
// encoding (gin renders it as a nested object).
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
HealthState string `gorm:"size:16;not null;default:'unknown';index" json:"health_state"`
LastCheckAt *time.Time `json:"last_check_at"`
Uptime24h float64 `gorm:"not null;default:1.0" json:"uptime_24h"`
Uptime30d float64 `gorm:"not null;default:1.0" json:"uptime_30d"`
Monitors []Monitor `gorm:"many2many:monitor_servers;joinForeignKey:server_id;joinReferences:monitor_id;" json:"monitors,omitempty"`
Workers []WorkerNode `gorm:"foreignKey:ServerID" json:"workers,omitempty"`
concerns.Timestamped
Audited
}
// TableName provides functionality.
func (Server) TableName() string { return "servers" }
// MonitorServer is the join row for the N:M relation between Monitor and
// Server. One Monitor can be hosted on many Servers (multi-region
// failover); one Server can host many Monitors.
type MonitorServer struct {
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id) ON DELETE CASCADE;primaryKey" json:"monitor_id"`
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;primaryKey" json:"server_id"`
Position int `gorm:"not null;default:0" json:"position"`
concerns.Timestamped
}
// TableName provides functionality.
func (MonitorServer) TableName() string { return "monitor_servers" }
// ServerMetric is the last-N-point cache written alongside VictoriaMetrics.
// The full time-series lives in TSDB; Postgres only keeps the most recent
// row per (server, source) for fast health badges and "last seen" cells.
// See docs/plans/servers-and-hardware-metrics.md §5.3.
type ServerMetric struct {
concerns.Model
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
WorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"worker_id"`
Source string `gorm:"size:32;not null;default:'worker'" json:"source"`
CPUPercent *float64 `json:"cpu_percent"`
MemUsed *int64 `json:"mem_used"`
MemTotal *int64 `json:"mem_total"`
DiskUsed *int64 `json:"disk_used"`
DiskTotal *int64 `json:"disk_total"`
NetRx *int64 `json:"net_rx"`
NetTx *int64 `json:"net_tx"`
HostUptimeSec *int64 `json:"host_uptime_sec"`
Load1 *float64 `json:"load1"`
Load5 *float64 `json:"load5"`
Load15 *float64 `json:"load15"`
ProcessCount *int `json:"process_count"`
Processes datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"processes"`
Networks datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"networks"`
concerns.Timestamped
}
// ServerAlertRule defines one account-owned threshold for a server metric.
// ClearThreshold implements hysteresis: a firing rule only recovers after the
// value drops below it, avoiding alert flapping around Threshold.
type ServerAlertRule struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
Metric string `gorm:"size:32;not null" json:"metric"`
Threshold float64 `gorm:"not null" json:"threshold"`
ClearThreshold float64 `gorm:"not null" json:"clear_threshold"`
DurationSec int `gorm:"not null;default:300" json:"duration_sec"`
NotificationID int64 `gorm:"type:bigint REFERENCES notifications(id) ON DELETE CASCADE;not null" json:"notification_id"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
State string `gorm:"size:16;not null;default:'ok'" json:"state"`
BreachSince *time.Time `json:"breach_since"`
LastValue *float64 `json:"last_value"`
LastMetricID *int64 `json:"last_metric_id"`
LastFiredAt *time.Time `json:"last_fired_at"`
concerns.Timestamped
}
func (ServerAlertRule) TableName() string { return "server_alert_rules" }
// ServerAlertEvent is the durable dedupe/audit record for threshold changes.
type ServerAlertEvent struct {
concerns.Model
RuleID int64 `gorm:"type:bigint REFERENCES server_alert_rules(id) ON DELETE CASCADE;not null;index" json:"rule_id"`
State string `gorm:"size:16;not null" json:"state"`
Value float64 `gorm:"not null" json:"value"`
concerns.Timestamped
}
func (ServerAlertEvent) TableName() string { return "server_alert_events" }
// TableName provides functionality.
func (ServerMetric) TableName() string { return "server_metrics" }
// ValidateEnvironment returns nil iff env is in the allow list.
func ValidateEnvironment(env string) error {
for _, e := range ServerEnvironments {
if env == e {
return nil
}
}
return errors.New("invalid environment")
}
// Slugify turns a server name into a URL-safe slug.
func Slugify(name string) string {
slug := strings.ToLower(strings.TrimSpace(name))
var b strings.Builder
for _, r := range slug {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == ' ', r == '_', r == '-', r == '.':
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
out = "server"
}
return out
}
// AssignMonitors replaces the full set of monitors for a server. Saves
// the join table explicitly because gorm:association_autoupdate is
// disabled globally (CLAUDE.md GORM Conventions).
func (s *Server) AssignMonitors(tx *gorm.DB, monitorIDs []uint) error {
if tx == nil {
tx = DB()
}
if err := tx.Exec("DELETE FROM monitor_servers WHERE server_id = ?", s.ID).Error; err != nil {
return err
}
if len(monitorIDs) == 0 {
return nil
}
seen := make(map[uint]struct{}, len(monitorIDs))
for i, mid := range monitorIDs {
if _, ok := seen[mid]; ok {
continue
}
seen[mid] = struct{}{}
row := MonitorServer{ServerID: s.ID, MonitorID: int64(mid), Position: i}
if err := tx.Create(&row).Error; err != nil {
return err
}
}
return nil
}
// RollupHealthState computes the worst-of-monitor-states. Returns one of
// ServerHealth{Down,Warn,Up,Paused,Unknown}.
func (s *Server) RollupHealthState(monitors []Monitor) string {
if s.Paused {
return ServerHealthPaused
}
if len(monitors) == 0 {
return ServerHealthUnknown
}
allPaused := true
worst := ServerHealthUp
for i := range monitors {
m := &monitors[i]
if m.Enabled {
allPaused = false
}
switch m.State {
case stateERR, stateFail:
return ServerHealthDown
case stateWARN:
worst = ServerHealthWarn
}
}
if allPaused {
return ServerHealthPaused
}
return worst
}
// SortedTagList returns tags sorted ascending; helper for stable JSON.
func (s *Server) SortedTagList() []string {
out := make([]string, len(s.Tags))
copy(out, s.Tags)
sort.Strings(out)
return out
}
// HealthForServer recomputes and persists health_state + last_check_at
// for one server. Called by the distworker health ticker.
func HealthForServer(serverID int64) error {
server := Server{}
if err := DB().First(&server, serverID).Error; err != nil {
return err
}
var monitors []Monitor
if err := DB().Joins("JOIN monitor_servers ms ON ms.monitor_id = monitors.id").
Where("ms.server_id = ?", serverID).Find(&monitors).Error; err != nil {
return err
}
state := server.RollupHealthState(monitors)
now := time.Now()
updates := map[string]interface{}{"health_state": state}
if len(monitors) > 0 {
updates["last_check_at"] = &now
}
return DB().Model(&server).Updates(updates).Error
}
// LatestServerMetric returns the newest accepted worker snapshot for a server.
func LatestServerMetric(serverID int64) (*ServerMetric, error) {
metric := ServerMetric{}
err := DB().Where("server_id = ?", serverID).Order("id DESC").First(&metric).Error
if err != nil {
return nil, err
}
return &metric, nil
}
// FindServerByToken returns the Server whose token column equals
// the given hex value, or nil with gorm.ErrRecordNotFound when no
// row matches. Used by the deploymentd receiver middleware.
func FindServerByToken(token string) (*Server, error) {
var s Server
err := DB().Where("token = ?", token).First(&s).Error
if err != nil {
return nil, err
}
return &s, nil
}
// GenerateServerToken returns a 32-byte random hex string. Caller
// stores the plaintext exactly once (via /servers/:id/rotate-token)
// and updates Server.Token; the old value is no longer recoverable.
func GenerateServerToken() string {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
// crypto/rand should not fail on Linux; panic keeps the
// contract simple for callers in the rare fatal case.
panic(err)
}
return hex.EncodeToString(b)
}

43
app/models/server_health_ticker.go Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package models
import (
"context"
"log"
"sync"
"time"
)
// HealthTickInterval is exported so focused tests can exercise the same
// lifecycle with a short interval.
var HealthTickInterval = time.Minute
var serverHealthTickerOnce sync.Once
// StartServerHealthTicker periodically refreshes derived server health. The
// first production tick is delayed to keep CLI migration paths side-effect
// free; assignment/pause paths recompute synchronously.
func StartServerHealthTicker(parent context.Context) {
serverHealthTickerOnce.Do(func() {
go func() {
ticker := time.NewTicker(HealthTickInterval)
defer ticker.Stop()
for {
select {
case <-parent.Done():
return
case <-ticker.C:
var ids []int64
if err := DB().Model(&Server{}).Pluck("id", &ids).Error; err != nil {
log.Printf("server health: list: %v", err)
continue
}
for _, id := range ids {
if err := HealthForServer(id); err != nil {
log.Printf("server health %d: %v", id, err)
}
}
}
}
}()
})
}

36
app/models/server_ip.go Обычный файл
Просмотреть файл

@@ -0,0 +1,36 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// ServerIp is a single IP (v4 or v6) bound to a Server. The
// canonical source for these rows is the deploymentd server-inventory
// receiver (M1) and the network-diagnostics partial plan; today the
// only writer is operator-entered via /api/v1/servers/:id/ips.
//
// `address` is Postgres `inet` so range queries (`<<` / `>>`) work
// without parsing text. One row per (server_id, address). The
// `is_primary` flag is set when more than one IP exists and the
// deploymentd payload signals a primary; otherwise the first row wins.
//
//nolint:revive // ServerIP rename deferred to M3; rstuff schema uses ServerIp verbatim and parity test depends on it.
type ServerIp struct {
concerns.Model
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
Server *Server `json:"-"`
// Address is mapped to Postgres inet via raw SQL in Migrate(); the
// GORM `type:` tag is not enough because gorm.io/driver/postgres
// does not register `inet` in its type map. Field is stored as a
// string and validated by the API layer (see
// app/controllers/api/server.go ServerIPsAdd).
Address string `gorm:"type:inet;not null" json:"address"`
IsPrimary bool `gorm:"not null;default:false" json:"is_primary"`
RelatedSitesCount int `gorm:"not null;default:0" json:"related_sites_count"`
concerns.Timestamped
}
// TableName provides functionality.
func (ServerIp) TableName() string { return "server_ips" }

98
app/models/site.go Обычный файл
Просмотреть файл

@@ -0,0 +1,98 @@
package models
import (
"strings"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Site represents a customer-facing website or app. One Site can be
// hosted on zero or one Server (server_id nullable) and exposes one
// or more Deployments (compose services or nginx vhosts) plus zero
// or more Repos (via site_repos). The RSMon slice mirrors rstuff's
// `sites` table verbatim — see docs/parity/rstuff-inventory.md §2
// and docs/plans/inventory-management.md §4.
type Site struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Server *Server `json:"-"`
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
Name string `gorm:"size:120;not null" json:"name"`
Slug string `gorm:"size:120;not null;index" json:"slug"`
URL *string `gorm:"type:text" json:"url,omitempty"`
Description *string `gorm:"type:text" json:"description,omitempty"`
// Kind is a free-text label (not the PG enum) so we can absorb
// rstuff additions without a migration. Default "production".
Kind string `gorm:"size:32;not null;default:'production'" json:"kind"`
IsActive bool `gorm:"not null;default:true" json:"is_active"`
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
Deployments []Deployment `gorm:"foreignKey:SiteID" json:"deployments,omitempty"`
Repos []Repo `gorm:"many2many:site_repos;" json:"repos,omitempty"`
concerns.Timestamped
Audited
}
// TableName returns the table name used for Site. Matches rstuff's
// `sites` plural exactly so the parity test stays trivial.
func (Site) TableName() string { return "sites" }
// SiteSlugify turns a name into a URL-safe slug. Mirrors the rules
// in app/models/server.go:Slugify so /sites/:slug looks the same as
// /servers/:slug. Consecutive separators collapse to a single dash;
// non-ASCII letters are stripped (the same rule as Slugify — we
// don't transliterate in v1, see docs/plans/inventory-management.md
// §11 for transliteration as a future hardening item).
func SiteSlugify(name string) string {
slug := strings.ToLower(strings.TrimSpace(name))
var b strings.Builder
prevDash := false
for _, r := range slug {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
prevDash = false
case r == ' ', r == '_', r == '-', r == '.':
if !prevDash && b.Len() > 0 {
b.WriteByte('-')
prevDash = true
}
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
out = "site"
}
return out
}
// FindOrCreateSiteBySlug returns the site with the given slug for the
// given account, creating an empty row (Name=slug, Kind=production,
// IsActive=true) when no match exists. The caller's tx wraps the
// operation so docker payload ingestion stays atomic.
// See docs/plans/inventory-management.md §7.2 — Docker receiver.
func FindOrCreateSiteBySlug(tx *gorm.DB, accountID int64, slug string) (*Site, error) {
if tx == nil {
tx = DB()
}
var site Site
err := tx.Where("account_id = ? AND slug = ?", accountID, slug).First(&site).Error
if err == nil {
return &site, nil
}
if err != gorm.ErrRecordNotFound {
return nil, err
}
site = Site{AccountID: accountID, Slug: slug, Name: slug, Kind: "production", IsActive: true}
if err := tx.Create(&site).Error; err != nil {
return nil, err
}
return &site, nil
}

120
app/models/stats_data.go Обычный файл
Просмотреть файл

@@ -0,0 +1,120 @@
package models
// StatsData provides functionality.
type StatsData struct {
MonitorID *int64 `json:"monitor_id,omitempty"`
Up30d *float64 `json:"up_30d"`
Up7d *float64 `json:"up_7d"`
Up1d *float64 `json:"up_1d"`
}
// Process provides functionality.
func (data *StatsData) Process() {
if data.Up1d == nil {
dv := 100.0
data.Up1d = &dv
}
if data.Up7d == nil {
dv := 100.0
data.Up7d = &dv
}
if data.Up30d == nil {
dv := 100.0
data.Up30d = &dv
}
}
// 86400 seconds / 1d
// 604800 seconds / 7d
// 2592000 seconds / 30d
// UptimeSQL provides functionality.
const UptimeSQL = `
SELECT
monitors.id AS monitor_id,
round(up1.up::numeric, 3) AS up1d,
round(up7.up::numeric, 3) AS up7d,
round(up30.up::numeric, 3) AS up30d
FROM monitors
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '1' day
group by monitor_id
) up1 ON monitors.id = up1.monitor_id
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '7' day
group by monitor_id
) up7 ON monitors.id = up7.monitor_id
LEFT JOIN (
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '30' day
group by monitor_id
) up30 ON monitors.id = up30.monitor_id
`
// UptimeAllSQL provides functionality.
const UptimeAllSQL = `
SELECT
round(up1.up::numeric, 3) AS up1d,
round(up7.up::numeric, 3) AS up7d,
round(up30.up::numeric, 3) AS up30d
FROM (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '1' day AND events.monitor_id IN (?)
) up1, (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '7' day AND events.monitor_id IN (?)
) up7, (
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
join (
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
) lt ON lt.id = events.monitor_id
where start_time > NOW() - interval '30' day AND events.monitor_id IN (?)
) up30
`
// MonitorStats provides functionality.
func MonitorStats(monitors *[]Monitor) error {
ids := make([]int64, len(*monitors))
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
ids[i] = m.ID
}
rows, err := DB().Raw(UptimeSQL+"WHERE monitors.id IN (?)", ids).Rows()
if err != nil {
return err
}
stats := make(map[int64]StatsData, 0)
defer rows.Close() //nolint:errcheck
for rows.Next() {
data := StatsData{}
_ = DB().ScanRows(rows, &data)
data.Process()
stats[*data.MonitorID] = data
}
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
(*monitors)[i].StatsData = stats[m.ID]
}
return nil
}

461
app/models/status_page.go Обычный файл
Просмотреть файл

@@ -0,0 +1,461 @@
// Package models — status page subsystem (docs/plans/status-pages.md).
//
// M0 ships the schema for status_pages and its five related tables
// (subscribers, incidents, maintenance, domains). The M0 milestone is
// read-only at the dashboard level — no editor and no public render yet —
// but landing the schema now lets downstream milestones wire public
// routes, editor flows, and the notifier→subscriber bridge without
// further ALTER TABLE churn. M5 (custom domain) only fills in
// status_page_domains rows; the table itself is reserved here so the
// M5 migration is just data, not DDL.
//
// All tables follow the existing RSMon conventions: concerns.Model +
// concerns.Timestamped + Audited mixins, gorm.DeletedAt for soft delete
// on the top-level status_pages row, pq.Int64Array for the monitor_ids
// bigint[] join columns (same shape as sites and the Check.Warnings
// slice). Partial-unique indexes (slug, subscriber email) are added via
// raw SQL in app/models/migrate.go because the GORM tag language cannot
// express a WHERE deleted_at IS NULL predicate.
package models
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"net"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/lib/pq"
"golang.org/x/net/idna"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
"rsgit.ru/rsmon/rsmon/config/credis"
)
// Status page color defaults. Matches the existing landing-page primary
// green and the accent blue used in the /settings UI, so a freshly
// created page already blends in with the rest of the app.
const (
statusPageDefaultPrimaryColor = "#62c600"
statusPageDefaultAccentColor = "#1a73e8"
statusPageDefaultHistoryDays = 90
statusPageMaxSlugLen = 64
statusPageMaxNameLen = 120
)
// Hex color regex — accepts #RGB and #RRGGBB. Centralized so the
// controller/model validation agree on the same shape.
var hexColorRegex = regexp.MustCompile(`^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$`)
// StatusPageSlugRegex mirrors the slug format enforced in
// StatusPage.NormalizeSlug / ValidateSlug. Lowercase alphanumerics and
// dashes, must start and end with an alphanumeric. Length is checked
// separately so the regex stays readable.
var statusPageSlugRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`)
// StatusPage represents a public status page owned by an account. One
// account may own many pages (gated by plan in M2+); slugs are globally
// unique because public URLs do not contain the account ID. Each page exposes
// a curated subset of the account's monitors and a recent-incidents
// feed. Soft-deleted rows remain in the table so the partial-unique
// index on (account_id, slug) WHERE deleted_at IS NULL still rejects
// duplicate slugs against historical records — see the comment on
// StatusPagesAccountSlugUnique in migrate.go.
type StatusPage struct {
concerns.Model
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
Account *Account `json:"-"`
Slug string `gorm:"size:64;not null;index" json:"slug"`
Name string `gorm:"size:120;not null" json:"name"`
Description *string `gorm:"type:text" json:"description,omitempty"`
LogoURL *string `gorm:"size:255" json:"logo_url,omitempty"`
PrimaryColor string `gorm:"size:7;not null;default:'#62c600'" json:"primary_color"`
AccentColor string `gorm:"size:7;not null;default:'#1a73e8'" json:"accent_color"`
// MonitorIDs is the curated subset of account monitors the page
// exposes. Order is preserved so the dashboard list and the public
// render show the same ordering. Stored as bigint[] to keep
// monitor-to-page mapping lookup-free on the read path; M2 will
// add a UI to maintain this set.
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
ShowUptimeBars bool `gorm:"not null;default:true" json:"show_uptime_bars"`
ShowResponseTimes bool `gorm:"not null;default:true" json:"show_response_times"`
ShowHistoryDays int `gorm:"not null;default:90" json:"show_history_days"`
// PasswordHash is populated in M4 only. Stored at length 255 so
// a future bcrypt cost bump does not need a column resize.
PasswordHash *string `gorm:"size:255" json:"-"`
// GATrackingID — Google Analytics 4 measurement ID; emitted by
// the public renderer in M4.
GATrackingID *string `gorm:"size:32" json:"ga_tracking_id,omitempty"`
// NoIndex emits <meta name="robots" content="noindex"> so the
// page can be staged without polluting search indexes.
NoIndex bool `gorm:"not null;default:false" json:"no_index"`
// IsPublished gates the public /status/:slug render. Until M2
// ships the editor the default value keeps M0 pages invisible.
IsPublished bool `gorm:"not null;default:false" json:"is_published"`
// AutoOpenIncidents is opt-in so publishing a page does not change
// existing alert behavior until an owner explicitly enables it.
AutoOpenIncidents bool `gorm:"not null;default:false" json:"auto_open_incidents"`
concerns.Timestamped
Audited
// DeletedAt is the GORM soft-delete marker. Using gorm.DeletedAt
// rather than concerns.SoftDelete because the latter adds a
// DeleterID users(id) FK that we do not yet need on status_pages.
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
}
// TableName returns the explicit table name so GORM does not try to
// pluralize to "status_pages" via inflection. The plural is already
// correct; we declare it anyway for clarity.
func (StatusPage) TableName() string { return "status_pages" }
// StatusPageSubscriberKind — values stored in status_page_subscribers.kind.
// "alert" subscribes to incident-driven notifications; "digest_daily"
// receives the morning summary (M3). New kinds should be appended so
// the JSON serializations stay stable.
const (
StatusPageSubscriberKindAlert = "alert"
StatusPageSubscriberKindDigestDaily = "digest_daily"
)
// StatusPageSubscriber is a row in status_page_subscribers. Email is
// stored verbatim (no citext) because the codebase already persists
// contact emails as-is; case-insensitive uniqueness is enforced via
// the partial unique index in migrate.go using lower(email).
type StatusPageSubscriber struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
// ContactID is an internal delivery endpoint. It is never returned by public
// subscription APIs; tasks use it to preserve the normal worker email path.
ContactID *int64 `gorm:"type:bigint REFERENCES contacts(id) ON DELETE SET NULL;index" json:"-"`
Email string `gorm:"size:255;not null" json:"email"`
Kind string `gorm:"size:16;not null;default:'alert'" json:"kind"`
ConfirmTokenHash string `gorm:"size:64" json:"-"`
// LegacyConfirmToken is retained only for confirmation links issued before
// token hashing shipped. It is cleared on first use or resend.
LegacyConfirmToken *string `gorm:"column:confirm_token;size:255" json:"-"`
TokenExpiresAt time.Time `json:"-"`
UnsubscribeTokenHash string `gorm:"size:64;default:''" json:"-"`
ConfirmedAt *time.Time `json:"confirmed_at,omitempty"`
UnsubscribedAt *time.Time `json:"unsubscribed_at,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_subscribers table name.
func (StatusPageSubscriber) TableName() string { return "status_page_subscribers" }
// StatusPageIncident severity values. info = heads-up notices, warn =
// degradation, crit = full outage. Used for color-coding in the public
// render (M1) and for filtering in the dashboard list (M0).
const (
StatusPageIncidentSeverityInfo = "info"
StatusPageIncidentSeverityWarn = "warn"
StatusPageIncidentSeverityCrit = "crit"
)
// StatusPageIncident represents a single incident entry on a status
// page. event_id is a soft link back to the existing Event model so
// "auto-open on monitor error" can be wired later without a second
// migration. posted_by_user_id is nullable so external integrations
// can write incidents anonymously.
type StatusPageIncident struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
EventID *int64 `gorm:"type:bigint REFERENCES events(id) ON DELETE SET NULL;index" json:"event_id,omitempty"`
Title string `gorm:"size:200;not null" json:"title"`
BodyMD string `gorm:"type:text" json:"body_md,omitempty"`
Severity string `gorm:"size:16;not null;default:'info'" json:"severity"`
StartedAt time.Time `gorm:"not null;index" json:"started_at"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
PostedByUserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"posted_by_user_id,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_incidents table name.
func (StatusPageIncident) TableName() string { return "status_page_incidents" }
// StatusPageMaintenance is a scheduled maintenance window. The
// monitor_ids column is the set of monitors the window covers; empty
// means "all monitors on the page".
type StatusPageMaintenance struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
Title string `gorm:"size:200;not null" json:"title"`
Description string `gorm:"type:text" json:"description,omitempty"`
StartsAt time.Time `gorm:"not null;index" json:"starts_at"`
EndsAt time.Time `gorm:"not null" json:"ends_at"`
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
NotifySubscribers bool `gorm:"not null;default:true" json:"notify_subscribers"`
concerns.Timestamped
}
// TableName returns the explicit status_page_maintenance table name.
func (StatusPageMaintenance) TableName() string { return "status_page_maintenance" }
// StatusPageDomain is the M5 custom-domain mapping. Reserved in M0 so
// the table does not need to be created at M5 — only rows are written
// then. domain is unique globally (CNAMEs are hostnames, they cannot
// be reused across pages), txt_token is the value the user adds as a
// DNS TXT record to prove ownership.
type StatusPageDomain struct {
concerns.Model
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
StatusPage *StatusPage `json:"-"`
Domain string `gorm:"size:255;not null;uniqueIndex" json:"domain"`
VerifiedAt *time.Time `json:"verified_at,omitempty"`
TXTToken string `gorm:"size:64;not null" json:"txt_token,omitempty"`
VerifyError string `gorm:"type:text" json:"verify_error,omitempty"`
concerns.Timestamped
}
// TableName returns the explicit status_page_domains table name.
func (StatusPageDomain) TableName() string { return "status_page_domains" }
// NormalizeStatusPageDomain accepts a hostname only. URLs, ports, IP literals,
// wildcard names, and invalid IDNA are deliberately rejected before DNS work.
func NormalizeStatusPageDomain(in string) (string, error) {
domain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(in)), ".")
if domain == "" || len(domain) > 253 || strings.ContainsAny(domain, "/:@") || net.ParseIP(domain) != nil {
return "", errStatusPage("domain must be a hostname")
}
ascii, err := idna.Lookup.ToASCII(domain)
if err != nil || ascii == "" || len(ascii) > 253 || !strings.Contains(ascii, ".") {
return "", errStatusPage("domain must be a valid hostname")
}
for _, label := range strings.Split(ascii, ".") {
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
return "", errStatusPage("domain must be a valid hostname")
}
for _, r := range label {
if r != '-' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
return "", errStatusPage("domain must be a valid hostname")
}
}
}
return ascii, nil
}
// HashStatusPageToken keeps bearer-style subscription URLs out of the database.
func HashStatusPageToken(token string) string {
sum := sha256.Sum256([]byte(token))
return fmt.Sprintf("%x", sum[:])
}
func (s *StatusPageSubscriber) TokenMatches(token string) bool {
return s != nil && s.TokenExpiresAt.After(time.Now()) && s.ConfirmTokenHash == HashStatusPageToken(token)
}
func (s *StatusPageSubscriber) UnsubscribeTokenMatches(token string) bool {
return s != nil && s.UnsubscribeTokenHash != "" && s.UnsubscribeTokenHash == HashStatusPageToken(token)
}
// StatusPagePlatformDomain and StatusPagePublicIPs are deployment-owned DNS
// targets. Customer domains must point here before they can become routable.
func StatusPagePlatformDomain() string {
return strings.TrimSuffix(strings.ToLower(os.Getenv("STATUS_PAGE_PLATFORM_DOMAIN")), ".")
}
func StatusPagePublicIPs() []string {
return strings.FieldsFunc(os.Getenv("STATUS_PAGE_PUBLIC_IPS"), func(r rune) bool { return r == ',' || r == ' ' })
}
var (
statusPageLookupCNAME = net.LookupCNAME
statusPageLookupHost = net.LookupHost
statusPageLookupTXT = net.LookupTXT
)
// VerifyStatusPageDomain performs the BYO-DNS preflight. A customer must keep
// the ownership TXT record and point either a CNAME at the platform hostname or
// an A/AAAA record at one of the explicitly configured public addresses.
func VerifyStatusPageDomain(domain *StatusPageDomain) error {
if domain == nil {
return errStatusPage("domain is required")
}
want, err := NormalizeStatusPageDomain(domain.Domain)
if err != nil {
return err
}
platform := StatusPagePlatformDomain()
publicIPs := StatusPagePublicIPs()
if platform == "" && len(publicIPs) == 0 {
return errStatusPage("custom domain verification is not configured")
}
matchedTarget := false
if platform != "" {
if cname, lookupErr := statusPageLookupCNAME(want); lookupErr == nil {
matchedTarget = strings.TrimSuffix(strings.ToLower(cname), ".") == platform
}
}
if !matchedTarget && len(publicIPs) > 0 {
if hosts, lookupErr := statusPageLookupHost(want); lookupErr == nil {
for _, host := range hosts {
for _, allowed := range publicIPs {
if host == allowed {
matchedTarget = true
}
}
}
}
}
if !matchedTarget {
return errStatusPage("DNS must contain the configured CNAME or public A/AAAA address")
}
txt, lookupErr := statusPageLookupTXT(want)
if lookupErr != nil {
return errStatusPage("ownership TXT record was not found")
}
for _, value := range txt {
if value == "rsmon-verify="+domain.TXTToken {
return nil
}
}
return errStatusPage("ownership TXT record does not match")
}
// NormalizeStatusPageSlug lowercases and trims a candidate slug so the
// global partial unique index on slug is satisfied
// regardless of how the caller capitalizes the input. Returns the
// empty string when the result would be unusable as a URL path;
// callers should fall back to a name-derived slug in that case.
func NormalizeStatusPageSlug(in string) string {
slug := strings.ToLower(strings.TrimSpace(in))
return slug
}
// ValidateStatusPageSlug enforces the slug format we expose to users:
// lowercase alphanumeric plus dash, must start and end with an
// alphanumeric, max length 64. Used by the editor before save (M2).
// Returns nil when the slug is acceptable.
func ValidateStatusPageSlug(slug string) error {
if slug == "" {
return errStatusPage("slug is required")
}
if len(slug) > statusPageMaxSlugLen {
return errStatusPage("slug is too long")
}
if !statusPageSlugRegex.MatchString(slug) {
return errStatusPage("slug must be lowercase alphanumeric with dashes")
}
return nil
}
// ValidateStatusPageColors returns an error if either color is set but
// not a valid CSS hex string. Empty strings fall back to the model
// defaults when written via BeforeSave hooks.
func ValidateStatusPageColors(primary, accent string) error {
if primary != "" && !hexColorRegex.MatchString(primary) {
return errStatusPage("primary_color must be #RGB or #RRGGBB")
}
if accent != "" && !hexColorRegex.MatchString(accent) {
return errStatusPage("accent_color must be #RGB or #RRGGBB")
}
return nil
}
// BeforeSave is the GORM hook that fills in the canonical defaults
// (colors, history days) so callers can pass an empty struct and still
// get a usable page. Hook is also the single place where slugs are
// normalized, so the unique index never has to chase trailing spaces.
// The gorm.DB parameter is required by the hook signature but unused —
// the validation here is purely local to the model.
func (p *StatusPage) BeforeSave(_ *gorm.DB) error {
if p == nil {
return nil
}
p.Slug = NormalizeStatusPageSlug(p.Slug)
if err := ValidateStatusPageSlug(p.Slug); err != nil {
return err
}
if p.PrimaryColor == "" {
p.PrimaryColor = statusPageDefaultPrimaryColor
}
if p.AccentColor == "" {
p.AccentColor = statusPageDefaultAccentColor
}
if p.ShowHistoryDays == 0 {
p.ShowHistoryDays = statusPageDefaultHistoryDays
}
return ValidateStatusPageColors(p.PrimaryColor, p.AccentColor)
}
// IsPublishedNow reports whether the page is publicly visible. M0
// always returns false because the editor (M2) is the only thing that
// flips IsPublished to true; this helper centralizes that contract.
func (p *StatusPage) IsPublishedNow() bool {
return p != nil && p.IsPublished && !p.DeletedAt.Valid
}
// errStatusPage builds a validation error carrying the message. The
// returned error is a plain error; controllers translate it into a
// 422 response.
func errStatusPage(msg string) error {
if msg == "" {
return errors.New("status_page: invalid")
}
return fmt.Errorf("status_page: %s", msg)
}
// IsActiveSubscriber returns true when the subscriber has confirmed and
// has not unsubscribed. Used by the M3 incident-notification loop.
func (s *StatusPageSubscriber) IsActiveSubscriber() bool {
if s == nil {
return false
}
return s.ConfirmedAt != nil && s.UnsubscribedAt == nil
}
func (s *StatusPageSubscriber) BeforeCreate(_ *gorm.DB) error {
if s.TokenExpiresAt.IsZero() {
s.TokenExpiresAt = time.Now().Add(24 * time.Hour)
}
if s.ConfirmTokenHash == "" {
s.ConfirmTokenHash = HashStatusPageToken(fmt.Sprintf("legacy-%d-%s", time.Now().UnixNano(), s.Email))
}
return nil
}
// InvalidateStatusPageCache removes the public HTML cache without making
// Redis availability part of the monitor or management write path.
func InvalidateStatusPageCache(pageID int64) {
if credis.Redis != nil {
_ = credis.Redis.Del(context.Background(), "statuspage:html:"+strconv.FormatInt(pageID, 10)).Err()
}
}

408
app/models/status_page_delivery.go Обычный файл
Просмотреть файл

@@ -0,0 +1,408 @@
package models
import (
"context"
"encoding/json"
"errors"
"fmt"
"html"
"log"
"strings"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/internal/wire"
)
const (
statusPageDigestHour = 9
statusPageOutboxPending = "pending"
statusPageOutboxDispatched = "dispatched"
statusPageOutboxCanceled = "canceled"
)
// StatusPageDelivery is an outbox row. It does not retain email addresses;
// confirmation links are the only transient body data and are redacted on
// cancellation. The subscriber/contact data is read while dispatching.
type StatusPageDelivery struct {
ID int64 `gorm:"primarykey"`
StatusPageID int64 `gorm:"not null;index"`
SubscriberID int64 `gorm:"not null;index"`
IncidentID *int64 `gorm:"index"`
Kind string `gorm:"size:32;not null"`
Version string `gorm:"size:64;not null"`
LocalDate string `gorm:"size:10"`
State string `gorm:"size:16;not null;index"`
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
MessageID *int64 `gorm:"index"`
TaskID *int64 `gorm:"index"`
LastError string `gorm:"type:text"`
Subject string `gorm:"type:text"`
BodyText string `gorm:"type:text"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
type StatusPageDigestSchedule struct {
ID int64 `gorm:"primarykey"`
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
DueAt time.Time `gorm:"not null;index"`
CreatedAt time.Time
}
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
if subscriber.ContactID != nil {
return nil
}
accountID := page.AccountID
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
if err := tx.Create(contact).Error; err != nil {
return err
}
subscriber.ContactID = &contact.ID
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
}
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
}
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
}
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
}
func valueOrZero(v *int64) int64 {
if v == nil {
return 0
}
return *v
}
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
// transaction, so an incident can never become visible without its delivery
// intent being recoverable by the dispatcher.
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
return err
}
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
for _, subscriber := range subscribers {
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
return err
}
}
return nil
}
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
var notification Notification
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(&notification).Error
if err == nil {
return &notification, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
return &notification, tx.Create(&notification).Error
}
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
// Message and Task rows in the same transaction, and preserves a pending row on
// transient worker/capability failure for the next tick.
func DispatchStatusPageDeliveries(now time.Time) error {
var rows []StatusPageDelivery
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
return err
}
var errs []error
for _, row := range rows {
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func dispatchStatusPageDelivery(id int64, now time.Time) error {
err := DB().Transaction(func(tx *gorm.DB) error {
var outbox StatusPageDelivery
if err := tx.First(&outbox, id).Error; err != nil {
return err
}
var subscriber StatusPageSubscriber
var page StatusPage
// Subscriber then outbox is the global lock order shared with
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
}
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
return err
}
if outbox.State != statusPageOutboxPending {
return nil
}
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
}
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
return err
}
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
return err
}
notification, err := statusPageNotificationTx(tx, page.AccountID)
if err != nil {
return err
}
subject, text := outbox.Subject, outbox.BodyText
if subject == "" {
subject, text = "["+page.Name+"] status update", "Status update"
}
if outbox.IncidentID != nil {
var incident StatusPageIncident
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
return err
}
subject = "[" + page.Name + "] " + incident.Title
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
} else if outbox.BodyText == "" {
text = "Open incidents daily digest"
}
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
if err := tx.Create(message).Error; err != nil {
return err
}
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
if err != nil {
return err
}
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
if err != nil {
return err
}
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
})
if err != nil {
// This update intentionally runs after rollback: diagnostic state must not
// disappear with the failed message/task transaction.
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
}
return err
}
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
return err
}
if outbox.TaskID != nil {
// Redact every task, including terminal audit rows. State is preserved for
// terminal rows, while the payload can no longer disclose the address.
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
return err
}
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
return err
}
}
if outbox.MessageID != nil {
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
}
return nil
}
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
var subscriber StatusPageSubscriber
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
var rows []StatusPageDelivery
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
return err
}
}
return nil
}
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
// retained status-page lineage without touching unrelated account messages.
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
return err
}
for i := range rows {
row := &rows[i]
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if row.TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if row.MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
var rows []StatusPageDelivery
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
return err
}
for i := range rows {
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
return err
}
if rows[i].TaskID != nil {
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
return err
}
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
return err
}
}
if rows[i].MessageID != nil {
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
return err
}
}
}
return nil
}
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
// endpoint inert and irreversibly removing subscriber PII.
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
return err
}
var subscribers []StatusPageSubscriber
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
return err
}
now := time.Now()
for i := range subscribers {
s := &subscribers[i]
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
return err
}
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
return err
}
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
return err
}
if s.ContactID != nil {
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
return err
}
}
}
return tx.Delete(page).Error
}
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
func EnqueueStatusPageDailyDigests(now time.Time) error {
var subscribers []StatusPageSubscriber
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
return err
}
var errs []error
for i := range subscribers {
s := &subscribers[i]
if s.StatusPage == nil || s.StatusPage.Account == nil {
continue
}
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
if err != nil {
loc = time.UTC
}
localNow := now.In(loc)
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
// Only persist today's actual due date. Older pending outbox rows are
// retried by DispatchStatusPageDeliveries; never invent history from a
// current incident snapshot after downtime.
if now.Before(due) || s.ConfirmedAt.After(due) {
continue
}
var incidents []StatusPageIncident
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
errs = append(errs, err)
continue
}
if len(incidents) == 0 {
continue
} // No digest is better than a misleading outage summary.
lines := make([]string, 0, len(incidents))
for _, incident := range incidents {
lines = append(lines, "- "+incident.Title)
}
err = DB().Transaction(func(tx *gorm.DB) error {
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
return err
}
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
})
if err != nil {
errs = append(errs, err)
}
}
if err := DispatchStatusPageDeliveries(now); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func StartStatusPageDigestScheduler(ctx context.Context) {
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
log.Printf("status-page delivery dispatch: %v", err)
}
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
log.Printf("status-page digest: %v", err)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
}

401
app/models/status_page_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,401 @@
package models_test
import (
"strings"
"testing"
"time"
"gorm.io/gorm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// statusPageSeedAcc returns a fresh account for status-page tests.
func statusPageSeedAcc(t *testing.T) *models.Account {
t.Helper()
acc := &models.Account{Name: "status-page-" + strings.ReplaceAll(t.Name(), "/", "_")}
require.NoError(t, models.DB().Create(acc).Error)
return acc
}
// TestStatusPage_Defaults asserts that creating a StatusPage with the
// minimum required fields fills in the documented defaults (colors,
// history days, soft-delete marker) and leaves IsPublished false.
func TestStatusPage_Defaults(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{
AccountID: acc.ID,
Slug: "acme-status",
Name: "Acme Status",
}
require.NoError(t, models.DB().Create(page).Error)
got := models.StatusPage{}
require.NoError(t, models.DB().First(&got, page.ID).Error)
assert.Equal(t, "#62c600", got.PrimaryColor, "default primary color")
assert.Equal(t, "#1a73e8", got.AccentColor, "default accent color")
assert.Equal(t, 90, got.ShowHistoryDays, "default history days")
assert.True(t, got.ShowUptimeBars, "show_uptime_bars defaults true")
assert.True(t, got.ShowResponseTimes, "show_response_times defaults true")
assert.False(t, got.IsPublished, "is_published defaults false")
assert.False(t, got.IsPublishedNow(), "IsPublishedNow() returns false until publish")
assert.True(t, got.DeletedAt.Valid == false, "no soft-delete timestamp on fresh row")
}
// TestStatusPage_SlugUniquenessSoftDelete verifies the partial unique
// index allows recreating a slug after soft-deleting the previous row.
func TestStatusPage_SlugUniquenessSoftDelete(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
first := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout",
}
require.NoError(t, models.DB().Create(first).Error)
// Second live row with the same slug must fail.
dup := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout copy",
}
err := models.DB().Create(dup).Error
require.Error(t, err, "duplicate slug on live rows must fail")
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"unexpected error: %v", err,
)
// Soft-delete the first row. After that the slug is reusable.
require.NoError(t, models.DB().Delete(first).Error)
again := &models.StatusPage{
AccountID: acc.ID,
Slug: "rollout",
Name: "Rollout v2",
}
require.NoError(t, models.DB().Create(again).Error,
"recreating a slug after a soft-delete must succeed (partial index)")
assert.NotEqual(t, first.ID, again.ID)
}
// TestStatusPage_SlugIsGlobal verifies public URLs are unambiguous: the
// account is not part of /status/:slug, so another account cannot claim it.
func TestStatusPage_SlugIsGlobal(t *testing.T) {
models.Drop()
models.Migrate()
accA := statusPageSeedAcc(t)
accB := &models.Account{Name: "status-page-other"}
require.NoError(t, models.DB().Create(accB).Error)
a := &models.StatusPage{AccountID: accA.ID, Slug: "shared", Name: "A"}
require.NoError(t, models.DB().Create(a).Error)
b := &models.StatusPage{AccountID: accB.ID, Slug: "shared", Name: "B"}
require.Error(t, models.DB().Create(b).Error,
"different accounts must not be able to reuse a public slug")
}
func TestStatusPageIncidentDeliveryIsIdempotentAndAccountScoped(t *testing.T) {
models.Drop()
models.Migrate()
account := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: account.ID, Slug: "delivery-page", Name: "Delivery"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "subscriber@example.test", Kind: models.StatusPageSubscriberKindAlert, ConfirmedAt: &now}
require.NoError(t, models.DB().Create(subscriber).Error)
incident := &models.StatusPageIncident{StatusPageID: page.ID, Title: "API unavailable", Severity: models.StatusPageIncidentSeverityCrit, StartedAt: now}
require.NoError(t, models.DB().Create(incident).Error)
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
}))
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
}))
var deliveries []models.StatusPageDelivery
require.NoError(t, models.DB().Where("idempotency_key LIKE ?", "status-page:opened:%").Find(&deliveries).Error)
require.Len(t, deliveries, 1)
assert.Equal(t, page.ID, deliveries[0].StatusPageID)
}
func TestStatusPageDailyDigestIsTimezoneDateIdempotent(t *testing.T) {
models.Drop()
models.Migrate()
account := statusPageSeedAcc(t)
account.Timezone = "UTC"
require.NoError(t, models.DB().Save(account).Error)
page := &models.StatusPage{AccountID: account.ID, Slug: "digest-page", Name: "Digest"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Date(2026, time.July, 13, 9, 15, 0, 0, time.UTC)
confirmedAt := now.Add(-time.Hour)
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "digest@example.test", Kind: models.StatusPageSubscriberKindDigestDaily, ConfirmedAt: &confirmedAt}
require.NoError(t, models.DB().Create(subscriber).Error)
require.NoError(t, models.DB().Create(&models.StatusPageIncident{StatusPageID: page.ID, Title: "Still open", Severity: models.StatusPageIncidentSeverityWarn, StartedAt: now}).Error)
_ = models.EnqueueStatusPageDailyDigests(now)
_ = models.EnqueueStatusPageDailyDigests(now.Add(30 * time.Second))
var count int64
require.NoError(t, models.DB().Model(&models.StatusPageDelivery{}).Where("idempotency_key LIKE ?", "status-page:digest:%").Count(&count).Error)
assert.GreaterOrEqual(t, count, int64(1), "catch-up may include earlier local due dates")
}
// TestStatusPage_NormalizeSlugAndValidate exercises the slug normalizer
// and validator — uppercase input becomes lowercase, and an invalid
// slug (leading dash) is rejected on save via BeforeSave.
func TestStatusPage_NormalizeSlugAndValidate(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
assert.Equal(t, "lower-case", models.NormalizeStatusPageSlug(" Lower-Case "))
assert.NoError(t, models.ValidateStatusPageSlug("acme-status"))
bad := []string{"", "-leading-dash", "trailing-dash-", "UPPER", "with spaces", "with_underscore"}
for _, slug := range bad {
err := models.ValidateStatusPageSlug(slug)
assert.Error(t, err, "expected error for slug %q", slug)
}
// Save enforces the same rules — uppercase gets normalized, invalid
// characters reject.
good := &models.StatusPage{AccountID: acc.ID, Slug: "ACME-Status", Name: "Acme"}
require.NoError(t, models.DB().Create(good).Error)
assert.Equal(t, "acme-status", good.Slug)
bad2 := &models.StatusPage{AccountID: acc.ID, Slug: "-bad", Name: "Bad"}
err := models.DB().Create(bad2).Error
require.Error(t, err, "invalid slug should be rejected by BeforeSave")
}
// TestStatusPage_ColorValidation rejects non-hex colors at save time.
func TestStatusPage_ColorValidation(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
bad := &models.StatusPage{
AccountID: acc.ID,
Slug: "badcolor",
Name: "BadColor",
PrimaryColor: "not-a-color",
}
err := models.DB().Create(bad).Error
require.Error(t, err, "non-hex color must be rejected")
good := &models.StatusPage{
AccountID: acc.ID,
Slug: "goodcolor",
Name: "GoodColor",
PrimaryColor: "#0a1b2c",
AccentColor: "#abc",
}
require.NoError(t, models.DB().Create(good).Error)
reloaded := models.StatusPage{}
require.NoError(t, models.DB().First(&reloaded, good.ID).Error)
assert.Equal(t, "#0a1b2c", reloaded.PrimaryColor)
assert.Equal(t, "#abc", reloaded.AccentColor)
}
// TestStatusPageSubscriber_UniqueActive verifies the partial unique
// index on (status_page_id, lower(email)) only fires while a
// subscriber is not unsubscribed.
func TestStatusPageSubscriber_UniqueActive(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{
AccountID: acc.ID,
Slug: "sub",
Name: "Sub",
}
require.NoError(t, models.DB().Create(page).Error)
mk := func(email string, unsub *time.Time) *models.StatusPageSubscriber {
s := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: email,
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-" + email),
}
s.UnsubscribedAt = unsub
return s
}
first := mk("Alice@Example.com", nil)
require.NoError(t, models.DB().Create(first).Error)
// Same email with mixed case must conflict (lower(email) is the
// unique key), so a true case-insensitive uniqueness is in place.
dup := mk("alice@example.com", nil)
err := models.DB().Create(dup).Error
require.Error(t, err, "active subscriber with same lowercased email must conflict")
// Soft-unsubscribe the original then re-add a fresh row with the
// same email — this must succeed because the partial index
// excludes unsubscribed rows.
now := time.Now()
first.UnsubscribedAt = &now
require.NoError(t, models.DB().Save(first).Error)
resurrected := mk("alice@example.com", nil)
require.NoError(t, models.DB().Create(resurrected).Error,
"creating a fresh subscriber after the previous one unsubscribed must succeed")
assert.NotEqual(t, first.ID, resurrected.ID)
// ActiveSubscriber reflects both confirmed and unsubscribed flags.
confirmed := time.Now()
active := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: "bob@example.com",
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-bob"),
ConfirmedAt: &confirmed,
}
require.NoError(t, models.DB().Create(active).Error)
assert.True(t, active.IsActiveSubscriber(), "confirmed and not unsubscribed → active")
pending := &models.StatusPageSubscriber{
StatusPageID: page.ID,
Email: "carol@example.com",
Kind: models.StatusPageSubscriberKindAlert,
ConfirmTokenHash: models.HashStatusPageToken("tok-carol"),
}
require.NoError(t, models.DB().Create(pending).Error)
assert.False(t, pending.IsActiveSubscriber(), "unconfirmed → not active")
}
// TestStatusPageIncident_SeverityAndEventFK checks that the FK to
// events is wired correctly and that severity defaults to "info".
func TestStatusPageIncident_SeverityAndEventFK(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "inc", Name: "Inc"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
inc := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "API slowdown",
BodyMD: "Investigating.",
Severity: models.StatusPageIncidentSeverityWarn,
StartedAt: now,
}
require.NoError(t, models.DB().Create(inc).Error)
got := models.StatusPageIncident{}
require.NoError(t, models.DB().First(&got, inc.ID).Error)
assert.Equal(t, "warn", got.Severity)
// Default severity is "info" when omitted.
auto := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "heads up",
StartedAt: now,
}
require.NoError(t, models.DB().Create(auto).Error)
gotAuto := models.StatusPageIncident{}
require.NoError(t, models.DB().First(&gotAuto, auto.ID).Error)
assert.Equal(t, models.StatusPageIncidentSeverityInfo, gotAuto.Severity)
// EventID stays nullable and the FK tolerates a NULL event.
var nilEvt *int64
noEvt := &models.StatusPageIncident{
StatusPageID: page.ID,
Title: "no event",
Severity: models.StatusPageIncidentSeverityCrit,
StartedAt: now,
EventID: nilEvt,
}
require.NoError(t, models.DB().Create(noEvt).Error)
}
// TestStatusPageMaintenance_BigintArray ensures the monitor_ids bigint[]
// column round-trips through the GORM pq.Int64Array driver correctly.
func TestStatusPageMaintenance_BigintArray(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "mt", Name: "Maintenance"}
require.NoError(t, models.DB().Create(page).Error)
now := time.Now()
maint := &models.StatusPageMaintenance{
StatusPageID: page.ID,
Title: "DB upgrade",
Description: "Rolling upgrade.",
StartsAt: now,
EndsAt: now.Add(2 * time.Hour),
MonitorIDs: models.Int64ArrayFromSlice([]int64{1, 7, 42}),
NotifySubscribers: true,
}
require.NoError(t, models.DB().Create(maint).Error)
got := models.StatusPageMaintenance{}
require.NoError(t, models.DB().First(&got, maint.ID).Error)
assert.Equal(t, []int64{1, 7, 42}, []int64(got.MonitorIDs))
assert.True(t, got.NotifySubscribers)
}
// TestStatusPageDomain_TableReservation verifies the M5 table is
// created with the right unique constraints even though M0 does not
// yet populate it.
func TestStatusPageDomain_TableReservation(t *testing.T) {
models.Drop()
models.Migrate()
acc := statusPageSeedAcc(t)
page := &models.StatusPage{AccountID: acc.ID, Slug: "dom", Name: "Domain"}
require.NoError(t, models.DB().Create(page).Error)
d := &models.StatusPageDomain{
StatusPageID: page.ID,
Domain: "status.example.com",
TXTToken: "rsmon-verify=abc123",
}
require.NoError(t, models.DB().Create(d).Error)
require.NoError(t, models.DB().Create(&models.StatusPageDomain{
StatusPageID: page.ID,
Domain: "www.status.example.com",
TXTToken: "rsmon-verify=second",
}).Error, "one status page may own multiple domains")
// Second page claiming the same domain must fail because domain
// is globally unique.
other := &models.StatusPage{AccountID: acc.ID, Slug: "dom2", Name: "Domain2"}
require.NoError(t, models.DB().Create(other).Error)
dup := &models.StatusPageDomain{
StatusPageID: other.ID,
Domain: "status.example.com",
TXTToken: "rsmon-verify=xyz789",
}
err := models.DB().Create(dup).Error
require.Error(t, err)
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "unique") ||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
"expected unique-constraint error, got %v", err)
}
func TestNormalizeStatusPageDomain(t *testing.T) {
got, err := models.NormalizeStatusPageDomain(" Status.Example.COM. ")
require.NoError(t, err)
assert.Equal(t, "status.example.com", got)
for _, input := range []string{"https://example.com", "example.com:443", "127.0.0.1", "*.example.com", "-bad.example.com"} {
_, err := models.NormalizeStatusPageDomain(input)
assert.Error(t, err, input)
}
}

61
app/models/subscription.go Обычный файл
Просмотреть файл

@@ -0,0 +1,61 @@
package models
import (
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
SubscriptionStatusTrialing = "trialing"
SubscriptionStatusActive = "active"
SubscriptionStatusPastDue = "past_due"
SubscriptionStatusPaused = "paused"
SubscriptionStatusCanceled = "canceled"
SubscriptionStatusExpired = "expired"
)
// Subscription is the current billable entitlement for one account. M0 only
// writes manual subscriptions; provider flows are deliberately deferred.
type Subscription struct {
concerns.Model
AccountID int64 `gorm:"not null;index" json:"account_id"`
Account *Account `json:"-"`
PlanID int64 `gorm:"not null;index" json:"plan_id"`
Plan *Plan `json:"plan,omitempty"`
Provider string `gorm:"size:16;not null;default:'manual'" json:"provider"`
ProviderSubscriptionID *string `gorm:"size:128" json:"provider_subscription_id,omitempty"`
ProviderCustomerID *string `gorm:"size:128" json:"provider_customer_id,omitempty"`
Status string `gorm:"size:24;not null;default:'active';index" json:"status"`
BillingCycle string `gorm:"size:8;not null;default:'monthly'" json:"billing_cycle"`
CurrentPeriodStart *time.Time `json:"current_period_start,omitempty"`
CurrentPeriodEnd *time.Time `gorm:"index" json:"current_period_end,omitempty"`
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
CancelAtPeriodEnd bool `gorm:"not null;default:false" json:"cancel_at_period_end"`
CanceledAt *time.Time `json:"canceled_at,omitempty"`
Currency string `gorm:"size:3;not null" json:"currency"`
AmountMinor int64 `gorm:"not null;default:0" json:"amount_minor"`
MetadataJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"metadata_json"`
concerns.Timestamped
}
// SubscriptionEvent is an append-only audit record for entitlement changes
// and provider deliveries. ProviderEventID makes future webhook processing
// idempotent without coupling the ledger to a specific PSP.
type SubscriptionEvent struct {
concerns.Model
SubscriptionID int64 `gorm:"not null;index" json:"subscription_id"`
AccountID int64 `gorm:"not null;index" json:"account_id"`
Provider string `gorm:"size:16;not null;default:'manual';uniqueIndex:idx_subscription_events_provider_event,priority:1" json:"provider"`
Kind string `gorm:"size:32;not null" json:"kind"`
FromPlanID *int64 `json:"from_plan_id,omitempty"`
ToPlanID *int64 `json:"to_plan_id,omitempty"`
AmountMinor *int64 `json:"amount_minor,omitempty"`
Currency string `gorm:"size:3" json:"currency,omitempty"`
ActorUserID *int64 `json:"actor_user_id,omitempty"`
ProviderEventID *string `gorm:"size:128;uniqueIndex:idx_subscription_events_provider_event,priority:2" json:"provider_event_id,omitempty"`
PayloadJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"payload_json"`
CreatedAt time.Time `json:"created_at"`
}

47
app/models/tag.go Обычный файл
Просмотреть файл

@@ -0,0 +1,47 @@
package models
import (
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Tag stores metadata for a single tag name within an account: the
// user-chosen color (hex string like "#FF5733") and icon (FontAwesome
// icon name like "faTag"). The (account_id, name) pair is unique.
//
// Tags are still attached to monitors via the monitors.tags text[]
// column (see Monitor.Tags). This table only holds the per-tag display
// metadata so the same tag renders consistently across the SPA — it
// does not affect monitor scoping or filtering.
//
// Rows are created lazily: the SPA POSTs a Tag the first time a user
// customizes its color/icon, and GET /tags?with_counts=1 LEFT JOINs
// the unnested monitors.tags array against this table to enrich the
// (name, count) pairs with display metadata. A Tag row may exist with
// zero monitors using it (count=0) — that happens when a user creates
// a tag from /settings/tags/new but has not yet applied it.
type Tag struct {
concerns.Model
AccountID int64 `gorm:"index;not null" json:"account_id"`
Account *Account `gorm:"foreignKey:AccountID" json:"account,omitempty"`
Name string `gorm:"type:varchar(255);not null" json:"name"`
Color string `gorm:"type:varchar(32);not null;default:'#6c757d'" json:"color"`
Icon string `gorm:"type:varchar(64);not null;default:'faTag'" json:"icon"`
Audited
concerns.Timestamped
}
// TagInfo is a tag with the number of monitors using it plus optional
// display metadata (color, icon) coming from the Tag table.
type TagInfo struct {
Name string `json:"name"`
Count int64 `json:"count"`
Color string `json:"color"`
Icon string `json:"icon"`
}
// DefaultTagColor is the hex color used when a Tag has no metadata row.
const DefaultTagColor = "#6c757d"
// DefaultTagIcon is the FontAwesome icon name used when a Tag has no
// metadata row. Matches the icon the SPA renders by default.
const DefaultTagIcon = "faTag"

135
app/models/tags_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,135 @@
package models_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
// seedTagsMonitors creates one group with four monitors:
//
// a.test [prod, web]
// b.test [prod, api]
// c.test [staging]
// d.test [] (NULL tags)
//
// and returns the group id. It mirrors the exact SQL the /tags rename, delete
// and with_counts endpoints run, pinning the contract.
func seedTagsMonitors(t *testing.T) int64 {
models.Drop()
models.Migrate()
plan := models.Plan{Name: "test", Default: true}
require.NoError(t, models.DB().Create(&plan).Error)
acc := models.Account{Name: "A", PlanID: &plan.ID}
require.NoError(t, models.DB().Create(&acc).Error)
group := models.Group{AccountID: acc.ID, Name: "A-default"}
require.NoError(t, models.DB().Create(&group).Error)
monitors := []models.Monitor{
{GroupID: group.ID, Host: "a.test", Tags: []string{"prod", "web"}},
{GroupID: group.ID, Host: "b.test", Tags: []string{"prod", "api"}},
{GroupID: group.ID, Host: "c.test", Tags: []string{"staging"}},
{GroupID: group.ID, Host: "d.test"},
}
for i := range monitors {
require.NoError(t, models.DB().Create(&monitors[i]).Error)
}
return group.ID
}
func TestTagsRenameSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
res := models.DB().Exec(
`UPDATE monitors SET tags = (
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", "production", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected, "only a.test and b.test contain prod")
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"production", "web"}, []string(a.Tags))
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.ElementsMatch(t, []string{"production", "api"}, []string(b.Tags))
var c models.Monitor
require.NoError(t, models.DB().Where("host = ?", "c.test").First(&c).Error)
assert.Equal(t, []string{"staging"}, []string(c.Tags), "staging-only monitor untouched")
}
func TestTagsRenameSQL_Dedup(t *testing.T) {
groupID := seedTagsMonitors(t)
// Rename prod -> web. a.test has both prod and web: must collapse to a
// single "web" entry (array_agg DISTINCT), not [web, web].
res := models.DB().Exec(
`UPDATE monitors SET tags = (
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", "web", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected)
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"web"}, []string(a.Tags), "duplicate must be deduped")
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.ElementsMatch(t, []string{"web", "api"}, []string(b.Tags))
}
func TestTagsDeleteSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
res := models.DB().Exec(
`UPDATE monitors SET tags = array_remove(tags, ?)
WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
"prod", []int64{groupID}, "prod",
)
require.NoError(t, res.Error)
assert.Equal(t, int64(2), res.RowsAffected)
var a models.Monitor
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
assert.Equal(t, []string{"web"}, []string(a.Tags))
var b models.Monitor
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
assert.Equal(t, []string{"api"}, []string(b.Tags))
}
func TestTagsCountSQL(t *testing.T) {
groupID := seedTagsMonitors(t)
type tagInfo struct {
Name string
Count int64
}
var tags []tagInfo
err := models.DB().Raw(
`SELECT tag AS name, COUNT(*) AS count
FROM (SELECT unnest(tags) AS tag FROM monitors
WHERE tags IS NOT NULL AND group_id IN (?)) sub
GROUP BY tag
ORDER BY tag`,
[]int64{groupID},
).Scan(&tags).Error
require.NoError(t, err)
got := map[string]int64{}
for _, tg := range tags {
got[tg.Name] = tg.Count
}
assert.Equal(t, map[string]int64{"prod": 2, "web": 1, "api": 1, "staging": 1}, got)
}

145
app/models/task.go Обычный файл
Просмотреть файл

@@ -0,0 +1,145 @@
package models
import (
"time"
"gorm.io/datatypes"
"gorm.io/gorm/clause"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Task kinds stored in the tasks.kind column. The plan (docs/plans/worker-notifier-mvp.md
// section 4.1) reserves the same enum for checks and notifications; this phase only
// emits notification rows but the enum is shared so the selector can stay a single
// function.
const (
TaskKindNotification = "notification"
TaskKindCheck = "check"
TaskKindServerMetric = "server_metric"
)
// Task states for the durable task envelope.
const (
TaskStateQueued = "queued"
TaskStateLeased = "leased"
TaskStateSucceeded = "succeeded"
TaskStateFailedRetry = "failed_retry"
TaskStateFailedPerm = "failed_perm"
TaskStateDead = "dead"
)
// Notification result statuses reported by the worker (mirrors the wire enum so the
// result handler can decode without re-typing the constants).
const (
NotificationResultDelivered = "delivered"
NotificationResultRetryable = "retryable"
NotificationResultPermanent = "permanent"
NotificationResultPartial = "partial"
)
// SkipLockedClause is the SELECT ... FOR UPDATE SKIP LOCKED clause used by
// every worker-pool selector (checks in check_jobs.ChecksForWorker and
// tasks in task_selector.TasksForWorker / TasksForWorkerNotification).
// Sharing the value keeps the SQL identical across selectors so goconst
// does not flag the literal, and a future change (e.g. NOWAIT) only has
// to touch one place.
var SkipLockedClause = clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}
// Task is the durable envelope for both check and notification work executed by the
// distributed worker pool. Selection uses FOR UPDATE SKIP LOCKED per worker poll so
// a single primary key or sequence never becomes the bottleneck.
//
// Phase 1 (this commit) only emits notification tasks. The `kind` discriminator and
// capability filters are designed to accept checks in phase 2 without a schema change.
type Task struct {
concerns.Model
JobID string `gorm:"uniqueIndex;size:64" json:"job_id"`
Kind string `gorm:"size:32;index" json:"kind"`
State string `gorm:"size:32;index" json:"state"`
LastError string `gorm:"type:text" json:"last_error"`
// Tenancy + audit anchor. AccountID is required for the capability match in
// TasksForWorker; monitor_id / message_id / contact_id are denormalized for
// fast admin queries.
AccountID int64 `gorm:"index" json:"account_id"`
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
CheckID *int64 `json:"check_id,omitempty"`
MessageID *int64 `json:"message_id,omitempty"`
ContactID *int64 `json:"contact_id,omitempty"`
// Payload is the kind-specific blob the worker needs to execute. For
// notifications the producer pre-renders subject/body so the worker does not
// need templating context (see RenderNotificationContent in internal/notifier).
Payload datatypes.JSON `gorm:"type:jsonb" json:"payload"`
// Scheduling + retry envelope. NotBefore is set to NOW() by the producer and
// bumped by the result handler on retryable failures. Deadline is a soft cap
// the selector can use to skip stale tasks.
NotBefore time.Time `json:"not_before"`
Deadline *time.Time `json:"deadline,omitempty"`
// LeaseOwner + LeaseExpiresAt are owned by the selector while the task is
// in state=leased. The reaper clears them when the lease expires.
LeaseOwner string `gorm:"size:128" json:"lease_owner"`
LeaseToken string `gorm:"size:64" json:"-"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
Attempts int `json:"attempts"`
MaxAttempts int `json:"max_attempts"`
// IdempotencyKey is unique per logical event so a retry of the producer's
// enqueue never produces a second Task row. See EnqueueNotificationTask.
IdempotencyKey string `gorm:"uniqueIndex;size:255" json:"idempotency_key"`
// Result holds the most recent worker result body (NotificationResultReport or
// CheckResultReport shape, depending on Kind). It is JSONB so the admin UI can
// pretty-print without a separate result table for transient lookups.
Result datatypes.JSON `gorm:"type:jsonb" json:"result"`
SucceededAt *time.Time `json:"succeeded_at,omitempty"`
concerns.Timestamped
}
// TaskReplay records the single operator-initiated replay of a dead task.
// OriginalTaskID is unique, making repeated clicks/API retries idempotent.
type TaskReplay struct {
concerns.Model
OriginalTaskID int64 `gorm:"uniqueIndex" json:"original_task_id"`
RequeuedTaskID int64 `gorm:"uniqueIndex" json:"requeued_task_id"`
OperatorUserID int64 `gorm:"index" json:"operator_user_id"`
concerns.Timestamped
}
// TableName overrides the default table name so pluralization stays consistent
// with the rest of the schema (tasks, not "task" or "taskses").
func (Task) TableName() string {
return "tasks"
}
// NotificationDelivery is the per-attempt audit row required by section 7.5 of the
// plan ("Audit rows: each successful or failed delivery writes a row in a new
// notification_deliveries table"). The result handler appends one row per result
// frame, which lets support answer "did the customer ever get this alert" without
// scanning application logs.
type NotificationDelivery struct {
concerns.Model
MessageID int64 `gorm:"index" json:"message_id"`
WorkerID string `gorm:"size:128;index" json:"worker_id"`
TaskID int64 `gorm:"index" json:"task_id"`
Status string `gorm:"size:32" json:"status"`
Error string `gorm:"type:text" json:"error"`
DurationMs int `json:"duration_ms"`
ProviderResponse string `gorm:"type:text" json:"provider_response"`
concerns.Timestamped
}
// TableName mirrors the plan's preferred lowercase plural.
func (NotificationDelivery) TableName() string {
return "notification_deliveries"
}

136
app/models/task_reaper.go Обычный файл
Просмотреть файл

@@ -0,0 +1,136 @@
package models
import (
"context"
"log"
"time"
"gorm.io/gorm"
)
// ReapExpiredTasks is the periodic cleanup function described in
// docs/plans/worker-notifier-mvp.md section 8.5:
//
// - tasks in state='leased' whose lease_expires_at is past are returned to
// state='queued' and have their lease_owner cleared, so the next selector
// poll can pick them up.
// - tasks in state='failed_retry' whose not_before is past AND
// attempts >= max_attempts are moved to state='dead' so they show up on
// the admin dead-letter page and stop consuming selector bandwidth.
//
// It returns the number of rows it touched so the caller can log a metric.
// Cheap enough to run from the web process every 30s.
func ReapExpiredTasks() (reaped int, deaded int, err error) {
now := time.Now()
err = DB().Transaction(func(tx *gorm.DB) error {
if err := expireQueuedNotificationTasksTx(tx, now); err != nil {
return err
}
var expired []Task
if err := tx.Where("state = ? AND lease_expires_at IS NOT NULL AND lease_expires_at < ?", TaskStateLeased, now).Find(&expired).Error; err != nil {
return err
}
for i := range expired {
if expired[i].Attempts >= expired[i].MaxAttempts {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateDead, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "last_error": "lease expired after max attempts", "updated_at": now})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 1 {
deaded++
if err := FinalizeNotificationTaskTx(tx, &expired[i], "dead", "lease expired after max attempts"); err != nil {
return err
}
}
continue
}
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateQueued, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "updated_at": now})
if result.Error != nil {
return result.Error
}
reaped += int(result.RowsAffected)
}
var exhausted []Task
if err := tx.Where("state = ? AND not_before <= ? AND attempts >= max_attempts", TaskStateFailedRetry, now).Find(&exhausted).Error; err != nil {
return err
}
for i := range exhausted {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", exhausted[i].ID, TaskStateFailedRetry).Updates(map[string]interface{}{"state": TaskStateDead, "updated_at": now})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 1 {
deaded++
if err := FinalizeNotificationTaskTx(tx, &exhausted[i], "dead", exhausted[i].LastError); err != nil {
return err
}
}
}
return nil
})
return reaped, deaded, err
}
// FinalizeNotificationTaskTx makes a terminal notification task customer-visible
// and auditable. The caller owns the task state transition in this transaction.
func FinalizeNotificationTaskTx(tx *gorm.DB, task *Task, status, reason string) error {
if task == nil || task.Kind != TaskKindNotification || task.MessageID == nil {
return nil
}
if err := tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *task.MessageID, []string{"sent", "error"}).Updates(map[string]interface{}{"state": "error", "error": reason}).Error; err != nil {
return err
}
return tx.Create(&NotificationDelivery{MessageID: *task.MessageID, TaskID: task.ID, Status: status, Error: reason}).Error
}
func expireQueuedNotificationTasksTx(tx *gorm.DB, now time.Time) error {
var tasks []Task
if err := tx.Where("state = ? AND kind = ? AND deadline IS NOT NULL AND deadline <= ?", TaskStateQueued, TaskKindNotification, now).Find(&tasks).Error; err != nil {
return err
}
for i := range tasks {
result := tx.Model(&Task{}).Where("id = ? AND state = ?", tasks[i].ID, TaskStateQueued).Updates(map[string]interface{}{
"state": TaskStateDead, "last_error": "notification deadline expired", "updated_at": now,
})
if result.Error != nil || result.RowsAffected == 0 {
if result.Error != nil {
return result.Error
}
continue
}
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "expired", "notification deadline expired"); err != nil {
return err
}
}
return nil
}
// StartTaskReaper launches a goroutine that runs ReapExpiredTasks on the given
// interval. It honors ctx.Done() so the caller can wind it down without
// leaking. The function is safe to call once per process; the control plane
// runs the reaper from main.init() so only one ticker ever exists in a single
// web process.
func StartTaskReaper(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 30 * time.Second
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
reaped, deaded, err := ReapExpiredTasks()
if err != nil {
log.Printf("task_reaper: error: %v", err)
continue
}
if reaped > 0 || deaded > 0 {
log.Printf("task_reaper: reaped=%d dead=%d", reaped, deaded)
}
}
}
}()
}

391
app/models/task_selector.go Обычный файл
Просмотреть файл

@@ -0,0 +1,391 @@
package models
import (
"errors"
"fmt"
"log"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// DefaultCheckTaskLeaseTTL is how long a leased check remains owned before the reaper
// returns it to the queue. It is deliberately larger than the worker's heartbeat
// (10s) so a healthy worker can finish a 30s check without the reaper stealing
// the lease, and deliberately smaller than the heartbeat timeout (2m) so a
// crashed worker sees its tasks reaped within one watchdog cycle.
const DefaultCheckTaskLeaseTTL = 60 * time.Second
// DefaultTaskLeaseTTL remains the check-task default for existing confirmation
// callers. Generic task selection must use TaskLeaseTTL so notification work is
// not reclaimed during its longer execution window.
const DefaultTaskLeaseTTL = DefaultCheckTaskLeaseTTL
// Notification execution is bounded by the worker runner at five minutes. The
// additional minute covers result serialization, websocket/HTTP transport, and
// a control-plane scheduling delay before the persisted lease may be reaped.
const (
DefaultNotificationExecutionTimeout = 5 * time.Minute
NotificationTaskReportMargin = time.Minute
DefaultNotificationTaskLeaseTTL = DefaultNotificationExecutionTimeout + NotificationTaskReportMargin
)
// TaskLeaseTTL returns the persisted lease lifetime for a task kind.
func TaskLeaseTTL(kind string) time.Duration {
if kind == TaskKindNotification {
return DefaultNotificationTaskLeaseTTL
}
return DefaultCheckTaskLeaseTTL
}
// DefaultTaskMaxAttempts is the retry budget for a task before it moves to dead.
const DefaultTaskMaxAttempts = 5
// DefaultNotificationTaskDeadline is assigned to manually replayed notification
// dead letters. Normal producer tasks may be deadline-free, but a replay must
// never inherit an already-expired deadline.
const DefaultNotificationTaskDeadline = 15 * time.Minute
// ErrNotificationMethodNotAuthorized is returned by EnqueueNotificationTask
// when the producer can prove no worker in the pool is authorized for the
// (method, account) pair. The caller may skip the enqueue or log + continue.
var ErrNotificationMethodNotAuthorized = errors.New("no worker authorized for method/account")
// EnqueueNotificationTaskInput is the pre-rendered envelope produced by the
// notifier producer. All slices are required; the selector never reads them.
type EnqueueNotificationTaskInput struct {
AccountID int64
NotificationID int64
ContactID int64
MessageID *int64
MonitorID *int64
CheckID *int64
EventIDs []int64
Method string // "email", "telegram", "webhook", "mattermost", "sms", "voice"
Subject string
BodyText string
BodyHTML string
BodyMarkdown string
Language string
MessageKind string // "down", "up", "exp", "test"
NotBefore time.Time
Deadline *time.Time
MaxAttempts int
Payload []byte // marshaled task-specific data
IdempotencyKey string // optional; manual test tasks use a unique key and do not have event IDs
}
// EnqueueNotificationTask writes one Task row keyed by a stable idempotency key.
// A second call with the same key (same notification/contact/event triple) is a
// no-op so the producer is safe to call more than once per pass.
//
// The capability precheck uses the same NotificationMethods + NotificationAccounts
// rule the selector does, so the producer can skip enqueueing work that no
// operated worker could ever pick up (sms/voice until phase 4).
func EnqueueNotificationTask(input *EnqueueNotificationTaskInput) (*Task, error) {
return EnqueueNotificationTaskTx(DB(), input)
}
// EnqueueNotificationTaskTx is the transactional form used by state machines
// that must commit their transition, audit event, message, and task together.
func EnqueueNotificationTaskTx(tx *gorm.DB, input *EnqueueNotificationTaskInput) (*Task, error) {
if tx == nil {
return nil, errors.New("enqueue: nil transaction")
}
if input.AccountID == 0 || input.ContactID == 0 {
return nil, errors.New("enqueue: account_id and contact_id are required")
}
idempotencyKey := input.IdempotencyKey
if idempotencyKey == "" {
if input.NotificationID == 0 || len(input.EventIDs) == 0 {
return nil, errors.New("enqueue: notification_id and event_ids are required without explicit idempotency_key")
}
idempotencyKey = notificationIdempotencyKey(input.NotificationID, input.ContactID, input.EventIDs[0])
}
// Fast path: row already exists from a previous producer tick. Returning
// the existing row is the idempotency guarantee — second calls return the
// same id, no second INSERT.
var existing Task
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
return &existing, nil
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
maxAttempts := input.MaxAttempts
if maxAttempts <= 0 {
maxAttempts = DefaultTaskMaxAttempts
}
if !anyWorkerCanDeliver(input.Method, input.AccountID) {
return nil, fmt.Errorf("%w: method=%s account=%d", ErrNotificationMethodNotAuthorized, input.Method, input.AccountID)
}
notBefore := input.NotBefore
if notBefore.IsZero() {
notBefore = time.Now()
}
now := time.Now()
task := &Task{
JobID: uuid.New().String(),
Kind: TaskKindNotification,
State: TaskStateQueued,
AccountID: input.AccountID,
MessageID: input.MessageID,
ContactID: &input.ContactID,
MonitorID: input.MonitorID,
CheckID: input.CheckID,
NotBefore: notBefore,
Deadline: input.Deadline,
Attempts: 0,
MaxAttempts: maxAttempts,
IdempotencyKey: idempotencyKey,
}
if len(input.Payload) > 0 {
task.Payload = input.Payload
}
task.CreatedAt = now
task.UpdatedAt = now
// ON CONFLICT DO NOTHING so a concurrent producer tick racing with us on
// the same idempotency_key loses the race but does not duplicate the row.
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(task).Error; err != nil {
return nil, err
}
if task.ID == 0 {
// Lost the race. Re-read and return the winner.
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err != nil {
return nil, err
}
return &existing, nil
}
return task, nil
}
// NotificationIdempotencyKey exposes the producer's idempotency key so the
// result handler and admin tooling can match a Task row back to the logical
// (notification, contact, event) tuple without re-deriving the format.
func NotificationIdempotencyKey(notificationID, contactID, eventID int64) string {
return notificationIdempotencyKey(notificationID, contactID, eventID)
}
func notificationIdempotencyKey(notificationID, contactID, eventID int64) string {
return fmt.Sprintf("notif:%d:contact:%d:event:%d", notificationID, contactID, eventID)
}
// anyWorkerCanDeliver returns true if at least one active worker in the pool is
// authorized to deliver the given (method, account) pair. Used by the producer
// to skip enqueues no worker could ever pick up.
func anyWorkerCanDeliver(method string, accountID int64) bool {
var nodes []WorkerNode
if err := DB().Where("status = ? AND last_seen > ?", "active", time.Now().Add(-WorkerHeartbeatFreshness)).Find(&nodes).Error; err != nil {
log.Printf("task_selector: cannot list workers: %v", err)
// Be permissive on lookup failure: the selector's own filter would still
// hold the lease back, so the worst case is a queued task nobody picks
// up — which the reaper eventually dead-letters.
return true
}
for i := range nodes {
if nodes[i].SupportsTaskEnvelope() && nodes[i].CanDeliverNotification(method, accountID) {
return true
}
}
return false
}
// TasksForWorker leases up to `limit` due tasks for the worker. The selection
// is one transaction so the FOR UPDATE SKIP LOCKED + UPDATE that flips state
// from queued to leased is atomic. Notification tasks are filtered by the worker's
// notification_methods + notification_accounts capability set; check tasks are
// filtered by check_types in their payload.
//
// The function is safe to call from multiple goroutines for different workers.
// Two workers that hit the DB at the same time will see disjoint task sets.
func TasksForWorker(worker *WorkerNode, limit int) ([]Task, error) {
if worker == nil {
return nil, errors.New("TasksForWorker: worker is nil")
}
if !worker.SupportsTaskEnvelope() {
return nil, nil
}
if limit <= 0 {
limit = 1
}
notifMethods := worker.NotificationMethods()
notifAccounts := worker.AccessibleAccountIDs()
hasNotif := len(notifMethods) > 0
tx := DB().Begin()
if tx.Error != nil {
return nil, tx.Error
}
defer func() {
if r := recover(); r != nil {
_ = tx.Rollback().Error
panic(r)
}
}()
now := time.Now()
var out []Task
// First pass: notification tasks the worker is authorized to deliver. We
// also bump attempts and flip state to leased in the same row so the
// outer selector+lease is atomic. The method filter is a JSONB extract on
// payload->>'method' so a single worker query can target one method list.
if hasNotif {
notifQuery := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'method' IN (?)", notifMethods).
Where("payload->>'method' <> ''")
if len(notifAccounts) > 0 {
notifQuery = notifQuery.Where("account_id IN (?)", notifAccounts)
}
var picked []Task
if err := notifQuery.Limit(limit).Find(&picked).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
for i := range picked {
row := picked[i]
newAttempts := row.Attempts + 1
leaseToken := uuid.NewString()
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
if err := tx.Model(&row).Updates(map[string]interface{}{
colState: TaskStateLeased,
"lease_owner": worker.WorkerID,
"lease_expires_at": leaseUntil,
"attempts": newAttempts,
"lease_token": leaseToken,
"updated_at": now,
}).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
row.State = TaskStateLeased
row.LeaseOwner = worker.WorkerID
row.LeaseExpiresAt = &leaseUntil
row.Attempts = newAttempts
row.LeaseToken = leaseToken
out = append(out, row)
}
}
remaining := limit - len(out)
if checkTypes := worker.CheckTypes(); remaining > 0 && len(checkTypes) > 0 {
checkQuery := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindCheck).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'kind' IN (?)", checkTypes)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
checkQuery = checkQuery.Where("account_id IN (?)", accounts)
}
var picked []Task
if err := checkQuery.Limit(remaining).Find(&picked).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
for i := range picked {
row := picked[i]
newAttempts := row.Attempts + 1
leaseToken := uuid.NewString()
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
if err := tx.Model(&row).Updates(map[string]interface{}{
colState: TaskStateLeased, "lease_owner": worker.WorkerID,
"lease_expires_at": leaseUntil, "attempts": newAttempts, "lease_token": leaseToken, "updated_at": now,
}).Error; err != nil {
_ = tx.Rollback().Error
return nil, err
}
row.State, row.LeaseOwner, row.LeaseExpiresAt, row.Attempts, row.LeaseToken = TaskStateLeased, worker.WorkerID, &leaseUntil, newAttempts, leaseToken
out = append(out, row)
}
}
if err := tx.Commit().Error; err != nil {
return nil, err
}
return out, nil
}
// AvailableWorkerTaskCapacity returns unoccupied local worker slots. Durable
// unexpired leases and the worker's heartbeat-reported active/queued workload
// describe the same work from different sides, so the larger value is used to
// avoid both over-dispatching and double-counting a healthy worker.
func AvailableWorkerTaskCapacity(worker *WorkerNode) (int, error) {
if worker == nil {
return 0, errors.New("worker capacity: worker is nil")
}
concurrency := worker.Concurrency
if concurrency < 1 {
concurrency = 1
}
var leased int64
if err := DB().Model(&Task{}).Where("state = ? AND lease_owner = ? AND lease_expires_at > ?", TaskStateLeased, worker.WorkerID, time.Now()).Count(&leased).Error; err != nil {
return 0, err
}
var confirmationLeases int64
if err := DB().Model(&CheckAttempt{}).Where("worker_node_id = ? AND kind = ? AND state = ? AND lease_expires_at > ?", worker.ID, AttemptKindConfirm, AttemptStateLeased, time.Now()).Count(&confirmationLeases).Error; err != nil {
return 0, err
}
used := int(leased + confirmationLeases)
if reported := worker.ReportedWorkload(); reported > used {
used = reported
}
if used >= concurrency {
return 0, nil
}
return concurrency - used, nil
}
// GetTaskByJobID returns one task row keyed by its unique job_id. The result
// handler uses this to validate that the incoming JobID exists and matches the
// calling worker before it mutates state.
func GetTaskByJobID(jobID string) (*Task, error) {
if jobID == "" {
return nil, errors.New("GetTaskByJobID: empty job_id")
}
var task Task
if err := DB().Where("job_id = ?", jobID).First(&task).Error; err != nil {
return nil, err
}
return &task, nil
}
// TasksForWorkerTx is the variant exposed for tests so a single SELECT inside a
// caller-provided transaction can be inspected without the auto-commit wrapper.
// Production code should use TasksForWorker.
func TasksForWorkerTx(tx *gorm.DB, worker *WorkerNode, limit int) ([]Task, error) {
if tx == nil {
return nil, errors.New("TasksForWorkerTx: nil tx")
}
notifMethods := worker.NotificationMethods()
if len(notifMethods) == 0 {
return nil, nil
}
now := time.Now()
var out []Task
// Use SKIP LOCKED to avoid contention between workers (mirror of ChecksForWorker).
q := tx.Clauses(SkipLockedClause).
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
Where("not_before <= ?", now).
Where("(deadline IS NULL OR deadline > ?)", now).
Where("payload->>'method' IN (?)", notifMethods)
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
q = q.Where("account_id IN (?)", accounts)
}
if err := q.Limit(limit).Find(&out).Error; err != nil {
return nil, err
}
return out, nil
}

681
app/models/task_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,681 @@
package models_test
import (
"encoding/json"
"errors"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/datatypes"
"gorm.io/gorm"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/config/database"
)
func init() {
database.Init()
}
// seedRegion creates a Region row before a worker fixture inserts, so the FK
// from worker_nodes -> regions holds. Idempotent: Drop() cleans up.
func seedRegion(t *testing.T, code string) models.Region {
t.Helper()
r := models.Region{}
err := models.DB().Where("code = ?", code).First(&r).Error
if err == nil {
return r
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("seed region lookup: %v", err)
}
r = models.Region{Code: code, Name: code, Enabled: true}
require.NoError(t, models.DB().Create(&r).Error)
return r
}
// seedAccountUserPlan returns an account with a default plan and the first user
// for the FK chain that contacts/notifications walk. Cleanup is the caller's
// responsibility (Drop() at end of test).
func seedAccountUserPlan(t *testing.T) (models.Account, models.User) {
t.Helper()
plan := models.Plan{Name: "test-plan", Default: false}
if err := models.DB().Create(&plan).Error; err != nil {
t.Fatalf("seed plan: %v", err)
}
user := models.User{Name: "test-user", Email: taskStringPtr("test-" + uuid.NewString() + "@example.com"), Timezone: "UTC"}
if err := models.DB().Create(&user).Error; err != nil {
t.Fatalf("seed user: %v", err)
}
account := models.Account{Name: "test-account", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
if err := models.DB().Create(&account).Error; err != nil {
t.Fatalf("seed account: %v", err)
}
return account, user
}
func seedNotification(t *testing.T, accountID int64) models.Notification {
t.Helper()
n := models.Notification{Name: "default", AccountID: accountID, Enabled: true, NotifyDown: true, NotifyRestore: true}
require.NoError(t, models.DB().Create(&n).Error)
return n
}
func seedEmailContact(t *testing.T, accountID int64) models.Contact {
t.Helper()
c := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &accountID}
require.NoError(t, models.DB().Create(&c).Error)
return c
}
func taskStringPtr(s string) *string { return &s }
func TestTaskSchemaMigration(t *testing.T) {
models.Drop()
models.Migrate()
require.True(t, models.DB().Migrator().HasTable(&models.Task{}), "tasks table must exist after Migrate()")
require.True(t, models.DB().Migrator().HasTable(&models.NotificationDelivery{}), "notification_deliveries table must exist after Migrate()")
}
func TestEnqueueNotificationTask_Idempotency(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
// Producer precheck requires at least one worker authorized for the
// (method, account) pair. Add an operated-style email worker.
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-idempotency-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
LastSeen: &now,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
payload := []byte(`{"method":"email","subject":"[rsmon] x is down","body_text":"down","body_html":"<p>down</p>","body_markdown":"**down**","language":"en","message_kind":"down"}`)
input := models.EnqueueNotificationTaskInput{
AccountID: account.ID,
NotificationID: notification.ID,
ContactID: contact.ID,
Method: "email",
Subject: "[rsmon] x is down",
BodyText: "down",
BodyHTML: "<p>down</p>",
Language: "en",
MessageKind: "down",
EventIDs: []int64{42},
Payload: payload,
}
first, err := models.EnqueueNotificationTask(&input)
require.NoError(t, err)
require.NotZero(t, first.ID)
// Second call with the same logical event must not create a duplicate row.
second, err := models.EnqueueNotificationTask(&input)
require.NoError(t, err)
assert.Equal(t, first.ID, second.ID, "idempotency: second enqueue should return the same row")
var count int64
require.NoError(t, models.DB().Model(&models.Task{}).Where("idempotency_key = ?", first.IdempotencyKey).Count(&count).Error)
assert.EqualValues(t, 1, count, "exactly one Task row per idempotency key")
}
// TestEnqueueNotificationTask_AuthorizationSkip makes sure the producer can
// observe ErrNotificationMethodNotAuthorized when no worker is eligible for
// the (method, account) pair. The producer uses this to avoid enqueueing work
// no operated worker could ever pick up.
func TestEnqueueNotificationTask_AuthorizationSkip(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
// Register a worker that only knows telegram. An email task must fail the
// precheck.
w := &models.WorkerNode{
WorkerID: "worker-tg-only-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"telegram"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
_, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
AccountID: account.ID,
NotificationID: notification.ID,
ContactID: contact.ID,
Method: "email",
EventIDs: []int64{1},
Payload: []byte(`{"method":"email"}`),
})
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
}
func TestEnqueueNotificationTask_RequiresTaskEnvelopeWorker(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
worker := &models.WorkerNode{WorkerID: "legacy-notify-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"}, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
}))}
require.NoError(t, models.DB().Create(worker).Error)
require.NoError(t, models.DB().Model(&models.WorkerNode{}).Where("id <> ?", worker.ID).Update("status", "dead").Error)
input := &models.EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notification.ID, ContactID: contact.ID, Method: "email", EventIDs: []int64{77}, Payload: []byte(`{"method":"email"}`)}
_, err := models.EnqueueNotificationTask(input)
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
var count int64
require.NoError(t, models.DB().Model(&models.Task{}).Count(&count).Error)
assert.Zero(t, count)
worker.Capabilities = datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"}, "task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
}))
require.NoError(t, models.DB().Save(worker).Error)
task, err := models.EnqueueNotificationTask(input)
require.NoError(t, err)
assert.Equal(t, models.TaskStateQueued, task.State)
}
func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-email-only-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
LastSeen: &now,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
// Enqueue 3 tasks of different methods; only the email ones should be
// picked up by the worker. We use the raw helper because the producer's
// precheck would refuse the telegram row when no worker handles telegram —
// the selector test must exercise the SELECT-side filter, not the
// producer-side authorization.
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{1})
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "telegram", []int64{2})
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{3})
picked, err := models.TasksForWorker(w, 10)
require.NoError(t, err)
require.Len(t, picked, 2, "only email tasks should be leased")
for _, p := range picked {
assert.Equal(t, models.TaskStateLeased, p.State)
assert.Equal(t, w.WorkerID, p.LeaseOwner)
assert.NotNil(t, p.LeaseExpiresAt)
assert.Equal(t, 1, p.Attempts)
}
// A second call must not return the same rows.
picked2, err := models.TasksForWorker(w, 10)
require.NoError(t, err)
assert.Empty(t, picked2, "second selector poll should see an empty queue while leased")
}
func TestEnqueueDueCheckTasks_UsesGenericEnvelope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
account, _ := seedAccountUserPlan(t)
group := models.Group{Name: "checks", AccountID: account.ID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: "example.com", Enabled: true}
require.NoError(t, models.DB().Create(&monitor).Error)
enabled := true
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
require.NoError(t, models.DB().Create(&check).Error)
worker := &models.WorkerNode{
WorkerID: "generic-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
require.NoError(t, models.DB().Create(worker).Error)
require.NoError(t, models.EnqueueDueCheckTasks(worker, []string{"http"}, 1))
picked, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
require.Len(t, picked, 1)
assert.Equal(t, models.TaskKindCheck, picked[0].Kind)
assert.Equal(t, models.TaskStateLeased, picked[0].State)
assert.Equal(t, check.ID, *picked[0].CheckID)
}
func TestTasksForWorker_ChecksRespectPrivateAccountScope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
accountA, _ := seedAccountUserPlan(t)
accountB, _ := seedAccountUserPlan(t)
accountID := accountA.ID
private := &models.WorkerNode{
WorkerID: "private-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
platform := &models.WorkerNode{
WorkerID: "platform-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(),
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
}
require.NoError(t, models.DB().Create(private).Error)
require.NoError(t, models.DB().Create(platform).Error)
task := models.Task{JobID: uuid.NewString(), Kind: models.TaskKindCheck, State: models.TaskStateQueued, AccountID: accountB.ID, Payload: datatypes.JSON([]byte(`{"kind":"http"}`)), NotBefore: time.Now().Add(-time.Second), MaxAttempts: 5, IdempotencyKey: "cross-account-" + uuid.NewString()}
require.NoError(t, models.DB().Create(&task).Error)
picked, err := models.TasksForWorker(private, 1)
require.NoError(t, err)
assert.Empty(t, picked)
picked, err = models.TasksForWorker(platform, 1)
require.NoError(t, err)
require.Len(t, picked, 1)
assert.Equal(t, task.ID, picked[0].ID)
}
func TestChecksForWorker_RespectsPrivateAccountScope(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
accountA, _ := seedAccountUserPlan(t)
accountB, _ := seedAccountUserPlan(t)
makeCheck := func(accountID int64, host string) models.Check {
group := models.Group{Name: host, AccountID: accountID}
require.NoError(t, models.DB().Create(&group).Error)
monitor := models.Monitor{GroupID: group.ID, Host: host, Enabled: true}
require.NoError(t, models.DB().Create(&monitor).Error)
enabled := true
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
require.NoError(t, models.DB().Create(&check).Error)
return check
}
owned := makeCheck(accountA.ID, "owned.example")
_ = makeCheck(accountB.ID, "other.example")
accountID := accountA.ID
private := &models.WorkerNode{
WorkerID: "private-legacy-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
}
require.NoError(t, models.DB().Create(private).Error)
checks := models.ChecksForWorker(private, []string{"http"}, 10)
require.Len(t, checks, 1)
assert.Equal(t, owned.ID, checks[0].ID)
}
func TestTasksForWorker_SkipsExpiredDeadline(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
worker := &models.WorkerNode{
WorkerID: "worker-deadline-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
}
require.NoError(t, models.DB().Create(worker).Error)
expired := time.Now().Add(-time.Second)
task := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{91})
require.NoError(t, models.DB().Model(&task).Update("deadline", expired).Error)
picked, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
assert.Empty(t, picked)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateQueued, stored.State)
assert.Equal(t, 0, stored.Attempts)
}
func TestReapExpiredTasksTerminatesExpiredQueuedNotification(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
require.NoError(t, models.DB().Create(&message).Error)
deadline := time.Now().Add(-time.Minute)
// This reaper test intentionally has no eligible worker; insert directly
// so it tests deadline handling rather than producer capability validation.
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{92})
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{"message_id": message.ID, "deadline": deadline}).Error)
_, _, err := models.ReapExpiredTasks()
require.NoError(t, err)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateDead, stored.State)
assert.Equal(t, "notification deadline expired", stored.LastError)
var storedMessage models.Message
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
assert.Equal(t, "error", storedMessage.State)
var auditCount int64
require.NoError(t, models.DB().Model(&models.NotificationDelivery{}).Where("task_id = ? AND status = ?", task.ID, "expired").Count(&auditCount).Error)
assert.EqualValues(t, 1, auditCount)
}
func TestReapExpiredTasks_RecyclesLeasesAndDeadsExhaustedRetries(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
now := time.Now()
w := &models.WorkerNode{
WorkerID: "worker-reap-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
LastSeen: &now,
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"task_envelope": true,
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w).Error)
// 1) A leased task whose lease expired — should go back to queued.
expiredLease := time.Now().Add(-time.Minute)
leased := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{10})
require.NoError(t, models.DB().Model(&leased).Updates(map[string]interface{}{
"state": models.TaskStateLeased,
"lease_owner": w.WorkerID,
"lease_expires_at": expiredLease,
"attempts": 1,
}).Error)
// 2) A failed_retry task past its not_before and at max_attempts — should move to dead.
failedRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{11})
require.NoError(t, models.DB().Model(&failedRetry).Updates(map[string]interface{}{
"state": models.TaskStateFailedRetry,
"attempts": 5,
"max_attempts": 5,
"not_before": time.Now().Add(-time.Minute),
}).Error)
// 3) A failed_retry task past not_before but attempts < max_attempts — must stay failed_retry.
pendingRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{12})
require.NoError(t, models.DB().Model(&pendingRetry).Updates(map[string]interface{}{
"state": models.TaskStateFailedRetry,
"attempts": 2,
"max_attempts": 5,
"not_before": time.Now().Add(-time.Minute),
}).Error)
reaped, deaded, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, reaped, "one expired lease should be returned to queue")
assert.Equal(t, 1, deaded, "one exhausted retry should move to dead")
var leasedRow models.Task
require.NoError(t, models.DB().First(&leasedRow, leased.ID).Error)
assert.Equal(t, models.TaskStateQueued, leasedRow.State)
assert.Empty(t, leasedRow.LeaseOwner)
assert.Nil(t, leasedRow.LeaseExpiresAt)
var deadRow models.Task
require.NoError(t, models.DB().First(&deadRow, failedRetry.ID).Error)
assert.Equal(t, models.TaskStateDead, deadRow.State)
var pendingRow models.Task
require.NoError(t, models.DB().First(&pendingRow, pendingRetry.ID).Error)
assert.Equal(t, models.TaskStateFailedRetry, pendingRow.State)
}
func TestReapExpiredTasks_ExpiredNotificationLeaseExhaustionFinalizesMessage(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
require.NoError(t, models.DB().Create(&message).Error)
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{99})
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{
"message_id": message.ID, "state": models.TaskStateLeased, "attempts": 5, "max_attempts": 5,
"lease_expires_at": time.Now().Add(-time.Minute),
}).Error)
_, deaded, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, deaded)
var storedTask models.Task
require.NoError(t, models.DB().First(&storedTask, task.ID).Error)
assert.Equal(t, models.TaskStateDead, storedTask.State)
assert.Equal(t, "lease expired after max attempts", storedTask.LastError)
var storedMessage models.Message
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
assert.Equal(t, "error", storedMessage.State)
require.NotNil(t, storedMessage.Error)
assert.Equal(t, "lease expired after max attempts", *storedMessage.Error)
var delivery models.NotificationDelivery
require.NoError(t, models.DB().Where("task_id = ?", task.ID).First(&delivery).Error)
assert.Equal(t, "dead", delivery.Status)
assert.Equal(t, "lease expired after max attempts", delivery.Error)
}
func TestNotificationTaskLeaseOutlivesExecutionTimeoutAndReapsAfterExpiry(t *testing.T) {
models.Drop()
models.Migrate()
account, _ := seedAccountUserPlan(t)
notification := seedNotification(t, account.ID)
contact := seedEmailContact(t, account.ID)
seedRegion(t, "test")
worker := &models.WorkerNode{
WorkerID: "notification-lease-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
}
require.NoError(t, models.DB().Create(worker).Error)
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{101})
leased, err := models.TasksForWorker(worker, 1)
require.NoError(t, err)
require.Len(t, leased, 1)
require.NotNil(t, leased[0].LeaseExpiresAt)
assert.GreaterOrEqual(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationTaskLeaseTTL-time.Second)
assert.Greater(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationExecutionTimeout)
withinExecution := time.Now().Add(models.DefaultNotificationExecutionTimeout)
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", withinExecution).Error)
reaped, _, err := models.ReapExpiredTasks()
require.NoError(t, err)
assert.Zero(t, reaped)
var stored models.Task
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateLeased, stored.State)
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error)
reaped, _, err = models.ReapExpiredTasks()
require.NoError(t, err)
assert.Equal(t, 1, reaped)
require.NoError(t, models.DB().First(&stored, task.ID).Error)
assert.Equal(t, models.TaskStateQueued, stored.State)
}
func mustEnqueue(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
t.Helper()
payload := []byte(`{"method":"` + method + `"}`)
task, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
AccountID: accountID,
NotificationID: notificationID,
ContactID: contactID,
Method: method,
EventIDs: eventIDs,
Payload: payload,
NotBefore: time.Now().Add(-time.Second),
})
require.NoError(t, err)
require.NotZero(t, task.ID)
return *task
}
// mustEnqueueRaw inserts a Task row directly without going through the producer
// precheck. The selector test deliberately mixes methods (email + telegram) on a
// worker that only handles email; the producer would refuse the telegram row,
// which is the wrong thing to assert about in a selector test.
func mustEnqueueRaw(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
t.Helper()
payload := datatypes.JSON([]byte(`{"method":"` + method + `"}`))
contact := contactID
task := models.Task{
JobID: uuid.New().String(),
Kind: models.TaskKindNotification,
State: models.TaskStateQueued,
AccountID: accountID,
ContactID: &contact,
Payload: payload,
NotBefore: time.Now().Add(-time.Second),
Attempts: 0,
MaxAttempts: 5,
IdempotencyKey: models.NotificationIdempotencyKey(notificationID, contactID, eventIDs[0]),
}
require.NoError(t, models.DB().Create(&task).Error)
require.NotZero(t, task.ID)
return task
}
func mustJSON(t *testing.T, v interface{}) []byte {
t.Helper()
b, err := json.Marshal(v)
require.NoError(t, err)
return b
}
// TestWorkerNodeCapabilities_NotificationFlags confirms that the JSON-backed
// capabilities blob correctly exposes the notification_methods and
// notification_accounts arrays that the selector and credential push depend on.
func TestWorkerNodeCapabilities_NotificationFlags(t *testing.T) {
models.Drop()
models.Migrate()
seedRegion(t, "test")
w := &models.WorkerNode{
WorkerID: "caps-worker-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"email", "telegram"},
"notification_accounts": []int64{7, 8},
})),
}
require.NoError(t, models.DB().Create(w).Error)
got := models.WorkerNode{}
require.NoError(t, models.DB().First(&got, w.ID).Error)
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
assert.Equal(t, []int64{7, 8}, got.NotificationAccounts())
assert.True(t, got.CanDeliverNotification("email", 7))
assert.False(t, got.CanDeliverNotification("email", 9), "account 9 is not in the allowed list")
assert.False(t, got.CanDeliverNotification("mattermost", 7), "method not authorized")
// Operated-style worker: empty accounts list means "all accounts".
w2 := &models.WorkerNode{
WorkerID: "ops-worker-" + uuid.NewString(),
RegionCode: "test",
Status: "active",
AuthToken: uuid.NewString(),
Concurrency: 4,
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
"check_types": []string{"http"},
"notification_methods": []string{"email"},
"notification_accounts": []int64{},
})),
}
require.NoError(t, models.DB().Create(w2).Error)
got2 := models.WorkerNode{}
require.NoError(t, models.DB().First(&got2, w2.ID).Error)
assert.True(t, got2.CanDeliverNotification("email", 9999), "empty accounts list means all accounts")
}
func TestWorkerNodeReportedWorkloadUsesDisjointHeartbeatFields(t *testing.T) {
w := &models.WorkerNode{Capabilities: datatypes.JSON([]byte(`{"active_checks":2,"queue_depth":3,"active_notifications":5,"notification_queue_depth":7}`))}
assert.Equal(t, 17, w.ReportedWorkload())
}
// TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults verifies that
// EnsureConfiguredWorkerNode (the in-cluster worker provisioner) populates
// notification_methods / notification_accounts on the JSON blob so the new
// selector and the new credential push work out of the box for the bundled
// worker.
func TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults(t *testing.T) {
models.Drop()
models.Migrate()
t.Setenv("WORKER_AUTH_TOKEN", "secret-token-for-test-xyz")
t.Setenv("DEPLOY_ENV", "test-env")
t.Setenv("RSMON_WORKER_ID", "worker-test")
t.Setenv("NOTIFICATION_METHODS", "email,telegram")
t.Setenv("NOTIFICATION_ACCOUNTS", "11,22")
models.EnsureConfiguredWorkerNode()
var got models.WorkerNode
require.NoError(t, models.DB().Where("worker_id = ?", "worker-test").First(&got).Error)
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
assert.Equal(t, []int64{11, 22}, got.NotificationAccounts())
}
// TestNotificationIdempotencyKeyFormat pins the producer-side key shape so the
// result handler can re-derive it for matching without depending on internal
// package state.
func TestNotificationIdempotencyKeyFormat(t *testing.T) {
key := models.NotificationIdempotencyKey(7, 9, 13)
assert.Equal(t, "notif:7:contact:9:event:13", key)
}
// guard against uuid being accidentally dropped from the imports.
var _ = uuid.New

77
app/models/telegram_bot.go Обычный файл
Просмотреть файл

@@ -0,0 +1,77 @@
package models
import (
"time"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
const (
// TelegramBotMessageReceived marks inbound bot messages.
TelegramBotMessageReceived = "received"
// TelegramBotMessageSent marks outbound bot replies.
TelegramBotMessageSent = "sent"
// TelegramBotStatusMain is the singleton status row name for the bot.
TelegramBotStatusMain = "main"
)
// TelegramBotMessage stores a Telegram bot chat message for admin history.
type TelegramBotMessage struct {
concerns.Model
Direction string `gorm:"size:16;index" json:"direction"`
ChatID int64 `gorm:"index" json:"chat_id"`
ChatType string `gorm:"size:32" json:"chat_type"`
Username string `gorm:"size:255" json:"username"`
Text string `gorm:"type:text" json:"text"`
Command string `gorm:"size:64" json:"command"`
ContactID *int64 `gorm:"index" json:"contact_id,omitempty"`
Error string `gorm:"type:text" json:"error,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
// TableName overrides the default table name.
func (TelegramBotMessage) TableName() string {
return "telegram_bot_messages"
}
// TelegramBotStatus stores the current Telegram bot heartbeat/status.
type TelegramBotStatus struct {
concerns.Model
Name string `gorm:"uniqueIndex;size:64;not null" json:"name"`
Username string `gorm:"size:255" json:"username"`
Online bool `gorm:"not null;default:false" json:"online"`
LastSeen *time.Time `json:"last_seen,omitempty"`
LastError string `gorm:"type:text" json:"last_error,omitempty"`
concerns.Timestamped
}
// TableName overrides the default table name.
func (TelegramBotStatus) TableName() string {
return "telegram_bot_statuses"
}
// RecentTelegramBotMessages returns the latest Telegram bot messages, capped to a safe limit.
func RecentTelegramBotMessages(limit int) ([]TelegramBotMessage, error) {
if limit <= 0 || limit > 500 {
limit = 100
}
messages := []TelegramBotMessage{}
err := DB().Order("id DESC").Limit(limit).Find(&messages).Error
return messages, err
}
// TelegramBotCurrentStatus returns the singleton Telegram bot status row.
func TelegramBotCurrentStatus() (*TelegramBotStatus, error) {
status := TelegramBotStatus{}
if err := DB().Where("name = ?", TelegramBotStatusMain).First(&status).Error; err != nil {
return nil, err
}
if status.LastSeen == nil || time.Since(*status.LastSeen) > 2*time.Minute {
status.Online = false
}
return &status, nil
}

24
app/models/telegram_bot_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,24 @@
package models_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"rsgit.ru/rsmon/rsmon/app/models"
)
func TestTelegramBotCurrentStatusMarksStaleOffline(t *testing.T) {
models.Drop()
models.Migrate()
seen := time.Now().Add(-3 * time.Minute)
status := models.TelegramBotStatus{Name: models.TelegramBotStatusMain, Online: true, LastSeen: &seen}
require.NoError(t, models.DB().Create(&status).Error)
got, err := models.TelegramBotCurrentStatus()
require.NoError(t, err)
assert.False(t, got.Online)
}

233
app/models/user.go Обычный файл
Просмотреть файл

@@ -0,0 +1,233 @@
package models
import (
"crypto/md5"
"fmt"
"log"
"time"
"github.com/lib/pq"
"github.com/pkg/errors"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// User represents a platform user.
//
// A User is a global identity that may belong to many tenants via the
// Access join table (see Access). Contacts created by or assigned to a
// User carry that UserID; admins reach them via the cross-tenant
// /admin/users page, while per-account management is via
// /settings/users (which shows Accesses preloaded with User + Invite).
//
// Authentication state (password, confirmation, lock, recover) lives
// here, not on Access, because those attributes are account-independent.
type User struct {
// concerns.Model
ID int64 `gorm:"primarykey" json:"id"`
Email *string `gorm:"uniqueIndex;size:255" json:"email" validate:"required"`
Name string `json:"name"`
Enabled bool `gorm:"not null;default:true" json:"-"`
// Operator grants platform-wide operational access. Account ownership alone
// must never grant cross-tenant task inspection or replay.
Operator bool `gorm:"not null;default:false" json:"operator"`
Timezone string `json:"timezone"`
Language string `gorm:"default:ru" json:"language"`
// Settings holds small per-user UI preferences. It intentionally stays
// separate from account settings because the sidebar is a personal view.
Settings datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"settings"`
Phone string `gorm:"index,size:255" json:"phone"`
TelegramID *int64 `gorm:"uniqueIndex" json:"telegram_id"`
TelegramUsername string `gorm:"size:255" json:"telegram_username"`
Accesses []Access `json:"-"`
Contacts []Contact `json:"-"`
LastActiveAt *time.Time `json:"last_active_at"`
LastActiveIP *string `json:"last_active_ip"`
EncryptedPassword *string `json:"-"`
PasswordSetAt *time.Time `json:"-"`
// Confirm
ConfirmationToken *string `json:"-"`
Confirmed bool `json:"confirmed"`
// Lock
AttemptCount int `json:"-"`
LastAttempt *time.Time `json:"-"`
LockedAt *time.Time `json:"-"`
// Recover
RecoverToken *string `json:"-"`
RecoverTokenAt *time.Time `json:"-"`
RememberTokens pq.StringArray `gorm:"index;type:varchar(100)[]" json:"-"`
// DeletionRequestedAt is set when the user requests account deletion.
// During the 7-day grace period the user can cancel the deletion
// (clearing this field). After 7 days the user and all related data
// are hard-deleted by a scheduled job.
DeletionRequestedAt *time.Time `json:"deletion_requested_at"`
concerns.Timestamped `json:"-"`
}
// DeletionPending returns true if the user has requested deletion and is
// still within the 7-day grace period.
func (u *User) DeletionPending() bool {
return u.DeletionRequestedAt != nil
}
// GetLabel returns info label for user.
func (u *User) GetLabel() string {
return u.DisplayName()
}
// DisplayName implements qor.CurrentUser for admin.
func (u *User) DisplayName() string {
if u.Email != nil {
return u.Name + " " + *u.Email
}
return u.Name
}
// AfterSocialLogin is a callback after social login.
func (u *User) AfterSocialLogin(inviteID int64) (*User, error) {
oldUser := User{}
DB().Where("email = ?", u.Email).Where("id != ?", u.ID).First(&oldUser)
if oldUser.ID > 0 {
log.Println("new user", u.ID, "has same email", u.Email, "as old user", oldUser.ID, "so replacing")
err := DB().Model(&authidentity.AuthIdentity{}).Where("user_id = ?", u.ID).Updates(
authidentity.Basic{
UserID: &oldUser.ID,
},
).Error
return &oldUser, err
}
err := u.AfterRegister(inviteID)
return u, err
}
// AfterInvite is a callback after invite acceptance.
func (u *User) AfterInvite(invite *Invite) error {
invite.InviteeID = &u.ID
err := DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
if err != nil {
return errors.Wrap(err, "invite: failed to add accesses to invited user")
}
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{
"confirmed_at": time.Now(),
}).Error
if err != nil {
return errors.Wrap(err, "invite: failed set user as confirmed")
}
eml := invite.Email
u.Email = &eml
if invite.Name != "" {
u.Name = invite.Name
}
err = DB().Save(&u).Error
if err != nil {
return errors.Wrap(err, "invite: failed to save user")
}
if invite.Name == "" {
log.Println("set name", u.Name)
invite.Name = u.Name
}
invite.State = stateOK
err = DB().Save(&invite).Error
if err != nil {
return errors.Wrap(err, "invite: failed to save invite")
}
invite.Invitee = u
return nil
}
// AfterRegister is a callback after registration.
func (u *User) AfterRegister(inviteID int64) error {
var err error
if inviteID > 0 {
invite := Invite{}
err := DB().First(&invite, inviteID).Error
if err != nil {
return errors.Wrap(err, "invite: failed to find invite")
}
err = u.AfterInvite(&invite)
if err != nil {
return err
}
}
_, err = CreateAccountForUser("", u)
if err != nil {
return err
}
return nil
}
// AfterLogin is a callback after login.
func (u *User) AfterLogin(inviteID int64) error {
var err error
if inviteID > 0 {
invite := Invite{}
err = DB().First(&invite, inviteID).Error
if err == nil {
invite.InviteeID = &u.ID
invite.State = stateOK
err = DB().Save(&invite).Error
if err != nil {
return errors.Wrap(err, "failed to save invite")
}
err = DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
if err != nil {
return errors.Wrap(err, "failed to add accesses to invited user")
}
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{ //nolint:lll
"confirmed_at": time.Now(),
}).Error
if err != nil {
return errors.Wrap(err, "failed set user as confirmed")
}
}
}
return nil
}
// Gravatar returns the Gravatar URL for the user.
func (u *User) Gravatar(size int) string {
if u.Email == nil {
return ""
}
hash := md5.Sum([]byte(*u.Email))
return fmt.Sprintf("https://www.gravatar.com/avatar/%x?s=%d&d=blank", hash, size)
}
// AsJSON returns a JSON representation of user.
func (u User) AsJSON() map[string]interface{} { //nolint:gocritic // hugeParam: accepted for interface compatibility
r := map[string]interface{}{
"id": u.ID,
"email": u.Email,
"avatar": u.Gravatar(32),
"deletion_requested_at": u.DeletionRequestedAt,
}
return r
}

23
app/models/whois.go Обычный файл
Просмотреть файл

@@ -0,0 +1,23 @@
package models
import (
"time"
"github.com/lib/pq"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// Whois provides functionality.
type Whois struct {
concerns.Model
MonitorID *int64
Monitor *Monitor
Tld string
Nameservers pq.StringArray `gorm:"type:varchar(255)[]"`
Expires *time.Time
UpdatedAt *time.Time
Data string
}

31
app/models/worker_log_event.go Обычный файл
Просмотреть файл

@@ -0,0 +1,31 @@
package models
import (
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// WorkerLogEvent stores critical log events sent by worker nodes. The same
// payload may also be forwarded to VictoriaLogs; Postgres keeps a compact audit
// copy so the control plane can show recent critical events even if external log
// storage is temporarily unavailable.
type WorkerLogEvent struct {
concerns.Model
WorkerID int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL;index" json:"worker_id"`
WorkerNodeID string `gorm:"size:100;not null;index" json:"worker_node_id"`
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
Level string `gorm:"size:16;not null;index" json:"level"`
Message string `gorm:"type:text;not null" json:"message"`
Source string `gorm:"size:64" json:"source"`
Payload datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"payload"`
OccurredAt time.Time `gorm:"not null;index" json:"occurred_at"`
concerns.Timestamped
}
// TableName provides functionality.
func (WorkerLogEvent) TableName() string { return "worker_log_events" }

452
app/models/worker_node.go Обычный файл
Просмотреть файл

@@ -0,0 +1,452 @@
package models
import (
"encoding/json"
"log"
"os"
"strconv"
"strings"
"time"
"gorm.io/datatypes"
"rsgit.ru/rsmon/rsmon/app/models/concerns"
)
// WorkerNode represents a distributed monitoring worker
type WorkerNode struct {
concerns.Model
WorkerID string `gorm:"uniqueIndex;size:100;not null" json:"worker_id"` // UUID or configured ID
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
Status string `gorm:"not null;default:'registered'" json:"status"` // registered, active, inactive, dead
AuthToken string `gorm:"uniqueIndex;size:64;not null" json:"-"`
LastSeen *time.Time `json:"last_seen"`
Version string `gorm:"size:50" json:"version"`
URL string `gorm:"size:500" json:"url"` //nolint:lll // publicly-advertised URL; may differ from bind host:port when behind Traefik
Capabilities datatypes.JSON `gorm:"not null;default:'{}'" json:"capabilities"` // {"check_types": ["http","ssl","dns",...]}
Concurrency int `gorm:"not null;default:20" json:"concurrency"`
NetworkProblems bool `gorm:"not null;default:false;index" json:"network_problems"`
NetworkProblemsUntil *time.Time `json:"network_problems_until,omitempty"`
LastFailureCount int `gorm:"not null;default:0" json:"last_failure_count"`
LastTotalCount int `gorm:"not null;default:0" json:"last_total_count"`
// Capability flags (see docs/plans/inventory-management.md §3).
// All default to true so an existing worker row that pre-dates
// this migration keeps running checks. Toggle from the admin UI
// or POST a boolean to /api/v1/workers to disable any one of
// them; the distworker client re-reads these flags on every
// task poll.
//
// Pointer types so GORM can distinguish "client didn't supply
// the key, fall back to the DB default" from "client explicitly
// set false". A plain `bool` would be silently re-overwritten
// by the column default on Save (default:true kicks in when
// GORM sees the zero value, regardless of whether the handler
// asked for false). See the controller tests for the
// partial-update case.
RunChecks *bool `gorm:"not null;default:true" json:"run_checks"`
CollectMetrics *bool `gorm:"not null;default:true" json:"collect_metrics"`
DetectProjects *bool `gorm:"not null;default:true" json:"detect_projects"`
// ServerID is the optional inventory Server this worker daemon
// is running on (see docs/plans/servers-and-hardware-metrics.md
// §3 and docs/plans/inventory-management.md §1). Nullable so
// legacy "no server assigned" rows keep working. Indexed because
// the distworker health ticker joins servers→workers frequently.
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
// AccountID scopes a worker to a single customer account (private
// worker per docs/distributed/private-workers.md). NULL means a
// platform-operated worker eligible to serve any account; non-NULL
// is a customer-operated worker pinned to one account. ON DELETE
// SET NULL keeps an operated worker valid if its account row is
// ever removed without an explicit private-worker cleanup.
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE SET NULL;index" json:"account_id,omitempty"`
LLMs []LLM `json:"llms,omitempty" gorm:"many2many:worker_llms;"`
concerns.Timestamped
}
const WorkerHeartbeatFreshness = 2 * time.Minute
func (w *WorkerNode) NetworkProblemActive(now time.Time) bool {
return w != nil && w.NetworkProblems && (w.NetworkProblemsUntil == nil || w.NetworkProblemsUntil.After(now))
}
// WorkerStatuses provides functionality.
// WorkerStatus represents the possible worker statuses
var WorkerStatuses = []string{"registered", "active", "inactive", "dead"}
// AllWorkerCheckKinds is the complete distributed-worker capability set.
func AllWorkerCheckKinds() []string {
return []string{kindHTTP, kindSSL, kindDNS, kindSSH, kindFTP, kindWhois, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
}
// NormalizeWorkerCapabilities expands aliases and removes unknown or duplicate capabilities.
func NormalizeWorkerCapabilities(capabilities []string) []string {
allowed := make(map[string]bool)
for _, kind := range AllWorkerCheckKinds() {
allowed[kind] = true
}
seen := make(map[string]bool)
normalized := make([]string, 0, len(capabilities))
for _, capability := range capabilities {
capability = strings.TrimSpace(strings.ToLower(capability))
if capability == "all" || capability == "*" {
return AllWorkerCheckKinds()
}
if !allowed[capability] || seen[capability] {
continue
}
seen[capability] = true
normalized = append(normalized, capability)
}
if len(normalized) == 0 {
return AllWorkerCheckKinds()
}
return normalized
}
// IsAlive returns true if the worker is considered alive based on last_seen
func (w *WorkerNode) IsAlive() bool {
if w.LastSeen == nil {
return false
}
// Worker is considered dead if no heartbeat for 2 minutes
return w.LastSeen.After(time.Now().Add(-WorkerHeartbeatFreshness))
}
// NotificationMethods returns the notification methods the worker is authorized
// to deliver (e.g. ["email", "telegram"]). Empty slice means "no notification
// delivery authorized". See docs/plans/worker-notifier-mvp.md section 4.3.
func (w *WorkerNode) NotificationMethods() []string {
caps := w.capabilitiesMap()
if caps == nil {
return nil
}
raw, ok := caps["notification_methods"]
if !ok {
return nil
}
return parseStringList(raw)
}
// NotificationAccounts returns the account IDs the worker is authorized to
// serve for notifications. Empty slice means "owned by RSMon, all accounts".
// Customer-hosted workers (phase 4) ship a non-empty slice to scope credentials.
func (w *WorkerNode) NotificationAccounts() []int64 {
caps := w.capabilitiesMap()
if caps == nil {
return nil
}
raw, ok := caps["notification_accounts"]
if !ok {
return nil
}
return parseInt64List(raw)
}
// AccessibleAccountIDs returns the accounts this worker may access. Empty means
// RSMon-operated/global worker. It combines the legacy single AccountID field
// with the newer notification_accounts capability list.
func (w *WorkerNode) AccessibleAccountIDs() []int64 {
if w == nil {
return nil
}
if w.AccountID != nil && *w.AccountID > 0 {
// A private worker cannot widen its account scope through a mutable
// capability JSON blob.
return []int64{*w.AccountID}
}
seen := map[int64]bool{}
out := []int64{}
for _, id := range w.NotificationAccounts() {
if id > 0 && !seen[id] {
seen[id] = true
out = append(out, id)
}
}
return out
}
// CanDeliverNotification returns true when the worker is allowed to deliver
// the given method for the given account. An empty NotificationAccounts slice
// means the worker is RSMon-operated and may serve any account.
func (w *WorkerNode) CanDeliverNotification(method string, accountID int64) bool {
methods := w.NotificationMethods()
if len(methods) == 0 {
return false
}
if !containsString(methods, method) {
return false
}
accounts := w.AccessibleAccountIDs()
if len(accounts) == 0 {
return true
}
return containsInt64(accounts, accountID)
}
// CheckTypes returns the check kinds this worker may execute.
func (w *WorkerNode) CheckTypes() []string {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return nil
}
return parseStringList(capabilities["check_types"])
}
// SupportsTaskEnvelope is an explicit protocol capability. Version labels are
// build metadata (and may be "latest" or a commit SHA), not a wire contract.
// Rows created before this capability existed intentionally remain v1.
func (w *WorkerNode) SupportsTaskEnvelope() bool {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return false
}
supported, _ := capabilities["task_envelope"].(bool)
return supported
}
// ReportedWorkload is the worker's local active and queued work across both
// checks and notifications. It is advisory; durable leases remain authoritative.
func (w *WorkerNode) ReportedWorkload() int {
capabilities := w.capabilitiesMap()
if capabilities == nil {
return 0
}
keys := []string{"active_checks", "queue_depth", "active_notifications", "notification_queue_depth"}
total := 0
for _, key := range keys {
switch value := capabilities[key].(type) {
case float64:
if value > 0 {
total += int(value)
}
case int:
if value > 0 {
total += value
}
}
}
return total
}
func (w *WorkerNode) capabilitiesMap() map[string]interface{} {
if w == nil || len(w.Capabilities) == 0 {
return nil
}
var out map[string]interface{}
if err := json.Unmarshal(w.Capabilities, &out); err != nil {
return nil
}
return out
}
func parseStringList(raw interface{}) []string {
switch v := raw.(type) {
case []interface{}:
out := make([]string, 0, len(v))
for _, item := range v {
if s, ok := item.(string); ok && s != "" {
out = append(out, s)
}
}
return out
case []string:
out := make([]string, 0, len(v))
for _, s := range v {
if s != "" {
out = append(out, s)
}
}
return out
}
return nil
}
func parseInt64List(raw interface{}) []int64 {
switch v := raw.(type) {
case []interface{}:
out := make([]int64, 0, len(v))
for _, item := range v {
switch n := item.(type) {
case float64:
out = append(out, int64(n))
case int64:
out = append(out, n)
}
}
return out
case []int64:
return v
}
return nil
}
func containsString(haystack []string, needle string) bool {
for _, s := range haystack {
if s == needle {
return true
}
}
return false
}
func containsInt64(haystack []int64, needle int64) bool {
for _, n := range haystack {
if n == needle {
return true
}
}
return false
}
// EnsureConfiguredWorkerNode creates or updates the bundled Docker Compose worker from environment variables.
func EnsureConfiguredWorkerNode() {
token := envFirst("WORKER_AUTH_TOKEN", "RSMON_AUTH_TOKEN")
if token == "" {
return
}
deployEnv := envDefault("DEPLOY_ENV", "production")
workerID := envDefault("RSMON_WORKER_ID", "worker-"+deployEnv+"-01")
regionCode := envDefault("RSMON_REGION_CODE", deployEnv)
version := envDefault("RSMON_WORKER_VERSION", envDefault("IMAGE_TAG", "latest"))
concurrency := envInt("WORKER_CONCURRENCY", 20)
workerURL := strings.TrimSpace(os.Getenv("WORKER_URL"))
capabilities := NormalizeWorkerCapabilities(splitEnvList(envDefault(
"RSMON_CAPABILITIES",
"http,ssl,dns,ssh,ftp,whois,bssl,llm,llm-http,ping,tcp,udp",
)))
region := Region{}
if err := DB().Where("code = ?", regionCode).First(&region).Error; err != nil {
region = Region{Code: regionCode, Name: regionCode, Enabled: true}
if err := DB().Create(&region).Error; err != nil {
log.Printf("worker: failed to create configured worker region %s: %v", regionCode, err)
return
}
}
capJSON, err := json.Marshal(map[string]interface{}{
"check_types": capabilities,
"task_envelope": true,
"notification_methods": parseStringList(notificationMethodsFromEnv()),
"notification_accounts": parseInt64List(notificationAccountsFromEnv()),
})
if err != nil {
log.Printf("worker: failed to marshal configured worker capabilities: %v", err)
return
}
worker := WorkerNode{}
DB().Where("worker_id = ? OR auth_token = ?", workerID, token).First(&worker)
created := worker.ID == 0
worker.WorkerID = workerID
worker.RegionCode = regionCode
worker.Status = "registered"
worker.AuthToken = token
worker.Version = version
worker.URL = workerURL
worker.Capabilities = datatypes.JSON(capJSON)
worker.Concurrency = concurrency
if err := DB().Save(&worker).Error; err != nil {
log.Printf("worker: failed to provision configured worker %s: %v", workerID, err)
return
}
if created {
log.Printf("worker: provisioned configured worker %s in region %s", workerID, regionCode)
} else {
log.Printf("worker: updated configured worker %s in region %s", workerID, regionCode)
}
}
func splitEnvList(value string) []string {
parts := strings.Split(value, ",")
items := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
items = append(items, part)
}
}
return items
}
func envFirst(keys ...string) string {
for _, key := range keys {
if value := os.Getenv(key); value != "" {
return value
}
}
return ""
}
func envDefault(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
func envInt(key string, defaultValue int) int {
value, err := strconv.Atoi(os.Getenv(key))
if err != nil || value <= 0 {
return defaultValue
}
return value
}
// Notification method constants used across the package. Centralized here so
// the literal does not appear three or more times (goconst).
const (
methodEmail = "email"
methodTelegram = "telegram"
methodWebhook = "webhook"
methodMattermost = "mattermost"
)
// defaultNotificationMethods is the operated-worker default notification method
// list. Lives at package scope so goconst does not flag the literal across
// the package (account.go and user.go already reference "email").
var defaultNotificationMethods = []string{methodEmail, methodTelegram, methodWebhook, methodMattermost}
// notificationMethodsFromEnv reads the optional NOTIFICATION_METHODS env var.
// Empty result yields the default "all four" list so the operated worker can
// deliver email / telegram / webhook / mattermost out of the box.
func notificationMethodsFromEnv() []string {
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_METHODS"))
if raw == "" {
return append([]string{}, defaultNotificationMethods...)
}
out := make([]string, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
// notificationAccountsFromEnv reads the optional NOTIFICATION_ACCOUNTS env var.
// Empty result means "all accounts allowed" (the RSMon-operated default).
func notificationAccountsFromEnv() []int64 {
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_ACCOUNTS"))
if raw == "" {
return nil
}
out := make([]int64, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
n, err := strconv.ParseInt(part, 10, 64)
if err != nil || n <= 0 {
continue
}
out = append(out, n)
}
return out
}

687
checks/calls/init.go Обычный файл
Просмотреть файл

@@ -0,0 +1,687 @@
// Package calls provides functionality.
package calls
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"strings"
"time"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/chromedp"
"github.com/google/uuid"
"github.com/minio/minio-go/v7"
"github.com/openai/openai-go/v3"
"github.com/openai/openai-go/v3/option"
"rsgit.ru/rsmon/rsmon/app/models"
"rsgit.ru/rsmon/rsmon/internal/checkresult"
"rsgit.ru/rsmon/rsmon/storage"
)
const (
verdictError = "error"
verdictWarning = "warning"
verdictNormal = "normal"
)
// Result is a result of an AI health check
type Result struct {
checkresult.CheckResult
// Captured data
Screenshot string `json:"screenshot"` // base64 encoded screenshot
DOM string `json:"dom"` // HTML content
NetworkLogs string `json:"network_logs"` // JSON array of network logs
ConsoleLogs string `json:"console_logs"` // JSON array of console logs
URL string `json:"url"` // Final URL after redirects
StatusCode int `json:"status_code"` // HTTP status code
Title string `json:"title"` // Page title
// LLM analysis
LLMResponse string `json:"llm_response"` // Full LLM response
LLMVerdict string `json:"llm_verdict"` // "normal", "error", "warning"
LLMReasoning string `json:"llm_reasoning"` // LLM's explanation
HasChanges bool `json:"has_changes"` // Whether changes were detected from previous
// Previous screenshot comparison
PreviousURL string `json:"previous_url"` // URL of previous screenshot (if any)
PreviousTime string `json:"previous_time"` // Timestamp of previous screenshot
// Metadata
CheckID uint `json:"check_id"` // For database reference
ScreenshotID uint `json:"screenshot_id"` // ID of stored screenshot
}
// NetworkLog represents a network request log
type NetworkLog struct {
URL string `json:"url"`
Method string `json:"method"`
StatusCode int `json:"status_code"`
Type string `json:"type"`
Size int64 `json:"size"`
Time float64 `json:"time"`
}
// ConsoleLog represents a console log entry
type ConsoleLog struct {
Type string `json:"type"`
Value string `json:"value"`
}
// ScreenshotRecord represents a stored screenshot in the database
type ScreenshotRecord struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time `json:"created_at"`
CheckID uint `json:"check_id"`
MonitorID uint `json:"monitor_id"`
URL string `json:"url"`
// ScreenshotPath stores the S3 object path (e.g., "screenshots/monitor_123/2025-01-15_123456.jpg")
ScreenshotPath string `json:"screenshot_path" gorm:"type:varchar(500)"`
DOM string `gorm:"type:longtext" json:"dom"`
NetworkLogs string `gorm:"type:longtext" json:"network_logs"`
ConsoleLogs string `gorm:"type:longtext" json:"console_logs"`
StatusCode int `json:"status_code"`
Title string `json:"title"`
LLMVerdict string `json:"llm_verdict"`
LLMReasoning string `json:"llm_reasoning"`
}
// Perform executes the AI health check
func Perform(c *models.Check) *Result {
result := &Result{
CheckID: uint(c.ID),
}
result.Warnings = []string{}
result.Infos = []string{}
start := time.Now()
// Get the target URL
targetURL := getTargetURL(c)
result.URL = targetURL
// Step 1: Capture screenshot, DOM, and logs
screenshotData, domData, networkLogs, consoleLogs, statusCode, title, err := captureWebData(targetURL)
if err != nil {
result.Error = fmt.Errorf("failed to capture web data: %w", err)
result.Duration = time.Since(start)
result.State = "FAIL"
return result
}
result.Screenshot = screenshotData
result.DOM = domData
result.NetworkLogs = networkLogs
result.ConsoleLogs = consoleLogs
result.StatusCode = statusCode
result.Title = title
// Step 2: Store current screenshot in database (if available)
var screenshotID uint
if isDBAvailable() {
screenshotID, err = storeScreenshot(uint(c.MonitorID), uint(c.ID), result)
if err != nil {
result.Warnings = append(result.Warnings, "Failed to store screenshot: "+err.Error())
log.Printf("[ai-check] Failed to store screenshot: %v", err)
} else {
result.ScreenshotID = screenshotID
result.Infos = append(result.Infos, "Screenshot stored for reference")
}
} else {
result.Infos = append(result.Infos, "Database not available - running in standalone mode")
}
// Step 3: Get previous screenshots for comparison (if DB available)
var previousScreenshots []ScreenshotRecord
if isDBAvailable() {
previousScreenshots, err = getPreviousScreenshots(uint(c.MonitorID), 3)
if err != nil {
result.Warnings = append(result.Warnings, "Could not retrieve previous screenshots: "+err.Error())
}
}
if len(previousScreenshots) > 0 {
result.PreviousURL = previousScreenshots[0].URL
result.PreviousTime = previousScreenshots[0].CreatedAt.Format(time.RFC3339)
result.Infos = append(result.Infos, fmt.Sprintf("Comparing with %d previous screenshot(s)", len(previousScreenshots)))
}
// Step 4: Analyze with LLM
llmVerdict, llmReasoning, hasChanges, err := analyzeWithLLM(targetURL, result, previousScreenshots)
if err != nil {
result.Error = fmt.Errorf("LLM analysis failed: %w", err)
result.LLMResponse = err.Error()
result.Duration = time.Since(start)
result.State = "FAIL"
return result
}
result.LLMVerdict = llmVerdict
result.LLMReasoning = llmReasoning
result.HasChanges = hasChanges
// Update the screenshot record with LLM analysis
if screenshotID > 0 {
_ = updateScreenshotWithLLM(screenshotID, llmVerdict, llmReasoning)
}
// Determine final state based on LLM verdict
switch llmVerdict {
case verdictError:
result.State = "ERR"
result.Error = errors.New(llmReasoning)
case verdictWarning:
result.State = "WARN"
result.Warnings = append(result.Warnings, llmReasoning)
default:
result.State = "OK"
if hasChanges {
result.Infos = append(result.Infos, "Visual changes detected from previous screenshot")
}
}
result.Duration = time.Since(start)
return result
}
// getTargetURL constructs the target URL from the check
func getTargetURL(c *models.Check) string {
if c.URL != nil && *c.URL != "" {
return *c.URL
}
// Construct from monitor host
host := c.Monitor.Host
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
return "https://" + host
}
return host
}
// captureWebData captures screenshot, DOM, network and console logs using chromedp
func captureWebData(targetURL string) (screenshot, dom, networkLogsJSON, consoleLogsJSON string, statusCode int, title string, err error) {
// Create a context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Create chromedp context with options for headless chrome
allocCtx, allocCancel := chromedp.NewExecAllocator(ctx,
chromedp.NoDefaultBrowserCheck,
chromedp.NoFirstRun,
chromedp.DisableGPU,
chromedp.IgnoreCertErrors,
chromedp.Flag("headless", "new"),
chromedp.Flag("no-sandbox", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("hide-scrollbars", true),
chromedp.Flag("mute-audio", true),
chromedp.WindowSize(1920, 1080),
)
defer allocCancel()
taskCtx, taskCancel := chromedp.NewContext(allocCtx)
defer taskCancel()
// Collect network logs
networkLogs := []NetworkLog{}
consoleLogs := []ConsoleLog{}
var finalStatusCode int
// Listen for network events
chromedp.ListenTarget(taskCtx, func(ev interface{}) {
switch e := ev.(type) {
case *network.EventRequestWillBeSent:
// Track the main request
if e.Type == network.ResourceTypeDocument {
networkLogs = append(networkLogs, NetworkLog{
URL: e.Request.URL,
Method: e.Request.Method,
Type: string(e.Type),
})
}
case *network.EventResponseReceived:
// Update the main request with status code
for i, log := range networkLogs {
if log.URL == e.Response.URL && log.Method == "" { // Not yet updated
networkLogs[i].StatusCode = int(e.Response.Status)
networkLogs[i].Size = int64(e.Response.EncodedDataLength)
// Capture final status code for main document
if e.Type == network.ResourceTypeDocument && finalStatusCode == 0 {
finalStatusCode = int(e.Response.Status)
}
break
}
}
case *runtime.EventConsoleAPICalled:
// Capture console logs
value := ""
for _, arg := range e.Args {
// Get the string representation of the argument
value += arg.Value.String()
}
if value != "" && value != "undefined" {
consoleLogs = append(consoleLogs, ConsoleLog{
Type: "console." + string(e.Type),
Value: strings.TrimSpace(value),
})
}
}
})
// Run the browser tasks
var buf []byte
var html string
var pageTitle string
tasks := chromedp.Tasks{
network.Enable(),
chromedp.Navigate(targetURL),
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.Sleep(2 * time.Second), // Wait for page to fully render
chromedp.Title(&pageTitle),
chromedp.OuterHTML(":root", &html, chromedp.ByQueryAll),
chromedp.FullScreenshot(&buf, 90), // 90% quality JPEG
}
if err := chromedp.Run(taskCtx, tasks); err != nil {
// Check if it's a timeout error
if errors.Is(err, context.DeadlineExceeded) {
return "", "", "", "", 0, "", fmt.Errorf("page load timeout after 30s")
}
return "", "", "", "", 0, "", fmt.Errorf("chromedp error: %w", err)
}
// Encode screenshot as base64
screenshot = base64.StdEncoding.EncodeToString(buf)
dom = html
// Convert logs to JSON
networkBytes, _ := json.Marshal(networkLogs)
networkLogsJSON = string(networkBytes)
consoleBytes, _ := json.Marshal(consoleLogs)
consoleLogsJSON = string(consoleBytes)
// Set status code if we got it from network events
if finalStatusCode == 0 {
finalStatusCode = 200 // Assume OK if we got this far
}
return screenshot, dom, networkLogsJSON, consoleLogsJSON, finalStatusCode, pageTitle, nil
}
// storeScreenshot stores the screenshot data in S3 and metadata in the database
func storeScreenshot(monitorID, checkID uint, result *Result) (uint, error) {
// Initialize storage if not already initialized
if !storage.IsAvailable() {
if err := storage.Init(); err != nil {
return 0, fmt.Errorf("failed to initialize storage: %w", err)
}
}
// Generate a unique filename for the screenshot
// Format: screenshots/monitor_<monitor_id>/YYYY-MM-DD/<uuid>.jpg
timestamp := time.Now().Format("2006-01-02")
screenshotID := uuid.New().String()
objectPath := fmt.Sprintf("screenshots/monitor_%d/%s/%s.jpg", monitorID, timestamp, screenshotID)
// Decode base64 screenshot data
screenshotBytes, err := base64.StdEncoding.DecodeString(result.Screenshot)
if err != nil {
return 0, fmt.Errorf("failed to decode screenshot: %w", err)
}
// Upload screenshot to S3
ctx := context.Background()
reader := bytes.NewReader(screenshotBytes)
objectSize := int64(len(screenshotBytes))
_, err = storage.PutObject(ctx, objectPath, reader, objectSize, minio.PutObjectOptions{
ContentType: "image/jpeg",
})
if err != nil {
return 0, fmt.Errorf("failed to upload screenshot to S3: %w", err)
}
log.Printf("[ai-check] Stored screenshot at S3 path: %s (size: %d bytes)", objectPath, objectSize)
// Store metadata in database
record := &ScreenshotRecord{
MonitorID: monitorID,
CheckID: checkID,
URL: result.URL,
ScreenshotPath: objectPath,
DOM: truncateString(result.DOM, 50000), // Limit DOM size
NetworkLogs: result.NetworkLogs,
ConsoleLogs: result.ConsoleLogs,
StatusCode: result.StatusCode,
Title: result.Title,
}
// Auto migrate the table
if err := models.DB().AutoMigrate(&ScreenshotRecord{}); err != nil {
return 0, fmt.Errorf("failed to auto-migrate: %w", err)
}
if err := models.DB().Create(record).Error; err != nil {
return 0, fmt.Errorf("failed to insert screenshot metadata: %w", err)
}
return record.ID, nil
}
// loadScreenshotData loads screenshot data from S3 for a given record
func loadScreenshotData(record *ScreenshotRecord) (string, error) {
if record.ScreenshotPath == "" {
return "", errors.New("no screenshot path in record")
}
if !storage.IsAvailable() {
if err := storage.Init(); err != nil {
return "", fmt.Errorf("failed to initialize storage: %w", err)
}
}
ctx := context.Background()
object, err := storage.GetObject(ctx, record.ScreenshotPath, minio.GetObjectOptions{})
if err != nil {
return "", fmt.Errorf("failed to get screenshot from S3: %w", err)
}
defer object.Close() //nolint:errcheck // accepted lint exception
// Read the object data
var buf bytes.Buffer
_, err = io.Copy(&buf, object)
if err != nil {
return "", fmt.Errorf("failed to read screenshot data: %w", err)
}
// Encode as base64 for LLM analysis
return base64.StdEncoding.EncodeToString(buf.Bytes()), nil
}
// updateScreenshotWithLLM updates a screenshot record with LLM analysis
func updateScreenshotWithLLM(screenshotID uint, verdict, reasoning string) error {
return models.DB().Model(&ScreenshotRecord{}).
Where("id = ?", screenshotID).
Updates(map[string]interface{}{
"llm_verdict": verdict,
"llm_reasoning": reasoning,
}).Error
}
// isDBAvailable checks if the database is available
func isDBAvailable() bool {
return models.IsDBAvailable()
}
// getPreviousScreenshots retrieves previous screenshots for comparison
func getPreviousScreenshots(monitorID uint, limit int) ([]ScreenshotRecord, error) {
var records []ScreenshotRecord
err := models.DB().Model(&ScreenshotRecord{}).
Where("monitor_id = ? AND llm_verdict != ''", monitorID).
Order("created_at DESC").
Limit(limit).
Find(&records).Error
if err != nil {
return nil, err
}
return records, nil
}
// truncateString truncates a string to a maximum length
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "... (truncated)"
}
// analyzeWithLLM sends the screenshot and data to the LLM for analysis
func analyzeWithLLM(targetURL string, current *Result, previous []ScreenshotRecord) (verdict, reasoning string, hasChanges bool, err error) { //nolint:lll
// Get LLM configuration from environment
llmKey := firstEnv("LLM_APIKEY", "LLAMA_KEY")
llmURL := firstEnv("LLM_URL", "LLAMA_URL")
llmModel := firstEnv("LLM_MODEL", "LLAMA_MODEL")
if llmKey == "" || llmURL == "" {
return "", "", false, errors.New("LLM credentials not configured (LLAMA_KEY, LLAMA_URL)")
}
if llmModel == "" {
llmModel = "llama3.2-vision" // Default model
}
// Create OpenAI client with custom base URL
client := openai.NewClient(
option.WithBaseURL(llmURL),
option.WithAPIKey(llmKey),
)
// Build the system prompt
systemPrompt := buildSystemPrompt(len(previous) > 0)
// Build user message with current screenshot
userContent := buildUserMessage(targetURL, current, previous)
// Prepare messages
messages := []openai.ChatCompletionMessageParamUnion{
openai.SystemMessage(systemPrompt),
openai.UserMessage(userContent),
}
// Call LLM with timeout
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
params := openai.ChatCompletionNewParams{
Messages: messages,
Model: llmModel,
MaxTokens: openai.Int(2000),
Temperature: openai.Float(0.3), // Lower temperature for more consistent analysis
}
completion, err := client.Chat.Completions.New(ctx, params)
if err != nil {
return "", "", false, fmt.Errorf("LLM request failed: %w", err)
}
if len(completion.Choices) == 0 {
return "", "", false, errors.New("LLM returned no choices")
}
response := completion.Choices[0].Message.Content
// Parse the LLM response to extract verdict and reasoning
return parseLLMResponse(response)
}
func firstEnv(keys ...string) string {
for _, key := range keys {
if value := os.Getenv(key); value != "" {
return value
}
}
return ""
}
// buildSystemPrompt creates the system prompt for the LLM
func buildSystemPrompt(hasPrevious bool) string {
basePrompt := `
You are a web application health monitoring assistant. Your task is to analyze website screenshots
and determine if the website appears to be functioning normally or if there are issues.
Consider the following aspects:
1. Visual layout - Is the page rendered correctly? Are elements properly aligned?
2. Error messages - Are there any visible error messages, 404s, 500s, or similar?
3. Loading states - Is the page stuck loading or showing spinners?
4. Broken elements - Are there broken images, missing styles, or layout shifts?
5. Content availability - Is the expected content visible?
Respond in the following JSON format:
{
"verdict": "normal" | "warning" | "error",
"reasoning": "Brief explanation of your assessment",
"has_changes": true | false
}
Verdict guidelines:
- "error": Page is clearly broken (error messages, completely failed to load, blank page)
- "warning": Page loads but has issues (partial content, broken elements, degraded appearance)
- "normal": Page appears to be functioning correctly`
if hasPrevious {
basePrompt += `
You will also be shown previous screenshots of the same website. Compare the current screenshot with previous ones and report any significant visual changes that might indicate problems (layout shifts, missing elements, color changes, etc.).` //nolint:lll
}
return basePrompt
}
// buildUserMessage creates the user message with screenshot data
func buildUserMessage(targetURL string, current *Result, previous []ScreenshotRecord) string {
var sb strings.Builder
_, _ = fmt.Fprintf(&sb, "Analyze the screenshot of: %s\n\n", targetURL)
_, _ = fmt.Fprintf(&sb, "HTTP Status Code: %d\n", current.StatusCode)
_, _ = fmt.Fprintf(&sb, "Page Title: %s\n\n", current.Title)
// Add console errors if any
var consoleErrors []ConsoleLog
json.Unmarshal([]byte(current.ConsoleLogs), &consoleErrors) //nolint:errcheck // accepted lint exception
errorCount := 0
for _, log := range consoleErrors {
if strings.Contains(log.Type, "error") || strings.Contains(log.Type, "warn") {
errorCount++
}
}
if errorCount > 0 {
_, _ = fmt.Fprintf(&sb, "Console Errors/Warnings: %d\n\n", errorCount)
}
// Add network errors
var networkLogs []NetworkLog
json.Unmarshal([]byte(current.NetworkLogs), &networkLogs) //nolint:errcheck // accepted lint exception
failedRequests := 0
for _, req := range networkLogs {
if req.StatusCode >= 400 {
failedRequests++
}
}
if failedRequests > 0 {
_, _ = fmt.Fprintf(&sb, "Failed Network Requests: %d\n\n", failedRequests)
}
sb.WriteString("Current screenshot (base64):\n")
sb.WriteString(current.Screenshot)
sb.WriteString("\n\n")
// Add previous screenshots if available (load from S3)
if len(previous) > 0 {
_, _ = fmt.Fprintf(&sb, "For comparison, here are %d previous screenshot(s):\n\n", len(previous))
for i, prev := range previous { //nolint:gocritic // range copy is acceptable here
// Load screenshot data from S3
screenshotData, err := loadScreenshotData(&prev)
if err != nil {
log.Printf("[ai-check] Failed to load previous screenshot %d: %v", i+1, err)
_, _ = fmt.Fprintf(&sb, "Previous screenshot #%d (from %s): [error loading screenshot]\n\n",
i+1, prev.CreatedAt.Format("2006-01-02 15:04:05"))
continue
}
_, _ = fmt.Fprintf(&sb, "Previous screenshot #%d (from %s):\n", i+1, prev.CreatedAt.Format("2006-01-02 15:04:05"))
sb.WriteString(screenshotData)
sb.WriteString("\n\n")
}
}
return sb.String()
}
// parseLLMResponse parses the LLM response to extract structured data
func parseLLMResponse(response string) (verdict, reasoning string, hasChanges bool, err error) {
// Try to extract JSON from the response
response = strings.TrimSpace(response)
// Look for JSON block
jsonStart := strings.Index(response, "{")
jsonEnd := strings.LastIndex(response, "}")
if jsonStart == -1 || jsonEnd == -1 {
// No JSON found, try to parse text response
return parseTextResponse(response)
}
jsonStr := response[jsonStart : jsonEnd+1]
var parsed struct {
Verdict string `json:"verdict"`
Reasoning string `json:"reasoning"`
HasChanges bool `json:"has_changes"`
}
if err := json.Unmarshal([]byte(jsonStr), &parsed); err != nil {
// JSON parse failed, fall back to text parsing
return parseTextResponse(response)
}
// Validate verdict
switch parsed.Verdict {
case "normal", "ok", "healthy":
return verdictNormal, parsed.Reasoning, parsed.HasChanges, nil
case "warning", "warn":
return verdictWarning, parsed.Reasoning, parsed.HasChanges, nil
case "error", "fail", "unhealthy":
return verdictError, parsed.Reasoning, parsed.HasChanges, nil
default:
// Unknown verdict, default to normal with warning
if parsed.Verdict != "" {
return verdictWarning, parsed.Reasoning, parsed.HasChanges, nil
}
return verdictNormal, "Unable to determine specific issues from analysis", false, nil
}
}
// parseTextResponse parses a non-JSON response
func parseTextResponse(response string) (verdict, reasoning string, hasChanges bool, err error) {
responseLower := strings.ToLower(response)
// Look for keywords
hasError := strings.Contains(responseLower, "error") ||
strings.Contains(responseLower, "broken") ||
strings.Contains(responseLower, "failed") ||
strings.Contains(responseLower, "not working")
hasWarning := strings.Contains(responseLower, "warning") ||
strings.Contains(responseLower, "issue") ||
strings.Contains(responseLower, "problem") ||
strings.Contains(responseLower, "degraded")
hasChange := strings.Contains(responseLower, "change") ||
strings.Contains(responseLower, "different") ||
strings.Contains(responseLower, "modified")
if hasError {
return verdictError, response, hasChange, nil
} else if hasWarning {
return verdictWarning, response, hasChange, nil
}
return verdictNormal, response, hasChange, nil
}

11
checks/calls/init_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
package calls
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIsDBAvailableWithoutInitializedDatabase(t *testing.T) {
assert.False(t, isDBAvailable())
}

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше