feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
8
.dockerignore
Обычный файл
8
.dockerignore
Обычный файл
@@ -0,0 +1,8 @@
|
||||
.git
|
||||
.github
|
||||
.env
|
||||
bin
|
||||
dist
|
||||
rsmon-worker
|
||||
*.log
|
||||
*.db*
|
||||
26
.env.example
Обычный файл
26
.env.example
Обычный файл
@@ -0,0 +1,26 @@
|
||||
# Required: create the worker at https://rsmon.ru/settings/workers first.
|
||||
RSMON_URL=https://rsmon.ru
|
||||
RSMON_TOKEN=replace-with-worker-token
|
||||
|
||||
# Local operator console. Both credentials are required while the web UI is on.
|
||||
WORKER_HOST=0.0.0.0
|
||||
WORKER_PORT=27401
|
||||
WORKER_BIND_IP=127.0.0.1
|
||||
WORKER_URL=
|
||||
WORKER_LOGIN=admin
|
||||
WORKER_PASSWORD=replace-with-a-long-random-password
|
||||
|
||||
# Persistent web application state.
|
||||
RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp
|
||||
|
||||
# Optional Raft worker cluster.
|
||||
WORKER_CLUSTER_ENABLED=false
|
||||
WORKER_CLUSTER_ID=
|
||||
WORKER_CLUSTER_HOST=0.0.0.0
|
||||
WORKER_CLUSTER_PORT=37401
|
||||
WORKER_CLUSTER_PEERS=
|
||||
WORKER_CLUSTER_DATA_DIR=/var/lib/rsmon-worker/cluster
|
||||
WORKER_CLUSTER_BOOTSTRAP=false
|
||||
|
||||
# Optional update page API endpoint.
|
||||
WORKER_RELEASE_URL=
|
||||
27
.github/workflows/ci.yml
поставляемый
Обычный файл
27
.github/workflows/ci.yml
поставляемый
Обычный файл
@@ -0,0 +1,27 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.26.x'
|
||||
cache: true
|
||||
- name: Check module files
|
||||
run: |
|
||||
go mod tidy
|
||||
git diff --exit-code -- go.mod go.sum
|
||||
- name: Test worker packages
|
||||
run: make test
|
||||
- name: Build binary
|
||||
run: make build
|
||||
79
.github/workflows/docker.yml
поставляемый
Обычный файл
79
.github/workflows/docker.yml
поставляемый
Обычный файл
@@ -0,0 +1,79 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags:
|
||||
- v*
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
name: Build and publish worker image
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
HARBOR_REGISTRY: ${{ secrets.HARBOR_REGISTRY }}
|
||||
HARBOR_PROJECT: rsmon
|
||||
IMAGE_NAME: rsmon-worker
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Harbor
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.HARBOR_REGISTRY }}
|
||||
username: ${{ secrets.HARBOR_USER }}
|
||||
password: ${{ secrets.HARBOR_PASSWORD }}
|
||||
|
||||
- name: Compute image metadata
|
||||
id: vars
|
||||
env:
|
||||
REF_NAME: ${{ gitea.ref_name }}
|
||||
REF_TYPE: ${{ gitea.ref_type }}
|
||||
SHA: ${{ gitea.sha }}
|
||||
run: |
|
||||
IMAGE="${HARBOR_REGISTRY}/${HARBOR_PROJECT}/${IMAGE_NAME}"
|
||||
SHORT_SHA="$(printf '%s' "${SHA}" | cut -c1-12)"
|
||||
TAGS="${IMAGE}:sha-${SHORT_SHA}"
|
||||
if [ "${REF_TYPE}" = "branch" ] && [ "${REF_NAME}" = "master" ]; then
|
||||
TAGS="${TAGS},${IMAGE}:latest"
|
||||
fi
|
||||
if [ "${REF_TYPE}" = "tag" ]; then
|
||||
TAGS="${TAGS},${IMAGE}:${REF_NAME}"
|
||||
fi
|
||||
{
|
||||
echo "image=${IMAGE}"
|
||||
echo "tags=${TAGS}"
|
||||
echo "build_date=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
pull: true
|
||||
push: true
|
||||
tags: ${{ steps.vars.outputs.tags }}
|
||||
labels: |
|
||||
org.opencontainers.image.source=${{ gitea.server_url }}/${{ gitea.repository }}
|
||||
org.opencontainers.image.revision=${{ gitea.sha }}
|
||||
org.opencontainers.image.version=${{ gitea.ref_name }}
|
||||
org.opencontainers.image.licenses=LicenseRef-RSMon-Worker-Source-Available-1.0
|
||||
build-args: |
|
||||
VERSION=${{ gitea.ref_name }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
BUILD_DATE=${{ steps.vars.outputs.build_date }}
|
||||
cache-from: type=registry,ref=${{ steps.vars.outputs.image }}:buildcache
|
||||
cache-to: type=registry,ref=${{ steps.vars.outputs.image }}:buildcache,mode=max
|
||||
sbom: true
|
||||
provenance: mode=max
|
||||
8
.gitignore
поставляемый
Обычный файл
8
.gitignore
поставляемый
Обычный файл
@@ -0,0 +1,8 @@
|
||||
/bin/
|
||||
/dist/
|
||||
/rsmon-worker
|
||||
/.env
|
||||
*.log
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
37
Dockerfile
Обычный файл
37
Dockerfile
Обычный файл
@@ -0,0 +1,37 @@
|
||||
FROM reg.rsxx.ru/library/golang:1-trixie AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download && go mod verify
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG COMMIT=unknown
|
||||
ARG BUILD_DATE=unknown
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.buildDate=${BUILD_DATE}" \
|
||||
-o /out/rsmon-worker ./cmd/rsmon-worker
|
||||
|
||||
FROM reg.rsxx.ru/library/debian:13-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates chromium tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 rsmon-worker \
|
||||
&& useradd --uid 10001 --gid rsmon-worker --home-dir /var/lib/rsmon-worker --create-home --shell /usr/sbin/nologin rsmon-worker
|
||||
|
||||
COPY --from=builder /out/rsmon-worker /usr/local/bin/rsmon-worker
|
||||
|
||||
ENV HOME=/var/lib/rsmon-worker \
|
||||
RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp
|
||||
|
||||
VOLUME ["/var/lib/rsmon-worker"]
|
||||
EXPOSE 27401 37401
|
||||
USER rsmon-worker
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
|
||||
CMD ["/usr/local/bin/rsmon-worker", "health"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/rsmon-worker"]
|
||||
62
LICENSE
Обычный файл
62
LICENSE
Обычный файл
@@ -0,0 +1,62 @@
|
||||
RSMon Worker Source-Available License 1.0
|
||||
|
||||
Copyright (c) 2026 RSMon. All rights reserved.
|
||||
|
||||
This software is not open source. No rights are granted except those expressly
|
||||
stated in this license.
|
||||
|
||||
1. Permitted use
|
||||
|
||||
Subject to this license, RSMon grants you a limited, non-exclusive,
|
||||
non-transferable, revocable license to:
|
||||
|
||||
* inspect and evaluate the source code;
|
||||
* use the source code for private research and evaluation; and
|
||||
* modify and build the software solely to operate an RSMon worker connected to
|
||||
the monitoring service hosted at rsmon.ru.
|
||||
|
||||
2. Restrictions
|
||||
|
||||
You may not, without prior written permission from RSMon:
|
||||
|
||||
* use the software with any service other than rsmon.ru;
|
||||
* use the software to provide a competing or commercial monitoring service;
|
||||
* redistribute, publish, sell, sublicense, lease, or otherwise make the source
|
||||
code, modified source code, binaries, or container images available to any
|
||||
third party;
|
||||
* remove or alter copyright, license, attribution, or proprietary notices;
|
||||
* use RSMon names, logos, or trademarks except to identify compatibility with
|
||||
rsmon.ru; or
|
||||
* use the software or source code for any purpose not expressly permitted by
|
||||
section 1.
|
||||
|
||||
3. Modifications
|
||||
|
||||
Modifications and derivative works are subject to this license. You must keep
|
||||
this license and all copyright notices with every permitted copy. RSMon is not
|
||||
obligated to support, accept, or maintain modifications.
|
||||
|
||||
4. Ownership
|
||||
|
||||
RSMon and its licensors retain all right, title, and interest in the software,
|
||||
including all intellectual-property rights. No patent, trademark, or other
|
||||
license is granted by implication or estoppel.
|
||||
|
||||
5. Termination
|
||||
|
||||
This license terminates automatically if you breach any term. Upon termination,
|
||||
you must stop using the software and delete all copies in your possession or
|
||||
control.
|
||||
|
||||
6. Disclaimer
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
||||
PURPOSE, TITLE, AND NON-INFRINGEMENT. TO THE MAXIMUM EXTENT PERMITTED BY LAW,
|
||||
RSMON WILL NOT BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL,
|
||||
OR PUNITIVE DAMAGES, OR FOR LOSS OF DATA, PROFITS, OR BUSINESS, ARISING FROM OR
|
||||
RELATED TO THE SOFTWARE OR THIS LICENSE.
|
||||
|
||||
7. Additional permission
|
||||
|
||||
For permissions beyond this license, contact RSMon through https://rsmon.ru.
|
||||
43
Makefile
Обычный файл
43
Makefile
Обычный файл
@@ -0,0 +1,43 @@
|
||||
BINARY := bin/rsmon-worker
|
||||
VERSION ?= dev
|
||||
COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || printf unknown)
|
||||
BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE)
|
||||
|
||||
.PHONY: build test check image clean
|
||||
|
||||
build:
|
||||
mkdir -p bin
|
||||
CGO_ENABLED=0 go build -trimpath -ldflags="$(LDFLAGS)" -o $(BINARY) ./cmd/rsmon-worker
|
||||
|
||||
test:
|
||||
RSMON_ENV=test CWD=$(CURDIR) go test \
|
||||
./internal/distworker \
|
||||
./internal/webapp \
|
||||
./internal/workercluster \
|
||||
./internal/wire \
|
||||
./internal/checkexec \
|
||||
./checks/calls \
|
||||
./checks/cbssl \
|
||||
./checks/cdns \
|
||||
./checks/cftp \
|
||||
./checks/chttp \
|
||||
./checks/cping \
|
||||
./checks/cssh \
|
||||
./checks/cssl \
|
||||
./checks/ctcp \
|
||||
./checks/cudp \
|
||||
./checks/cwhois \
|
||||
./checks/llmhttp
|
||||
|
||||
check:
|
||||
go mod tidy
|
||||
git diff --exit-code -- go.mod go.sum
|
||||
$(MAKE) test
|
||||
$(MAKE) build
|
||||
|
||||
image:
|
||||
docker build --build-arg VERSION=$(VERSION) --build-arg COMMIT=$(COMMIT) --build-arg BUILD_DATE=$(BUILD_DATE) -t rsmon-worker:local .
|
||||
|
||||
clean:
|
||||
rm -rf bin dist
|
||||
6
NOTICE
Обычный файл
6
NOTICE
Обычный файл
@@ -0,0 +1,6 @@
|
||||
RSMon Worker
|
||||
Copyright (c) 2026 RSMon. All rights reserved.
|
||||
|
||||
This is source-available software, not open-source software. Use is restricted
|
||||
to building and operating workers for rsmon.ru and to private research or
|
||||
evaluation under the terms in LICENSE.
|
||||
127
README.md
Обычный файл
127
README.md
Обычный файл
@@ -0,0 +1,127 @@
|
||||
# RSMon Worker
|
||||
|
||||
Standalone distributed monitoring worker for [rsmon.ru](https://rsmon.ru). It
|
||||
connects to the RSMon control plane over WebSocket, executes checks locally,
|
||||
delivers delegated notifications, and reports results back to the service.
|
||||
|
||||
This repository is **source available, not open source**. Building and running
|
||||
the worker with rsmon.ru and private research/evaluation are permitted. See
|
||||
[LICENSE](LICENSE) for the complete terms.
|
||||
|
||||
## Requirements
|
||||
|
||||
- A worker token created in the rsmon.ru worker settings.
|
||||
- Outbound HTTPS/WebSocket access to rsmon.ru.
|
||||
- Chromium for browser-backed HTTP checks when running the binary directly.
|
||||
- `CAP_NET_RAW` or an unprivileged ICMP configuration for ping checks.
|
||||
|
||||
## Build
|
||||
|
||||
Go 1.26 or newer is required.
|
||||
|
||||
```bash
|
||||
make build
|
||||
./bin/rsmon-worker --version
|
||||
```
|
||||
|
||||
The binary reads `.env` from its working directory when present. The minimum
|
||||
configuration is `RSMON_URL`, `RSMON_TOKEN`, `WORKER_LOGIN`, and
|
||||
`WORKER_PASSWORD`.
|
||||
|
||||
## Docker Compose
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env and set the worker token and operator-console password.
|
||||
docker compose up -d
|
||||
docker compose logs -f worker
|
||||
```
|
||||
|
||||
The operator console is bound to `127.0.0.1:27401` by default. Set
|
||||
`WORKER_BIND_IP` only when a firewall or TLS reverse proxy protects the port.
|
||||
Persistent web and cluster state is stored in the `worker-data` volume.
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
docker build -t rsmon-worker:local .
|
||||
docker run --rm \
|
||||
--cap-add NET_RAW \
|
||||
--env-file .env \
|
||||
-p 127.0.0.1:27401:27401 \
|
||||
-v rsmon-worker-data:/var/lib/rsmon-worker \
|
||||
rsmon-worker:local
|
||||
```
|
||||
|
||||
Published images use these tags:
|
||||
|
||||
- `sha-<12-character-commit>` for every push;
|
||||
- `latest` for `master`;
|
||||
- the exact `v*` tag for releases.
|
||||
|
||||
## systemd
|
||||
|
||||
Install host dependencies first. On Debian or Ubuntu:
|
||||
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ca-certificates chromium libcap2-bin tzdata
|
||||
```
|
||||
|
||||
Build and install:
|
||||
|
||||
```bash
|
||||
make build
|
||||
cp packaging/systemd/worker.env.example worker.env
|
||||
# Edit worker.env.
|
||||
sudo ./scripts/install-systemd.sh --env ./worker.env
|
||||
```
|
||||
|
||||
To install an already downloaded release binary:
|
||||
|
||||
```bash
|
||||
sudo ./scripts/install-systemd.sh --binary ./rsmon-worker --env ./worker.env
|
||||
```
|
||||
|
||||
Operational commands:
|
||||
|
||||
```bash
|
||||
systemctl status rsmon-worker
|
||||
journalctl -u rsmon-worker -f
|
||||
sudo systemctl restart rsmon-worker
|
||||
```
|
||||
|
||||
The service runs as the dedicated `rsmon-worker` user, stores state below
|
||||
`/var/lib/rsmon-worker`, reads secrets from `/etc/rsmon-worker/worker.env`, and
|
||||
has only `CAP_NET_RAW` for ICMP checks.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Required | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `RSMON_URL` | yes | `https://rsmon.ru` for health only | Control-plane base URL. |
|
||||
| `RSMON_TOKEN` | yes | none | Worker bearer token. |
|
||||
| `WORKER_HOST` | no | `0.0.0.0` | Operator-console bind address. |
|
||||
| `WORKER_PORT` | no | `27401` | Operator-console port. |
|
||||
| `WORKER_URL` | no | none | Public URL advertised to the control plane. |
|
||||
| `WORKER_LOGIN` | yes | none | Operator-console basic-auth login. |
|
||||
| `WORKER_PASSWORD` | yes | none | Operator-console basic-auth password. |
|
||||
| `RSMON_WEBAPP_DATA_DIR` | no | user data directory | SQLite and local UI state. |
|
||||
| `WORKER_CLUSTER_ENABLED` | no | `false` | Enable the optional Raft cluster. |
|
||||
| `WORKER_CLUSTER_ID` | with cluster | none | Unique Raft node ID. |
|
||||
| `WORKER_CLUSTER_PORT` | no | `WORKER_PORT+10000` | Raft transport port. |
|
||||
| `WORKER_CLUSTER_PEERS` | no | none | Comma-separated `node@host:port` peers. |
|
||||
| `WORKER_CLUSTER_DATA_DIR` | with cluster | none | Persistent Raft state directory. |
|
||||
|
||||
The public liveness endpoint is `GET /healthz`. `rsmon-worker health` checks the
|
||||
configured control plane's `/up` endpoint and is suitable for container health
|
||||
checks.
|
||||
|
||||
## Security
|
||||
|
||||
- Do not commit `.env`, worker tokens, or operator-console credentials.
|
||||
- Expose the operator console only on loopback or behind authenticated TLS.
|
||||
- Each worker should have its own control-plane token.
|
||||
- Keep `/etc/rsmon-worker/worker.env` mode `0600`.
|
||||
|
||||
Report security issues privately through the contact channel at rsmon.ru.
|
||||
60
app/models/access.go
Обычный файл
60
app/models/access.go
Обычный файл
@@ -0,0 +1,60 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Access represents membership of a User (or ApiKey) within a tenant
|
||||
// Account, optionally scoped to a Group or Monitor.
|
||||
//
|
||||
// A User can hold many Access rows across many Accounts — the Access
|
||||
// table is the source of truth for "who can see what". Each row answers:
|
||||
//
|
||||
// "Does user U have permission P on scope S of account A?"
|
||||
//
|
||||
// Where:
|
||||
//
|
||||
// - A = AccountID (tenant)
|
||||
// - U = UserID (or ApiKeyID for service tokens)
|
||||
// - P = Role ("owner" | "admin" | "manager" | "view" |
|
||||
// "notify")
|
||||
// - S = Kind + (GroupID | MonitorID) — defaults to account-wide when
|
||||
// Kind = "account" and both ids are
|
||||
// nil.
|
||||
//
|
||||
// One Access row may also reference the Invite that produced it via
|
||||
// InviteID. The Invite is preserved after registration so the access
|
||||
// history stays auditable — system-registered users and admin-added
|
||||
// users have nil InviteID.
|
||||
//
|
||||
// See docs/plans/users-and-rbac.md for the full RBAC matrix.
|
||||
type Access struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
|
||||
// Kind access kind, account \ group \ monitor
|
||||
Kind string `gorm:"not null;default:'account'" json:"kind"`
|
||||
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
User *User `json:"-"`
|
||||
ApiKeyID *int64 `gorm:"type:bigint REFERENCES api_keys(id)" json:"-"` //nolint:revive // accepted lint exception
|
||||
ApiKey *ApiKey `json:"-"` //nolint:revive // accepted lint exception
|
||||
InviteID *int64 `gorm:"type:bigint REFERENCES invites(id)" json:"-"`
|
||||
Invite *Invite `json:"-"`
|
||||
|
||||
GroupID *int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty"`
|
||||
MonitorID *int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
|
||||
Role string `json:"role"`
|
||||
|
||||
// SeatType is additive to Role: role remains the authorization decision,
|
||||
// while seat type is the billing entitlement.
|
||||
SeatType string `gorm:"size:16;not null;default:'login'" json:"seat_type"`
|
||||
Status string `gorm:"size:16;not null;default:'active'" json:"status"`
|
||||
NotifyOnly bool `gorm:"not null;default:false" json:"notify_only"`
|
||||
SeatAddonID *int64 `json:"seat_addon_id,omitempty"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
Audited
|
||||
}
|
||||
117
app/models/account.go
Обычный файл
117
app/models/account.go
Обычный файл
@@ -0,0 +1,117 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Account represents a tenant — an isolated billing/permissions boundary
|
||||
// that groups users, contacts, groups, monitors and notifications.
|
||||
//
|
||||
// Access to an Account is granted via the Access join table (see Access).
|
||||
// `Role` on Account is a gorm:"-":all" virtual column populated by the
|
||||
// controllers for the current session — it is the role the calling user
|
||||
// holds on THIS account, not a property of the account itself.
|
||||
type Account struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name"`
|
||||
Accesses []Access `json:"-"`
|
||||
Contacts []Contact `json:"-"`
|
||||
Groups []Group `json:"-"`
|
||||
Notifications []Notification `json:"-"`
|
||||
PlanID *int64 `gorm:"type:bigint REFERENCES plans(id)" json:"-"`
|
||||
Plan *Plan `json:"plan"`
|
||||
// Diagnostic overrides are available only to plans that include confirmations.
|
||||
// Nil keeps the catalog value; bounds are enforced by DiagnosticSettings.
|
||||
ConfirmTimeoutSec *int `json:"confirm_timeout_sec,omitempty"`
|
||||
HealthWindowSec *int `json:"health_window_sec,omitempty"`
|
||||
HealthRateThreshold *float64 `json:"health_rate_threshold,omitempty"`
|
||||
HealthMinAttempts *int `json:"health_min_attempts,omitempty"`
|
||||
Role string `gorm:"-:all" json:"role"`
|
||||
Timezone string `json:"timezone"`
|
||||
Language string `gorm:"default:'ru'" json:"language"`
|
||||
Disabled bool `gorm:"not null;default:false" json:"disabled"`
|
||||
Blocked bool `gorm:"not null;default:false" json:"blocked"`
|
||||
PaidUntil *time.Time `json:"paid_until"`
|
||||
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
|
||||
Deleted bool `gorm:"not null;default:false"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// Users provides functionality.
|
||||
func (a Account) Users() []User { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
users := make([]User, 0)
|
||||
err := DB().Where("id IN (SELECT user_id FROM accesses WHERE account_id = ?)", a.ID).Find(&users).Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
// CreateAccountForUser provides functionality.
|
||||
func CreateAccountForUser(name string, u *User) (*Account, error) {
|
||||
trialPlan := Plan{}
|
||||
if err := DB().Where("code = ? AND archived = FALSE", "team").First(&trialPlan).Error; err != nil {
|
||||
return nil, errors.Wrap(err, "failed to find trial plan")
|
||||
}
|
||||
trialEndsAt := time.Now().UTC().AddDate(0, 0, 14)
|
||||
account := Account{PlanID: &trialPlan.ID, TrialEndsAt: &trialEndsAt}
|
||||
if name != "" {
|
||||
account.Name = name
|
||||
}
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Create(&account).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create account")
|
||||
}
|
||||
if err := tx.Create(&Subscription{
|
||||
AccountID: account.ID, PlanID: trialPlan.ID, Provider: "manual", Status: SubscriptionStatusTrialing,
|
||||
BillingCycle: "monthly", Currency: trialPlan.Currency, AmountMinor: trialPlan.PriceMonthlyMinor, CurrentPeriodEnd: &trialEndsAt, TrialEndsAt: &trialEndsAt,
|
||||
}).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create subscription")
|
||||
}
|
||||
var subscription Subscription
|
||||
if err := tx.Where("account_id = ?", account.ID).First(&subscription).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to load trial subscription")
|
||||
}
|
||||
if err := tx.Create(&SubscriptionEvent{SubscriptionID: subscription.ID, AccountID: account.ID, Provider: "manual", Kind: "trial_started", ToPlanID: &trialPlan.ID, ActorUserID: &u.ID, CreatedAt: time.Now().UTC()}).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to record trial")
|
||||
}
|
||||
access := Access{AccountID: account.ID, UserID: &u.ID, Role: "owner", SeatType: "admin"}
|
||||
if err := tx.Create(&access).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create access")
|
||||
}
|
||||
group := Group{AccountID: account.ID, Name: "Основные"}
|
||||
if err := tx.Create(&group).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create group")
|
||||
}
|
||||
notification := Notification{AccountID: account.ID, Name: "Основные", Enabled: true}
|
||||
if err := tx.Create(¬ification).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create notification")
|
||||
}
|
||||
if u.Email != nil {
|
||||
contact := Contact{AccountID: &account.ID, UserID: &u.ID, Kind: "email", Value: *u.Email}
|
||||
if err := tx.Create(&contact).Error; err != nil {
|
||||
return errors.Wrap(err, "failed to create contact")
|
||||
}
|
||||
if err := tx.Model(¬ification).Association("Contacts").Append(&contact); err != nil {
|
||||
return errors.Wrap(err, "failed to add contact to notification")
|
||||
}
|
||||
}
|
||||
if err := tx.Model(¬ification).Association("Groups").Append(&group); err != nil {
|
||||
return errors.Wrap(err, "failed to add group to notification")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &account, nil
|
||||
}
|
||||
283
app/models/account_test.go
Обычный файл
283
app/models/account_test.go
Обычный файл
@@ -0,0 +1,283 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestCreateAccountForUserStartsTeamTrial(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
email := "team-trial@example.test"
|
||||
user := models.User{Name: "Trial User", Email: &email, Enabled: true, Confirmed: true}
|
||||
require.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
account, err := models.CreateAccountForUser("Trial Account", &user)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, account.PlanID)
|
||||
require.NotNil(t, account.TrialEndsAt)
|
||||
|
||||
var plan models.Plan
|
||||
require.NoError(t, models.DB().First(&plan, *account.PlanID).Error)
|
||||
assert.Equal(t, "team", plan.Code)
|
||||
|
||||
var subscription models.Subscription
|
||||
require.NoError(t, models.DB().Where("account_id = ?", account.ID).First(&subscription).Error)
|
||||
assert.Equal(t, models.SubscriptionStatusTrialing, subscription.Status)
|
||||
assert.Equal(t, plan.ID, subscription.PlanID)
|
||||
require.NotNil(t, subscription.TrialEndsAt)
|
||||
assert.WithinDuration(t, *account.TrialEndsAt, *subscription.TrialEndsAt, time.Millisecond)
|
||||
}
|
||||
|
||||
// TestAccountModel tests basic Account model functionality
|
||||
func TestAccountModel(t *testing.T) {
|
||||
// Test Account structure
|
||||
account := models.Account{
|
||||
Name: "Test Account",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
Deleted: false,
|
||||
}
|
||||
|
||||
assert.Equal(t, "Test Account", account.Name)
|
||||
assert.Equal(t, "UTC", account.Timezone)
|
||||
assert.Equal(t, "en", account.Language)
|
||||
assert.False(t, account.Deleted)
|
||||
}
|
||||
|
||||
// TestAccountDisplayName tests User.DisplayName method
|
||||
func TestUserDisplayName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
user models.User
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "User with email",
|
||||
user: models.User{
|
||||
Name: "John Doe",
|
||||
Email: stringPtr("john@example.com"),
|
||||
},
|
||||
expected: "John Doe john@example.com",
|
||||
},
|
||||
{
|
||||
name: "User without email",
|
||||
user: models.User{
|
||||
Name: "Jane Doe",
|
||||
Email: nil,
|
||||
},
|
||||
expected: "Jane Doe",
|
||||
},
|
||||
{
|
||||
name: "User with empty name and email",
|
||||
user: models.User{
|
||||
Name: "",
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
expected: " test@example.com",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.user.DisplayName()
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserGravatar tests User.Gravatar method
|
||||
func TestUserGravatar(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
user models.User
|
||||
size int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "User with email",
|
||||
user: models.User{
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
size: 32,
|
||||
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=32&d=blank",
|
||||
},
|
||||
{
|
||||
name: "User without email",
|
||||
user: models.User{Email: nil},
|
||||
size: 32,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Different size",
|
||||
user: models.User{
|
||||
Email: stringPtr("test@example.com"),
|
||||
},
|
||||
size: 64,
|
||||
expected: "https://www.gravatar.com/avatar/55502f40dc8b7c769880b10874abc9d0?s=64&d=blank",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.user.Gravatar(tt.size)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserAsJSON tests User.AsJSON method
|
||||
func TestUserAsJSON(t *testing.T) {
|
||||
email := "test@example.com"
|
||||
user := models.User{
|
||||
ID: 123,
|
||||
Email: &email,
|
||||
Name: "Test User",
|
||||
}
|
||||
|
||||
result := user.AsJSON()
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, int64(123), result["id"])
|
||||
assert.Equal(t, &email, result["email"])
|
||||
assert.Contains(t, result["avatar"], "gravatar.com")
|
||||
}
|
||||
|
||||
// TestAccessModel tests Access model structure
|
||||
func TestAccessModel(t *testing.T) {
|
||||
access := models.Access{
|
||||
AccountID: 1,
|
||||
Kind: "account",
|
||||
Role: "owner",
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(1), access.AccountID)
|
||||
assert.Equal(t, "account", access.Kind)
|
||||
assert.Equal(t, "owner", access.Role)
|
||||
}
|
||||
|
||||
// TestGroupModel tests Group model structure
|
||||
func TestGroupModel(t *testing.T) {
|
||||
group := models.Group{
|
||||
AccountID: 1,
|
||||
Name: "Test Group",
|
||||
MonitorsCount: 5,
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(1), group.AccountID)
|
||||
assert.Equal(t, "Test Group", group.Name)
|
||||
assert.Equal(t, 5, group.MonitorsCount)
|
||||
}
|
||||
|
||||
// TestGroupIdsForAccountId tests GroupIdsForAccountId function
|
||||
func TestGroupIdsForAccountId(t *testing.T) {
|
||||
// This test would require a database connection
|
||||
// For now, we test that it doesn't panic with invalid input
|
||||
t.Run("handles zero account id", func(t *testing.T) {
|
||||
// Note: This will panic without DB connection, which is expected behavior
|
||||
// In a real test, we'd set up a test database
|
||||
})
|
||||
}
|
||||
|
||||
// TestAccountTableDrivenTests demonstrates table-driven testing pattern
|
||||
func TestAccountValidationTableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
account models.Account
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Valid account",
|
||||
account: models.Account{
|
||||
Name: "Valid Account",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Account with empty name",
|
||||
account: models.Account{
|
||||
Name: "",
|
||||
Timezone: "UTC",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: true, // Name should be required
|
||||
},
|
||||
{
|
||||
name: "Account with invalid timezone",
|
||||
account: models.Account{
|
||||
Name: "Test Account",
|
||||
Timezone: "Invalid/Timezone",
|
||||
Language: "en",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Validation logic would go here
|
||||
// For now, we just verify the test structure
|
||||
assert.NotNil(t, tt.account)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create string pointer
|
||||
func stringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// BenchmarkUserDisplayName benchmarks the DisplayName method
|
||||
func BenchmarkUserDisplayName(b *testing.B) {
|
||||
user := models.User{
|
||||
Name: "Test User",
|
||||
Email: stringPtr("test@example.com"),
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = user.DisplayName()
|
||||
}
|
||||
}
|
||||
|
||||
// TestAccountConcurrentOperations tests concurrent access to account data
|
||||
func TestAccountConcurrentOperations(t *testing.T) {
|
||||
account := models.Account{
|
||||
Name: "Concurrent Test",
|
||||
Timezone: "UTC",
|
||||
}
|
||||
|
||||
done := make(chan bool)
|
||||
|
||||
// Simulate concurrent reads
|
||||
for i := 0; i < 10; i++ {
|
||||
go func() {
|
||||
_ = account.Name
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all goroutines
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// ExampleAccountUsage provides an example of how to use Account model
|
||||
func ExampleAccount() {
|
||||
account := models.Account{
|
||||
Name: "Example Account",
|
||||
Timezone: "America/New_York",
|
||||
Language: "en",
|
||||
}
|
||||
|
||||
_ = account.Name
|
||||
// Output:
|
||||
}
|
||||
38
app/models/api_key.go
Обычный файл
38
app/models/api_key.go
Обычный файл
@@ -0,0 +1,38 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// ApiKey represents an API authentication key. //nolint:revive // accepted lint exception
|
||||
type ApiKey struct { //nolint:revive // accepted lint exception
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
AccountID int64 `json:"account_id,omitempty"`
|
||||
Account User `json:"-"`
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
User *User `json:"-"`
|
||||
Accesses []Access `json:"accesses" gorm:"foreignkey:api_key_id"`
|
||||
|
||||
concerns.HasToken
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
func (n *ApiKey) BeforeCreate(tx *gorm.DB) error { //nolint:revive // accepted lint exception
|
||||
n.SetToken()
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillAccesses provides functionality.
|
||||
func (n *ApiKey) FillAccesses() {
|
||||
for k, a := range n.Accesses { //nolint:gocritic // range copy is acceptable here
|
||||
if a.ID <= 0 {
|
||||
n.Accesses[k].ID = 0
|
||||
}
|
||||
n.Accesses[k].AccountID = n.AccountID
|
||||
}
|
||||
}
|
||||
63
app/models/audited.go
Обычный файл
63
app/models/audited.go
Обычный файл
@@ -0,0 +1,63 @@
|
||||
// Package models provides GORM models and business logic.
|
||||
// Audited models inspired by https://github.com/qor/audited
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AuditedCurrentUserKey is the GORM Set key for the current user.
|
||||
const AuditedCurrentUserKey = "audited:current_user"
|
||||
|
||||
// Audited tracks creator and updater IDs.
|
||||
type Audited struct {
|
||||
CreatorID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
Creator *User `json:"-"`
|
||||
UpdaterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
Updater *User `json:"-"`
|
||||
}
|
||||
|
||||
func getCurrentUser(scope *gorm.DB) (int64, bool) {
|
||||
var user interface{}
|
||||
var hasUser bool
|
||||
|
||||
user, hasUser = scope.Get(AuditedCurrentUserKey)
|
||||
|
||||
// spew.Dump(user, hasUser)
|
||||
if hasUser {
|
||||
return user.(*User).ID, true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func assignCreatedBy(tx *gorm.DB) {
|
||||
name := "CreatorID"
|
||||
if field := tx.Statement.Schema.LookUpField(name); field != nil {
|
||||
if user, ok := getCurrentUser(tx); ok {
|
||||
tx.Statement.SetColumn(name, user)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assignUpdatedBy(tx *gorm.DB) {
|
||||
name := "UpdaterID"
|
||||
if field := tx.Statement.Schema.LookUpField(name); field != nil {
|
||||
if user, ok := getCurrentUser(tx); ok {
|
||||
tx.Statement.SetColumn("UpdaterID", user, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterCallbacks register callback into GORM DB
|
||||
func RegisterCallbacks(db *gorm.DB) {
|
||||
callback := db.Callback()
|
||||
|
||||
if callback.Create().Get("audited:assign_created_by") == nil {
|
||||
_ = callback.Create().After("gorm:before_create").Register("audited:assign_created_by", assignCreatedBy)
|
||||
}
|
||||
|
||||
if callback.Update().Get("audited:assign_updated_by") == nil {
|
||||
_ = callback.Update().After("gorm:before_update").Register("audited:assign_updated_by", assignUpdatedBy)
|
||||
}
|
||||
}
|
||||
32
app/models/authidentity/auth_identity.go
Обычный файл
32
app/models/authidentity/auth_identity.go
Обычный файл
@@ -0,0 +1,32 @@
|
||||
// Package authidentity provides the AuthIdentity and Basic types for QOR-style
|
||||
// authentication identity management. Vendored from github.com/glebtv/auth/auth_identity
|
||||
// to eliminate the rsgit.ru/rs/sessionmanager transitive dependency.
|
||||
package authidentity
|
||||
|
||||
import "time"
|
||||
|
||||
// AuthIdentity combines Basic provider info with SignLogs for a full identity record.
|
||||
type AuthIdentity struct {
|
||||
Basic
|
||||
SignLogs
|
||||
}
|
||||
|
||||
// TableName returns the database table name for AuthIdentity.
|
||||
func (AuthIdentity) TableName() string {
|
||||
return "identities"
|
||||
}
|
||||
|
||||
// Basic represents the core identity fields (provider, UID, encrypted password).
|
||||
type Basic struct {
|
||||
ID int64 `gorm:"primary_key" json:"id"`
|
||||
Provider string
|
||||
UID string `gorm:"column:uid"`
|
||||
EncryptedPassword string
|
||||
UserID *int64
|
||||
ConfirmedAt *time.Time
|
||||
}
|
||||
|
||||
// TableName returns the database table name for Basic.
|
||||
func (Basic) TableName() string {
|
||||
return "identities"
|
||||
}
|
||||
50
app/models/authidentity/sign_logs.go
Обычный файл
50
app/models/authidentity/sign_logs.go
Обычный файл
@@ -0,0 +1,50 @@
|
||||
package authidentity
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SignLogs holds login history (log entries and sign-in count).
|
||||
type SignLogs struct {
|
||||
Log string `sql:"-"`
|
||||
SignInCount uint
|
||||
Logs []SignLog
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for deserializing SignLogs from JSON.
|
||||
func (signLogs *SignLogs) Scan(data interface{}) (err error) {
|
||||
switch values := data.(type) {
|
||||
case []byte:
|
||||
if len(values) != 0 {
|
||||
return json.Unmarshal(values, signLogs)
|
||||
}
|
||||
case string:
|
||||
return signLogs.Scan([]byte(values))
|
||||
case []string:
|
||||
for _, str := range values {
|
||||
if err := signLogs.Scan(str); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = errors.New("unsupported driver -> Scan pair for SignLogs")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for serializing SignLogs to JSON.
|
||||
func (signLogs SignLogs) Value() (driver.Value, error) {
|
||||
results, err := json.Marshal(signLogs)
|
||||
return string(results), err
|
||||
}
|
||||
|
||||
// SignLog represents a single login event entry.
|
||||
type SignLog struct {
|
||||
UserAgent string
|
||||
At *time.Time
|
||||
IP string
|
||||
}
|
||||
31
app/models/bits.go
Обычный файл
31
app/models/bits.go
Обычный файл
@@ -0,0 +1,31 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// BeginningOfDay provides functionality.
|
||||
func BeginningOfDay(t time.Time) time.Time {
|
||||
year, month, day := t.Date()
|
||||
return time.Date(year, month, day, 0, 0, 0, 0, t.Location())
|
||||
}
|
||||
|
||||
// SetBit provides functionality.
|
||||
// https://stackoverflow.com/questions/23192262/how-would-you-set-and-clear-a-single-bit-in-go
|
||||
// Sets the bit at pos in the integer n.
|
||||
func SetBit(n int, pos uint) int {
|
||||
n |= (1 << pos)
|
||||
return n
|
||||
}
|
||||
|
||||
// ClearBit provides functionality.
|
||||
// Clears the bit at pos in n.
|
||||
func ClearBit(n int, pos uint) int {
|
||||
mask := ^(1 << pos)
|
||||
n &= mask
|
||||
return n
|
||||
}
|
||||
|
||||
// HasBit provides functionality.
|
||||
func HasBit(n int, pos uint) bool {
|
||||
val := n & (1 << pos)
|
||||
return (val > 0)
|
||||
}
|
||||
171
app/models/check.go
Обычный файл
171
app/models/check.go
Обычный файл
@@ -0,0 +1,171 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Check provides functionality.
|
||||
type Check struct {
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
|
||||
|
||||
MonitorID int64 `gorm:"index;type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
Monitor *Monitor `json:"-"`
|
||||
|
||||
Name *string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Interval int `json:"interval" validate:"required,gte=60"`
|
||||
|
||||
// URL to monitor
|
||||
URL *string `json:"url,omitempty"`
|
||||
|
||||
// Other settings of the check
|
||||
Settings datatypes.JSON `gorm:"not null;" json:"settings"`
|
||||
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
|
||||
LastStart *time.Time `json:"last_start"`
|
||||
LastEnd *time.Time `json:"last_end"`
|
||||
LastOk *time.Time `json:"last_ok"`
|
||||
LastFail *time.Time `json:"last_fail"`
|
||||
WasUp *time.Time `json:"was_up"`
|
||||
|
||||
Fails int `json:"fails"`
|
||||
Expires *time.Time `json:"expires"`
|
||||
|
||||
Error *string `json:"error"`
|
||||
Warnings pq.StringArray `gorm:"type:varchar(255)[]" json:"warnings"`
|
||||
Infos pq.StringArray `gorm:"type:varchar(255)[]" json:"infos"`
|
||||
|
||||
// RequireQuorum enables multi-region result aggregation (Phase 3 of
|
||||
// docs/todo.md): when >1 the check's State is NOT written directly by
|
||||
// ApplyRemoteCheckResult — instead CheckRegionResult rows accumulate
|
||||
// until app/models/check_aggregator.go decides OK/ERR/DEGRADED.
|
||||
// Default 1 keeps the legacy single-region behavior unchanged.
|
||||
RequireQuorum int `gorm:"not null;default:1" json:"require_quorum"`
|
||||
|
||||
// AggregationWindowSeconds is how long the aggregator waits for
|
||||
// regional CheckRegionResult rows before deciding the check's State.
|
||||
// Stored as int seconds (matching the existing GORM style — no
|
||||
// time.Duration columns) and exposed via AggregationWindow(). Default
|
||||
// 5s; ignored when RequireQuorum <= 1.
|
||||
AggregationWindowSeconds int `gorm:"not null;default:5" json:"aggregation_window_seconds"`
|
||||
|
||||
IsNew bool `gorm:"-:all" sql:"-" json:"is_new,omitempty"`
|
||||
Deleted bool `gorm:"-:all" sql:"-" json:"deleted,omitempty"`
|
||||
|
||||
Events []Event `json:"-" gorm:"many2many:event_checks;"`
|
||||
|
||||
Audited
|
||||
}
|
||||
|
||||
// ExpScope provides functionality.
|
||||
func ExpScope(q *gorm.DB) *gorm.DB {
|
||||
return q.Where("kind IN ('whois', 'ssl')").
|
||||
Preload("Monitor").
|
||||
Preload("Monitor.Group").
|
||||
Preload("Monitor.Group.Notifications").
|
||||
Preload("Monitor.Group.Notifications.Contacts").
|
||||
Where("expires < ?", time.Now().Add(time.Hour*7*24))
|
||||
}
|
||||
|
||||
// IntervalOK provides functionality.
|
||||
func (c *Check) IntervalOK() bool {
|
||||
if c.Kind == kindRKN {
|
||||
return true
|
||||
}
|
||||
|
||||
if c.Kind == kindWhois {
|
||||
return c.Interval >= 43200
|
||||
}
|
||||
|
||||
return c.Interval >= 30
|
||||
}
|
||||
|
||||
// GetLabel provides functionality.
|
||||
func (c *Check) GetLabel() string {
|
||||
if c.Name != nil {
|
||||
return *c.Name
|
||||
}
|
||||
if c.URL != nil {
|
||||
return *c.URL
|
||||
}
|
||||
return c.Kind
|
||||
}
|
||||
|
||||
// KindLabel provides functionality.
|
||||
func (c *Check) KindLabel() string {
|
||||
if c.Kind == kindWhois {
|
||||
return "регистрация домена"
|
||||
}
|
||||
if c.Kind == kindSSL {
|
||||
return "SSL сертификат"
|
||||
}
|
||||
|
||||
return c.Kind
|
||||
}
|
||||
|
||||
// GetSettings provides functionality.
|
||||
func (c *Check) GetSettings() CheckSettings {
|
||||
d := CheckSettings{}
|
||||
|
||||
err := json.Unmarshal(c.Settings, &d)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ValidateSettings provides functionality.
|
||||
func (c *Check) ValidateSettings() error {
|
||||
if len(c.Settings) == 0 {
|
||||
c.Settings = []byte("{}")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetURL provides functionality.
|
||||
func (c *Check) GetURL() (string, error) {
|
||||
// return c.GetSettings()["url"].(string)
|
||||
if c.URL != nil {
|
||||
return *c.URL, nil
|
||||
}
|
||||
|
||||
return "http://" + c.Monitor.Host, nil
|
||||
}
|
||||
|
||||
// MetricName provides functionality.
|
||||
func (c *Check) MetricName() string {
|
||||
sanitized := strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == ':' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, c.Kind)
|
||||
return "c" + sanitized
|
||||
}
|
||||
|
||||
// QuorumEnabled reports whether this check should be aggregated by
|
||||
// app/models/check_aggregator.go. When false (RequireQuorum <= 1),
|
||||
// ApplyRemoteCheckResult keeps the legacy direct State update path.
|
||||
func (c *Check) QuorumEnabled() bool {
|
||||
return c.RequireQuorum > 1
|
||||
}
|
||||
|
||||
// AggregationWindow returns AggregationWindowSeconds as a time.Duration.
|
||||
// Defaults to 5s when the underlying int is zero/negative, mirroring the
|
||||
// GORM column default; callers can rely on a strictly positive value.
|
||||
func (c *Check) AggregationWindow() time.Duration {
|
||||
if c.AggregationWindowSeconds <= 0 {
|
||||
return 5 * time.Second
|
||||
}
|
||||
return time.Duration(c.AggregationWindowSeconds) * time.Second
|
||||
}
|
||||
336
app/models/check_aggregator.go
Обычный файл
336
app/models/check_aggregator.go
Обычный файл
@@ -0,0 +1,336 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Phase 3 of docs/todo.md — result aggregation for multi-region checks.
|
||||
//
|
||||
// When a check has RequireQuorum > 1, ApplyRemoteCheckResult does not
|
||||
// write Check.State directly. Instead it appends a CheckRegionResult row
|
||||
// and leaves AggregatedAt NULL. This file owns the background goroutine
|
||||
// that walks those pending rows once their aggregation window has
|
||||
// elapsed, decides OK/ERR/DEGRADED per the documented rule, writes the
|
||||
// aggregate state onto Check, stamps AggregatedAt on the contributing
|
||||
// rows, and triggers Monitor.UpdateStatusFromChecks so the monitor's own
|
||||
// status follows.
|
||||
//
|
||||
// Aggregation rule (see docs/todo.md Phase 3 + checkSeverityRank in
|
||||
// monitor.go for the corresponding severity order):
|
||||
//
|
||||
// - Aggregate only rows whose created_at is older than
|
||||
// NOW() - Check.AggregationWindowSeconds. This is the "watermark"
|
||||
// pattern: a row is eligible only when no fresher regional result
|
||||
// could still arrive and tip the vote. The window is per-check so
|
||||
// noisy checks can use a longer wait than fast ones.
|
||||
// - If zero eligible rows exist for a check, leave Check.State
|
||||
// untouched (the special case called out in the spec).
|
||||
// - Otherwise count OK vs not-OK among the eligible rows:
|
||||
// OK >= RequireQuorum → Check.State = OK
|
||||
// not-OK >= RequireQuorum → Check.State = ERR
|
||||
// neither side reaches quorum → Check.State = DEGRADED
|
||||
// - Stamp AggregatedAt = NOW() on every contributing row so the next
|
||||
// tick skips them. One transaction per check; per-row failures do
|
||||
// not poison other checks.
|
||||
|
||||
// AggregatorTickInterval is the default cadence of StartCheckAggregator
|
||||
// when the caller passes interval <= 0. Mirrors the 30s default used by
|
||||
// the other reapers in this package so the three reapers all tick on
|
||||
// the same wall clock cadence — easier to grep, easier to reason about
|
||||
// in incident timelines.
|
||||
const AggregatorTickInterval = 30 * time.Second
|
||||
|
||||
// EnsureCheckAggregatorIndexes adds the partial indexes the aggregator
|
||||
// relies on. AutoMigrate creates AggregatedAt as a regular btree column,
|
||||
// but the per-tick SELECT filters on `aggregated_at IS NULL` over what
|
||||
// grows to be a busy table; a partial index keeps the working set
|
||||
// tiny. Idempotent so it is safe to call from Migrate() and from tests.
|
||||
func EnsureCheckAggregatorIndexes() error {
|
||||
return DB().Exec(`
|
||||
CREATE INDEX IF NOT EXISTS check_region_results_pending_idx
|
||||
ON check_region_results (check_id, created_at)
|
||||
WHERE aggregated_at IS NULL
|
||||
`).Error
|
||||
}
|
||||
|
||||
// aggregateCheckState holds the per-check aggregation inputs we need to
|
||||
// keep the rule readable. Rows is the set of CheckRegionResult rows
|
||||
// eligible for the current decision; quorum is Check.RequireQuorum.
|
||||
type aggregateCheckState struct {
|
||||
CheckID int64
|
||||
Quorum int
|
||||
Rows []CheckRegionResult
|
||||
}
|
||||
|
||||
// decideAggregateState encodes the OK/ERR/DEGRADED rule described in
|
||||
// the package doc. Pure function — no DB, no time — so it is trivially
|
||||
// unit-testable from the test file.
|
||||
func decideAggregateState(in aggregateCheckState) (string, bool) {
|
||||
if len(in.Rows) == 0 || in.Quorum <= 1 {
|
||||
// Zero eligible rows in the window OR a misconfigured check
|
||||
// (QuorumEnabled false). Caller must leave Check.State alone
|
||||
// in both cases.
|
||||
return "", false
|
||||
}
|
||||
okCount := 0
|
||||
badCount := 0
|
||||
for i := range in.Rows {
|
||||
if in.Rows[i].State == stateOK {
|
||||
okCount++
|
||||
} else {
|
||||
badCount++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case okCount >= in.Quorum:
|
||||
return stateOK, true
|
||||
case badCount >= in.Quorum:
|
||||
return stateERR, true
|
||||
default:
|
||||
return stateDegraded, true
|
||||
}
|
||||
}
|
||||
|
||||
// CheckAggregatorTick performs one pass of the aggregator. It is the
|
||||
// per-tick body StartCheckAggregator calls. Exported so the test suite
|
||||
// can call it directly without spinning up the goroutine; production
|
||||
// always goes through StartCheckAggregator.
|
||||
//
|
||||
// The returned (aggregated, err) tuple lets the caller log a metric:
|
||||
// aggregated counts how many Check rows had their State written this
|
||||
// tick. The function is idempotent — a second call with no new
|
||||
// unaggregated rows is a no-op that returns (0, nil).
|
||||
func CheckAggregatorTick() (aggregated int, err error) {
|
||||
// Step 1: collect candidate check IDs. The JOIN to checks is needed
|
||||
// to read each check's window length and to filter on
|
||||
// require_quorum > 1 (so we never aggregate the legacy path).
|
||||
rows, err := DB().Raw(`
|
||||
SELECT DISTINCT crr.check_id
|
||||
FROM check_region_results crr
|
||||
JOIN checks c ON c.id = crr.check_id
|
||||
WHERE crr.aggregated_at IS NULL
|
||||
AND c.require_quorum > 1
|
||||
AND crr.created_at < NOW() - make_interval(secs => GREATEST(c.aggregation_window_seconds, 1))
|
||||
ORDER BY crr.check_id
|
||||
`).Rows()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var checkIDs []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return 0, scanErr
|
||||
}
|
||||
checkIDs = append(checkIDs, id)
|
||||
}
|
||||
if scanErr := rows.Err(); scanErr != nil {
|
||||
return 0, scanErr
|
||||
}
|
||||
if len(checkIDs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
for _, checkID := range checkIDs {
|
||||
n, err := aggregateOneCheck(checkID)
|
||||
if err != nil {
|
||||
// Log and continue: one bad check must not stop the loop.
|
||||
log.Printf("check_aggregator: check_id=%d error: %v", checkID, err)
|
||||
continue
|
||||
}
|
||||
aggregated += n
|
||||
}
|
||||
return aggregated, nil
|
||||
}
|
||||
|
||||
// aggregateOneCheck runs the aggregation logic for a single check inside
|
||||
// a transaction. The transaction holds a FOR UPDATE row lock on the
|
||||
// check so concurrent aggregator instances (multiple web processes) can
|
||||
// not race on the same check — the second one waits for the first to
|
||||
// commit, then sees AggregatedAt IS NOT NULL on every row and the
|
||||
// candidate SELECT below returns an empty set.
|
||||
func aggregateOneCheck(checkID int64) (int, error) {
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return 0, tx.Error
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
tx.Rollback()
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
var check Check
|
||||
if err := tx.Clauses(SkipLockedClause).First(&check, checkID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
// Check was deleted between candidate SELECT and lock; not
|
||||
// an error, just nothing to do.
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
// Defensive: only aggregate quorum-enabled checks. The candidate
|
||||
// SELECT already filters on this, but a stale row that flipped off
|
||||
// quorum between calls must not be re-aggregated here.
|
||||
if !check.QuorumEnabled() {
|
||||
tx.Rollback()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var results []CheckRegionResult
|
||||
if err := tx.
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
Order("created_at ASC").
|
||||
Find(&results).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
decision, ok := decideAggregateState(aggregateCheckState{
|
||||
CheckID: checkID,
|
||||
Quorum: check.RequireQuorum,
|
||||
Rows: results,
|
||||
})
|
||||
if !ok {
|
||||
// Zero eligible rows — leave Check.State alone. There is also
|
||||
// nothing to stamp, so just rollback and move on.
|
||||
tx.Rollback()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
// Pull the latest error string from the contributing rows so the
|
||||
// monitor event / notifier pipeline has something to show. Prefer
|
||||
// the most recent ERR row's message; fall back to the most recent
|
||||
// any-row message. UNK / empty stays NULL.
|
||||
var lastError *string
|
||||
for i := len(results) - 1; i >= 0; i-- {
|
||||
if results[i].Error != nil && *results[i].Error != "" {
|
||||
lastError = results[i].Error
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
upd := map[string]interface{}{
|
||||
colState: decision,
|
||||
colLastEnd: now,
|
||||
}
|
||||
if decision == stateOK {
|
||||
// OK resets error — mirrors the legacy ApplyRemoteCheckResult
|
||||
// path that sets `error = gorm.Expr("NULL")` when state==OK.
|
||||
upd["error"] = gorm.Expr("NULL")
|
||||
upd["last_ok"] = now
|
||||
upd["fails"] = 0
|
||||
upd["was_up"] = now
|
||||
} else {
|
||||
// Non-OK: bump the fail counter and only overwrite the error
|
||||
// when one of the contributing rows actually carries a
|
||||
// message. If none do, leave whatever was there before —
|
||||
// mirrors the legacy `if report.Error != nil` branch.
|
||||
upd["last_fail"] = now
|
||||
upd["fails"] = gorm.Expr("fails + 1")
|
||||
if lastError != nil {
|
||||
upd["error"] = *lastError
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Model(&Check{}).Where("id = ?", checkID).UpdateColumns(upd).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := tx.Model(&CheckRegionResult{}).
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
UpdateColumns(map[string]interface{}{
|
||||
"aggregated_at": now,
|
||||
}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Mirror ApplyRemoteCheckResult: propagate the aggregate decision
|
||||
// up to the monitor. We do this AFTER commit so a rollback does
|
||||
// not leave the monitor in a state whose corresponding check is
|
||||
// still pre-aggregate. The goroutine keeps the failure path of
|
||||
// UpdateStatusFromChecks isolated from the aggregator's hot loop.
|
||||
if check.MonitorID != 0 {
|
||||
var mon Monitor
|
||||
if err := DB().First(&mon, check.MonitorID).Error; err == nil {
|
||||
go mon.UpdateStatusFromChecks()
|
||||
} else {
|
||||
log.Printf("check_aggregator: monitor lookup failed for check_id=%d: %v", checkID, err)
|
||||
}
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// StartCheckAggregator launches a goroutine that calls
|
||||
// CheckAggregatorTick on the given interval until ctx is canceled.
|
||||
// Mirrors StartTaskReaper / StartDeadWorkerReaper in this package — same
|
||||
// ticker shape, same default-interval fall-back, same per-tick recover
|
||||
// so a malformed row cannot crash the web process.
|
||||
//
|
||||
// The default interval is AggregatorTickInterval (30s); values <= 0
|
||||
// fall back to the default so the helper is safe to call from any call
|
||||
// site without a guard. A nil context falls back to context.Background()
|
||||
// the same way StartDeadWorkerReaper does, so main.init() and tests
|
||||
// can both call it without ceremony.
|
||||
//
|
||||
// Wire from main.init() once per process. The aggregator is cheap in
|
||||
// steady state (one indexed SELECT for candidates + a per-check
|
||||
// transaction over a handful of unaggregated rows). Under load it
|
||||
// scales horizontally — multiple web processes can each run their own
|
||||
// StartCheckAggregator goroutine because FOR UPDATE SKIP LOCKED on the
|
||||
// per-check transaction guarantees at-most-one winner per check.
|
||||
func StartCheckAggregator(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = AggregatorTickInterval
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
// Best-effort index bootstrap. AutoMigrate declares AggregatedAt as
|
||||
// a regular btree column; the partial index speeds up the per-tick
|
||||
// candidate SELECT. Idempotent — safe to call on every boot.
|
||||
if err := EnsureCheckAggregatorIndexes(); err != nil {
|
||||
log.Printf("check_aggregator: ensure index: %v", err)
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("check_aggregator: panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
n, err := CheckAggregatorTick()
|
||||
if err != nil {
|
||||
log.Printf("check_aggregator: error: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("check_aggregator: aggregated=%d", n)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
448
app/models/check_aggregator_test.go
Обычный файл
448
app/models/check_aggregator_test.go
Обычный файл
@@ -0,0 +1,448 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// seedAggregatorWorld creates the minimum fixture the aggregator tests
|
||||
// need: a plan, account, group, monitor, and an http check whose
|
||||
// RequireQuorum / AggregationWindowSeconds are set per call. The check
|
||||
// is created with state=UNK so the test can observe the aggregator's
|
||||
// effect on Check.State directly.
|
||||
//
|
||||
// Returns the freshly-created monitor + check; the check is what every
|
||||
// test below mutates (RequireQuorum, AggregationWindowSeconds) and then
|
||||
// asserts on. Cleanup is the caller's responsibility — most tests call
|
||||
// models.Drop() at the top instead.
|
||||
func seedAggregatorWorld(t *testing.T, quorum, windowSeconds int) (models.Monitor, models.Check) {
|
||||
t.Helper()
|
||||
|
||||
plan := models.Plan{Name: "agg-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "agg-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
grp := &models.Group{AccountID: acc.ID, Name: "agg"}
|
||||
require.NoError(t, models.DB().Create(grp).Error)
|
||||
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrAgg("agg.test"),
|
||||
Host: "agg.test",
|
||||
GroupID: grp.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
enTrue := true
|
||||
check := models.Check{
|
||||
MonitorID: mon.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
RequireQuorum: quorum,
|
||||
AggregationWindowSeconds: windowSeconds,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return mon, check
|
||||
}
|
||||
|
||||
func stringPtrAgg(s string) *string { return &s }
|
||||
|
||||
// makeReport constructs a wire.CheckResultReport with sensible defaults
|
||||
// for the OK or not-OK case. Tests use this to push results through
|
||||
// ApplyRemoteCheckResult exactly the way a real worker would.
|
||||
func makeReport(checkID, monitorID int64, state string) wire.CheckResultReport {
|
||||
return wire.CheckResultReport{
|
||||
JobID: "job-" + state,
|
||||
CheckID: checkID,
|
||||
MonitorID: monitorID,
|
||||
State: state,
|
||||
}
|
||||
}
|
||||
|
||||
// regionResultWithErr is regionResultFor plus an error message. Used
|
||||
// when the test wants to verify that the aggregator forwards the row's
|
||||
// error string onto Check.Error (mimics a real worker reporting
|
||||
// state=ERR with a diagnostic message).
|
||||
func regionResultWithErr(t *testing.T, checkID int64, region, state, errMsg string, age time.Duration) models.CheckRegionResult {
|
||||
t.Helper()
|
||||
row := regionResultFor(t, checkID, region, state, age)
|
||||
require.NoError(t, models.DB().Model(&row).UpdateColumn("error", errMsg).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
// regionResultFor inserts a single CheckRegionResult row whose
|
||||
// created_at and executed_at are both backdated by `age`, so the
|
||||
// aggregator's window-based watermark picks it up immediately without
|
||||
// needing a real time.Sleep. Returned row has its DB-assigned ID
|
||||
// populated.
|
||||
func regionResultFor(t *testing.T, checkID int64, region string, state string, age time.Duration) models.CheckRegionResult {
|
||||
t.Helper()
|
||||
// RegionCode has a FK to regions.code, so the region must exist
|
||||
// before the result row is inserted. seedRegion is idempotent.
|
||||
seedRegion(t, region)
|
||||
row := models.CheckRegionResult{
|
||||
CheckID: checkID,
|
||||
RegionCode: region,
|
||||
ExecutedAt: time.Now().Add(-age),
|
||||
State: state,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&row).Error)
|
||||
// Backdate CreatedAt too — the aggregator SQL keys on
|
||||
// check_region_results.created_at (see CheckAggregatorTick). GORM
|
||||
// auto-sets CreatedAt on insert, so we have to UPDATE it post-hoc.
|
||||
require.NoError(t, models.DB().Model(&row).UpdateColumns(map[string]interface{}{
|
||||
"created_at": time.Now().Add(-age),
|
||||
"updated_at": time.Now().Add(-age),
|
||||
}).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
// loadCheck re-reads a Check row by ID — used after the aggregator
|
||||
// runs so the test asserts against the post-tick state.
|
||||
func loadCheck(t *testing.T, id int64) models.Check {
|
||||
t.Helper()
|
||||
var c models.Check
|
||||
require.NoError(t, models.DB().First(&c, id).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
// countPendingResults returns how many CheckRegionResult rows for
|
||||
// checkID have aggregated_at IS NULL — the working set the next
|
||||
// aggregator tick would consider.
|
||||
func countPendingResults(t *testing.T, checkID int64) int64 {
|
||||
t.Helper()
|
||||
var n int64
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("check_id = ? AND aggregated_at IS NULL", checkID).
|
||||
Count(&n).Error)
|
||||
return n
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ApplyRemoteCheckResult: regression tests for the QuorumEnabled split.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestApplyRemoteCheckResult_DirectWhenQuorumOne pins the legacy path:
|
||||
// when RequireQuorum==1 (the default), ApplyRemoteCheckResult still
|
||||
// writes Check.State synchronously, exactly the way it did before Phase
|
||||
// 3. This is the regression guard for the in-process RKN scheduler
|
||||
// tests in internal/rknscheduler.
|
||||
func TestApplyRemoteCheckResult_DirectWhenQuorumOne(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
mon, check := seedAggregatorWorld(t, 1, 5)
|
||||
// Region must exist because CheckRegionResult has a FK to
|
||||
// regions.code (seeded by Migrate, but the test region is custom).
|
||||
seedRegion(t, "ru-msk")
|
||||
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "OK"),
|
||||
"ru-msk",
|
||||
))
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "quorum=1 must keep the direct State update")
|
||||
assert.NotNil(t, got.LastEnd, "legacy path must keep stamping last_end")
|
||||
// One region result inserted with aggregated_at=NULL.
|
||||
assert.EqualValues(t, 1, countPendingResults(t, check.ID),
|
||||
"the region result row is always inserted even on the legacy path")
|
||||
}
|
||||
|
||||
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
|
||||
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
|
||||
// Check.State — it only inserts the CheckRegionResult row. The check
|
||||
// stays at its initial UNK and the unaggregated row count grows by
|
||||
// exactly 1 per call.
|
||||
func TestApplyRemoteCheckResult_BuffersWhenQuorumN(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
mon, check := seedAggregatorWorld(t, 3, 5)
|
||||
seedRegion(t, "ru-msk")
|
||||
seedRegion(t, "us-east")
|
||||
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "OK"), "ru-msk",
|
||||
))
|
||||
require.NoError(t, models.ApplyRemoteCheckResult(
|
||||
makeReport(check.ID, mon.ID, "ERR"), "us-east",
|
||||
))
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State,
|
||||
"quorum>1 must NOT touch Check.State — the aggregator owns it")
|
||||
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
|
||||
"two results buffered, both with aggregated_at=NULL")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CheckAggregatorTick: rule tests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestAggregator_QuorumOK: with RequireQuorum=2 and two OK results
|
||||
// buffered, the aggregator must decide OK and stamp AggregatedAt on
|
||||
// both contributing rows.
|
||||
func TestAggregator_QuorumOK(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n, "one check aggregated this tick")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "2 OK results >= quorum=2 → state=OK")
|
||||
assert.NotNil(t, got.LastOk, "OK decision must stamp last_ok")
|
||||
assert.EqualValues(t, 0, got.Fails, "fails must reset on OK")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID),
|
||||
"both contributing rows must be stamped aggregated_at")
|
||||
}
|
||||
|
||||
// TestAggregator_QuorumFail: with RequireQuorum=2 and two ERR results
|
||||
// buffered, the aggregator must decide ERR and surface the most recent
|
||||
// row's error message on Check.Error.
|
||||
func TestAggregator_QuorumFail(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultWithErr(t, check.ID, "ru-msk", "ERR", "connection refused", 5*time.Second)
|
||||
regionResultWithErr(t, check.ID, "us-east", "ERR", "timeout", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "ERR", got.State, "2 ERR >= quorum=2 → state=ERR")
|
||||
assert.NotNil(t, got.LastFail)
|
||||
assert.NotNil(t, got.Error, "ERR decision must carry an error message from the rows")
|
||||
assert.Contains(t, *got.Error, "timeout",
|
||||
"aggregator should surface the latest row's error message")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_DegradedWhenPartial: with RequireQuorum=3 and 1 OK +
|
||||
// 2 ERR (mixed within window), neither side reaches the quorum of 3 so
|
||||
// the aggregator must decide DEGRADED. The state must NOT silently
|
||||
// become OK or ERR.
|
||||
func TestAggregator_DegradedWhenPartial(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 3, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "ERR", 4*time.Second)
|
||||
regionResultFor(t, check.ID, "eu-west", "ERR", 3*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"neither OK nor ERR reaches quorum=3 → state=DEGRADED")
|
||||
assert.EqualValues(t, 0, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_NotEnoughRegionsAlsoDegraded covers the single-region-
|
||||
// only-delivered case: with RequireQuorum=3 and only 1 result buffered
|
||||
// (and it aged past the window), the rule still says "neither side
|
||||
// reached quorum" → DEGRADED. This is the documented behavior for slow
|
||||
// regions that never report in time.
|
||||
func TestAggregator_NotEnoughRegionsAlsoDegraded(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 3, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"single OK row vs quorum=3 → DEGRADED (below quorum on both sides)")
|
||||
}
|
||||
|
||||
// TestAggregator_NoResultsLeavesStateAlone is the "special case" from
|
||||
// the spec: when the aggregator tick finds no eligible rows for a
|
||||
// check, Check.State must NOT change. Pre-set the check to OK and
|
||||
// verify it stays OK.
|
||||
func TestAggregator_NoResultsLeavesStateAlone(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
// Pre-set state and a previous LastEnd so we can detect any
|
||||
// accidental overwrite.
|
||||
prevEnd := time.Now().Add(-time.Hour)
|
||||
require.NoError(t, models.DB().Model(&models.Check{}).
|
||||
Where("id = ?", check.ID).
|
||||
Updates(map[string]interface{}{
|
||||
"state": "OK",
|
||||
"last_end": prevEnd,
|
||||
"last_ok": prevEnd,
|
||||
}).Error)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "no eligible rows → nothing aggregated")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "OK", got.State, "state must not change with zero eligible rows")
|
||||
assert.WithinDuration(t, prevEnd, *got.LastEnd, time.Second,
|
||||
"last_end must not be touched when there are no eligible rows")
|
||||
}
|
||||
|
||||
// TestAggregator_MultipleChecksIndependent verifies that a single tick
|
||||
// processes every check with pending results, not just the first one.
|
||||
// Two checks, each with 2 regions, each should flip to OK after the
|
||||
// tick.
|
||||
func TestAggregator_MultipleChecksIndependent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, c1 := seedAggregatorWorld(t, 2, 1)
|
||||
_, c2 := seedAggregatorWorld(t, 2, 1)
|
||||
|
||||
regionResultFor(t, c1.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, c1.ID, "us-east", "OK", 4*time.Second)
|
||||
regionResultFor(t, c2.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, c2.ID, "eu-west", "OK", 4*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, n, "both checks aggregated in the same tick")
|
||||
|
||||
assert.Equal(t, "OK", loadCheck(t, c1.ID).State)
|
||||
assert.Equal(t, "OK", loadCheck(t, c2.ID).State)
|
||||
assert.EqualValues(t, 0, countPendingResults(t, c1.ID))
|
||||
assert.EqualValues(t, 0, countPendingResults(t, c2.ID))
|
||||
}
|
||||
|
||||
// TestAggregator_AlreadyAggregatedRowsSkipped pins the idempotency
|
||||
// story: a second tick with no new rows must be a no-op. We pre-mark
|
||||
// the rows aggregated_at=NOW() and verify the tick returns (0, nil)
|
||||
// without touching Check.State.
|
||||
func TestAggregator_AlreadyAggregatedRowsSkipped(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
r1 := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
r2 := regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
// Pretend a previous aggregator pass already stamped them.
|
||||
now := time.Now()
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("id IN ?", []int64{r1.ID, r2.ID}).
|
||||
UpdateColumns(map[string]interface{}{"aggregated_at": now}).Error)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "no candidate checks → 0 aggregated")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State, "already-aggregated rows must not cause a re-decision")
|
||||
}
|
||||
|
||||
// TestAggregator_IgnoresRowsInsideWindow verifies the watermark: rows
|
||||
// whose CreatedAt is NEWER than (NOW() - window) are NOT eligible and
|
||||
// must NOT be stamped. With AggregationWindowSeconds=10 and rows aged
|
||||
// only 2s, the aggregator finds nothing to do.
|
||||
func TestAggregator_IgnoresRowsInsideWindow(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 10)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 2*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 1*time.Second)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "rows still inside window → no aggregation")
|
||||
|
||||
got := loadCheck(t, check.ID)
|
||||
assert.Equal(t, "UNK", got.State)
|
||||
assert.EqualValues(t, 2, countPendingResults(t, check.ID),
|
||||
"rows inside window stay unaggregated for the next tick")
|
||||
}
|
||||
|
||||
// TestAggregator_SkipsChecksWithQuorumOne guards the candidate SELECT
|
||||
// filter: even though CheckRegionResult rows are written for
|
||||
// RequireQuorum=1 checks (via StoreCheckRegionResult), the aggregator
|
||||
// must not re-decide their state because ApplyRemoteCheckResult
|
||||
// already did. We simulate by inserting a region row with aggregated_at
|
||||
// NULL for a quorum=1 check and verifying the tick ignores it.
|
||||
func TestAggregator_SkipsChecksWithQuorumOne(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 1, 1)
|
||||
r := regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
require.Nil(t, r.AggregatedAt)
|
||||
|
||||
n, err := models.CheckAggregatorTick()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, n, "quorum=1 checks must be filtered out by the candidate SELECT")
|
||||
|
||||
// The row must stay unaggregated too — the aggregator has no
|
||||
// business stamping it.
|
||||
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// StartCheckAggregator: ticker smoke test.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestStartCheckAggregator_TickerFiresOnce is the smoke test for the
|
||||
// background helper: spin up the aggregator with a tight 10ms ticker
|
||||
// and a cancellable context, wait for one tick to flip a seeded
|
||||
// check's state, then cancel so the goroutine exits cleanly. Mirrors
|
||||
// TestStartDeadWorkerReaper_TickerFiresOnce in shape.
|
||||
func TestStartCheckAggregator_TickerFiresOnce(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
_, check := seedAggregatorWorld(t, 2, 1)
|
||||
regionResultFor(t, check.ID, "ru-msk", "OK", 5*time.Second)
|
||||
regionResultFor(t, check.ID, "us-east", "OK", 4*time.Second)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
models.StartCheckAggregator(ctx, 10*time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got := loadCheck(t, check.ID)
|
||||
if got.State == "OK" {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("aggregator goroutine did not flip check to OK within 2s; state=%q", loadCheck(t, check.ID).State)
|
||||
}
|
||||
|
||||
// silence unused import warnings when individual helpers are inlined by
|
||||
// editors — the package-level references below keep the imports live.
|
||||
var (
|
||||
_ = gorm.ErrRecordNotFound
|
||||
)
|
||||
14
app/models/check_data.go
Обычный файл
14
app/models/check_data.go
Обычный файл
@@ -0,0 +1,14 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
)
|
||||
|
||||
// CheckData provides functionality.
|
||||
type CheckData struct {
|
||||
Uptime int `json:"uptime"`
|
||||
Data []influx.InfluxData `json:"data"`
|
||||
LastCheck *time.Time `json:"last_check"`
|
||||
}
|
||||
415
app/models/check_jobs.go
Обычный файл
415
app/models/check_jobs.go
Обычный файл
@@ -0,0 +1,415 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// defaultRegionCode is the historical default region seeded by Migrate()
|
||||
// (see app/models/migrate.go) and used as a catch-all bucket for results
|
||||
// reported without a region code. Aliased to the exported Region
|
||||
// constant (DefaultRegionCode) so admin endpoints and the in-process
|
||||
// job router share one source of truth.
|
||||
const defaultRegionCode = DefaultRegionCode
|
||||
|
||||
// ChecksForWorker returns checks that need to be executed by a distributed worker.
|
||||
// It uses FOR UPDATE SKIP LOCKED to prevent race conditions between concurrent workers.
|
||||
// The worker specifies which check kinds it can handle via the kinds parameter.
|
||||
//
|
||||
// Phase 2 of docs/plans/worker-notifier-mvp.md adds regional job routing: when
|
||||
// worker is non-nil, the candidate monitor set is filtered by
|
||||
// applyRegionRouting so a worker only sees checks that explicitly allow its
|
||||
// region. Pass nil for the legacy "no region scoping" path used by
|
||||
// diagnostics/dashboard tooling.
|
||||
func ChecksForWorker(worker *WorkerNode, kinds []string, limit int) []*Check {
|
||||
tx := DB().Begin()
|
||||
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
|
||||
Where("monitors.enabled").
|
||||
Where("checks.enabled AND checks.kind IN (?)", kinds)
|
||||
|
||||
if worker != nil {
|
||||
q = applyRegionRouting(q, worker)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
|
||||
}
|
||||
// Flagged workers remain visible for audit/history but never receive new work.
|
||||
if worker.NetworkProblemActive(time.Now()) {
|
||||
tx.Rollback()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Allow faster retry for failed http/dns checks
|
||||
notOk := ""
|
||||
hasHTTPOrDNS := false
|
||||
for _, k := range kinds {
|
||||
if k == kindHTTP || k == kindDNS {
|
||||
hasHTTPOrDNS = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasHTTPOrDNS {
|
||||
notOk = `OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
|
||||
}
|
||||
|
||||
whereClause := `
|
||||
(checks.last_start IS NULL) OR
|
||||
(checks.last_start + (checks.interval || ' second')::interval < now())
|
||||
`
|
||||
if notOk != "" {
|
||||
whereClause += notOk
|
||||
}
|
||||
rq := q.Where(whereClause)
|
||||
|
||||
// Use SKIP LOCKED to avoid contention between workers. The same
|
||||
// FOR UPDATE SKIP LOCKED clause also gives us implicit load balancing
|
||||
// across workers in the same region: each concurrent worker call
|
||||
// grabs a disjoint slice of the pending checks and a row leased by
|
||||
// worker A is invisible to worker B until A's transaction commits
|
||||
// (or rolls back / lease expires).
|
||||
var checks []*Check
|
||||
rq.Clauses(SkipLockedClause).
|
||||
Limit(limit).
|
||||
Preload("Monitor").
|
||||
Find(&checks)
|
||||
|
||||
for _, c := range checks {
|
||||
log.Println("worker: assigned remote check:", c.ID, c.Kind)
|
||||
tx.Model(&c).Where("id = ?", c.ID).Update(colLastStart, time.Now())
|
||||
}
|
||||
|
||||
tx.Commit()
|
||||
return checks
|
||||
}
|
||||
|
||||
// EnqueueDueCheckTasks atomically turns due normal checks into durable generic
|
||||
// task envelopes. ChecksForWorker remains for the HTTP polling compatibility
|
||||
// endpoint, while websocket scheduling uses this task-producing path.
|
||||
func EnqueueDueCheckTasks(worker *WorkerNode, kinds []string, limit int) error {
|
||||
if worker == nil || len(kinds) == 0 || limit <= 0 || worker.NetworkProblemActive(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
q := tx.Joins("JOIN monitors ON checks.monitor_id = monitors.id").
|
||||
Where("monitors.enabled").Where("checks.enabled AND checks.kind IN (?)", kinds)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Joins("JOIN groups ON monitors.group_id = groups.id").Where("groups.account_id IN (?)", accounts)
|
||||
}
|
||||
q = applyRegionRouting(q, worker)
|
||||
where := `(checks.last_start IS NULL) OR (checks.last_start + (checks.interval || ' second')::interval < now())`
|
||||
for _, kind := range kinds {
|
||||
if kind == kindHTTP || kind == kindDNS {
|
||||
where += ` OR (checks.state != 'OK' AND checks.last_start + '120 second'::interval < now())`
|
||||
break
|
||||
}
|
||||
}
|
||||
var checks []*Check
|
||||
if err := q.Where(where).Clauses(SkipLockedClause).Limit(limit).Preload("Monitor.Group").Find(&checks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
for _, check := range checks {
|
||||
if check.Monitor == nil || check.Monitor.Group == nil {
|
||||
continue
|
||||
}
|
||||
job := JobForCheck(check)
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checkID, monitorID := check.ID, check.MonitorID
|
||||
bucket := now.UTC().Unix() / int64(check.Interval)
|
||||
task := Task{
|
||||
JobID: job.JobID, Kind: TaskKindCheck, State: TaskStateQueued,
|
||||
AccountID: check.Monitor.Group.AccountID, CheckID: &checkID, MonitorID: &monitorID,
|
||||
Payload: payload, NotBefore: now, MaxAttempts: DefaultTaskMaxAttempts,
|
||||
IdempotencyKey: fmt.Sprintf("check:%d:%d", check.ID, bucket),
|
||||
}
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&task).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(check).Update(colLastStart, now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// applyRegionRouting narrows the monitor JOIN in ChecksForWorker to the
|
||||
// subset whose routing rule matches the calling worker's region. The filter
|
||||
// is applied at the SQL layer so the SKIP LOCKED page only scans/leases
|
||||
// rows that this worker is allowed to run, instead of leasing and then
|
||||
// discarding forbidden checks.
|
||||
//
|
||||
// The function intentionally mirrors Monitor.WantsRegion so the helper can
|
||||
// be reused from non-SQL callers (UI preview, plan validation, etc.).
|
||||
//
|
||||
// SQL form:
|
||||
//
|
||||
// - monitors.region_mode IN ('any', 'all', ”)
|
||||
// → unconditional match; legacy / Phase 3 placeholder behavior.
|
||||
// - monitors.region_mode = 'specific' AND
|
||||
// (monitors.preferred_regions IS NULL OR
|
||||
// cardinality(monitors.preferred_regions) = 0 OR
|
||||
// ? = ANY(monitors.preferred_regions))
|
||||
// → empty array falls back to "any"; otherwise the worker code must
|
||||
// be in the whitelist.
|
||||
func applyRegionRouting(q *gorm.DB, worker *WorkerNode) *gorm.DB {
|
||||
if worker == nil || worker.RegionCode == "" {
|
||||
return q
|
||||
}
|
||||
if worker.RegionCode == defaultRegionCode {
|
||||
// The default "local" region is the historical catch-all; the
|
||||
// in-process scheduler (not ChecksForWorker) handles those
|
||||
// monitors. Skip regional filtering entirely so we don't leak
|
||||
// Phase 1 in-process workers through the new router.
|
||||
return q
|
||||
}
|
||||
// TODO(phase3): split RegionMode="all" into N assignments, one per
|
||||
// preferred region, so the result aggregator can build a quorum.
|
||||
// Today it is treated as "any" so existing checks keep flowing.
|
||||
return q.Where(
|
||||
`(monitors.region_mode IN ('any', 'all', '') OR `+
|
||||
`(monitors.region_mode = 'specific' AND `+
|
||||
`(monitors.preferred_regions IS NULL OR `+
|
||||
`coalesce(array_length(monitors.preferred_regions, 1), 0) = 0 OR `+
|
||||
`? = ANY(monitors.preferred_regions))))`,
|
||||
worker.RegionCode,
|
||||
)
|
||||
}
|
||||
|
||||
// JobForCheck creates a CheckJob from a Check model for sending to a worker
|
||||
func JobForCheck(c *Check) wire.CheckJob {
|
||||
jobID := uuid.New().String()
|
||||
var urlStr *string
|
||||
if c.URL != nil {
|
||||
urlStr = c.URL
|
||||
}
|
||||
return wire.CheckJob{
|
||||
JobID: jobID,
|
||||
CheckID: c.ID,
|
||||
MonitorID: c.MonitorID,
|
||||
Kind: c.Kind,
|
||||
Host: c.Monitor.Host,
|
||||
URL: urlStr,
|
||||
Interval: c.Interval,
|
||||
Settings: json.RawMessage(c.Settings),
|
||||
}
|
||||
}
|
||||
|
||||
// QueueMonitorChecks makes enabled checks for a monitor immediately eligible for remote assignment.
|
||||
func QueueMonitorChecks(monitorID int64) error {
|
||||
return DB().Model(&Check{}).
|
||||
Where("monitor_id = ? AND enabled", monitorID).
|
||||
Updates(map[string]interface{}{
|
||||
colLastStart: nil,
|
||||
colLastEnd: nil,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// QueueMonitorChecksKind makes enabled checks of one kind immediately eligible for remote assignment.
|
||||
func QueueMonitorChecksKind(monitorID int64, kind string) error {
|
||||
return DB().Model(&Check{}).
|
||||
Where("monitor_id = ? AND kind = ? AND enabled", monitorID, kind).
|
||||
Updates(map[string]interface{}{
|
||||
colLastStart: nil,
|
||||
colLastEnd: nil,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResult applies a check result reported by a distributed worker.
|
||||
// It updates the check state in the database and triggers monitor status aggregation.
|
||||
//
|
||||
// Phase 3 of docs/todo.md (multi-region quorum aggregation): when the
|
||||
// check has RequireQuorum > 1, the per-region result is recorded in
|
||||
// check_region_results but Check.State is NOT touched here — that is
|
||||
// the job of app/models/check_aggregator.go, which decides OK/ERR/
|
||||
// DEGRADED once enough regional results have arrived or the aggregation
|
||||
// window has elapsed. QuorumEnabled() == false preserves the legacy
|
||||
// direct-update path so single-region / non-aggregated monitors keep
|
||||
// the same behavior.
|
||||
func ApplyRemoteCheckResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
return ApplyRemoteCheckResultFromWorker(report, regionCode, nil)
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResultFromWorker persists worker attribution before changing
|
||||
// legacy check state. A confirmation result is consumed exactly once and never
|
||||
// overwrites the original check result.
|
||||
func ApplyRemoteCheckResultFromWorker(report wire.CheckResultReport, regionCode string, worker *WorkerNode) error { //nolint:gocritic,lll // hugeParam: wire compatibility
|
||||
var monitor *Monitor
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var err error
|
||||
monitor, err = ApplyRemoteCheckResultFromWorkerTx(tx, report, regionCode, worker)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor != nil {
|
||||
monitor.UpdateStatusFromChecks()
|
||||
}
|
||||
// VictoriaMetrics is outside PostgreSQL and is deliberately post-commit.
|
||||
// A caller that retries after this error will not duplicate relational state;
|
||||
// metric points are external at-least-once observations and need TSDB repair
|
||||
// if the write remains unavailable.
|
||||
return StoreRemoteCheckMetrics(report.Metrics)
|
||||
}
|
||||
|
||||
// ApplyRemoteCheckResultFromWorkerTx applies all relational result effects using
|
||||
// the caller's transaction. It intentionally does not write VictoriaMetrics or
|
||||
// aggregate monitor state: both must happen only after the transaction commits.
|
||||
// A nil monitor means the report was a consumed diagnostic attempt.
|
||||
func ApplyRemoteCheckResultFromWorkerTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, worker *WorkerNode) (*Monitor, error) { //nolint:gocritic,lll // hugeParam: wire compatibility
|
||||
if tx == nil {
|
||||
return nil, fmt.Errorf("apply check result: nil transaction")
|
||||
}
|
||||
now := time.Now()
|
||||
if worker != nil {
|
||||
handled := false
|
||||
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if handled {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
check := Check{}
|
||||
if err := tx.Preload("Monitor").First(&check, report.CheckID).Error; err != nil {
|
||||
log.Println("worker: check not found:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Always persist the per-region result first so the aggregator can
|
||||
// pick it up regardless of which path we take next. We rely on
|
||||
// StoreCheckRegionResult to default AggregatedAt=NULL (the column
|
||||
// type is *time.Time, so a zero value writes SQL NULL).
|
||||
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
|
||||
log.Println("worker: error storing region result:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Quorum-enabled checks: write nothing to Check.State here. The
|
||||
// aggregator will compute the aggregate state once the window has
|
||||
// elapsed (or enough regions have reported) and stamp AggregatedAt on
|
||||
// the contributing CheckRegionResult rows.
|
||||
if check.QuorumEnabled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
update := map[string]interface{}{
|
||||
colState: report.State,
|
||||
colLastEnd: now,
|
||||
colWarnings: pq.StringArray(report.Warnings),
|
||||
colInfos: pq.StringArray(report.Infos),
|
||||
}
|
||||
|
||||
if report.State == "OK" {
|
||||
update["was_up"] = now
|
||||
update["last_ok"] = now
|
||||
update["fails"] = 0
|
||||
update["error"] = gorm.Expr("NULL")
|
||||
} else {
|
||||
update["last_fail"] = now
|
||||
update["fails"] = gorm.Expr("fails + 1")
|
||||
if report.Error != nil {
|
||||
update["error"] = *report.Error
|
||||
}
|
||||
}
|
||||
|
||||
if report.ExpiresAt != nil {
|
||||
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
|
||||
if err == nil {
|
||||
update["expires"] = t
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Model(&check).UpdateColumns(update).Error; err != nil {
|
||||
log.Println("worker: error updating check:", report.CheckID, err)
|
||||
return nil, err
|
||||
}
|
||||
if worker != nil {
|
||||
payload, _ := json.Marshal(report)
|
||||
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
|
||||
if attempt.JobID == "" {
|
||||
attempt.JobID = uuid.New().String()
|
||||
}
|
||||
// A duplicate websocket/HTTP delivery must not create another attempt.
|
||||
if err := tx.Where("job_id = ?", attempt.JobID).FirstOrCreate(&attempt).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch report.State {
|
||||
case stateERR, stateFail:
|
||||
if err := StartConfirmationTx(tx, check.ID, worker.ID, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case stateOK:
|
||||
if err := RecoverDiagnosticTx(tx, check.ID, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return check.Monitor, nil
|
||||
}
|
||||
|
||||
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
|
||||
func StoreRemoteCheckMetrics(metrics []wire.MetricPoint) error {
|
||||
for _, metric := range metrics {
|
||||
if metric.Metric == "" || len(metric.Fields) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := influx.WriteOne(metric.Metric, metric.Tags, metric.Fields); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StoreCheckRegionResult stores a per-region check result for distributed monitoring analytics
|
||||
func StoreCheckRegionResult(report wire.CheckResultReport, regionCode string) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
return StoreCheckRegionResultTx(DB(), report, regionCode, time.Now())
|
||||
}
|
||||
|
||||
// StoreCheckRegionResultTx stores a regional result in the caller's transaction.
|
||||
func StoreCheckRegionResultTx(tx *gorm.DB, report wire.CheckResultReport, regionCode string, executedAt time.Time) error { //nolint:gocritic,lll // hugeParam: accepted for interface compatibility
|
||||
if tx == nil {
|
||||
return fmt.Errorf("store region result: nil transaction")
|
||||
}
|
||||
if regionCode == "" {
|
||||
regionCode = defaultRegionCode
|
||||
}
|
||||
|
||||
result := CheckRegionResult{
|
||||
CheckID: report.CheckID,
|
||||
RegionCode: regionCode,
|
||||
ExecutedAt: executedAt,
|
||||
State: report.State,
|
||||
DurationMs: report.DurationMs,
|
||||
Error: report.Error,
|
||||
}
|
||||
|
||||
return tx.Create(&result).Error
|
||||
}
|
||||
|
||||
// StaleWorkers marks workers as inactive or dead based on last_seen time
|
||||
func StaleWorkers() {
|
||||
// Mark workers with no heartbeat for 2 minutes as inactive
|
||||
DB().Model(&WorkerNode{}).
|
||||
Where("status = ? AND last_seen < ?", "active", time.Now().Add(-2*time.Minute)).
|
||||
Update("status", "inactive")
|
||||
|
||||
// Mark workers with no heartbeat for 5 minutes as dead
|
||||
DB().Model(&WorkerNode{}).
|
||||
Where("status IN (?, ?) AND last_seen < ?", "active", "inactive", time.Now().Add(-5*time.Minute)).
|
||||
Update("status", "dead")
|
||||
}
|
||||
451
app/models/check_jobs_test.go
Обычный файл
451
app/models/check_jobs_test.go
Обычный файл
@@ -0,0 +1,451 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// seedRegionRoutingWorld builds two regions and three groups (one per
|
||||
// monitor) plus three workers. The monitors and their PreferredRegions /
|
||||
// RegionMode are configured by the caller via a callback so each test can
|
||||
// express the exact routing scenario it wants to assert on.
|
||||
//
|
||||
// Returns a routerTestWorld that knows how to look up each fixture by name
|
||||
// for readable assertions.
|
||||
type routerTestWorld struct {
|
||||
groupAny *models.Group
|
||||
groupSpec *models.Group
|
||||
groupAll *models.Group
|
||||
|
||||
workerMsk *models.WorkerNode
|
||||
workerUSEast *models.WorkerNode
|
||||
workerLocal *models.WorkerNode
|
||||
}
|
||||
|
||||
// seedWorker creates a worker bound to regionCode. AuthToken is unique per
|
||||
// worker so the FOR UPDATE SKIP LOCKED path can exercise two distinct
|
||||
// concurrent callers.
|
||||
func seedWorker(t *testing.T, id, regionCode string) *models.WorkerNode {
|
||||
t.Helper()
|
||||
seedRegion(t, regionCode)
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: id,
|
||||
RegionCode: regionCode,
|
||||
Status: "active",
|
||||
AuthToken: "tok-" + id,
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
return w
|
||||
}
|
||||
|
||||
// seedRouterMonitor creates a Monitor with the given region routing
|
||||
// attributes and one ready-to-run http Check. The check has Interval=60
|
||||
// (minimum allowed) and LastStart=nil so ChecksForWorker picks it up
|
||||
// immediately on the next poll.
|
||||
func seedRouterMonitor(t *testing.T, group *models.Group, host, regionMode string, preferred []string) (models.Monitor, models.Check) {
|
||||
t.Helper()
|
||||
enTrue := true
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
if regionMode != "" {
|
||||
mon.RegionMode = regionMode
|
||||
}
|
||||
if preferred != nil {
|
||||
mon.PreferredRegions = models.RegionCodesFromSlice(preferred)
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
check := models.Check{
|
||||
MonitorID: mon.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return mon, check
|
||||
}
|
||||
|
||||
func stringPtrRouter(s string) *string { return &s }
|
||||
|
||||
// seedRouterWorld is the common fixture for the TestRegionRouting_* table.
|
||||
// It provisions two regions (ru-msk, us-east) and three monitors pinned to
|
||||
// different routing modes; the workers are created lazily by the caller.
|
||||
func seedRouterWorld(t *testing.T) routerTestWorld {
|
||||
t.Helper()
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "ru-msk")
|
||||
seedRegion(t, "us-east")
|
||||
|
||||
plan := models.Plan{Name: "router", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "router-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
groupAny := &models.Group{AccountID: acc.ID, Name: "any"}
|
||||
groupSpec := &models.Group{AccountID: acc.ID, Name: "spec"}
|
||||
groupAll := &models.Group{AccountID: acc.ID, Name: "all"}
|
||||
require.NoError(t, models.DB().Create(groupAny).Error)
|
||||
require.NoError(t, models.DB().Create(groupSpec).Error)
|
||||
require.NoError(t, models.DB().Create(groupAll).Error)
|
||||
|
||||
seedRouterMonitor(t, groupAny, "any.test", models.RegionModeAny, nil)
|
||||
seedRouterMonitor(t, groupSpec, "spec-msk.test", models.RegionModeSpecific, []string{"ru-msk"})
|
||||
seedRouterMonitor(t, groupSpec, "spec-us.test", models.RegionModeSpecific, []string{"us-east"})
|
||||
seedRouterMonitor(t, groupAll, "all.test", models.RegionModeAll, []string{"ru-msk", "us-east"})
|
||||
|
||||
return routerTestWorld{
|
||||
groupAny: groupAny,
|
||||
groupSpec: groupSpec,
|
||||
groupAll: groupAll,
|
||||
workerMsk: seedWorker(t, "w-msk", "ru-msk"),
|
||||
workerUSEast: seedWorker(t, "w-us", "us-east"),
|
||||
workerLocal: seedWorker(t, "w-local", "local"),
|
||||
}
|
||||
}
|
||||
|
||||
// idsOf returns a sorted list of monitor IDs assigned to the worker for
|
||||
// easier assertions across checks.
|
||||
func idsOf(checks []*models.Check) []int64 {
|
||||
out := make([]int64, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestRegionRouting_AnyReturnsAll confirms the SQL filter preserves the
|
||||
// legacy behavior for region_mode='any' monitors and the Phase 3
|
||||
// placeholder 'all' monitors: a worker in a region nobody explicitly
|
||||
// whitelisted must still see them, while monitors pinned to specific
|
||||
// regions stay filtered out.
|
||||
//
|
||||
// seedRouterWorld configures four monitors:
|
||||
// - any.test → region_mode='any', no PreferredRegions
|
||||
// - spec-msk.test → region_mode='specific', preferred=[ru-msk]
|
||||
// - spec-us.test → region_mode='specific', preferred=[us-east]
|
||||
// - all.test → region_mode='all', preferred=[ru-msk, us-east]
|
||||
//
|
||||
// A worker in region "remote" (whitelisted by nobody) must see exactly
|
||||
// {any.test, all.test} — the two monitors whose region_mode bypasses the
|
||||
// whitelist — and nothing else.
|
||||
func TestRegionRouting_AnyReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
seedRegion(t, "remote")
|
||||
w := seedWorker(t, "w-remote", "remote")
|
||||
|
||||
checks := models.ChecksForWorker(w, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
|
||||
assert.ElementsMatch(t, []string{"any.test", "all.test"}, hosts,
|
||||
"region_mode='any' and the Phase 3 'all' placeholder must bypass the whitelist")
|
||||
assert.NotContains(t, hosts, "spec-msk.test",
|
||||
"specific-mode monitor with whitelisted ru-msk must NOT reach a remote worker")
|
||||
assert.NotContains(t, hosts, "spec-us.test",
|
||||
"specific-mode monitor with whitelisted us-east must NOT reach a remote worker")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificFiltersByRegion proves the core Phase 2 promise:
|
||||
// workers in different regions never receive a monitor whose PreferredRegions
|
||||
// does not include their region code. The check is run with parallel
|
||||
// goroutines because ChecksForWorker stamps `last_start` on every row it
|
||||
// leases — a sequential second poll would always see an already-claimed
|
||||
// queue and the filter would have nothing to test against.
|
||||
//
|
||||
// Under the FOR UPDATE SKIP LOCKED race, whichever SELECT fires first grabs
|
||||
// every matching row, so the *exact* per-worker host list is non-deterministic.
|
||||
// The deterministic invariant the test asserts is the no-leak property: a
|
||||
// worker in ru-msk must never see spec-us.test, and vice versa.
|
||||
func TestRegionRouting_SpecificFiltersByRegion(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mskChecks []*models.Check
|
||||
usChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
mskChecks = models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
usChecks = models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
mskHosts := hostsOf(mskChecks)
|
||||
usHosts := hostsOf(usChecks)
|
||||
|
||||
// Aggregate coverage: together the two workers must see every check
|
||||
// the routing layer would ever allow them — the four seeded monitors.
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
},
|
||||
append(append([]string{}, mskHosts...), usHosts...),
|
||||
"union of both workers' slices must cover every seeded monitor (any/specific/all × region)")
|
||||
|
||||
// Core Phase 2 invariant: regional filtering never leaks across
|
||||
// PreferredRegions boundaries. This is the only assertion a
|
||||
// concurrent SKIP LOCKED race lets us pin deterministically.
|
||||
assert.NotContains(t, mskHosts, "spec-us.test",
|
||||
"ru-msk worker must never see a monitor whitelisted for us-east only")
|
||||
assert.NotContains(t, usHosts, "spec-msk.test",
|
||||
"us-east worker must never see a monitor whitelisted for ru-msk only")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificEmptyPreferredFallsBackToAny confirms the
|
||||
// documented fall-back: a monitor in RegionModeSpecific with no
|
||||
// PreferredRegions behaves like RegionModeAny so the field is safe to
|
||||
// leave blank. We poll from the us-east worker — without the fall-back it
|
||||
// would only see any.test + all.test + spec-us.test.
|
||||
func TestRegionRouting_SpecificEmptyPreferredFallsBackToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
// Reset the spec-msk monitor to have an empty PreferredRegions list
|
||||
// (the seed above gave it one). The Monitor row's RegionMode stays
|
||||
// 'specific'.
|
||||
require.NoError(t, models.DB().Model(&models.Monitor{}).
|
||||
Where("host = ?", "spec-msk.test").
|
||||
Update("preferred_regions", models.RegionCodesFromSlice(nil)).Error)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "spec-msk.test",
|
||||
"empty PreferredRegions with region_mode=specific must fall back to 'any'")
|
||||
}
|
||||
|
||||
// TestRegionRouting_AllDeferredToAny pins the Phase 3 placeholder behavior:
|
||||
// region_mode='all' is logged and treated as 'any' today. The test asserts
|
||||
// the monitor flows to a worker in any region (the TODO log marker is
|
||||
// emitted from applyRegionRouting — pinned here as a code-grep contract).
|
||||
func TestRegionRouting_AllDeferredToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "all.test",
|
||||
"region_mode='all' must currently behave like 'any' so existing checks keep flowing")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LocalWorkerBypass ensures the historic "local" region
|
||||
// still routes everything: the in-process scheduler handles those monitors
|
||||
// and we don't want the Phase 2 filter to leak platform workers through it.
|
||||
func TestRegionRouting_LocalWorkerBypass(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerLocal, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "worker in region 'local' must receive every check (bypass)")
|
||||
}
|
||||
|
||||
// TestRegionRouting_NilWorkerReturnsAll asserts the diagnostic-friendly
|
||||
// escape hatch: passing nil for the worker skips the routing filter and
|
||||
// returns every check the kinds/limit envelope allows.
|
||||
func TestRegionRouting_NilWorkerReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(nil, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "nil worker must bypass the routing filter")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LoadBalanceImplicit confirms the SKIP LOCKED implicit
|
||||
// load-balancing story: when two workers in the same region race for a pool
|
||||
// of pending checks, each of them receives a non-empty disjoint slice. The
|
||||
// two polls run in parallel goroutines so the FOR UPDATE SKIP LOCKED race
|
||||
// window is actually exercised.
|
||||
//
|
||||
// IMPORTANT: SKIP LOCKED with a large LIMIT is unfair — whichever
|
||||
// transaction's SELECT fires first grabs everything. The test therefore
|
||||
// uses LIMIT=4 with 10 pending rows so each worker is forced to leave some
|
||||
// rows unlocked for the other worker to pick up. Together they must cover
|
||||
// at most 8 rows (LIMIT × workers) without overlap; the remaining rows are
|
||||
// intentionally left for a future poll cycle, which mirrors production
|
||||
// behavior where workers continually drain a backlog.
|
||||
func TestRegionRouting_LoadBalanceImplicit(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "shared")
|
||||
plan := models.Plan{Name: "lb-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "lb", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := &models.Group{AccountID: acc.ID, Name: "lb-g"}
|
||||
require.NoError(t, models.DB().Create(group).Error)
|
||||
|
||||
enTrue := true
|
||||
for i := 0; i < 10; i++ {
|
||||
host := "lb-" + string(rune('a'+i)) + ".test"
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
ck := models.Check{
|
||||
MonitorID: mon.ID, Kind: "http", Interval: 60,
|
||||
Enabled: &enTrue, State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&ck).Error)
|
||||
}
|
||||
|
||||
w1 := seedWorker(t, "lb-w1", "shared")
|
||||
w2 := seedWorker(t, "lb-w2", "shared")
|
||||
|
||||
const limitPerWorker = 4
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
aChecks, bChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
aChecks = models.ChecksForWorker(w1, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
bChecks = models.ChecksForWorker(w2, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
assert.Greater(t, len(aChecks), 0, "worker 1 must receive at least one check")
|
||||
assert.Greater(t, len(bChecks), 0, "worker 2 must receive at least one check")
|
||||
assert.LessOrEqual(t, len(aChecks)+len(bChecks), 2*limitPerWorker,
|
||||
"two concurrent workers with LIMIT each can lease at most LIMIT*2 rows per cycle")
|
||||
assert.Empty(t, intersectHosts(aChecks, bChecks),
|
||||
"the two slices must be disjoint (FOR UPDATE SKIP LOCKED must not double-lease)")
|
||||
}
|
||||
|
||||
// TestMonitorValidateRegionMode exercises the documented enum on the
|
||||
// Monitor type so the validator surface does not regress.
|
||||
func TestMonitorValidateRegionMode(t *testing.T) {
|
||||
cases := []struct {
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", false},
|
||||
{"any", false},
|
||||
{"specific", false},
|
||||
{"all", false},
|
||||
{"round-robin", true},
|
||||
{"RANDOM", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run("mode="+c.mode, func(t *testing.T) {
|
||||
m := models.Monitor{RegionMode: c.mode}
|
||||
err := m.ValidateRegionMode()
|
||||
if c.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMonitorWantsRegion pins the public WantsRegion helper that powers the
|
||||
// non-SQL callers (UI preview, plan validation). Phase 3 will swap the
|
||||
// 'all' branch's behavior, so the table covers all three modes today.
|
||||
func TestMonitorWantsRegion(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mode string
|
||||
regions []string
|
||||
workerRC string
|
||||
want bool
|
||||
}{
|
||||
{"any_always_true", "any", []string{"ru-msk"}, "us-east", true},
|
||||
{"any_empty_pref_still_true", "any", nil, "us-east", true},
|
||||
{"specific_match", "specific", []string{"ru-msk", "eu-west"}, "ru-msk", true},
|
||||
{"specific_no_match", "specific", []string{"ru-msk", "eu-west"}, "us-east", false},
|
||||
{"specific_empty_pref_fallback", "specific", nil, "us-east", true},
|
||||
{"all_placeholder_true", "all", []string{"ru-msk", "us-east"}, "ru-msk", true},
|
||||
{"all_placeholder_foreign_region", "all", []string{"ru-msk", "us-east"}, "eu-west", true},
|
||||
{"empty_mode_defaults_to_any", "", nil, "us-east", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
m := models.Monitor{
|
||||
RegionMode: c.mode,
|
||||
PreferredRegions: models.RegionCodesFromSlice(c.regions),
|
||||
}
|
||||
assert.Equal(t, c.want, m.WantsRegion(c.workerRC))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hostsOf extracts the hostnames from the assigned checks for readable
|
||||
// assertions in table-driven tests.
|
||||
func hostsOf(checks []*models.Check) []string {
|
||||
out := make([]string, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// intersectHosts returns the hostnames present in both slices — used to
|
||||
// prove two concurrent workers did not lease the same check twice.
|
||||
func intersectHosts(a, b []*models.Check) []string {
|
||||
set := make(map[string]struct{}, len(a))
|
||||
for _, c := range a {
|
||||
if c.Monitor != nil {
|
||||
set[c.Monitor.Host] = struct{}{}
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, c := range b {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[c.Monitor.Host]; ok {
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
42
app/models/check_metric_test.go
Обычный файл
42
app/models/check_metric_test.go
Обычный файл
@@ -0,0 +1,42 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckMetricName(t *testing.T) {
|
||||
tests := []struct {
|
||||
kind string
|
||||
want string
|
||||
}{
|
||||
{"http", "chttp"},
|
||||
{"ssl", "cssl"},
|
||||
{"bssl", "cbssl"},
|
||||
{"ssh", "cssh"},
|
||||
{"ftp", "cftp"},
|
||||
{"dns", "cdns"},
|
||||
{"whois", "cwhois"},
|
||||
{"rkn", "crkn"},
|
||||
{"llm", "cllm"},
|
||||
{"llm-http", "cllm_http"},
|
||||
{"weird-kind", "cweird_kind"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.kind, func(t *testing.T) {
|
||||
c := &Check{Kind: tt.kind, ID: 1}
|
||||
got := c.MetricName()
|
||||
assert.Equal(t, tt.want, got, "MetricName() for kind %q", tt.kind)
|
||||
|
||||
metric := got
|
||||
for _, ch := range metric {
|
||||
if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' && ch != ':' {
|
||||
t.Errorf("metric name %q contains invalid character %q", metric, ch)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
33
app/models/check_region_result.go
Обычный файл
33
app/models/check_region_result.go
Обычный файл
@@ -0,0 +1,33 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// CheckRegionResult stores check results from distributed workers by region
|
||||
type CheckRegionResult struct {
|
||||
concerns.Model
|
||||
CheckID int64 `gorm:"index;not null" json:"check_id"`
|
||||
Check *Check `gorm:"foreignKey:CheckID" json:"check,omitempty"`
|
||||
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
|
||||
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id"`
|
||||
WorkerNode *WorkerNode `gorm:"foreignKey:WorkerNodeID" json:"worker_node,omitempty"`
|
||||
ExecutedAt time.Time `gorm:"not null" json:"executed_at"`
|
||||
State string `gorm:"size:10;not null" json:"state"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Error *string `json:"error"`
|
||||
|
||||
// AggregatedAt is set by app/models/check_aggregator.go once a row has
|
||||
// been folded into a Check.State decision. NULL means "still waiting
|
||||
// for the aggregator"; non-NULL means "this row has already been
|
||||
// counted in a quorum decision and must not be re-counted". The
|
||||
// column is indexed (see check_aggregator.go index helper) so the
|
||||
// per-tick SELECT that finds unaggregated rows is O(matching rows)
|
||||
// rather than scanning the whole table.
|
||||
AggregatedAt *time.Time `gorm:"index" json:"aggregated_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
155
app/models/check_settings.go
Обычный файл
155
app/models/check_settings.go
Обычный файл
@@ -0,0 +1,155 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CheckHeader provides functionality.
|
||||
type CheckHeader struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// CheckSettings provides functionality.
|
||||
type CheckSettings struct {
|
||||
// HTTP Basic Auth Username
|
||||
HTTPUsername string `json:"http_username,omitempty"`
|
||||
// HTTP Basic Auth Password
|
||||
HTTPPassword string `json:"http_password,omitempty"`
|
||||
|
||||
// ExpectedAnswer (default, redirect, custom)
|
||||
ExpectedAnswer string `json:"expected_answer,omitempty"`
|
||||
// Expected redirect location
|
||||
ExpectedLocation string `json:"expected_location,omitempty"`
|
||||
// Expected HTTP status codes for custom
|
||||
ExpectedHTTPCode int `json:"expected_http_code,omitempty"`
|
||||
|
||||
// Keyword search type (off, present or absent)
|
||||
KeywordType string `json:"keyword_type,omitempty"`
|
||||
// Keyword to search for
|
||||
KeywordValue string `json:"keyword_value,omitempty"`
|
||||
|
||||
SlowTime int `json:"slow_time"`
|
||||
Timeout int `json:"timeout"`
|
||||
// Request Settings
|
||||
CheckIp bool `json:"checkip"` //nolint:revive // accepted lint exception
|
||||
CheckIPv6 bool `json:"checkipv6"`
|
||||
RequestHeader []CheckHeader `json:"request_headers"`
|
||||
RequestMethod string `json:"request_method"`
|
||||
RequestType string `json:"request_type"`
|
||||
RequestContent string `json:"request_content"`
|
||||
// SSH Settings
|
||||
Port string `json:"port"`
|
||||
|
||||
// Host optionally overrides the monitor host for checks that do
|
||||
// not naturally target the Monitor.Host (e.g. an ICMP/TCP/UDP
|
||||
// probe to a separate machine, or a different IP family). Used by
|
||||
// cping/ctcp/cudp.
|
||||
Host string `json:"host,omitempty"`
|
||||
|
||||
// Count is the per-check packet count for ping. Defaults to 1 in
|
||||
// cping when zero or negative.
|
||||
Count int `json:"count,omitempty"`
|
||||
|
||||
// PacketSize is the ICMP payload size for ping (bytes). Defaults
|
||||
// to 56 in cping when zero or negative.
|
||||
PacketSize int `json:"packet_size,omitempty"`
|
||||
|
||||
// Distributed marks this check as eligible for execution on the
|
||||
// distributed worker pool (multi-region, multi-worker) instead of
|
||||
// only the in-process scheduler. Reserved for paid plans; the
|
||||
// controller layer enforces the plan check.
|
||||
Distributed bool `json:"distributed"`
|
||||
}
|
||||
|
||||
// redirectCodes contains all HTTP redirect status codes.
|
||||
var redirectCodes = []int{300, 301, 302, 303, 307, 308}
|
||||
|
||||
// CheckKeyword provides functionality.
|
||||
func (s *CheckSettings) CheckKeyword(body []byte, warns []string) ([]string, error) {
|
||||
var err error
|
||||
|
||||
switch s.KeywordType {
|
||||
case "", "off":
|
||||
return warns, nil
|
||||
case "present":
|
||||
if !strings.Contains(string(body), s.KeywordValue) {
|
||||
err = errors.New("expected keyword " + s.KeywordValue + " not found")
|
||||
}
|
||||
case "absent":
|
||||
if strings.Contains(string(body), s.KeywordValue) {
|
||||
err = errors.New("unexpected keyword " + s.KeywordValue + " found")
|
||||
}
|
||||
}
|
||||
|
||||
return warns, err
|
||||
}
|
||||
|
||||
// CheckAnswer provides functionality.
|
||||
func (s *CheckSettings) CheckAnswer(resp *http.Response, body []byte) ([]string, error) {
|
||||
// log.Println("check answer, settings:")
|
||||
// spew.Dump(s)
|
||||
|
||||
isRedirect := false
|
||||
|
||||
for _, rc := range redirectCodes {
|
||||
if resp.StatusCode == rc {
|
||||
isRedirect = true
|
||||
}
|
||||
}
|
||||
|
||||
location := resp.Header.Get("location")
|
||||
|
||||
// log.Println("redirect?", isRedirect, location)
|
||||
|
||||
switch s.ExpectedAnswer {
|
||||
case "", "default":
|
||||
if isRedirect {
|
||||
return s.CheckKeyword(body, []string{"redirect: " + location})
|
||||
} else if resp.StatusCode != 200 {
|
||||
return []string{}, fmt.Errorf("bad status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
case "redirect":
|
||||
if !isRedirect {
|
||||
return []string{}, fmt.Errorf("bad status code: %d (expected redirect)", resp.StatusCode)
|
||||
}
|
||||
|
||||
if s.ExpectedLocation != "" {
|
||||
if location != s.ExpectedLocation {
|
||||
return []string{}, fmt.Errorf(
|
||||
"bad location: %s (expected %s)",
|
||||
location,
|
||||
s.ExpectedLocation,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case "custom":
|
||||
if s.ExpectedHTTPCode == 0 {
|
||||
s.ExpectedHTTPCode = 200
|
||||
}
|
||||
if resp.StatusCode != s.ExpectedHTTPCode {
|
||||
return []string{}, fmt.Errorf("bad status code: %d (expected %d)", resp.StatusCode, s.ExpectedHTTPCode)
|
||||
}
|
||||
|
||||
if s.ExpectedLocation != "" {
|
||||
if location != s.ExpectedLocation {
|
||||
return []string{}, fmt.Errorf(
|
||||
"bad location: %s (expected %s)",
|
||||
location,
|
||||
s.ExpectedLocation,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
panic("bad expectedAnswer")
|
||||
}
|
||||
|
||||
// return []string{}, nil
|
||||
return s.CheckKeyword(body, []string{})
|
||||
}
|
||||
60
app/models/check_settings_test.go
Обычный файл
60
app/models/check_settings_test.go
Обычный файл
@@ -0,0 +1,60 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckSettingsDistributed_Default(t *testing.T) {
|
||||
s := CheckSettings{}
|
||||
assert.False(t, s.Distributed, "new CheckSettings should default to Distributed=false")
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_JSON(t *testing.T) {
|
||||
original := CheckSettings{
|
||||
HTTPUsername: "user",
|
||||
HTTPPassword: "pass",
|
||||
ExpectedAnswer: "default",
|
||||
Timeout: 30,
|
||||
Distributed: true,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(original)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, string(data), `"distributed":true`)
|
||||
|
||||
var decoded CheckSettings
|
||||
err = json.Unmarshal(data, &decoded)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, original, decoded)
|
||||
assert.True(t, decoded.Distributed)
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_OmitFalse(t *testing.T) {
|
||||
s := CheckSettings{HTTPUsername: "user"}
|
||||
data, err := json.Marshal(s)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, string(data), `"distributed":false`)
|
||||
}
|
||||
|
||||
func TestCheckSettingsDistributed_CheckUnmarshal(t *testing.T) {
|
||||
c := &Check{
|
||||
Settings: []byte(`{"distributed": true, "timeout": 60}`),
|
||||
}
|
||||
got := c.GetSettings()
|
||||
assert.True(t, got.Distributed)
|
||||
assert.Equal(t, 60, got.Timeout)
|
||||
}
|
||||
|
||||
func TestPlanAllowsDistributed(t *testing.T) {
|
||||
var nilPlan *Plan
|
||||
assert.False(t, nilPlan.AllowsDistributed(), "nil plan must not allow distributed")
|
||||
|
||||
free := &Plan{Price: 0}
|
||||
assert.False(t, free.AllowsDistributed(), "free plan must not allow distributed")
|
||||
|
||||
paid := &Plan{Price: 100}
|
||||
assert.True(t, paid.AllowsDistributed(), "paid plan must allow distributed")
|
||||
}
|
||||
181
app/models/cleanup_stale_test.go
Обычный файл
181
app/models/cleanup_stale_test.go
Обычный файл
@@ -0,0 +1,181 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// makeStaleUser inserts a user whose LastActiveAt is older than the
|
||||
// 3-month cutoff used by the cleanup filter. The returned user is what
|
||||
// the candidates query should pick up.
|
||||
func makeStaleUser(t *testing.T, email string, lastActive *time.Time) *models.User {
|
||||
t.Helper()
|
||||
u := &models.User{
|
||||
Email: &email,
|
||||
Name: "stale " + email,
|
||||
Enabled: true,
|
||||
Confirmed: true,
|
||||
LastActiveAt: lastActive,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(u).Error)
|
||||
return u
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_EmptyWhenNoCandidates checks the obvious
|
||||
// negative case: a fresh account with an active owner is not eligible.
|
||||
func TestFindStaleAccounts_EmptyWhenNoCandidates(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "fresh", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
// Group + owner access + active user
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
recent := time.Now().Add(-1 * time.Hour)
|
||||
user := makeStaleUser(t, "active@example.com", &recent)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "an account with an active owner is not stale")
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_PicksStaleEmptyAccount checks the happy path:
|
||||
// account with no monitors + single user + last login > 3 months ago.
|
||||
func TestFindStaleAccounts_PicksStaleEmptyAccount(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "ghost", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
neverLoggedIn := (*time.Time)(nil)
|
||||
user := makeStaleUser(t, "ghost@example.com", neverLoggedIn)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1, "the empty stale account should be picked up")
|
||||
assert.Equal(t, acc.ID, got[0].AccountID)
|
||||
assert.Equal(t, user.ID, got[0].UserID)
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_SkipsAccountWithMonitors makes sure the
|
||||
// "zero monitors" gate is enforced.
|
||||
func TestFindStaleAccounts_SkipsAccountWithMonitors(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "active", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
group := models.Group{AccountID: acc.ID, Name: "default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
// One monitor → account is NOT eligible even if the user is stale.
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "example.com"}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
stale := time.Now().Add(-365 * 24 * time.Hour)
|
||||
user := makeStaleUser(t, "owner@example.com", &stale)
|
||||
access := models.Access{AccountID: acc.ID, UserID: &user.ID, Kind: "account", Role: "owner"}
|
||||
require.NoError(t, models.DB().Create(&access).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
|
||||
// TestFindStaleAccounts_SkipsUserWithMultipleAccounts verifies that a
|
||||
// user holding two accounts disqualifies BOTH accounts.
|
||||
func TestFindStaleAccounts_SkipsUserWithMultipleAccounts(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc1 := models.Account{Name: "acc1", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc1).Error)
|
||||
acc2 := models.Account{Name: "acc2", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc2).Error)
|
||||
models.DB().Create(&models.Group{AccountID: acc1.ID, Name: "g1"})
|
||||
models.DB().Create(&models.Group{AccountID: acc2.ID, Name: "g2"})
|
||||
|
||||
stale := time.Now().Add(-365 * 24 * time.Hour)
|
||||
user := makeStaleUser(t, "shared@example.com", &stale)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc1.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: acc2.ID, UserID: &user.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
got, err := models.FindStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "user with two accounts disqualifies both accounts")
|
||||
}
|
||||
|
||||
// TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans verifies the
|
||||
// end-to-end cleanup: matching account + user are removed, and
|
||||
// recently-active accounts survive.
|
||||
func TestCleanupStaleAccounts_HardDeletesEligibleAndOrphans(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
// Stale account with a stale user that has only this one account.
|
||||
staleAcc := models.Account{Name: "ghost", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&staleAcc).Error)
|
||||
staleGroup := models.Group{AccountID: staleAcc.ID, Name: "g"}
|
||||
require.NoError(t, models.DB().Create(&staleGroup).Error)
|
||||
staleUser := makeStaleUser(t, "ghost@example.com", nil)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: staleAcc.ID, UserID: &staleUser.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
// Active account with a recent user — must NOT be touched.
|
||||
freshAcc := models.Account{Name: "fresh", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&freshAcc).Error)
|
||||
require.NoError(t, models.DB().Create(&models.Group{AccountID: freshAcc.ID, Name: "g"}).Error)
|
||||
recent := time.Now().Add(-1 * time.Hour)
|
||||
freshUser := makeStaleUser(t, "fresh@example.com", &recent)
|
||||
require.NoError(t, models.DB().Create(&models.Access{AccountID: freshAcc.ID, UserID: &freshUser.ID, Kind: "account", Role: "owner"}).Error)
|
||||
|
||||
deleted, err := models.CleanupStaleAccounts()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, deleted, "only the stale empty account should be deleted")
|
||||
|
||||
// Stale account is gone.
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", staleAcc.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(0), count)
|
||||
|
||||
// Stale user is orphaned → also hard-deleted by the cleanup pass.
|
||||
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", staleUser.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(0), count)
|
||||
|
||||
// Fresh account and user survive.
|
||||
require.NoError(t, models.DB().Model(&models.Account{}).Where("id = ?", freshAcc.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
require.NoError(t, models.DB().Model(&models.User{}).Where("id = ?", freshUser.ID).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
41
app/models/concerns/has_token.go
Обычный файл
41
app/models/concerns/has_token.go
Обычный файл
@@ -0,0 +1,41 @@
|
||||
// Package concerns provides functionality.
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
)
|
||||
|
||||
// HasToken provides functionality.
|
||||
type HasToken struct {
|
||||
Token string `json:"-" gorm:"unique_index"`
|
||||
}
|
||||
|
||||
// SetToken provides functionality.
|
||||
func (m *HasToken) SetToken() {
|
||||
tk := RandomToken(32)
|
||||
m.Token = base64.RawURLEncoding.EncodeToString(tk)
|
||||
if m.Token == "" {
|
||||
panic("RandomToken failed: token not set")
|
||||
}
|
||||
if len(m.Token) < 32 {
|
||||
panic("RandomToken failed: token too short")
|
||||
}
|
||||
}
|
||||
|
||||
// RandomToken provides functionality.
|
||||
func RandomToken(tokenLen int) []byte {
|
||||
b := make([]byte, tokenLen)
|
||||
n, err := rand.Read(b)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if n != tokenLen {
|
||||
panic("RandomToken failed: bad len")
|
||||
}
|
||||
if bytes.Equal(b, make([]byte, tokenLen)) {
|
||||
panic("RandomToken failed: generated empty token")
|
||||
}
|
||||
return b
|
||||
}
|
||||
14
app/models/concerns/model.go
Обычный файл
14
app/models/concerns/model.go
Обычный файл
@@ -0,0 +1,14 @@
|
||||
// Source: https://gorm.io/gorm/blob/master/model.go
|
||||
// Use 64 bit keys
|
||||
|
||||
package concerns
|
||||
|
||||
// Model base model definition, including fields `ID`, `CreatedAt`, `UpdatedAt`, `DeletedAt`, which could be embedded in your models
|
||||
// type User struct {
|
||||
// gorm.Model
|
||||
// }
|
||||
|
||||
// Model provides functionality.
|
||||
type Model struct {
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
}
|
||||
7
app/models/concerns/renderable.go
Обычный файл
7
app/models/concerns/renderable.go
Обычный файл
@@ -0,0 +1,7 @@
|
||||
package concerns
|
||||
|
||||
// Renderable provides functionality.
|
||||
type Renderable struct {
|
||||
Raw string
|
||||
Rendered string
|
||||
}
|
||||
12
app/models/concerns/soft_delete.go
Обычный файл
12
app/models/concerns/soft_delete.go
Обычный файл
@@ -0,0 +1,12 @@
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// SoftDelete provides functionality.
|
||||
type SoftDelete struct {
|
||||
DeletedAt *time.Time `gorm:"index" json:"-"`
|
||||
DeleterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"-"`
|
||||
// Deleter *User `json:"-"`
|
||||
}
|
||||
11
app/models/concerns/timestamped.go
Обычный файл
11
app/models/concerns/timestamped.go
Обычный файл
@@ -0,0 +1,11 @@
|
||||
package concerns
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Timestamped provides functionality.
|
||||
type Timestamped struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
86
app/models/contact.go
Обычный файл
86
app/models/contact.go
Обычный файл
@@ -0,0 +1,86 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Contact represents a notification contact.
|
||||
//
|
||||
// Ownership / tenancy:
|
||||
//
|
||||
// - A Contact belongs to exactly one Account (tenant). AccountID may be
|
||||
// nil for system-level admin contacts (see SystemContacts).
|
||||
// - A Contact may belong to at most one User. UserID is set when the
|
||||
// contact was created on behalf of a specific user (the typical
|
||||
// case for self-service "My email" / "My Telegram" contacts) and is
|
||||
// nil for account-wide contacts.
|
||||
//
|
||||
// `User` is omitempty because most list payloads don't preload it; the
|
||||
// `/settings/users` page loads it server-side via the AccountUserRow
|
||||
// payload.
|
||||
type Contact struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id)"`
|
||||
Account *Account `json:"-"`
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
|
||||
User *User `json:"user,omitempty"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
Kind string `json:"kind" gorm:"not null;index:contact_value"`
|
||||
Value string `json:"value" gorm:"index:contact_value"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
Data datatypes.JSON `json:"data"`
|
||||
IsSystem *bool `json:"is_system" gorm:"default:false"`
|
||||
Notifications []Notification `json:"-" gorm:"many2many:notification_contacts;"`
|
||||
NotificationsCount int `gorm:"-:all" json:"notifications_count"`
|
||||
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
|
||||
Messages []Message `json:"-"`
|
||||
|
||||
concerns.Timestamped
|
||||
concerns.HasToken
|
||||
Audited
|
||||
}
|
||||
|
||||
// SystemContacts returns all contacts marked as system/admin (is_system=true).
|
||||
// These are the contacts distributed workers notify directly when the main
|
||||
// API is unreachable (see docs/worker-protocol.md "System Selfcheck").
|
||||
func SystemContacts() ([]Contact, error) {
|
||||
var contacts []Contact
|
||||
err := DB().Where("is_system = ? AND enabled = ?", true, true).Find(&contacts).Error
|
||||
return contacts, err
|
||||
}
|
||||
|
||||
// ContactsCounts fills NotificationsCount and MonitorsCount for each contact.
|
||||
func ContactsCounts(contacts *[]Contact) {
|
||||
groupIDs := make(map[int64]bool, 0)
|
||||
groupCount := make(map[int64]int, 0)
|
||||
|
||||
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
|
||||
(*contacts)[i].NotificationsCount = len(c.Notifications)
|
||||
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
|
||||
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
groupIDs[g.ID] = true
|
||||
groupCount[g.ID] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gids := make([]int64, 0, len(groupIDs))
|
||||
for g := range groupIDs {
|
||||
gids = append(gids, g)
|
||||
}
|
||||
|
||||
CountGroups(gids, &groupCount)
|
||||
|
||||
for i, c := range *contacts { //nolint:gocritic // range copy is acceptable here
|
||||
cnt := 0
|
||||
for _, n := range c.Notifications { //nolint:gocritic // range copy is acceptable here
|
||||
for _, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
cnt += groupCount[g.ID]
|
||||
}
|
||||
}
|
||||
(*contacts)[i].MonitorsCount = cnt
|
||||
}
|
||||
}
|
||||
65
app/models/contact_test.go
Обычный файл
65
app/models/contact_test.go
Обычный файл
@@ -0,0 +1,65 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestSystemContacts verifies that SystemContacts returns only contacts with
|
||||
// is_system=true and ignores contacts with is_system=false or nil.
|
||||
func TestSystemContacts(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account := &models.Account{Name: "test-account"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
accountID := account.ID
|
||||
|
||||
trueVal, falseVal := true, false
|
||||
|
||||
systemContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "system-admin",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
regularContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "regular",
|
||||
Kind: "email",
|
||||
Value: "user@example.com",
|
||||
IsSystem: &falseVal,
|
||||
}
|
||||
nilSystemContact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "nil-system",
|
||||
Kind: "email",
|
||||
Value: "nil@example.com",
|
||||
}
|
||||
|
||||
require.NoError(t, models.DB().Create(systemContact).Error)
|
||||
require.NoError(t, models.DB().Create(regularContact).Error)
|
||||
require.NoError(t, models.DB().Create(nilSystemContact).Error)
|
||||
|
||||
got, err := models.SystemContacts()
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []int64
|
||||
var names []string
|
||||
for _, c := range got {
|
||||
ids = append(ids, c.ID)
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
|
||||
assert.Contains(t, names, "system-admin")
|
||||
assert.NotContains(t, names, "regular")
|
||||
assert.NotContains(t, names, "nil-system")
|
||||
assert.Contains(t, ids, systemContact.ID)
|
||||
assert.NotContains(t, ids, regularContact.ID)
|
||||
assert.NotContains(t, ids, nilSystemContact.ID)
|
||||
}
|
||||
97
app/models/credential_crypto.go
Обычный файл
97
app/models/credential_crypto.go
Обычный файл
@@ -0,0 +1,97 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/config/secrets"
|
||||
)
|
||||
|
||||
const credentialKeyEnv = "RSMON_CRED_KEY"
|
||||
|
||||
// encryptSecret encrypts plaintext with AES-GCM. If no key is configured
|
||||
// (dev/test), stores the value with a "plain:" prefix. Production deployments
|
||||
// must set RSMON_CRED_KEY or config/secrets.yml crypto.pepper.
|
||||
func encryptSecret(plaintext string) (string, error) {
|
||||
key := credentialKey()
|
||||
if key == "" {
|
||||
return "plain:" + plaintext, nil
|
||||
}
|
||||
block, err := aes.NewCipher(deriveKey(key))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return "", err
|
||||
}
|
||||
sealed := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
|
||||
return "enc:" + base64.StdEncoding.EncodeToString(sealed), nil
|
||||
}
|
||||
|
||||
// decryptSecret decrypts a value produced by encryptSecret. Values without a
|
||||
// known prefix are returned as-is for backward compatibility with legacy
|
||||
// plaintext records.
|
||||
func decryptSecret(stored string) (string, error) {
|
||||
if stored == "" {
|
||||
return "", nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(stored, "plain:"):
|
||||
return strings.TrimPrefix(stored, "plain:"), nil
|
||||
case strings.HasPrefix(stored, "enc:"):
|
||||
key := credentialKey()
|
||||
if key == "" {
|
||||
return "", errors.New("credential key not configured but secret is encrypted")
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(stored, "enc:"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
block, err := aes.NewCipher(deriveKey(key))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(raw) < gcm.NonceSize() {
|
||||
return "", errors.New("ciphertext too short")
|
||||
}
|
||||
nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
|
||||
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(plaintext), nil
|
||||
default:
|
||||
return stored, nil
|
||||
}
|
||||
}
|
||||
|
||||
func deriveKey(s string) []byte {
|
||||
h := sha256.Sum256([]byte(s))
|
||||
return h[:32]
|
||||
}
|
||||
|
||||
func credentialKey() string {
|
||||
if k := os.Getenv(credentialKeyEnv); k != "" {
|
||||
return k
|
||||
}
|
||||
if secrets.Crypto != nil && secrets.Crypto.Pepper != "" {
|
||||
return secrets.Crypto.Pepper
|
||||
}
|
||||
return ""
|
||||
}
|
||||
120
app/models/dead_worker_reaper.go
Обычный файл
120
app/models/dead_worker_reaper.go
Обычный файл
@@ -0,0 +1,120 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeadWorkerHeartbeatTimeout is the threshold for ReapDeadWorkers — a worker
|
||||
// whose last_seen is older than this is considered dead and any leased tasks
|
||||
// it owns are reassigned to the pool. Five minutes mirrors the StaleWorkers()
|
||||
// check in check_jobs.go so the two reapers cannot disagree about who is
|
||||
// dead. See docs/todo.md Phase 4 §5.
|
||||
const DeadWorkerHeartbeatTimeout = 5 * time.Minute
|
||||
|
||||
// ReapDeadWorkers marks any non-dead worker whose last_seen is older than
|
||||
// DeadWorkerHeartbeatTimeout as "dead", then reassigns its leased tasks
|
||||
// back to the queued pool so other workers (or freshly registered ones)
|
||||
// can pick them up. It mirrors the structure of ReapExpiredTasks — two
|
||||
// short UPDATE statements, cheap enough to run from the web process
|
||||
// every 30s.
|
||||
//
|
||||
// The returned tuple is (reaped, reassigned, err): reaped counts the
|
||||
// workers that flipped to dead during this call; reassigned counts the
|
||||
// leased tasks that were given back to the pool. A zero count on either
|
||||
// is normal — the reaper is idempotent and the call is silent when
|
||||
// nothing is due.
|
||||
func ReapDeadWorkers() (reaped int, reassigned int, err error) {
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-DeadWorkerHeartbeatTimeout)
|
||||
|
||||
// First flip the workers to dead so the second UPDATE can match the
|
||||
// freshly-stamped ids without having to re-derive them in Go.
|
||||
r := DB().Exec(`
|
||||
UPDATE worker_nodes
|
||||
SET status = ?, updated_at = ?
|
||||
WHERE status <> ? AND last_seen IS NOT NULL AND last_seen < ?`,
|
||||
"dead", now, "dead", cutoff,
|
||||
)
|
||||
if r.Error != nil {
|
||||
return 0, 0, r.Error
|
||||
}
|
||||
reaped = int(r.RowsAffected)
|
||||
|
||||
// Nothing flipped → no tasks to return. Cheaper than running an
|
||||
// UPDATE that touches 0 rows on every tick when the fleet is healthy.
|
||||
if reaped == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
// Second: clear any leased tasks owned by the now-dead workers.
|
||||
// The selector stashes the worker's WorkerID string in lease_owner;
|
||||
// matching against the (now-stale) worker row's WorkerID is the
|
||||
// same identifier the selector uses, so we don't need an extra
|
||||
// join. Tasks in other states (queued, succeeded, dead, …) are
|
||||
// unaffected — only leased work the dead worker still owned has
|
||||
// to go back to the queue.
|
||||
r2 := DB().Exec(`
|
||||
UPDATE tasks
|
||||
SET state = ?, lease_owner = '', lease_expires_at = NULL, updated_at = ?
|
||||
WHERE state = ? AND lease_owner IN (
|
||||
SELECT worker_id FROM worker_nodes WHERE status = ?
|
||||
)`,
|
||||
TaskStateQueued, now, TaskStateLeased, "dead",
|
||||
)
|
||||
if r2.Error != nil {
|
||||
return reaped, 0, r2.Error
|
||||
}
|
||||
reassigned = int(r2.RowsAffected)
|
||||
return reaped, reassigned, nil
|
||||
}
|
||||
|
||||
// StartDeadWorkerReaper launches a goroutine that runs ReapDeadWorkers
|
||||
// on the given interval until ctx is canceled. Mirrors StartTaskReaper
|
||||
// in this package: same ticker pattern, same logging style, same
|
||||
// recover() safety net so a malformed row cannot crash the web process.
|
||||
//
|
||||
// The default interval is 30s; values <= 0 fall back to the default so
|
||||
// the helper is safe to call from any call site without a guard. Wire
|
||||
// from main.init() once per process — the reaper uses short row-level
|
||||
// locks and is cheap under load (two indexed UPDATEs of <= a few
|
||||
// hundred rows in steady state).
|
||||
//
|
||||
// Passing a nil context falls back to context.Background() so callers
|
||||
// can write `models.StartDeadWorkerReaper(nil, ...)` in one-liners
|
||||
// (main, tests) without having to import "context" first.
|
||||
func StartDeadWorkerReaper(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("dead_worker_reaper: panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
reaped, reassigned, err := ReapDeadWorkers()
|
||||
if err != nil {
|
||||
log.Printf("dead_worker_reaper: error: %v", err)
|
||||
return
|
||||
}
|
||||
if reaped > 0 || reassigned > 0 {
|
||||
log.Printf("dead_worker_reaper: reaped=%d reassigned=%d", reaped, reassigned)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
274
app/models/dead_worker_reaper_test.go
Обычный файл
274
app/models/dead_worker_reaper_test.go
Обычный файл
@@ -0,0 +1,274 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestReapDeadWorkers_SkipsHealthyAndKillsSilent mirrors the
|
||||
// reaper-shape test for ReapExpiredTasks in task_test.go: seed three
|
||||
// workers (heartbeat fresh / heartbeat stale / already dead) plus two
|
||||
// leased tasks owned by the stale worker and one leased task on a
|
||||
// healthy worker (which must NOT be touched). Then assert:
|
||||
//
|
||||
// - the fresh worker is left active
|
||||
// - the silent worker flips to "dead"
|
||||
// - the already-dead worker is left dead (idempotent)
|
||||
// - the silent worker's leased tasks are reset to queued
|
||||
// - the healthy worker's leased task is untouched
|
||||
func TestReapDeadWorkers_SkipsHealthyAndKillsSilent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "test")
|
||||
|
||||
healthy := &models.WorkerNode{
|
||||
WorkerID: "w-healthy",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "tok-healthy",
|
||||
LastSeen: timePtr(time.Now()),
|
||||
}
|
||||
stale := &models.WorkerNode{
|
||||
WorkerID: "w-stale",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "tok-stale",
|
||||
LastSeen: timePtr(time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)),
|
||||
}
|
||||
alreadyDead := &models.WorkerNode{
|
||||
WorkerID: "w-dead",
|
||||
RegionCode: "test",
|
||||
Status: "dead",
|
||||
AuthToken: "tok-dead",
|
||||
LastSeen: timePtr(time.Now().Add(-time.Hour)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(healthy).Error)
|
||||
require.NoError(t, models.DB().Create(stale).Error)
|
||||
require.NoError(t, models.DB().Create(alreadyDead).Error)
|
||||
|
||||
// Two leased tasks on the stale worker — both must come back.
|
||||
task1 := mustLeaseTask(t, stale.WorkerID, "test-acct")
|
||||
task2 := mustLeaseTask(t, stale.WorkerID, "test-acct")
|
||||
// One leased task on the healthy worker — must stay leased.
|
||||
healthyTask := mustLeaseTask(t, healthy.WorkerID, "test-acct")
|
||||
// Already-dead worker with a leased task — not part of THIS reaper's
|
||||
// reaping set (it would only be touched by a fresh reaper pass), so
|
||||
// leave it leased to demonstrate that we don't accidentally reassign
|
||||
// from prior-dead leases too.
|
||||
deadPriorTask := mustLeaseTask(t, alreadyDead.WorkerID, "test-acct")
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped, "only the stale worker should flip (already-dead is left untouched)")
|
||||
// 3 tasks come back: the 2 leased by the stale worker (just flipped
|
||||
// to dead) + the 1 leased by the prior-dead worker, which had never
|
||||
// been cleaned up because no previous reaper ran. The reaper matches
|
||||
// dead workers by status, so any leased task on a dead worker is a
|
||||
// stranded lease and must be returned to the pool regardless of when
|
||||
// the worker flipped.
|
||||
assert.Equal(t, 3, reassigned)
|
||||
|
||||
var healthyRow, staleRow, deadRow models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&healthyRow, healthy.ID).Error)
|
||||
assert.Equal(t, "active", healthyRow.Status, "healthy worker must stay active")
|
||||
|
||||
require.NoError(t, models.DB().First(&staleRow, stale.ID).Error)
|
||||
assert.Equal(t, "dead", staleRow.Status, "stale worker should flip to dead")
|
||||
|
||||
require.NoError(t, models.DB().First(&deadRow, alreadyDead.ID).Error)
|
||||
assert.Equal(t, "dead", deadRow.Status, "already-dead worker should not be touched")
|
||||
|
||||
got := func(id int64) models.Task {
|
||||
var row models.Task
|
||||
require.NoError(t, models.DB().First(&row, id).Error)
|
||||
return row
|
||||
}
|
||||
|
||||
t1 := got(task1.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, t1.State, "stale worker task must come back to queued")
|
||||
assert.Empty(t, t1.LeaseOwner)
|
||||
assert.Nil(t, t1.LeaseExpiresAt)
|
||||
|
||||
t2 := got(task2.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, t2.State)
|
||||
assert.Empty(t, t2.LeaseOwner)
|
||||
assert.Nil(t, t2.LeaseExpiresAt)
|
||||
|
||||
ht := got(healthyTask.ID)
|
||||
assert.Equal(t, models.TaskStateLeased, ht.State, "healthy worker's lease must be untouched")
|
||||
assert.Equal(t, healthy.WorkerID, ht.LeaseOwner)
|
||||
|
||||
dt := got(deadPriorTask.ID)
|
||||
assert.Equal(t, models.TaskStateQueued, dt.State,
|
||||
"prior-dead task must also be returned to the pool — any leased task on a dead worker is a stranded lease")
|
||||
assert.Empty(t, dt.LeaseOwner)
|
||||
}
|
||||
|
||||
// TestReapDeadWorkers_NoOpOnHealthyFleet verifies the cheap path: when
|
||||
// no workers are stale the reaper returns (0, 0, nil) without doing any
|
||||
// work. Mirrors the "reaped = int(r.RowsAffected)" branch in
|
||||
// ReapExpiredTasks.
|
||||
func TestReapDeadWorkers_NoOpOnHealthyFleet(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-only-healthy",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: timePtr(time.Now()),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, reaped)
|
||||
assert.Equal(t, 0, reassigned)
|
||||
|
||||
var row models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&row, w.ID).Error)
|
||||
assert.Equal(t, "active", row.Status)
|
||||
}
|
||||
|
||||
// TestReapDeadWorkers_OnlyReassignsLeasedNotOthers ensures that the
|
||||
// reaper does not touch queued/succeeded/failed_retry tasks on the dead
|
||||
// worker — only leased ones need to be returned to the queue. Tasks in
|
||||
// other states either belong to no one (queued) or are terminal/semi-
|
||||
// terminal and have their own audit trail.
|
||||
func TestReapDeadWorkers_OnlyReassignsLeasedNotOthers(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now().Add(-models.DeadWorkerHeartbeatTimeout - time.Minute)
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-mix",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: &now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
leased := mustLeaseTask(t, w.WorkerID, "acct-mix")
|
||||
succeeded := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateSucceeded)
|
||||
failedRetry := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedRetry)
|
||||
failedPerm := mustInsertTask(t, w.WorkerID, "acct-mix", models.TaskStateFailedPerm)
|
||||
otherWorker := mustLeaseTask(t, "w-other", "acct-mix")
|
||||
|
||||
reaped, reassigned, err := models.ReapDeadWorkers()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
assert.Equal(t, 1, reassigned, "exactly the one leased task on the dead worker")
|
||||
|
||||
got := func(id int64) string {
|
||||
var row models.Task
|
||||
require.NoError(t, models.DB().First(&row, id).Error)
|
||||
return row.State
|
||||
}
|
||||
assert.Equal(t, models.TaskStateQueued, got(leased.ID))
|
||||
assert.Equal(t, models.TaskStateSucceeded, got(succeeded.ID), "succeeded must not move")
|
||||
assert.Equal(t, models.TaskStateFailedRetry, got(failedRetry.ID), "failed_retry must not move")
|
||||
assert.Equal(t, models.TaskStateFailedPerm, got(failedPerm.ID), "failed_perm must not move")
|
||||
assert.Equal(t, models.TaskStateLeased, got(otherWorker.ID),
|
||||
"tasks leased by another worker must not move")
|
||||
}
|
||||
|
||||
// TestStartDeadWorkerReaper_TickerFiresOnce is a smoke test for the
|
||||
// background helper: spin up the reaper with a tight 10ms ticker and
|
||||
// a cancellable context, wait for the first tick, then cancel the
|
||||
// context so the goroutine exits cleanly without leaking. Mirrors the
|
||||
// shape of how main.init() uses StartTaskReaper (it can't be torn
|
||||
// down, but for tests we always pass a cancellable context).
|
||||
func TestStartDeadWorkerReaper_TickerFiresOnce(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "w-ticker",
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
LastSeen: timePtr(time.Now().Add(-2 * models.DeadWorkerHeartbeatTimeout)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
models.StartDeadWorkerReaper(ctx, 10*time.Millisecond)
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var got models.WorkerNode
|
||||
for time.Now().Before(deadline) {
|
||||
require.NoError(t, models.DB().First(&got, w.ID).Error)
|
||||
if got.Status == "dead" {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("reaper goroutine did not flip worker to dead within 2s; last status=%q", got.Status)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// test helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func timePtr(t time.Time) *time.Time { return &t }
|
||||
|
||||
// mustLeaseTask inserts a leased Task row owned by workerID. The state
|
||||
// is the only field that matters for reaper tests; payload/idempotency
|
||||
// are stubs.
|
||||
func mustLeaseTask(t *testing.T, workerID, accountLabel string) models.Task {
|
||||
t.Helper()
|
||||
return mustInsertTask(t, workerID, accountLabel, models.TaskStateLeased)
|
||||
}
|
||||
|
||||
func mustInsertTask(t *testing.T, workerID, accountLabel string, state string) models.Task {
|
||||
t.Helper()
|
||||
leaseUntil := time.Now().Add(time.Hour)
|
||||
stamp := time.Now().UnixNano()
|
||||
jobID := fmt.Sprintf("reap-%s-%s-%d", state, workerID, stamp)
|
||||
// Cap at 64 chars (tasks.job_id VARCHAR(64)). The components above
|
||||
// already stay under the limit because mustLeaseTask keeps workerID
|
||||
// short ("w-mix", "w-stale", …) and state is bounded.
|
||||
if len(jobID) > 64 {
|
||||
jobID = jobID[:64]
|
||||
}
|
||||
idemp := fmt.Sprintf("reap:%s:%d", workerID, stamp)
|
||||
if len(idemp) > 255 {
|
||||
idemp = idemp[:255]
|
||||
}
|
||||
task := models.Task{
|
||||
JobID: jobID,
|
||||
Kind: models.TaskKindNotification,
|
||||
State: state,
|
||||
AccountID: 1,
|
||||
Payload: datatypes.JSON([]byte(`{"method":"email"}`)),
|
||||
NotBefore: time.Now().Add(-time.Minute),
|
||||
LeaseOwner: workerID,
|
||||
LeaseExpiresAt: &leaseUntil,
|
||||
Attempts: 1,
|
||||
MaxAttempts: 5,
|
||||
IdempotencyKey: idemp,
|
||||
}
|
||||
if state != models.TaskStateLeased {
|
||||
task.LeaseOwner = ""
|
||||
task.LeaseExpiresAt = nil
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
require.NotZero(t, task.ID)
|
||||
return task
|
||||
}
|
||||
582
app/models/deletion.go
Обычный файл
582
app/models/deletion.go
Обычный файл
@@ -0,0 +1,582 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// OnUserCacheInvalidate is called when a user is mutated in a way that
|
||||
// invalidates the cached representation in auth/cache.go. It is wired up
|
||||
// from the auth package during init() to avoid an import cycle.
|
||||
var OnUserCacheInvalidate func(userID int64)
|
||||
|
||||
// DeletionGracePeriod is the time between a user requesting account deletion
|
||||
// and the scheduled hard-delete. During this period the user can cancel the
|
||||
// request and monitoring is paused for all of the user's accounts.
|
||||
const DeletionGracePeriod = 7 * 24 * time.Hour
|
||||
|
||||
// RequestUserDeletion sets the user's DeletionRequestedAt to now,
|
||||
// starting the 7-day grace period.
|
||||
func RequestUserDeletion(user *User) error {
|
||||
if user == nil {
|
||||
return errors.New("nil user")
|
||||
}
|
||||
if user.DeletionPending() {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
user.DeletionRequestedAt = &now
|
||||
if err := DB().Save(user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if OnUserCacheInvalidate != nil {
|
||||
OnUserCacheInvalidate(user.ID)
|
||||
}
|
||||
log.Printf("user %d requested account deletion (grace until %s)", user.ID, now.Add(DeletionGracePeriod))
|
||||
return nil
|
||||
}
|
||||
|
||||
// CancelUserDeletion clears the user's DeletionRequestedAt field,
|
||||
// canceling the pending account deletion.
|
||||
func CancelUserDeletion(user *User) error {
|
||||
if user == nil {
|
||||
return errors.New("nil user")
|
||||
}
|
||||
if !user.DeletionPending() {
|
||||
return nil
|
||||
}
|
||||
user.DeletionRequestedAt = nil
|
||||
if err := DB().Save(user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if OnUserCacheInvalidate != nil {
|
||||
OnUserCacheInvalidate(user.ID)
|
||||
}
|
||||
log.Printf("user %d canceled account deletion", user.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// HardDeleteAccount removes an account and all of its data (monitors, checks,
|
||||
// events, messages, contacts, groups, notifications, etc.) in a single
|
||||
// transaction. Accesses and invites pointing at the account are also cleaned
|
||||
// up. Users themselves are kept (they may belong to other accounts). This is
|
||||
// the admin "purge" path used to remove spam/error accounts immediately.
|
||||
func HardDeleteAccount(accountID int64) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
// Remove accesses pointing at this account first (FK constraint).
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Access{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove invites for this account.
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Invite{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Delete all account data and the account row itself.
|
||||
return deleteAccountData(tx, accountID)
|
||||
})
|
||||
}
|
||||
|
||||
// HardDeleteUser removes the user and all of their owned data: accesses,
|
||||
// contacts, accounts, monitors, checks, events and messages. The deletion
|
||||
// is wrapped in a transaction to make sure partial failures don't leave
|
||||
// the database in a broken state.
|
||||
func HardDeleteUser(userID int64) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
user := User{}
|
||||
if err := tx.First(&user, userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1) Find all accounts the user has any access to.
|
||||
accesses := make([]Access, 0)
|
||||
if err := tx.Where("user_id = ?", userID).Find(&accesses).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
accountIDs := make([]int64, 0, len(accesses))
|
||||
for i := range accesses {
|
||||
accountIDs = append(accountIDs, accesses[i].AccountID)
|
||||
}
|
||||
|
||||
// 2) Find all contacts that belong to those accounts or directly to the user.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&Contact{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 3) For each account: delete the related data, then the account itself.
|
||||
for _, accountID := range accountIDs {
|
||||
if err := deleteAccountData(tx, accountID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 4) Remove accesses.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&Access{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 5) Remove invites issued by or for this user.
|
||||
if err := tx.Exec("DELETE FROM invites WHERE invitee_id = ? OR account_id IN (?)",
|
||||
userID, accountIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 6) Remove auth identities (password, social).
|
||||
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 7) Remove api keys owned by the user.
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 8) Remove sessions.
|
||||
if err := tx.Exec("DELETE FROM remember_tokens WHERE user_id = ?", userID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 9) Finally remove the user row.
|
||||
if err := tx.Delete(&user).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("user %d hard-deleted (cascade accounts=%v)", userID, accountIDs)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// deleteAccountData removes everything associated with a single account:
|
||||
// contacts, monitors, checks, events, messages, notifications, groups and
|
||||
// the account itself. The user accesses are removed separately.
|
||||
func deleteAccountData(tx *gorm.DB, accountID int64) error {
|
||||
// All contacts that reference this account (both account-scoped ones
|
||||
// with user_id IS NULL and per-user ones created in
|
||||
// CreateAccountForUser that set both account_id and user_id).
|
||||
// User-only contacts (account_id IS NULL) are independent of the
|
||||
// account and survive the deletion. contacts.account_id has a FK
|
||||
// to accounts(id) without ON DELETE CASCADE, so all referencing
|
||||
// rows must be removed before the account row goes away.
|
||||
contactIDs := make([]int64, 0)
|
||||
if err := tx.Model(&Contact{}).Where("account_id = ?", accountID).Pluck("id", &contactIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
notificationIDs := make([]int64, 0)
|
||||
if err := tx.Model(&Notification{}).Where("account_id = ?", accountID).Pluck("id", ¬ificationIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(contactIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM notification_contacts WHERE contact_id IN (?)", contactIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(notificationIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM notification_contacts WHERE notification_id IN (?)", notificationIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
messageQuery := tx.Model(&Message{})
|
||||
switch {
|
||||
case len(contactIDs) > 0 && len(notificationIDs) > 0:
|
||||
messageQuery = messageQuery.Where("contact_id IN (?) OR notification_id IN (?)", contactIDs, notificationIDs)
|
||||
case len(contactIDs) > 0:
|
||||
messageQuery = messageQuery.Where("contact_id IN (?)", contactIDs)
|
||||
case len(notificationIDs) > 0:
|
||||
messageQuery = messageQuery.Where("notification_id IN (?)", notificationIDs)
|
||||
default:
|
||||
messageQuery = nil
|
||||
}
|
||||
if messageQuery != nil {
|
||||
messageIDs := make([]int64, 0)
|
||||
if err := messageQuery.Pluck("id", &messageIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(messageIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("id IN (?)", messageIDs).Delete(&Message{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Contact{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Monitors (and their checks/events/messages via cascade below).
|
||||
monitors := make([]Monitor, 0)
|
||||
if err := tx.Joins("JOIN groups ON monitors.group_id = groups.id").
|
||||
Where("groups.account_id = ?", accountID).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
monitorIDs := make([]int64, 0, len(monitors))
|
||||
for i := range monitors {
|
||||
monitorIDs = append(monitorIDs, monitors[i].ID)
|
||||
}
|
||||
|
||||
if len(monitorIDs) > 0 {
|
||||
// Checks
|
||||
checks := make([]Check, 0)
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&checks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
checkIDs := make([]int64, 0, len(checks))
|
||||
for i := range checks {
|
||||
checkIDs = append(checkIDs, checks[i].ID)
|
||||
}
|
||||
|
||||
// Events
|
||||
events := make([]Event, 0)
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Find(&events).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
eventIDs := make([]int64, 0, len(events))
|
||||
for i := range events {
|
||||
eventIDs = append(eventIDs, events[i].ID)
|
||||
}
|
||||
|
||||
// Join tables first to avoid FK violations
|
||||
if len(checkIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_checks WHERE check_id IN (?)", checkIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// check_region_results.check_id has a FK to checks(id) without
|
||||
// ON DELETE CASCADE, so it must be purged before checks go away.
|
||||
if err := tx.Exec("DELETE FROM check_region_results WHERE check_id IN (?)", checkIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(eventIDs) > 0 {
|
||||
if err := tx.Exec("DELETE FROM event_messages WHERE event_id IN (?)", eventIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(checkIDs) > 0 {
|
||||
if err := tx.Where("id IN (?)", checkIDs).Delete(&Check{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(eventIDs) > 0 {
|
||||
if err := tx.Where("id IN (?)", eventIDs).Delete(&Event{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// DNS records
|
||||
if err := tx.Where("monitor_id IN (?)", monitorIDs).Delete(&DNSRecord{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Monitors themselves
|
||||
if err := tx.Where("id IN (?)", monitorIDs).Delete(&Monitor{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Notification <-> group links
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM notification_groups WHERE group_id IN (SELECT id FROM groups WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Notifications
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Notification{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// LLMs scoped to this account. worker_llms.llm_id has a FK to llms(id)
|
||||
// without ON DELETE CASCADE, so the join rows must go first.
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM worker_llms WHERE llm_id IN (SELECT id FROM llms WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&LLM{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Groups
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Group{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Inventory entities (docs/plans/inventory-management.md §6). All
|
||||
// four tables hold account_id FKs to accounts(id) without ON
|
||||
// DELETE CASCADE, so we wipe them in dependency order: deployments
|
||||
// and domains first (both FK into sites and servers), then
|
||||
// server_ips, then sites, then servers. Anything the account does
|
||||
// not own is left alone (e.g. shared infra servers are filtered
|
||||
// by account_id and survive).
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Deployment{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Domain{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM server_ips WHERE server_id IN (SELECT id FROM servers WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Site{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&Server{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Private workers scoped to this account (NULL account_id workers
|
||||
// are platform-operated and survive account deletion). The
|
||||
// worker_llms and check_region_results joins both FK to
|
||||
// worker_nodes(id) without ON DELETE CASCADE, so the join rows
|
||||
// must be cleared before the worker rows go away.
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM worker_llms WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM check_region_results WHERE worker_node_id IN (SELECT id FROM worker_nodes WHERE account_id = ?)",
|
||||
accountID,
|
||||
).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("account_id = ?", accountID).Delete(&WorkerNode{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The account itself
|
||||
return tx.Delete(&Account{}, accountID).Error
|
||||
}
|
||||
|
||||
// ProcessPendingDeletions hard-deletes users whose 7-day grace period has
|
||||
// elapsed. Designed to be called from a periodic scheduler.
|
||||
func ProcessPendingDeletions() (int, error) {
|
||||
cutoff := time.Now().Add(-DeletionGracePeriod)
|
||||
users := make([]User, 0)
|
||||
if err := DB().Where("deletion_requested_at IS NOT NULL AND deletion_requested_at < ?", cutoff).
|
||||
Find(&users).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
deleted := 0
|
||||
for i := range users {
|
||||
if err := HardDeleteUser(users[i].ID); err != nil {
|
||||
log.Printf("ProcessPendingDeletions: failed to delete user %d: %v", users[i].ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("ProcessPendingDeletions: hard-deleted %d user(s)", deleted)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// StaleAccountInactivity is the minimum inactivity window before a stale
|
||||
// account becomes eligible for admin cleanup. Picked at 3 months per the
|
||||
// /admin/accounts "Удалить старые" button.
|
||||
const StaleAccountInactivity = 90 * 24 * time.Hour
|
||||
|
||||
// StaleAccountCandidate describes an account that matched the
|
||||
// admin-cleanup eligibility filter but has not yet been deleted. The
|
||||
// snapshot is what the frontend shows in the confirmation dialog.
|
||||
type StaleAccountCandidate struct {
|
||||
AccountID int64 `json:"account_id"`
|
||||
AccountName string `json:"account_name"`
|
||||
UserID int64 `json:"user_id"`
|
||||
UserEmail *string `json:"user_email"`
|
||||
LastActiveAt *time.Time `json:"last_active_at"`
|
||||
}
|
||||
|
||||
// FindStaleAccounts returns the accounts that are eligible for the
|
||||
// admin "Удалить старые" cleanup:
|
||||
//
|
||||
// - the account has zero monitors configured (via group.account_id)
|
||||
// - every user with access to the account has exactly one account
|
||||
// membership total (so removing the account also orphans them)
|
||||
// - every such user's last_active_at is older than
|
||||
// StaleAccountInactivity (3 months). Users who have never logged in
|
||||
// (last_active_at IS NULL) are also eligible.
|
||||
func FindStaleAccounts() ([]StaleAccountCandidate, error) {
|
||||
cutoff := time.Now().Add(-StaleAccountInactivity)
|
||||
|
||||
// Step 1: account IDs that have zero monitors.
|
||||
accountsWithMonitors := make([]int64, 0)
|
||||
if err := DB().
|
||||
Table("monitors").
|
||||
Select("DISTINCT groups.account_id").
|
||||
Joins("JOIN groups ON groups.id = monitors.group_id").
|
||||
Scan(&accountsWithMonitors).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accounts := make([]Account, 0)
|
||||
q := DB().Order("id ASC")
|
||||
if len(accountsWithMonitors) > 0 {
|
||||
q = q.Where("id NOT IN (?)", accountsWithMonitors)
|
||||
}
|
||||
if err := q.Find(&accounts).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]StaleAccountCandidate, 0, len(accounts))
|
||||
for i := range accounts {
|
||||
acc := accounts[i]
|
||||
|
||||
// Step 2: every user with access to this account must have
|
||||
// exactly one account membership total.
|
||||
type userAccessCount struct {
|
||||
UserID int64
|
||||
Cnt int
|
||||
}
|
||||
counts := make([]userAccessCount, 0)
|
||||
err := DB().
|
||||
Table("accesses AS a1").
|
||||
Select("a1.user_id AS user_id, (SELECT COUNT(*) FROM accesses AS a2 WHERE a2.user_id = a1.user_id) AS cnt").
|
||||
Where("a1.account_id = ? AND a1.user_id IS NOT NULL", acc.ID).
|
||||
Group("a1.user_id").
|
||||
Scan(&counts).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
// An account with no user accesses is a config bug
|
||||
// (the owner access should always exist). Skip.
|
||||
continue
|
||||
}
|
||||
allSingle := true
|
||||
for _, c := range counts {
|
||||
if c.Cnt != 1 {
|
||||
allSingle = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allSingle {
|
||||
continue
|
||||
}
|
||||
|
||||
// Step 3: every such user must be inactive beyond the cutoff.
|
||||
userIDs := make([]int64, 0, len(counts))
|
||||
for _, c := range counts {
|
||||
userIDs = append(userIDs, c.UserID)
|
||||
}
|
||||
users := make([]User, 0)
|
||||
if err := DB().Where("id IN (?)", userIDs).Find(&users).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allStale := true
|
||||
for j := range users {
|
||||
u := users[j]
|
||||
if u.LastActiveAt != nil && u.LastActiveAt.After(cutoff) {
|
||||
allStale = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allStale {
|
||||
continue
|
||||
}
|
||||
|
||||
// All gates passed — emit one candidate per user so the UI
|
||||
// can list which specific accounts would be removed.
|
||||
for j := range users {
|
||||
out = append(out, StaleAccountCandidate{
|
||||
AccountID: acc.ID,
|
||||
AccountName: acc.Name,
|
||||
UserID: users[j].ID,
|
||||
UserEmail: users[j].Email,
|
||||
LastActiveAt: users[j].LastActiveAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CleanupStaleAccounts hard-deletes every account eligible for the
|
||||
// admin "Удалить старые" sweep. Returns the number of accounts that
|
||||
// were deleted. The matching users (each of whom only belonged to one
|
||||
// account) are deleted by HardDeleteAccount's cascading access cleanup
|
||||
// only if they no longer have any other account — that final teardown
|
||||
// is done here.
|
||||
func CleanupStaleAccounts() (int, error) {
|
||||
candidates, err := FindStaleAccounts()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
accountIDs := make([]int64, 0, len(candidates))
|
||||
seen := make(map[int64]bool, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if !seen[c.AccountID] {
|
||||
seen[c.AccountID] = true
|
||||
accountIDs = append(accountIDs, c.AccountID)
|
||||
}
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, accountID := range accountIDs {
|
||||
if err := HardDeleteAccount(accountID); err != nil {
|
||||
log.Printf("CleanupStaleAccounts: failed to delete account %d: %v", accountID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
// Users that no longer have any accesses after the cascade are
|
||||
// clearly orphaned — purge them so the auth layer doesn't keep
|
||||
// dangling rows around. We bypass HardDeleteUser here because the
|
||||
// account-level data (monitors, checks, groups, etc.) was already
|
||||
// removed by the HardDeleteAccount loop above, so only the user
|
||||
// row and the dangling identities / contacts need cleanup.
|
||||
if deleted > 0 {
|
||||
orphans := make([]int64, 0)
|
||||
err := DB().
|
||||
Table("users").
|
||||
Select("users.id").
|
||||
Joins("LEFT JOIN accesses ON accesses.user_id = users.id").
|
||||
Where("accesses.id IS NULL").
|
||||
Pluck("users.id", &orphans).Error
|
||||
if err != nil {
|
||||
log.Printf("CleanupStaleAccounts: orphan user scan failed: %v", err)
|
||||
return deleted, nil
|
||||
}
|
||||
for _, userID := range orphans {
|
||||
tx := DB().Begin()
|
||||
if err := tx.Exec("DELETE FROM identities WHERE user_id = ?", userID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: identities delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Where("user_id = ? AND account_id IS NULL", userID).Delete(&Contact{}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: contacts delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Where("user_id = ?", userID).Delete(&ApiKey{}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: api_keys delete failed for user %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Delete(&User{}, userID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
log.Printf("CleanupStaleAccounts: user delete failed for %d: %v", userID, err)
|
||||
continue
|
||||
}
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
log.Printf("CleanupStaleAccounts: commit failed for user %d: %v", userID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if deleted > 0 {
|
||||
log.Printf("CleanupStaleAccounts: hard-deleted %d stale account(s)", deleted)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
352
app/models/deletion_test.go
Обычный файл
352
app/models/deletion_test.go
Обычный файл
@@ -0,0 +1,352 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/icrowley/fake"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
var gormErrRecordNotFound = gorm.ErrRecordNotFound
|
||||
|
||||
func deletionStringPtr(s string) *string { return &s }
|
||||
func deletionBoolPtr(b bool) *bool { return &b }
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// TestUserDeletionPending tests the DeletionPending method
|
||||
func TestUserDeletionPending(t *testing.T) {
|
||||
t.Run("returns false when no deletion requested", func(t *testing.T) {
|
||||
user := models.User{}
|
||||
assert.False(t, user.DeletionPending())
|
||||
})
|
||||
|
||||
t.Run("returns true when deletion requested", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
user := models.User{DeletionRequestedAt: &now}
|
||||
assert.True(t, user.DeletionPending())
|
||||
})
|
||||
}
|
||||
|
||||
// TestUserAsJSONIncludesDeletionField verifies that the AsJSON output
|
||||
// contains the deletion_requested_at field so the frontend can react to it.
|
||||
func TestUserAsJSONIncludesDeletionField(t *testing.T) {
|
||||
email := "test@example.com"
|
||||
now := time.Now()
|
||||
user := models.User{
|
||||
ID: 42,
|
||||
Email: &email,
|
||||
Name: "Test User",
|
||||
DeletionRequestedAt: &now,
|
||||
}
|
||||
|
||||
result := user.AsJSON()
|
||||
assert.NotNil(t, result)
|
||||
assert.Contains(t, result, "deletion_requested_at")
|
||||
assert.Equal(t, &now, result["deletion_requested_at"])
|
||||
}
|
||||
|
||||
// TestRequestUserDeletionNilUser ensures nil-safety
|
||||
func TestRequestUserDeletionNilUser(t *testing.T) {
|
||||
err := models.RequestUserDeletion(nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestCancelUserDeletionNilUser ensures nil-safety
|
||||
func TestCancelUserDeletionNilUser(t *testing.T) {
|
||||
err := models.CancelUserDeletion(nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestCacheInvalidationHook_CalledByRequestDeletion verifies that
|
||||
// RequestUserDeletion calls OnUserCacheInvalidate with the correct user ID.
|
||||
func TestCacheInvalidationHook_CalledByRequestDeletion(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
email := fake.EmailAddress()
|
||||
user := models.User{Email: &email, Name: "CacheHookTest"}
|
||||
assert.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
var calledID int64
|
||||
originalHook := models.OnUserCacheInvalidate
|
||||
models.OnUserCacheInvalidate = func(userID int64) {
|
||||
calledID = userID
|
||||
}
|
||||
defer func() { models.OnUserCacheInvalidate = originalHook }()
|
||||
|
||||
assert.NoError(t, models.RequestUserDeletion(&user))
|
||||
assert.Equal(t, user.ID, calledID)
|
||||
|
||||
models.DB().Unscoped().Delete(&user)
|
||||
}
|
||||
|
||||
// TestCacheInvalidationHook_CalledByCancelUserDeletion verifies that
|
||||
// CancelUserDeletion calls OnUserCacheInvalidate with the correct user ID.
|
||||
func TestCacheInvalidationHook_CalledByCancelUserDeletion(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
email := fake.EmailAddress()
|
||||
now := time.Now()
|
||||
user := models.User{Email: &email, Name: "CacheHookCancel", DeletionRequestedAt: &now}
|
||||
assert.NoError(t, models.DB().Create(&user).Error)
|
||||
|
||||
var calledID int64
|
||||
originalHook := models.OnUserCacheInvalidate
|
||||
models.OnUserCacheInvalidate = func(userID int64) {
|
||||
calledID = userID
|
||||
}
|
||||
defer func() { models.OnUserCacheInvalidate = originalHook }()
|
||||
|
||||
assert.NoError(t, models.CancelUserDeletion(&user))
|
||||
assert.Equal(t, user.ID, calledID)
|
||||
|
||||
models.DB().Unscoped().Delete(&user)
|
||||
}
|
||||
|
||||
// TestHardDeleteAccount_FKCascadeRegression covers the FK regressions
|
||||
// that surfaced as a series of distinct SQL errors when an operator
|
||||
// purged an account that owned monitors with distributed-worker
|
||||
// activity or per-user contacts:
|
||||
//
|
||||
// 1. check_region_results.check_id → checks(id) had no ON DELETE
|
||||
// CASCADE, so deleting a check while it still had region-result
|
||||
// rows raised 23503.
|
||||
// 2. worker_nodes did not have an account_id column, so the
|
||||
// "Workers scoped to this account" delete raised 42703.
|
||||
// 3. contacts.account_id → accounts(id) had no ON DELETE CASCADE,
|
||||
// and the original delete filter required user_id IS NULL, so
|
||||
// per-user contacts created in CreateAccountForUser (both
|
||||
// account_id and user_id set) survived and blocked the account
|
||||
// delete with 23503.
|
||||
//
|
||||
// The test seeds an account with a monitor, a check with a region
|
||||
// result row, an account-scoped LLM, a per-user contact, and both a
|
||||
// private worker (with account_id) and an operated worker (NULL
|
||||
// account_id), then runs HardDeleteAccount and asserts that the
|
||||
// account, the private worker, the LLM, the monitor/check/region
|
||||
// result, and the per-user contact all disappear, while the operated
|
||||
// worker and an unrelated user-only contact survive.
|
||||
func TestHardDeleteAccount_FKCascadeRegression(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "hard-delete-regression"}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
account := models.Account{Name: "victim", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&account).Error)
|
||||
otherAccount := models.Account{Name: "survivor", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&otherAccount).Error)
|
||||
|
||||
owner := models.User{Name: "owner", Email: deletionStringPtr("owner@example.com"), Timezone: "UTC"}
|
||||
require.NoError(t, models.DB().Create(&owner).Error)
|
||||
stranger := models.User{Name: "stranger", Email: deletionStringPtr("stranger@example.com"), Timezone: "UTC"}
|
||||
require.NoError(t, models.DB().Create(&stranger).Error)
|
||||
|
||||
region := models.Region{}
|
||||
if err := models.DB().Where("code = ?", "test").First(®ion).Error; err != nil {
|
||||
require.NoError(t, models.DB().Create(&models.Region{Code: "test", Name: "test", Enabled: true}).Error)
|
||||
}
|
||||
|
||||
group := models.Group{Name: "g", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{
|
||||
Name: deletionStringPtr("m"),
|
||||
Host: "example.com",
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
check := models.Check{
|
||||
MonitorID: monitor.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
Enabled: deletionBoolPtr(true),
|
||||
State: "UNK",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
|
||||
// Per-user contact that references both the victim account and a
|
||||
// user. CreateAccountForUser writes a contact in this shape, and
|
||||
// the old "user_id IS NULL" filter would let it survive and block
|
||||
// the account delete with contacts_account_id_fkey 23503.
|
||||
ownerContact := models.Contact{
|
||||
AccountID: &account.ID,
|
||||
UserID: &owner.ID,
|
||||
Name: "owner-email",
|
||||
Kind: "email",
|
||||
Value: "owner@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&ownerContact).Error)
|
||||
// Account-only contact (user_id IS NULL) — also tied to the account
|
||||
// via FK and must be removed.
|
||||
accountContact := models.Contact{
|
||||
AccountID: &account.ID,
|
||||
Name: "ops",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&accountContact).Error)
|
||||
notification := models.Notification{AccountID: account.ID, Name: "alerts", Enabled: true}
|
||||
require.NoError(t, models.DB().Create(¬ification).Error)
|
||||
require.NoError(t, models.DB().Model(¬ification).Association("Contacts").Append(&accountContact))
|
||||
message := models.Message{
|
||||
NotificationID: notification.ID,
|
||||
ContactID: accountContact.ID,
|
||||
Kind: "test",
|
||||
State: "OK",
|
||||
CreatedAt: time.Now(),
|
||||
SentAt: time.Now(),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
// User-only contact on a stranger — must survive account deletion.
|
||||
userOnlyContact := models.Contact{
|
||||
UserID: &stranger.ID,
|
||||
Name: "stranger",
|
||||
Kind: "email",
|
||||
Value: "stranger@example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&userOnlyContact).Error)
|
||||
|
||||
// Region result row: this is the row that used to trigger the
|
||||
// fk_check_region_results_check FK violation when Check was
|
||||
// deleted. Without the fix the entire HardDeleteAccount would
|
||||
// fail here.
|
||||
privateWorker := &models.WorkerNode{
|
||||
WorkerID: "private-" + uuid.New().String(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "priv-tok-" + uuid.New().String(),
|
||||
AccountID: &account.ID,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(privateWorker).Error)
|
||||
operatedWorker := &models.WorkerNode{
|
||||
WorkerID: "operated-" + uuid.New().String(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: "op-tok-" + uuid.New().String(),
|
||||
AccountID: nil,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(operatedWorker).Error)
|
||||
require.NoError(t, models.DB().Create(&models.CheckRegionResult{
|
||||
CheckID: check.ID,
|
||||
RegionCode: "test",
|
||||
WorkerNodeID: &privateWorker.ID,
|
||||
ExecutedAt: time.Now(),
|
||||
State: "OK",
|
||||
}).Error)
|
||||
|
||||
// LLM scoped to the victim account + linked to the private worker.
|
||||
// worker_llms.llm_id has a FK to llms(id) without ON DELETE
|
||||
// CASCADE, so the join row used to block LLM deletion too.
|
||||
llm := models.LLM{
|
||||
AccountID: &account.ID,
|
||||
Name: "private-llm",
|
||||
URL: "https://llm.example.com",
|
||||
ModelName: "gpt-test",
|
||||
APIKey: "secret",
|
||||
Kind: "openai",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&llm).Error)
|
||||
require.NoError(t, models.DB().Exec(
|
||||
"INSERT INTO worker_llms (worker_node_id, llm_id) VALUES (?, ?)",
|
||||
privateWorker.ID, llm.ID,
|
||||
).Error)
|
||||
|
||||
// Inventory entities scoped to the victim account. Each has an
|
||||
// account_id FK to accounts(id) without ON DELETE CASCADE so they
|
||||
// must be removed before the account row goes away. The
|
||||
// shared-infra server belongs to another account and must
|
||||
// survive.
|
||||
victimServer := models.Server{Name: "victim-srv", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&victimServer).Error)
|
||||
victimServerIP := models.ServerIp{ServerID: victimServer.ID, Address: "10.0.0.1"}
|
||||
require.NoError(t, models.DB().Create(&victimServerIP).Error)
|
||||
victimSite := models.Site{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID,
|
||||
Slug: "victim-site", Name: "victim-site", Kind: "production", IsActive: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimSite).Error)
|
||||
victimDeployment := models.Deployment{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
|
||||
Kind: "production", Mode: "compose",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimDeployment).Error)
|
||||
victimDomain := models.Domain{
|
||||
AccountID: account.ID, ServerID: &victimServer.ID, SiteID: &victimSite.ID,
|
||||
Name: "victim.example.com",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&victimDomain).Error)
|
||||
otherServer := models.Server{Name: "shared-srv", AccountID: otherAccount.ID}
|
||||
require.NoError(t, models.DB().Create(&otherServer).Error)
|
||||
|
||||
require.NoError(t, models.HardDeleteAccount(account.ID))
|
||||
|
||||
// Account and all account-scoped rows must be gone.
|
||||
assert.ErrorIs(t, models.DB().First(&models.Account{}, account.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Check{}, check.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Monitor{}, monitor.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Group{}, group.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Notification{}, notification.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Message{}, message.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.LLM{}, llm.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Contact{}, ownerContact.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Contact{}, accountContact.ID).Error, gormErrRecordNotFound)
|
||||
var notificationContactCount int64
|
||||
require.NoError(t, models.DB().Table("notification_contacts").
|
||||
Where("notification_id = ? OR contact_id = ?", notification.ID, accountContact.ID).
|
||||
Count(¬ificationContactCount).Error)
|
||||
assert.Zero(t, notificationContactCount)
|
||||
|
||||
// Private worker must be gone; operated worker must survive.
|
||||
assert.ErrorIs(t, models.DB().First(&models.WorkerNode{}, privateWorker.ID).Error, gormErrRecordNotFound)
|
||||
var stillOperated models.WorkerNode
|
||||
require.NoError(t, models.DB().First(&stillOperated, operatedWorker.ID).Error)
|
||||
assert.Nil(t, stillOperated.AccountID, "operated worker account_id must remain NULL")
|
||||
|
||||
// User-only contact (no account_id) must survive.
|
||||
var stillUserOnly models.Contact
|
||||
require.NoError(t, models.DB().First(&stillUserOnly, userOnlyContact.ID).Error)
|
||||
|
||||
// Inventory entities scoped to the account must be gone.
|
||||
assert.ErrorIs(t, models.DB().First(&models.Server{}, victimServer.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Site{}, victimSite.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Deployment{}, victimDeployment.ID).Error, gormErrRecordNotFound)
|
||||
assert.ErrorIs(t, models.DB().First(&models.Domain{}, victimDomain.ID).Error, gormErrRecordNotFound)
|
||||
var leftover int64
|
||||
require.NoError(t, models.DB().Model(&models.ServerIp{}).
|
||||
Where("server_id = ?", victimServer.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "server_ips for the deleted server must be cleaned up")
|
||||
// Other-account inventory must survive.
|
||||
var stillOtherServer models.Server
|
||||
require.NoError(t, models.DB().First(&stillOtherServer, otherServer.ID).Error)
|
||||
|
||||
// Region-result rows for the deleted check must be gone.
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("check_id = ?", check.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "check_region_results must be cleaned up before checks")
|
||||
require.NoError(t, models.DB().Model(&models.CheckRegionResult{}).
|
||||
Where("worker_node_id = ?", privateWorker.ID).Count(&leftover).Error)
|
||||
assert.Zero(t, leftover, "check_region_results referencing a private worker must be cleaned up")
|
||||
require.NoError(t, models.DB().Raw(
|
||||
"SELECT COUNT(*) FROM worker_llms WHERE llm_id = ? OR worker_node_id = ?",
|
||||
llm.ID, privateWorker.ID,
|
||||
).Scan(&leftover).Error)
|
||||
assert.Zero(t, leftover, "worker_llms rows for the deleted LLM and private worker must be gone")
|
||||
|
||||
// Sanity: the other account and its data are untouched.
|
||||
var stillOther models.Account
|
||||
require.NoError(t, models.DB().First(&stillOther, otherAccount.ID).Error)
|
||||
}
|
||||
240
app/models/deployment.go
Обычный файл
240
app/models/deployment.go
Обычный файл
@@ -0,0 +1,240 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DeploymentKind is the rstuff-mirrored lifecycle label for a
|
||||
// deployment (production / production_prev / production_next /
|
||||
// internal / staging / old). See
|
||||
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable numeric
|
||||
// mapping (which Postgres stores alphabetically, not numerically).
|
||||
type DeploymentKind string
|
||||
|
||||
// Lifecycle labels for a Deployment row. See DeploymentKind for
|
||||
// the matching rstuff enum. The label set is closed; new values
|
||||
// require adding a Postgres enum value via app/models/migrate.go.
|
||||
const (
|
||||
// DeploymentKindProduction is the customer-facing "live" deployment.
|
||||
DeploymentKindProduction DeploymentKind = "production"
|
||||
DeploymentKindProductionPrev DeploymentKind = "production_prev"
|
||||
DeploymentKindProductionNext DeploymentKind = "production_next"
|
||||
// DeploymentKindInternal is for ops/admin tooling (not customer-facing).
|
||||
DeploymentKindInternal DeploymentKind = "internal"
|
||||
DeploymentKindStaging DeploymentKind = "staging"
|
||||
DeploymentKindOld DeploymentKind = "old"
|
||||
)
|
||||
|
||||
// DeploymentMode is the host-side lifecycle label (kubernetes /
|
||||
// compose / dedicated / vds / user). `dedicated` covers a single
|
||||
// nginx vhost; `compose` covers a Docker Compose project.
|
||||
type DeploymentMode string
|
||||
|
||||
// DeploymentMode values map onto rstuff's Deployment.mode enum.
|
||||
// `dedicated` covers a single nginx vhost; `compose` covers a
|
||||
// Docker Compose project; the others are reserved for future
|
||||
// v2 surfaces (Kubernetes, VDS, user-owned).
|
||||
const (
|
||||
DeploymentModeKubernetes DeploymentMode = "kubernetes"
|
||||
DeploymentModeCompose DeploymentMode = "compose"
|
||||
DeploymentModeDedicated DeploymentMode = "dedicated"
|
||||
DeploymentModeVDS DeploymentMode = "vds"
|
||||
DeploymentModeUser DeploymentMode = "user"
|
||||
)
|
||||
|
||||
// DeploymentAction is the reconciliation state. Updated by the
|
||||
// deploymentd receiver on every POST and by the 90s reconcile sweep
|
||||
// (see app/models/deployment.go ReconcileMissing).
|
||||
type DeploymentAction string
|
||||
|
||||
// DeploymentAction values. Pending/PendingMove/PendingDrop are
|
||||
// transient (operator or receiver-initiated); Deleted/Missing are
|
||||
// sticky until the deployment shows up again on a future POST.
|
||||
const (
|
||||
DeploymentActionOk DeploymentAction = "ok"
|
||||
DeploymentActionPending DeploymentAction = "pending"
|
||||
DeploymentActionPendingMove DeploymentAction = "pending_move"
|
||||
DeploymentActionPendingDrop DeploymentAction = "pending_drop"
|
||||
DeploymentActionDeleted DeploymentAction = "deleted"
|
||||
DeploymentActionMissing DeploymentAction = "missing"
|
||||
)
|
||||
|
||||
// Deployment represents a single host-side binding: one nginx
|
||||
// vhost, one Docker Compose service, or one Kubernetes service. The
|
||||
// shape mirrors rstuff's `deployments` table. See
|
||||
// docs/plans/inventory-management.md §4 / §6.1.
|
||||
type Deployment struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
ServiceName *string `gorm:"size:120" json:"service_name,omitempty"`
|
||||
|
||||
Kind DeploymentKind `gorm:"type:deployment_kind;not null;default:'production'" json:"kind"`
|
||||
Mode DeploymentMode `gorm:"type:deployment_mode;not null;default:'dedicated'" json:"mode"`
|
||||
Action DeploymentAction `gorm:"type:deployment_action;not null;default:'ok'" json:"action"`
|
||||
|
||||
URL *string `gorm:"type:text" json:"url,omitempty"`
|
||||
SSHUser *string `gorm:"size:64" json:"ssh_user,omitempty"`
|
||||
RootPath *string `gorm:"type:text" json:"root_path,omitempty"`
|
||||
ConfigPath *string `gorm:"type:text" json:"config_path,omitempty"`
|
||||
IP *string `gorm:"type:inet" json:"ip,omitempty"`
|
||||
Listen pq.StringArray `gorm:"type:varchar(64)[];not null;default:'{}'" json:"listen"`
|
||||
ServerName pq.StringArray `gorm:"type:varchar(255)[];not null;default:'{}'" json:"server_name"`
|
||||
Auth bool `gorm:"not null;default:false" json:"auth"`
|
||||
IsProxied bool `gorm:"not null;default:false" json:"is_proxied"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Deployment) TableName() string { return "deployments" }
|
||||
|
||||
// Scan implements sql.Scanner so a Postgres enum value can land in
|
||||
// our typed string alias without a code-generation step.
|
||||
func (d *DeploymentKind) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentKind(v)
|
||||
case []byte:
|
||||
*d = DeploymentKind(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_kind: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for the inverse direction.
|
||||
func (d DeploymentKind) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for DeploymentMode.
|
||||
func (d *DeploymentMode) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentMode(v)
|
||||
case []byte:
|
||||
*d = DeploymentMode(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_mode: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for DeploymentMode.
|
||||
func (d DeploymentMode) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for DeploymentAction.
|
||||
func (d *DeploymentAction) Scan(src any) error {
|
||||
if src == nil {
|
||||
*d = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*d = DeploymentAction(v)
|
||||
case []byte:
|
||||
*d = DeploymentAction(string(v))
|
||||
default:
|
||||
return fmt.Errorf("deployment_action: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for DeploymentAction.
|
||||
func (d DeploymentAction) Value() (driver.Value, error) {
|
||||
if d == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(d), nil
|
||||
}
|
||||
|
||||
// ReconcileMissingDeployments flips action='missing' on every Deployment for the
|
||||
// given server whose last_seen_at is older than cutoff. Called by
|
||||
// the deploymentd receiver after every successful upsert so the
|
||||
// "missing" badge appears within one POST cycle.
|
||||
//
|
||||
// Idempotent: re-running with the same cutoff is a no-op.
|
||||
func ReconcileMissingDeployments(serverID int64, mode DeploymentMode, cutoff time.Time) (int64, error) {
|
||||
res := DB().Model(&Deployment{}).
|
||||
Where("server_id = ? AND mode = ? AND action NOT IN ?", serverID, mode,
|
||||
[]DeploymentAction{DeploymentActionDeleted, DeploymentActionMissing, DeploymentActionPendingDrop}).
|
||||
Where("last_seen_at IS NULL OR last_seen_at < ?", cutoff).
|
||||
Update("action", DeploymentActionMissing)
|
||||
return res.RowsAffected, res.Error
|
||||
}
|
||||
|
||||
// UpsertNginxDeployment finds or creates a Deployment by
|
||||
// (server_id, config_path) for an nginx vhost. The caller fills in
|
||||
// the lifecycle fields (listen, server_name, etc.) after the upsert
|
||||
// returns. The return value is the row to mutate; the caller MUST
|
||||
// also touch last_seen_at and save.
|
||||
func UpsertNginxDeployment(tx *gorm.DB, accountID int64, serverID int64, configPath string) (*Deployment, error) {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
var d Deployment
|
||||
err := tx.Where("server_id = ? AND config_path = ?", serverID, configPath).First(&d).Error
|
||||
if err == nil {
|
||||
return &d, nil
|
||||
}
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
return nil, err
|
||||
}
|
||||
d = Deployment{
|
||||
AccountID: accountID,
|
||||
ServerID: &serverID,
|
||||
Kind: DeploymentKindProduction,
|
||||
Mode: DeploymentModeDedicated,
|
||||
Action: DeploymentActionOk,
|
||||
ConfigPath: &configPath,
|
||||
}
|
||||
if err := tx.Create(&d).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// RotateServerToken sets a new random token for a server and returns
|
||||
// the plaintext. Called by the operator-only
|
||||
// POST /api/v1/servers/:id/rotate-token endpoint. The plaintext is
|
||||
// returned exactly once — it is not stored anywhere recoverable.
|
||||
func RotateServerToken(tx *gorm.DB, serverID int64, newToken string) error {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
return tx.Model(&Server{}).Where("id = ?", serverID).Update("token", newToken).Error
|
||||
}
|
||||
80
app/models/dns_record.go
Обычный файл
80
app/models/dns_record.go
Обычный файл
@@ -0,0 +1,80 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/netaddr"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DNSRecord provides functionality.
|
||||
type DNSRecord struct {
|
||||
concerns.Model
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id,omitempty"`
|
||||
Monitor *Monitor `json:"-"`
|
||||
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Value netaddr.Inet `json:"value" gorm:"type:bytea;"`
|
||||
}
|
||||
|
||||
// SaveIps provides functionality.
|
||||
func SaveIps(m *Monitor, _ Check, ips []NSRecord) { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
// log.Println("save ips for monitor")
|
||||
// spew.Dump(m)
|
||||
var err error
|
||||
tx := DB().Begin()
|
||||
|
||||
currentRecords := make([]DNSRecord, 0)
|
||||
_ = tx.Model(&m).Association("DNSRecords").Find(¤tRecords)
|
||||
|
||||
recordHash := make(map[string]DNSRecord, 0)
|
||||
for _, record := range currentRecords {
|
||||
recordHash[record.Name] = record
|
||||
}
|
||||
nextRecords := make(map[string]bool, 0)
|
||||
|
||||
for _, ip := range ips {
|
||||
if record, ok := recordHash[ip.Name]; ok {
|
||||
// log.Println("old value:", record)
|
||||
record.MonitorID = m.ID
|
||||
record.Value = ip.Value
|
||||
record.Kind = ip.Kind
|
||||
err = tx.Model(&m).Association("DNSRecords").Replace(&record)
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
record = DNSRecord{
|
||||
MonitorID: m.ID,
|
||||
Name: ip.Name,
|
||||
Kind: ip.Kind,
|
||||
Value: ip.Value,
|
||||
}
|
||||
if _, ok := nextRecords[ip.Name]; !ok {
|
||||
nextRecords[ip.Name] = true
|
||||
err = tx.Model(&m).Association("DNSRecords").Append(&record)
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// spew.Dump(ips_hash)
|
||||
|
||||
err = tx.Commit().Error
|
||||
if err != nil {
|
||||
log.Println("fatal error in saveips", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// NSRecord provides functionality.
|
||||
type NSRecord struct {
|
||||
Name string
|
||||
Kind string
|
||||
Value netaddr.Inet
|
||||
}
|
||||
32
app/models/domain.go
Обычный файл
32
app/models/domain.go
Обычный файл
@@ -0,0 +1,32 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Domain represents a customer-facing DNS name. The shape mirrors
|
||||
// rstuff's `domains` table — see docs/parity/rstuff-inventory.md §2
|
||||
// and docs/plans/inventory-management.md §4.
|
||||
//
|
||||
// Distinct from RknDomain (the RKN blocklist cache, see
|
||||
// app/models/rkn_domain.go): RknDomain is read-only data about
|
||||
// blocked domains; Domain is the customer-side name→site/server
|
||||
// pointer that monitoring reasons about.
|
||||
type Domain struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
Name string `gorm:"size:255;not null" json:"name"`
|
||||
Env string `gorm:"size:32;not null;default:'production'" json:"env"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Domain) TableName() string { return "domains" }
|
||||
96
app/models/drop.go
Обычный файл
96
app/models/drop.go
Обычный файл
@@ -0,0 +1,96 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
)
|
||||
|
||||
// Drop removes all test data from the database.
|
||||
// It uses TRUNCATE ... CASCADE for join tables and deletes leaf-to-root for main tables.
|
||||
// Safe to call from multiple goroutines within a single test binary; concurrent calls
|
||||
// from separate test binaries are serialized by the advisory lock in Migrate().
|
||||
func Drop() {
|
||||
var dbname string
|
||||
if err := DB().Raw("SELECT current_database()").Scan(&dbname).Error; err != nil {
|
||||
panic(fmt.Sprintf("Drop: cannot read database name: %v", err))
|
||||
}
|
||||
if !strings.Contains(dbname, "test") {
|
||||
panic(fmt.Sprintf(
|
||||
"Drop() refused: database '%s' is not a test database. "+
|
||||
"Set DATABASE_NAME=rsmon_test to run tests safely.", dbname,
|
||||
))
|
||||
}
|
||||
|
||||
// Truncate many2many join tables first to avoid FK violations.
|
||||
DB().Exec("TRUNCATE event_checks, event_messages, notification_contacts, notification_groups RESTART IDENTITY CASCADE")
|
||||
|
||||
// Delete main tables in dependency order (leaf tables first).
|
||||
DB().Where("1=1").Delete(&NotificationDelivery{})
|
||||
DB().Where("1=1").Delete(&TaskReplay{})
|
||||
DB().Where("1=1").Delete(&TelegramBotMessage{})
|
||||
DB().Where("1=1").Delete(&TelegramBotStatus{})
|
||||
DB().Where("1=1").Delete(&Task{})
|
||||
DB().Where("1=1").Delete(&Access{})
|
||||
DB().Where("1=1").Delete(&Invite{})
|
||||
DB().Where("1=1").Delete(&Message{})
|
||||
DB().Where("1=1").Delete(&Event{})
|
||||
// Durable network-diagnostic rows reference checks, monitors, and workers.
|
||||
DB().Where("1=1").Delete(&DiagnosticAuditEvent{})
|
||||
DB().Where("1=1").Delete(&CheckAttempt{})
|
||||
// CheckRegionResult FKs check_id; it must be cleared before Check.
|
||||
DB().Where("1=1").Delete(&CheckRegionResult{})
|
||||
DB().Where("1=1").Delete(&Check{})
|
||||
DB().Where("1=1").Delete(&Notification{})
|
||||
// DNSRecord FKs monitor_id; it must be cleared before Monitor.
|
||||
DB().Where("1=1").Delete(&DNSRecord{})
|
||||
DB().Where("1=1").Delete(&Monitor{})
|
||||
DB().Where("1=1").Delete(&Group{})
|
||||
DB().Where("1=1").Delete(&Contact{})
|
||||
DB().Where("1=1").Delete(&NotificationCredential{})
|
||||
DB().Where("1=1").Delete(&WorkerLogEvent{})
|
||||
DB().Where("1=1").Delete(&WorkerNode{})
|
||||
DB().Where("1=1").Delete(&LLM{})
|
||||
DB().Where("1=1").Delete(&Region{})
|
||||
// Inventory (docs/plans/inventory-management.md §6): leaf tables
|
||||
// (sites, deployments, server_ips) reference accounts/servers, so
|
||||
// they must be cleared before Server is deleted.
|
||||
DB().Where("1=1").Delete(&Deployment{})
|
||||
DB().Where("1=1").Delete(&SiteRepo{})
|
||||
DB().Where("1=1").Delete(&Site{})
|
||||
DB().Where("1=1").Delete(&Repo{})
|
||||
DB().Where("1=1").Delete(&ServerIp{})
|
||||
DB().Where("1=1").Delete(&Domain{})
|
||||
DB().Where("1=1").Delete(&Server{})
|
||||
// Account-scoped tag metadata (account_id FK to accounts(id)).
|
||||
// Cleared before Account so a future cascade change cannot orphan
|
||||
// rows mid-truncate.
|
||||
DB().Where("1=1").Delete(&Tag{})
|
||||
// Status pages (docs/plans/status-pages.md §3.1–3.5). Children
|
||||
// reference status_pages(id) with ON DELETE CASCADE so GORM
|
||||
// ordering would already wipe them, but we delete them
|
||||
// explicitly so the test DB stays clean even if a future model
|
||||
// change drops the cascade.
|
||||
DB().Where("1=1").Delete(&StatusPageDomain{})
|
||||
DB().Unscoped().Where("1=1").Delete(&Maintenance{})
|
||||
DB().Where("1=1").Delete(&StatusPageMaintenance{})
|
||||
DB().Where("1=1").Delete(&StatusPageIncident{})
|
||||
DB().Where("1=1").Delete(&StatusPageDelivery{})
|
||||
DB().Where("1=1").Delete(&StatusPageDigestSchedule{})
|
||||
DB().Where("1=1").Delete(&StatusPageSubscriber{})
|
||||
DB().Unscoped().Where("1=1").Delete(&StatusPage{})
|
||||
DB().Where("1=1").Delete(&SubscriptionEvent{})
|
||||
DB().Where("1=1").Delete(&Subscription{})
|
||||
DB().Where("1=1").Delete(&Account{})
|
||||
|
||||
DB().Unscoped().Where("1=1").Delete(&authidentity.AuthIdentity{})
|
||||
// User has a unique email index, so test fixtures must be physically
|
||||
// removed rather than soft-deleted between tests.
|
||||
DB().Unscoped().Where("1=1").Delete(&User{})
|
||||
// A few audited tables added by feature migrations can retain a user FK that
|
||||
// is intentionally not modeled as an association. CASCADE keeps fixture
|
||||
// cleanup deterministic instead of silently leaving unique emails behind.
|
||||
DB().Exec("TRUNCATE users CASCADE")
|
||||
DB().Exec("TRUNCATE regions CASCADE")
|
||||
}
|
||||
83
app/models/event.go
Обычный файл
83
app/models/event.go
Обычный файл
@@ -0,0 +1,83 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/util"
|
||||
)
|
||||
|
||||
// Event provides functionality.
|
||||
type Event struct {
|
||||
concerns.Model
|
||||
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id)" json:"monitor_id"`
|
||||
Monitor *Monitor `json:"monitor,omitempty"`
|
||||
|
||||
StartTime *time.Time `json:"start_time"`
|
||||
EndTime *time.Time `json:"end_time"`
|
||||
Duration int `json:"duration"`
|
||||
Errors int `json:"errors"`
|
||||
Oks int `json:"oks"`
|
||||
State string `gorm:"index" json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
|
||||
Messages []Message `json:"messages" gorm:"many2many:event_messages;"`
|
||||
|
||||
ChecksDown pq.StringArray `gorm:"type:varchar(255)[]" json:"checks_down"`
|
||||
Checks []Check `json:"-" gorm:"many2many:event_checks;"`
|
||||
|
||||
ExpiresAt *time.Time `json:"-"`
|
||||
|
||||
Audited
|
||||
}
|
||||
|
||||
// EventScope provides functionality.
|
||||
func EventScope(q *gorm.DB) *gorm.DB {
|
||||
return q.Where("state IN ('current', 'ended')").
|
||||
Preload("Checks").
|
||||
Preload("Monitor").
|
||||
Preload("Monitor.Group").
|
||||
Preload("Monitor.Group.Notifications").
|
||||
Preload("Monitor.Group.Notifications.Contacts")
|
||||
}
|
||||
|
||||
// GetDuration provides functionality.
|
||||
func (e *Event) GetDuration(tn time.Time) int64 {
|
||||
endTime := e.EndTime
|
||||
if endTime == nil {
|
||||
endTime = &tn
|
||||
}
|
||||
return int64(endTime.Sub(*e.StartTime) / time.Second)
|
||||
}
|
||||
|
||||
// FormatDuration provides functionality.
|
||||
func (e *Event) FormatDuration() string {
|
||||
d := e.GetDuration(time.Now())
|
||||
return util.FormatDuration(d)
|
||||
}
|
||||
|
||||
// Inspect provides functionality.
|
||||
func (e *Event) Inspect() string {
|
||||
var st, et string
|
||||
if e.StartTime != nil {
|
||||
st = e.StartTime.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
if e.EndTime != nil {
|
||||
et = e.EndTime.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"Event<id: %d, monitor_id: %d, start_time: %s, end_time: %s, reason: %s, duration: %d>",
|
||||
e.ID,
|
||||
e.MonitorID,
|
||||
st,
|
||||
et,
|
||||
e.Reason,
|
||||
e.Duration,
|
||||
)
|
||||
}
|
||||
76
app/models/group.go
Обычный файл
76
app/models/group.go
Обычный файл
@@ -0,0 +1,76 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// Group represents a monitor group.
|
||||
type Group struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
IsSystem *bool `json:"is_system" gorm:"default:false"`
|
||||
MonitorsCount int `gorm:"-:all" json:"monitors_count"`
|
||||
Monitors []Monitor `json:"-"`
|
||||
Notifications []Notification `gorm:"many2many:notification_groups;" json:"-"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
Audited
|
||||
}
|
||||
|
||||
// SystemGroups returns all groups marked as system/internal (is_system=true).
|
||||
// These groups are converted to distributed monitors (system checks running
|
||||
// on the distributed worker pool).
|
||||
func SystemGroups() ([]Group, error) {
|
||||
var groups []Group
|
||||
err := DB().Where("is_system = ?", true).Find(&groups).Error
|
||||
return groups, err
|
||||
}
|
||||
|
||||
// GroupIdsForAccountId returns all group IDs for the given account.
|
||||
func GroupIdsForAccountId(accountID int64) []int64 { //nolint:revive // accepted lint exception
|
||||
rows, err := DB().Raw("SELECT id FROM groups WHERE account_id = ?", accountID).Rows()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // accepted lint exception
|
||||
var cid int64
|
||||
groupIDs := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
rows.Scan(&cid) //nolint:errcheck // accepted lint exception
|
||||
groupIDs = append(groupIDs, cid)
|
||||
}
|
||||
return groupIDs
|
||||
}
|
||||
|
||||
// CountGroups counts monitors per group.
|
||||
func CountGroups(groupIDs []int64, groupCount *map[int64]int) { //nolint:gocritic // ptrToRefParam: accepted pattern
|
||||
rows, err := DB().Raw("select group_id, count(id) from monitors where group_id IN (?) group by group_id ", groupIDs).Rows()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // accepted lint exception
|
||||
|
||||
var gid int64
|
||||
var count int
|
||||
for rows.Next() {
|
||||
rows.Scan(&gid, &count) //nolint:errcheck // accepted lint exception
|
||||
(*groupCount)[gid] = count
|
||||
}
|
||||
}
|
||||
|
||||
// GroupsCounts fills MonitorsCount for each group.
|
||||
func GroupsCounts(groups *[]Group) {
|
||||
groupIDs := make([]int64, len(*groups))
|
||||
groupCount := make(map[int64]int, len(*groups))
|
||||
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
|
||||
groupIDs[i] = g.ID
|
||||
groupCount[g.ID] = 0
|
||||
}
|
||||
|
||||
CountGroups(groupIDs, &groupCount)
|
||||
|
||||
for i, g := range *groups { //nolint:gocritic // range copy is acceptable here
|
||||
(*groups)[i].MonitorsCount = groupCount[g.ID]
|
||||
}
|
||||
}
|
||||
58
app/models/group_test.go
Обычный файл
58
app/models/group_test.go
Обычный файл
@@ -0,0 +1,58 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestSystemGroups verifies that SystemGroups returns only groups with
|
||||
// is_system=true and ignores groups with is_system=false or nil.
|
||||
func TestSystemGroups(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account := &models.Account{Name: "test-account"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
|
||||
trueVal, falseVal := true, false
|
||||
|
||||
systemGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "system-internal",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
regularGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "regular",
|
||||
IsSystem: &falseVal,
|
||||
}
|
||||
nilSystemGroup := &models.Group{
|
||||
AccountID: account.ID,
|
||||
Name: "nil-system",
|
||||
}
|
||||
|
||||
require.NoError(t, models.DB().Create(systemGroup).Error)
|
||||
require.NoError(t, models.DB().Create(regularGroup).Error)
|
||||
require.NoError(t, models.DB().Create(nilSystemGroup).Error)
|
||||
|
||||
got, err := models.SystemGroups()
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []int64
|
||||
var names []string
|
||||
for _, g := range got {
|
||||
ids = append(ids, g.ID)
|
||||
names = append(names, g.Name)
|
||||
}
|
||||
|
||||
assert.Contains(t, names, "system-internal")
|
||||
assert.NotContains(t, names, "regular")
|
||||
assert.NotContains(t, names, "nil-system")
|
||||
assert.Contains(t, ids, systemGroup.ID)
|
||||
assert.NotContains(t, ids, regularGroup.ID)
|
||||
assert.NotContains(t, ids, nilSystemGroup.ID)
|
||||
}
|
||||
30
app/models/init.go
Обычный файл
30
app/models/init.go
Обычный файл
@@ -0,0 +1,30 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/fatih/structs"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func init() {
|
||||
structs.DefaultTagName = "json"
|
||||
}
|
||||
|
||||
// db Gorm DB
|
||||
var db *gorm.DB
|
||||
|
||||
// DB provides functionality.
|
||||
func DB() *gorm.DB {
|
||||
return db.WithContext(context.TODO())
|
||||
}
|
||||
|
||||
// SetDB provides functionality.
|
||||
func SetDB(newDb *gorm.DB) {
|
||||
db = newDb
|
||||
}
|
||||
|
||||
// IsDBAvailable returns true if the database has been initialized
|
||||
func IsDBAvailable() bool {
|
||||
return db != nil
|
||||
}
|
||||
222
app/models/inventory_test.go
Обычный файл
222
app/models/inventory_test.go
Обычный файл
@@ -0,0 +1,222 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestServer_RotateToken(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "rotate-token-account"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "rotate-target",
|
||||
Slug: "rotate-target",
|
||||
Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
t1 := models.GenerateServerToken()
|
||||
require.NoError(t, models.RotateServerToken(nil, srv.ID, t1))
|
||||
|
||||
got, err := models.FindServerByToken(t1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srv.ID, got.ID)
|
||||
|
||||
t2 := models.GenerateServerToken()
|
||||
require.NoError(t, models.RotateServerToken(nil, srv.ID, t2))
|
||||
|
||||
// Old token no longer matches.
|
||||
_, err = models.FindServerByToken(t1)
|
||||
assert.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
||||
|
||||
got2, err := models.FindServerByToken(t2)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srv.ID, got2.ID)
|
||||
|
||||
// Generated tokens are hex-encoded 32 bytes (64 chars).
|
||||
assert.Len(t, t1, 64)
|
||||
assert.NotEqual(t, t1, t2)
|
||||
}
|
||||
|
||||
func TestServer_InventoryFields_Default(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "inv-defaults"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "fresh-server",
|
||||
Slug: "fresh-server",
|
||||
Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
got := models.Server{}
|
||||
require.NoError(t, models.DB().First(&got, srv.ID).Error)
|
||||
assert.Equal(t, models.ServerKindProduction, got.Kind)
|
||||
assert.Equal(t, 0, got.PriceCents)
|
||||
assert.False(t, got.Paused)
|
||||
}
|
||||
|
||||
func TestServer_KindEnum_OnlyAllowsValidValues(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "inv-enum"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID,
|
||||
Name: "kinder",
|
||||
Slug: "kinder",
|
||||
Region: "local",
|
||||
Kind: models.ServerKindStaging,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
got := models.Server{}
|
||||
require.NoError(t, models.DB().First(&got, srv.ID).Error)
|
||||
assert.Equal(t, models.ServerKindStaging, got.Kind)
|
||||
|
||||
// Inserting an invalid value via raw SQL fails the enum check.
|
||||
err := models.DB().Exec(
|
||||
"INSERT INTO servers (account_id, name, slug, region, kind) VALUES (?, ?, ?, ?, ?)",
|
||||
acc.ID, "bad-kinder", "bad-kinder", "local", "scrapped",
|
||||
).Error
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid input value for enum")
|
||||
}
|
||||
|
||||
func TestSite_Slugify(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"Cafe", "cafe"}, // cyrillic stripped (latin-only rule, see Slugify)
|
||||
{" Spaces Everywhere ", "spaces-everywhere"},
|
||||
{"dots.and-dashes_and spaces", "dots-and-dashes-and-spaces"},
|
||||
{"", "site"},
|
||||
{"-leading-and-trailing-", "leading-and-trailing"},
|
||||
{"mix_of.dots-dashes spaces", "mix-of-dots-dashes-spaces"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.in, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, models.SiteSlugify(c.in))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSite_FindOrCreateBySlug(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "site-foc"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
|
||||
got, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, got.ID, "row should be persisted")
|
||||
assert.Equal(t, "my-site", got.Slug)
|
||||
assert.Equal(t, "production", got.Kind)
|
||||
assert.True(t, got.IsActive)
|
||||
|
||||
// Second call returns the same row (idempotent).
|
||||
got2, err := models.FindOrCreateSiteBySlug(nil, acc.ID, "my-site")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, got.ID, got2.ID)
|
||||
}
|
||||
|
||||
func TestDeployment_UpsertNginx_MatchesByConfigPath(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "dep-upsert"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID, Name: "host1", Slug: "host1", Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
d1, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, models.DB().Save(d1).Error)
|
||||
|
||||
// Second call with the same config_path returns the existing row.
|
||||
d2, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/a.conf")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, d1.ID, d2.ID, "should reuse the same row on identical config_path")
|
||||
|
||||
// Different config_path creates a new row.
|
||||
d3, err := models.UpsertNginxDeployment(nil, acc.ID, srv.ID, "/etc/nginx/sites-enabled/b.conf")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, d1.ID, d3.ID)
|
||||
}
|
||||
|
||||
func TestDeployment_ReconcileMissing_FlipsAction(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
acc := &models.Account{Name: "reconcile"}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
srv := &models.Server{
|
||||
AccountID: acc.ID, Name: "rec", Slug: "rec", Region: "local",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(srv).Error)
|
||||
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
fresh := models.Deployment{
|
||||
AccountID: acc.ID,
|
||||
ServerID: &srv.ID,
|
||||
Kind: models.DeploymentKindProduction,
|
||||
Mode: models.DeploymentModeDedicated,
|
||||
Action: models.DeploymentActionOk,
|
||||
ConfigPath: ptr("/etc/nginx/old.conf"),
|
||||
LastSeenAt: &old,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&fresh).Error)
|
||||
|
||||
recent := models.Deployment{
|
||||
AccountID: acc.ID,
|
||||
ServerID: &srv.ID,
|
||||
Kind: models.DeploymentKindProduction,
|
||||
Mode: models.DeploymentModeDedicated,
|
||||
Action: models.DeploymentActionOk,
|
||||
ConfigPath: ptr("/etc/nginx/recent.conf"),
|
||||
LastSeenAt: ptrTime(time.Now()),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&recent).Error)
|
||||
|
||||
cutoff := time.Now().Add(-90 * time.Second)
|
||||
marked, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1, marked, "only the old row should flip")
|
||||
|
||||
var oldAfter models.Deployment
|
||||
require.NoError(t, models.DB().First(&oldAfter, fresh.ID).Error)
|
||||
assert.Equal(t, models.DeploymentActionMissing, oldAfter.Action)
|
||||
|
||||
var recentAfter models.Deployment
|
||||
require.NoError(t, models.DB().First(&recentAfter, recent.ID).Error)
|
||||
assert.Equal(t, models.DeploymentActionOk, recentAfter.Action, "fresh row stays ok")
|
||||
|
||||
// Re-running with the same cutoff is a no-op.
|
||||
marked2, err := models.ReconcileMissingDeployments(srv.ID, models.DeploymentModeDedicated, cutoff)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 0, marked2)
|
||||
}
|
||||
|
||||
func ptr(s string) *string { return &s }
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
52
app/models/invite.go
Обычный файл
52
app/models/invite.go
Обычный файл
@@ -0,0 +1,52 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Invite provides functionality.
|
||||
type Invite struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
InviterID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"inviter_id"`
|
||||
Inviter *User `json:"inviter"`
|
||||
InviteeID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"invitee_id"`
|
||||
Invitee *User `json:"invitee"`
|
||||
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
|
||||
// invite state, FAIL - failed to send, SENT - not regestired, OK - registered
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RegisteredAt time.Time `json:"registered_at"`
|
||||
SentAt time.Time `json:"sent_at"`
|
||||
|
||||
Accesses []Access `json:"accesses" gorm:"foreignkey:invite_id"`
|
||||
|
||||
concerns.HasToken
|
||||
Audited
|
||||
}
|
||||
|
||||
// BeforeCreate runs before creating an Invite record.
|
||||
func (i *Invite) BeforeCreate(_ *gorm.DB) error {
|
||||
i.SetToken()
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillAccesses provides functionality.
|
||||
func (i *Invite) FillAccesses() {
|
||||
for k, a := range i.Accesses { //nolint:gocritic // range copy is acceptable here
|
||||
if a.ID <= 0 {
|
||||
i.Accesses[k].ID = 0
|
||||
}
|
||||
i.Accesses[k].AccountID = i.AccountID
|
||||
}
|
||||
}
|
||||
17
app/models/llm.go
Обычный файл
17
app/models/llm.go
Обычный файл
@@ -0,0 +1,17 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// LLM stores an OpenAI-compatible LLM endpoint available to checks.
|
||||
type LLM struct {
|
||||
concerns.Model
|
||||
AccountID *int64 `json:"account_id" gorm:"type:bigint REFERENCES accounts(id);index"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
URL string `json:"url" gorm:"not null"`
|
||||
ModelName string `json:"model" gorm:"column:model;not null"`
|
||||
APIKey string `json:"-" gorm:"not null"`
|
||||
Kind string `json:"kind" gorm:"not null;default:'openai'"`
|
||||
Workers []WorkerNode `json:"-" gorm:"many2many:worker_llms;"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
324
app/models/maintenance.go
Обычный файл
324
app/models/maintenance.go
Обычный файл
@@ -0,0 +1,324 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
MaintenanceManual = "manual"
|
||||
MaintenanceSingle = "single"
|
||||
MaintenanceCron = "cron"
|
||||
MaintenanceRecurringInterval = "recurring-interval"
|
||||
MaintenanceRecurringWeekday = "recurring-weekday"
|
||||
MaintenanceRecurringDayOfMonth = "recurring-day-of-month"
|
||||
)
|
||||
|
||||
// Maintenance is account-owned planned downtime. Times are stored as UTC;
|
||||
// Timezone only defines how recurring wall-clock fields are interpreted.
|
||||
type Maintenance struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
Description string `gorm:"type:text;not null;default:''" json:"description"`
|
||||
Strategy string `gorm:"size:32;not null" json:"strategy"`
|
||||
Cron string `gorm:"type:text;not null;default:''" json:"cron"`
|
||||
DurationSec int `gorm:"not null;default:0" json:"duration_sec"`
|
||||
StartDate *time.Time `json:"start_date,omitempty"`
|
||||
EndDate *time.Time `json:"end_date,omitempty"`
|
||||
StartTime string `gorm:"size:5;not null;default:''" json:"start_time"`
|
||||
EndTime string `gorm:"size:5;not null;default:''" json:"end_time"`
|
||||
Weekdays pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"weekdays"`
|
||||
DaysOfMonth pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"days_of_month"`
|
||||
IntervalDay int `gorm:"not null;default:1" json:"interval_day"`
|
||||
Timezone string `gorm:"size:64;not null;default:'UTC'" json:"timezone"`
|
||||
Active bool `gorm:"not null;default:true" json:"active"`
|
||||
LastStartDate *time.Time `json:"last_start_date,omitempty"`
|
||||
LegacyStatusPageMaintenanceID *int64 `gorm:"uniqueIndex" json:"-"`
|
||||
ShowOnAllStatusPages bool `gorm:"not null;default:true" json:"show_on_all_status_pages"`
|
||||
Monitors []Monitor `gorm:"many2many:maintenance_monitors;constraint:OnDelete:CASCADE" json:"monitors,omitempty"`
|
||||
StatusPages []StatusPage `gorm:"many2many:maintenance_status_pages;constraint:OnDelete:CASCADE" json:"status_pages,omitempty"`
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
|
||||
}
|
||||
|
||||
func (Maintenance) TableName() string { return "maintenances" }
|
||||
|
||||
func (m *Maintenance) location() (*time.Location, error) {
|
||||
if m.Timezone == "" || m.Timezone == "SAME_AS_SERVER" {
|
||||
return time.UTC, nil
|
||||
}
|
||||
return time.LoadLocation(m.Timezone)
|
||||
}
|
||||
|
||||
func (m *Maintenance) generatedCron() (string, error) {
|
||||
if m.Strategy == MaintenanceCron {
|
||||
return m.Cron, nil
|
||||
}
|
||||
if m.Strategy == MaintenanceManual || m.Strategy == MaintenanceSingle {
|
||||
return "", nil
|
||||
}
|
||||
parts := strings.Split(m.StartTime, ":")
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("start_time must be HH:MM")
|
||||
}
|
||||
base := parts[1] + " " + parts[0]
|
||||
switch m.Strategy {
|
||||
case MaintenanceRecurringInterval:
|
||||
return "", nil
|
||||
case MaintenanceRecurringWeekday:
|
||||
if len(m.Weekdays) == 0 {
|
||||
return "", fmt.Errorf("at least one weekday is required")
|
||||
}
|
||||
values := make([]string, len(m.Weekdays))
|
||||
for i, day := range m.Weekdays {
|
||||
if day < 0 || day > 6 {
|
||||
return "", fmt.Errorf("weekday must be 0 through 6")
|
||||
}
|
||||
values[i] = fmt.Sprint(day)
|
||||
}
|
||||
return base + " * * " + strings.Join(values, ","), nil
|
||||
case MaintenanceRecurringDayOfMonth:
|
||||
if len(m.DaysOfMonth) == 0 {
|
||||
return "", fmt.Errorf("at least one day of month is required")
|
||||
}
|
||||
values := make([]string, 0, len(m.DaysOfMonth))
|
||||
for _, day := range m.DaysOfMonth {
|
||||
if day == "lastDay1" {
|
||||
values = append(values, "28-31")
|
||||
} else {
|
||||
values = append(values, day)
|
||||
}
|
||||
}
|
||||
return base + " " + strings.Join(values, ",") + " * *", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate normalizes generated schedules and rejects ambiguous or invalid
|
||||
// input before it can reach the scheduler.
|
||||
func (m *Maintenance) Validate() error {
|
||||
m.Title = strings.TrimSpace(m.Title)
|
||||
if m.Title == "" || len(m.Title) > 200 {
|
||||
return fmt.Errorf("title is required and must be at most 200 characters")
|
||||
}
|
||||
if _, err := m.location(); err != nil {
|
||||
return fmt.Errorf("invalid timezone: %w", err)
|
||||
}
|
||||
switch m.Strategy {
|
||||
case MaintenanceManual:
|
||||
return nil
|
||||
case MaintenanceSingle:
|
||||
if m.StartDate == nil || m.EndDate == nil || !m.EndDate.After(*m.StartDate) {
|
||||
return fmt.Errorf("single maintenance requires end_date after start_date")
|
||||
}
|
||||
m.DurationSec = int(m.EndDate.Sub(*m.StartDate).Seconds())
|
||||
return nil
|
||||
case MaintenanceRecurringInterval:
|
||||
if m.DurationSec <= 0 || m.IntervalDay <= 0 || (m.IntervalDay > 1 && m.StartDate == nil) {
|
||||
return fmt.Errorf("recurring interval requires positive duration_sec and interval_day; intervals over one day require start_date")
|
||||
}
|
||||
if _, _, err := parseMaintenanceTime(m.StartTime); err != nil {
|
||||
return err
|
||||
}
|
||||
m.Cron = ""
|
||||
return nil
|
||||
case MaintenanceCron, MaintenanceRecurringWeekday, MaintenanceRecurringDayOfMonth:
|
||||
if m.DurationSec <= 0 {
|
||||
return fmt.Errorf("duration_sec must be positive")
|
||||
}
|
||||
cronText, err := m.generatedCron()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := cron.ParseStandard(cronText); err != nil {
|
||||
return fmt.Errorf("invalid cron: %w", err)
|
||||
}
|
||||
m.Cron = cronText
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unknown maintenance strategy %q", m.Strategy)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Maintenance) BeforeSave(_ *gorm.DB) error { return m.Validate() }
|
||||
|
||||
// IsUnderMaintenance evaluates durable data only. This intentionally avoids
|
||||
// scheduler-owned state so a process restart and multiple web pods agree.
|
||||
func (m *Maintenance) IsUnderMaintenance(now time.Time) bool {
|
||||
if !m.Active {
|
||||
return false
|
||||
}
|
||||
if m.Strategy == MaintenanceManual {
|
||||
return true
|
||||
}
|
||||
if m.Strategy == MaintenanceSingle {
|
||||
return m.StartDate != nil && m.EndDate != nil && !now.Before(*m.StartDate) && now.Before(*m.EndDate)
|
||||
}
|
||||
if m.Strategy == MaintenanceRecurringInterval {
|
||||
return m.isUnderInterval(now)
|
||||
}
|
||||
if m.DurationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
schedule, err := cron.ParseStandard(m.Cron)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// Ask cron for each candidate since the earliest possible active start.
|
||||
// Cron is minute-granular, hence the extra minute catches exact boundaries.
|
||||
from := now.In(loc).Add(-time.Duration(m.DurationSec)*time.Second - time.Minute)
|
||||
to := now.In(loc)
|
||||
for candidate := schedule.Next(from); !candidate.After(to); candidate = schedule.Next(candidate) {
|
||||
if !m.allowsRecurringCandidate(candidate.In(loc)) {
|
||||
continue
|
||||
}
|
||||
start := candidate.UTC()
|
||||
if !now.Before(start) && now.Before(start.Add(time.Duration(m.DurationSec)*time.Second)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMaintenanceTime(value string) (int, int, error) {
|
||||
parsed, err := time.Parse("15:04", value)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("start_time must be HH:MM")
|
||||
}
|
||||
return parsed.Hour(), parsed.Minute(), nil
|
||||
}
|
||||
|
||||
func (m *Maintenance) intervalStartOn(date time.Time, loc *time.Location) (time.Time, bool) {
|
||||
if m.IntervalDay <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
hour, minute, err := parseMaintenanceTime(m.StartTime)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if m.IntervalDay == 1 && m.StartDate == nil {
|
||||
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
|
||||
}
|
||||
if m.StartDate == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
anchor := m.StartDate.In(loc)
|
||||
anchorDay := time.Date(anchor.Year(), anchor.Month(), anchor.Day(), 0, 0, 0, 0, loc)
|
||||
candidateDay := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc)
|
||||
// Compare civil dates rather than elapsed hours: a local day can be 23 or
|
||||
// 25 hours when the maintenance timezone crosses a DST boundary.
|
||||
days := civilDaysBetween(anchorDay, candidateDay)
|
||||
if days < 0 || days%m.IntervalDay != 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc), true
|
||||
}
|
||||
|
||||
func civilDaysBetween(from, to time.Time) int {
|
||||
fromDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
|
||||
toDay := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC)
|
||||
return int(toDay.Sub(fromDay) / (24 * time.Hour))
|
||||
}
|
||||
|
||||
func (m *Maintenance) isUnderInterval(now time.Time) bool {
|
||||
loc, err := m.location()
|
||||
if err != nil || m.DurationSec <= 0 {
|
||||
return false
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
for day := 0; day <= int(time.Duration(m.DurationSec)/24/time.Hour)+1; day++ {
|
||||
start, ok := m.intervalStartOn(localNow.AddDate(0, 0, -day), loc)
|
||||
if ok && !now.Before(start.UTC()) && now.Before(start.UTC().Add(time.Duration(m.DurationSec)*time.Second)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// robfig/cron cannot express "last day". The generated 28-31 range is only
|
||||
// a candidate generator; this final predicate makes lastDay1 exact.
|
||||
func (m *Maintenance) allowsRecurringCandidate(candidate time.Time) bool {
|
||||
if m.Strategy != MaintenanceRecurringDayOfMonth {
|
||||
return true
|
||||
}
|
||||
lastDay := candidate.AddDate(0, 0, 1).Month() != candidate.Month()
|
||||
for _, value := range m.DaysOfMonth {
|
||||
if value == "lastDay1" && lastDay {
|
||||
return true
|
||||
}
|
||||
if value == fmt.Sprint(candidate.Day()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Maintenance) NextRun(now time.Time) *time.Time {
|
||||
if !m.Active || m.Strategy == MaintenanceManual {
|
||||
return nil
|
||||
}
|
||||
if m.Strategy == MaintenanceSingle {
|
||||
if m.StartDate != nil && m.StartDate.After(now) {
|
||||
return m.StartDate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if m.Strategy == MaintenanceRecurringInterval {
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
for day := 0; day <= m.IntervalDay; day++ {
|
||||
if next, ok := m.intervalStartOn(localNow.AddDate(0, 0, day), loc); ok && next.After(localNow) {
|
||||
result := next.UTC()
|
||||
return &result
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
loc, err := m.location()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
s, err := cron.ParseStandard(m.Cron)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for candidate := s.Next(now.In(loc)); ; candidate = s.Next(candidate) {
|
||||
if m.allowsRecurringCandidate(candidate.In(loc)) {
|
||||
next := candidate.UTC()
|
||||
return &next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MonitorUnderMaintenance is the notifier/public-page lookup.
|
||||
func MonitorUnderMaintenance(monitorID int64, now time.Time) (bool, error) {
|
||||
var rows []Maintenance
|
||||
err := DB().Joins("JOIN maintenance_monitors mm ON mm.maintenance_id = maintenances.id").Where("mm.monitor_id = ? AND maintenances.active = TRUE", monitorID).Find(&rows).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for i := range rows {
|
||||
if rows[i].IsUnderMaintenance(now) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
33
app/models/maintenance_migration_test.go
Обычный файл
33
app/models/maintenance_migration_test.go
Обычный файл
@@ -0,0 +1,33 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMigratePreservesEveryLegacyStatusPageMaintenance(t *testing.T) {
|
||||
Drop()
|
||||
Migrate()
|
||||
plan := Plan{Name: "legacy migration plan"}
|
||||
require.NoError(t, DB().Create(&plan).Error)
|
||||
account := Account{Name: "legacy migration account", PlanID: &plan.ID}
|
||||
require.NoError(t, DB().Create(&account).Error)
|
||||
page := StatusPage{AccountID: account.ID, Slug: "legacy-maintenance-migration", Name: "Legacy"}
|
||||
require.NoError(t, DB().Create(&page).Error)
|
||||
start := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
|
||||
for i := 0; i < 2; i++ {
|
||||
row := StatusPageMaintenance{StatusPageID: page.ID, Title: "same", StartsAt: start, EndsAt: start.Add(time.Hour), MonitorIDs: pq.Int64Array{}}
|
||||
require.NoError(t, DB().Create(&row).Error)
|
||||
}
|
||||
|
||||
Migrate()
|
||||
var migrated []Maintenance
|
||||
require.NoError(t, DB().Where("legacy_status_page_maintenance_id IS NOT NULL").Find(&migrated).Error)
|
||||
require.Len(t, migrated, 2)
|
||||
var joins int64
|
||||
require.NoError(t, DB().Table("maintenance_status_pages").Where("status_page_id = ?", page.ID).Count(&joins).Error)
|
||||
require.EqualValues(t, 2, joins)
|
||||
}
|
||||
44
app/models/maintenance_notifications.go
Обычный файл
44
app/models/maintenance_notifications.go
Обычный файл
@@ -0,0 +1,44 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const maintenanceStartNotificationPrefix = "maintenance:%d:start:"
|
||||
|
||||
// MaintenanceStartNotificationKey uniquely identifies one contact's warning
|
||||
// for one schedule revision and occurrence. Updated windows get a new revision
|
||||
// while concurrent scheduler replicas share the same key.
|
||||
func MaintenanceStartNotificationKey(maintenanceID int64, revision, startsAt time.Time, timezone string, notificationID, contactID int64) string {
|
||||
loc, err := time.LoadLocation(timezone)
|
||||
if err != nil || timezone == "SAME_AS_SERVER" || timezone == "" {
|
||||
loc = time.UTC
|
||||
}
|
||||
// A fall-back hour can have two UTC instants for one wall-clock occurrence.
|
||||
// Warnings are once per civil occurrence, matching the recurrence editor.
|
||||
civilOccurrence := startsAt.In(loc).Format("200601021504")
|
||||
return fmt.Sprintf("maintenance:%d:start:%d:%s:notification:%d:contact:%d", maintenanceID, revision.UnixNano(), civilOccurrence, notificationID, contactID)
|
||||
}
|
||||
|
||||
// CancelMaintenanceStartNotificationsTx prevents queued warnings from being
|
||||
// delivered after an operator pauses, changes, or deletes the maintenance.
|
||||
// Leased work may already be executing and cannot be recalled from a worker.
|
||||
func CancelMaintenanceStartNotificationsTx(tx *gorm.DB, maintenanceID int64, reason string) error {
|
||||
prefix := fmt.Sprintf(maintenanceStartNotificationPrefix, maintenanceID) + "%"
|
||||
var tasks []Task
|
||||
if err := tx.Clauses(SkipLockedClause).Where("idempotency_key LIKE ? AND state IN ?", prefix, []string{TaskStateQueued, TaskStateFailedRetry}).Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range tasks {
|
||||
if err := tx.Model(&Task{}).Where("id = ? AND state IN ?", tasks[i].ID, []string{TaskStateQueued, TaskStateFailedRetry}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "canceled", "canceled: "+reason); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
105
app/models/maintenance_test.go
Обычный файл
105
app/models/maintenance_test.go
Обычный файл
@@ -0,0 +1,105 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestMaintenanceValidateGeneratesRecurringCron(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
m models.Maintenance
|
||||
want string
|
||||
}{
|
||||
{"interval", models.Maintenance{Title: "interval", Strategy: models.MaintenanceRecurringInterval, StartDate: maintenanceTimePtr(time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)), StartTime: "02:30", IntervalDay: 3, DurationSec: 60, Timezone: "UTC"}, ""},
|
||||
{"weekday", models.Maintenance{Title: "weekdays", Strategy: models.MaintenanceRecurringWeekday, StartTime: "02:30", Weekdays: []int64{1, 5}, DurationSec: 60, Timezone: "UTC"}, "30 02 * * 1,5"},
|
||||
{"month", models.Maintenance{Title: "month", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "02:30", DaysOfMonth: []string{"1", "lastDay1"}, DurationSec: 60, Timezone: "UTC"}, "30 02 1,28-31 * *"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) { require.NoError(t, test.m.Validate()); assert.Equal(t, test.want, test.m.Cron) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceIsUnderMaintenanceBoundariesAndTimezone(t *testing.T) {
|
||||
start := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
|
||||
end := start.Add(time.Hour)
|
||||
single := models.Maintenance{Title: "single", Strategy: models.MaintenanceSingle, StartDate: &start, EndDate: &end, Active: true, Timezone: "UTC"}
|
||||
assert.True(t, single.IsUnderMaintenance(start))
|
||||
assert.True(t, single.IsUnderMaintenance(end.Add(-time.Nanosecond)))
|
||||
assert.False(t, single.IsUnderMaintenance(end))
|
||||
cron := models.Maintenance{Title: "moscow", Strategy: models.MaintenanceCron, Cron: "0 12 * * *", DurationSec: 3600, Active: true, Timezone: "Europe/Moscow"}
|
||||
require.NoError(t, cron.Validate())
|
||||
assert.True(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 9, 30, 0, 0, time.UTC)), "12:30 Moscow is 09:30 UTC in July")
|
||||
assert.False(t, cron.IsUnderMaintenance(time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)))
|
||||
}
|
||||
|
||||
func TestMaintenanceValidationRejectsInvalidSchedules(t *testing.T) {
|
||||
badCron := models.Maintenance{Title: "bad", Strategy: models.MaintenanceCron, Cron: "not cron", DurationSec: 1, Timezone: "UTC"}
|
||||
badSingle := models.Maintenance{Title: "bad", Strategy: models.MaintenanceSingle, Timezone: "UTC"}
|
||||
assert.Error(t, badCron.Validate())
|
||||
assert.Error(t, badSingle.Validate())
|
||||
}
|
||||
|
||||
func TestMaintenanceLastDayIsNotEveryDayInExpandedCronRange(t *testing.T) {
|
||||
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 3600, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 3, 28, 12, 30, 0, 0, time.UTC)))
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 31, 12, 30, 0, 0, time.UTC)))
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalUsesAnchorAndIntervalDay(t *testing.T) {
|
||||
anchor := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
m := models.Maintenance{Title: "every three days", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "12:00", IntervalDay: 3, DurationSec: 3600, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 7, 4, 12, 30, 0, 0, time.UTC)))
|
||||
assert.False(t, m.IsUnderMaintenance(time.Date(2026, 7, 5, 12, 30, 0, 0, time.UTC)))
|
||||
next := m.NextRun(time.Date(2026, 7, 2, 13, 0, 0, 0, time.UTC))
|
||||
require.NotNil(t, next)
|
||||
assert.Equal(t, time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC), *next)
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalRequiresAnchorForMultiDayAndPositiveInterval(t *testing.T) {
|
||||
m := models.Maintenance{Title: "invalid", Strategy: models.MaintenanceRecurringInterval, StartTime: "12:00", IntervalDay: 2, DurationSec: 60, Timezone: "UTC"}
|
||||
assert.Error(t, m.Validate())
|
||||
}
|
||||
|
||||
func TestMaintenanceRecurringIntervalKeepsCivilDayAcrossDST(t *testing.T) {
|
||||
loc, err := time.LoadLocation("Europe/Berlin")
|
||||
require.NoError(t, err)
|
||||
// March 29, 2026 is the spring-forward day in Berlin. The second run is
|
||||
// still two civil days after the anchor, not one because a day was 23h.
|
||||
anchor := time.Date(2026, 3, 27, 0, 0, 0, 0, loc)
|
||||
m := models.Maintenance{Title: "DST", Strategy: models.MaintenanceRecurringInterval, StartDate: &anchor, StartTime: "03:30", IntervalDay: 2, DurationSec: 3600, Active: true, Timezone: "Europe/Berlin"}
|
||||
require.NoError(t, m.Validate())
|
||||
assert.True(t, m.IsUnderMaintenance(time.Date(2026, 3, 29, 4, 0, 0, 0, loc).UTC()))
|
||||
}
|
||||
|
||||
func TestMaintenanceStartNotificationKeyIncludesRevisionAndOccurrence(t *testing.T) {
|
||||
revision := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
start := revision.Add(5 * time.Minute)
|
||||
key := models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13)
|
||||
assert.Equal(t, key, models.MaintenanceStartNotificationKey(7, revision, start, "Europe/Berlin", 11, 13))
|
||||
assert.NotEqual(t, key, models.MaintenanceStartNotificationKey(7, revision.Add(time.Second), start, "Europe/Berlin", 11, 13))
|
||||
}
|
||||
|
||||
func TestMaintenanceStartNotificationKeyDeduplicatesDSTFallbackCivilOccurrence(t *testing.T) {
|
||||
revision := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
first := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC)
|
||||
second := first.Add(time.Hour)
|
||||
assert.Equal(t, models.MaintenanceStartNotificationKey(7, revision, first, "Europe/Berlin", 11, 13), models.MaintenanceStartNotificationKey(7, revision, second, "Europe/Berlin", 11, 13))
|
||||
}
|
||||
|
||||
func TestMaintenanceNextRunSkipsNonFinalLastDayCandidates(t *testing.T) {
|
||||
m := models.Maintenance{Title: "last", Strategy: models.MaintenanceRecurringDayOfMonth, StartTime: "12:00", DaysOfMonth: []string{"lastDay1"}, DurationSec: 60, Active: true, Timezone: "UTC"}
|
||||
require.NoError(t, m.Validate())
|
||||
next := m.NextRun(time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC))
|
||||
require.NotNil(t, next)
|
||||
assert.Equal(t, time.Date(2026, 4, 30, 12, 0, 0, 0, time.UTC), *next)
|
||||
}
|
||||
|
||||
func maintenanceTimePtr(value time.Time) *time.Time { return &value }
|
||||
48
app/models/message.go
Обычный файл
48
app/models/message.go
Обычный файл
@@ -0,0 +1,48 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Message info about performed notification
|
||||
type Message struct {
|
||||
concerns.Model
|
||||
|
||||
NotificationID int64 `json:"notification_id"`
|
||||
Notification *Notification `json:"notification,omitempty"`
|
||||
|
||||
ContactID int64 `gorm:"index;type:bigint REFERENCES contacts(id)" json:"contact_id"`
|
||||
Contact *Contact `json:"contact,omitempty"`
|
||||
|
||||
// Events are for up/down messages
|
||||
Events []Event `json:"events" gorm:"many2many:event_messages;"`
|
||||
|
||||
// Checks are for expires messages
|
||||
CheckID *int64 `gorm:"index;type:bigint REFERENCES checks(id)" json:"check_id"`
|
||||
Check *Check `json:"check,omitempty"`
|
||||
|
||||
Kind string `json:"kind"`
|
||||
State string `json:"state"`
|
||||
Error *string `json:"error"`
|
||||
Response *string `json:"response"`
|
||||
Tries int `json:"-"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
SentAt time.Time `json:"sent_at"`
|
||||
}
|
||||
|
||||
// MessageScope provides functionality.
|
||||
func MessageScope(q *gorm.DB) *gorm.DB {
|
||||
return q.
|
||||
Preload("Events").
|
||||
Preload("Events.Checks").
|
||||
Preload("Events.Monitor").
|
||||
Preload("Notification").
|
||||
Preload("Contact").
|
||||
Preload("Check").
|
||||
Preload("Check.Monitor")
|
||||
}
|
||||
750
app/models/migrate.go
Обычный файл
750
app/models/migrate.go
Обычный файл
@@ -0,0 +1,750 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
)
|
||||
|
||||
// isTypeExistsError returns true if the error is a Postgres "type already exists" error.
|
||||
// This happens when AutoMigrate is called concurrently from multiple test processes.
|
||||
func isTypeExistsError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
// SQLSTATE 42710 = duplicate_object (type already exists)
|
||||
return strings.Contains(s, "42710") ||
|
||||
strings.Contains(s, "already exists") ||
|
||||
strings.Contains(s, "pg_type_typname_nsp_index")
|
||||
}
|
||||
|
||||
// migrateOrIgnoreTypeExists runs AutoMigrate and ignores "type already exists" errors
|
||||
// that can occur when parallel test processes both try to create the same Postgres types.
|
||||
func migrateOrIgnoreTypeExists(models ...interface{}) {
|
||||
err := DB().AutoMigrate(models...)
|
||||
if err != nil && !isTypeExistsError(err) {
|
||||
panic(err)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("migrate: ignoring type-exists error (expected during parallel test runs): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ensureSingleCurrentEventInvariant pins cleanup and index creation to one
|
||||
// transaction/connection. The global migration lock is session-scoped through a
|
||||
// pool, so it is not sufficient for this multi-statement invariant by itself.
|
||||
func ensureSingleCurrentEventInvariant() error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", int64(1234567892)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`WITH ranked AS (
|
||||
SELECT id, row_number() OVER (PARTITION BY monitor_id ORDER BY start_time DESC NULLS LAST, id DESC) AS n
|
||||
FROM events WHERE state = 'current'
|
||||
) UPDATE events SET state = 'ended', end_time = COALESCE(end_time, now())
|
||||
FROM ranked WHERE events.id = ranked.id AND ranked.n > 1`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS events_one_current_per_monitor
|
||||
ON events (monitor_id) WHERE state = 'current'`).Error
|
||||
})
|
||||
}
|
||||
|
||||
// Migrate run db migration
|
||||
var migrateMu sync.Mutex
|
||||
|
||||
func Migrate() {
|
||||
migrateMu.Lock()
|
||||
defer migrateMu.Unlock()
|
||||
withMigrationAdvisoryLock(migrateLocked)
|
||||
}
|
||||
|
||||
// withMigrationAdvisoryLock holds the session lock on a dedicated connection
|
||||
// while migration work uses GORM's normal pool. Reusing the lock connection
|
||||
// for GORM transactions can leave its *sql.Conn closed after commit.
|
||||
func withMigrationAdvisoryLock(migrate func()) {
|
||||
const migrateAdvisoryLock = int64(1234567890)
|
||||
sqlDB, err := DB().DB()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("migrate: database handle: %v", err))
|
||||
}
|
||||
ctx := context.Background()
|
||||
conn, err := sqlDB.Conn(ctx)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("migrate: lock connection: %v", err))
|
||||
}
|
||||
defer conn.Close() //nolint:errcheck // closing releases the session lock after a migration panic
|
||||
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", migrateAdvisoryLock); err != nil {
|
||||
panic(fmt.Sprintf("migrate: advisory lock: %v", err))
|
||||
}
|
||||
unlocked := false
|
||||
defer func() {
|
||||
if unlocked {
|
||||
return
|
||||
}
|
||||
if _, unlockErr := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); unlockErr != nil {
|
||||
log.Printf("migrate: unlock after failure: %v", unlockErr)
|
||||
}
|
||||
}()
|
||||
|
||||
migrate()
|
||||
if _, err = conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock); err != nil {
|
||||
panic(fmt.Sprintf("migrate: unlock: %v", err))
|
||||
}
|
||||
unlocked = true
|
||||
}
|
||||
|
||||
func migrateLocked() {
|
||||
// M0 replaces the old flat plans table without rewriting historical rows.
|
||||
// Rename before AutoMigrate so GORM creates the canonical table rather than
|
||||
// adding columns to the incompatible legacy schema.
|
||||
if err := prepareCanonicalPlansTable(); err != nil {
|
||||
panic(fmt.Sprintf("migrate: prepare canonical plans: %v", err))
|
||||
}
|
||||
|
||||
// Step 0: create inventory Postgres enum types FIRST. The DO/EXCEPTION
|
||||
// blocks are idempotent so concurrent migrateOrIgnoreTypeExists
|
||||
// reruns from parallel test binaries are safe (the type already
|
||||
// exists → duplicate_object is swallowed). The enum types MUST
|
||||
// exist before any AutoMigrate below because GORM emits
|
||||
// `kind server_kind` literals in CREATE TABLE for the Server
|
||||
// struct (referenced transitively from Monitor.Site → Site → Server).
|
||||
for _, ddl := range []string{
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE server_kind AS ENUM ('production','staging','old');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_kind AS ENUM
|
||||
('production','production_prev','production_next','internal','staging','old');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_mode AS ENUM
|
||||
('kubernetes','compose','dedicated','vds','user');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
CREATE TYPE deployment_action AS ENUM
|
||||
('ok','pending','pending_move','pending_drop','deleted','missing');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
} {
|
||||
if err := DB().Exec(ddl).Error; err != nil {
|
||||
panic(fmt.Sprintf("migrate: enum creation: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Step 1: Migrate core models (User, Plan, Account, ApiKey, AuthIdentity, Invite)
|
||||
// Note: Access is moved to Step 2 because it has FKs to groups/monitors
|
||||
// Seed and remap plans before Account migration recreates its plan FK.
|
||||
migrateOrIgnoreTypeExists(&Plan{})
|
||||
if err := seedCanonicalPlansAndBackfill(true); err != nil {
|
||||
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
|
||||
}
|
||||
migrateOrIgnoreTypeExists(
|
||||
&User{},
|
||||
&Account{},
|
||||
&ApiKey{},
|
||||
&authidentity.AuthIdentity{}, // After User (has FK to users)
|
||||
&Invite{}, // After Account/User (has FKs to accounts/users)
|
||||
&Subscription{}, // After Account/Plan
|
||||
&SubscriptionEvent{}, // After Subscription
|
||||
)
|
||||
|
||||
// Monitor and metric rows reference worker_nodes, while worker_nodes itself
|
||||
// references servers. Create the two roots without their associations before
|
||||
// migrating Monitor/ServerMetric on a fresh database.
|
||||
migrateOrIgnoreTypeExists(&Region{}, &LLM{})
|
||||
if err = DB().Omit("Monitors", "Workers").AutoMigrate(&Server{}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
migrateOrIgnoreTypeExists(&WorkerNode{})
|
||||
|
||||
// Step 2: Migrate Group, Monitor, and Access (which has FKs to groups/monitors)
|
||||
// This ensures the groups table exists when GORM creates foreign keys
|
||||
migrateOrIgnoreTypeExists(
|
||||
&Group{},
|
||||
&Monitor{},
|
||||
&Check{},
|
||||
// Server is a customer-facing logical host, distinct from the
|
||||
// WorkerNode executor. Keep the join/cache models here so a fresh
|
||||
// database gets the complete server metrics schema in one migration.
|
||||
&Server{},
|
||||
&AccountMCPToken{},
|
||||
&MonitorServer{},
|
||||
&ServerMetric{},
|
||||
&ServerAlertRule{},
|
||||
&ServerAlertEvent{},
|
||||
&RknIP{},
|
||||
&RknDomain{},
|
||||
&DNSRecord{},
|
||||
&Contact{},
|
||||
&Whois{},
|
||||
&Payment{},
|
||||
&Message{},
|
||||
&TelegramBotMessage{},
|
||||
&TelegramBotStatus{},
|
||||
&Event{},
|
||||
&SelfCheck{},
|
||||
&Notification{}, // After Group/Monitor so notification_groups FK works
|
||||
&Access{}, // After Group/Monitor so access FKs work
|
||||
&NotificationCredential{}, // No FKs to other domain tables; safe here.
|
||||
)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS servers_account_slug_unique ON servers (account_id, slug)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS monitor_servers_position_idx ON monitor_servers (server_id, position)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_server_recent_idx ON server_metrics (server_id, id DESC)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_metrics_worker_idx ON server_metrics (worker_id)`)
|
||||
|
||||
// Step 3: Clean up orphaned references (now that all tables exist)
|
||||
// Fix typo in old table name (only present on DBs migrated from older versions)
|
||||
err = DB().Exec("ALTER TABLE IF EXISTS envent_checks RENAME TO event_checks;").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec("DROP TABLE IF EXISTS envent_messages;").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
// Defense-in-depth: make sure the columns that the in-process notifier
|
||||
// scheduler eagerly queries at startup exist, even if AutoMigrate above
|
||||
// was skipped or the column was dropped by a manual operation. Without
|
||||
// these, a fresh restore from a pre-soft-delete production dump will
|
||||
// panic the first time RunExp preloads Contacts or
|
||||
// ProcessPendingDeletions queries Users (see internal/notifier for the
|
||||
// defensive recover() that catches the resulting query errors).
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT TRUE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS disabled BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE accounts ADD COLUMN IF NOT EXISTS blocked BOOLEAN NOT NULL DEFAULT FALSE",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE users ADD COLUMN IF NOT EXISTS deletion_requested_at TIMESTAMPTZ",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS webhook_token VARCHAR(128)",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
var telegramCreds []NotificationCredential
|
||||
if err = DB().Where("kind = ? AND (webhook_token IS NULL OR webhook_token = '')", CredentialKindTelegram).Find(&telegramCreds).Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
for i := range telegramCreds {
|
||||
telegramCreds[i].EnsureWebhookToken()
|
||||
if err = DB().Save(&telegramCreds[i]).Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
err = DB().Exec("DROP INDEX IF EXISTS idx_notification_credentials_webhook_token").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
err = DB().Exec(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_notification_credentials_webhook_token ON notification_credentials (webhook_token) WHERE webhook_token IS NOT NULL AND webhook_token <> ''",
|
||||
).Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
// Replace the early M0 per-event-id index with provider-scoped webhook
|
||||
// idempotency: PSP event IDs are only unique inside a provider.
|
||||
if err = DB().Exec("DROP INDEX IF EXISTS idx_subscription_events_provider_event_id").Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if err = DB().Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_subscription_events_provider_event ON subscription_events (provider, provider_event_id) WHERE provider_event_id IS NOT NULL").Error; err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned event_checks
|
||||
err = DB().Exec("DELETE FROM event_checks where event_id NOT IN (select id from events)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM event_checks where check_id NOT IN (select id from checks)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned event_messages
|
||||
err = DB().Exec("DELETE FROM event_messages where event_id NOT IN (select id from events)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM event_messages where message_id NOT IN (select id from messages)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned notification_contacts
|
||||
err = DB().Exec("DELETE FROM notification_contacts where notification_id NOT IN (select id from notifications)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM notification_contacts where contact_id NOT IN (select id from contacts)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Clean up orphaned notification_groups (now safe - groups table exists)
|
||||
err = DB().Exec("DELETE FROM notification_groups where notification_id NOT IN (select id from notifications)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("DELETE FROM notification_groups where group_id NOT IN (select id from groups)").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS not_old_events on events (monitor_id, id) where state != 'old'")
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS current_events ON event (monitor_id, start_time) WHERE state = 'current'")
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS ended_events ON event (monitor_id, start_time) WHERE state = 'ended'")
|
||||
|
||||
DB().Raw("CREATE INDEX IF NOT EXISTS queued_messages ON message (id) WHERE state = 'queued'")
|
||||
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, account_id)")
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS access_accounts ON accesses (user_id, group_id)")
|
||||
|
||||
DB().Raw("CREATE UNIQUE INDEX IF NOT EXISTS invite_email ON invites (account_id, email)")
|
||||
|
||||
// RKN indexes — see app/models/rkn_ip.go EnsureRknIndexes. GORM
|
||||
// AutoMigrate above declared the uniqueIndex on RknDomain.Domain
|
||||
// and the cidr column type on RknIP, but GiST on rkn_ips.network
|
||||
// is not expressible via the GORM tag language; we add it here so
|
||||
// the (>>=) containment operator used by IsRknIPBlocked has an
|
||||
// index to back it.
|
||||
if err := EnsureRknIndexes(); err != nil {
|
||||
log.Printf("migrate: EnsureRknIndexes failed: %v", err)
|
||||
}
|
||||
|
||||
// Distributed worker models
|
||||
migrateOrIgnoreTypeExists(
|
||||
&WorkerLogEvent{},
|
||||
&CheckAttempt{},
|
||||
&DiagnosticAuditEvent{},
|
||||
&CheckRegionResult{},
|
||||
&Task{},
|
||||
&TaskReplay{},
|
||||
&NotificationDelivery{},
|
||||
&Tag{},
|
||||
)
|
||||
|
||||
// Tags — (account_id, name) is the unique key so a single account
|
||||
// cannot register two metadata rows for the same tag string. The
|
||||
// tag name itself is also the join key against monitors.tags, so
|
||||
// uniqueness is enforced at the table level (not just on the
|
||||
// metadata row).
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS tags_account_name_unique ON tags (account_id, name)`)
|
||||
|
||||
// Status pages use globally unique public slugs. A public URL has no
|
||||
// account component, so account-scoped uniqueness would make /status/:slug
|
||||
// ambiguous. Soft-deleted rows do not reserve their slug.
|
||||
// subscriber email index uses lower(email) for case-insensitive
|
||||
// matching (the codebase does not adopt citext). All five tables
|
||||
// are created together so M0 ships a consistent schema baseline
|
||||
// regardless of which milestone first writes rows.
|
||||
migrateOrIgnoreTypeExists(
|
||||
&StatusPage{},
|
||||
&StatusPageSubscriber{},
|
||||
&StatusPageIncident{},
|
||||
&StatusPageMaintenance{},
|
||||
&StatusPageDomain{},
|
||||
&StatusPageDelivery{},
|
||||
&StatusPageDigestSchedule{},
|
||||
&Maintenance{},
|
||||
)
|
||||
DB().Exec(`DROP INDEX IF EXISTS status_pages_account_slug_unique`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_pages_slug_unique
|
||||
ON status_pages (slug) WHERE deleted_at IS NULL`)
|
||||
if err := ensureSingleCurrentEventInvariant(); err != nil {
|
||||
panic(fmt.Sprintf("migrate: current event invariant: %v", err))
|
||||
}
|
||||
DB().Exec(
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS status_page_subscribers_active_email
|
||||
ON status_page_subscribers (status_page_id, lower(email))
|
||||
WHERE unsubscribed_at IS NULL`,
|
||||
)
|
||||
// Existing installations can already have subscriber rows. Keep the legacy
|
||||
// token column during the nullable transition: outstanding links remain
|
||||
// valid, while each resend/confirmation rotates it into a hash.
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS confirm_token_hash varchar(64)`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS token_expires_at timestamptz`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS unsubscribe_token_hash varchar(64)`)
|
||||
DB().Exec(`ALTER TABLE status_page_subscribers ADD COLUMN IF NOT EXISTS contact_id bigint REFERENCES contacts(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`UPDATE status_page_subscribers SET token_expires_at = created_at + interval '24 hours' WHERE token_expires_at IS NULL`)
|
||||
for {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := DB().Where("confirm_token_hash IS NULL AND confirm_token IS NOT NULL").Limit(500).Find(&subscribers).Error; err != nil || len(subscribers) == 0 {
|
||||
break
|
||||
}
|
||||
for i := range subscribers {
|
||||
subscribers[i].ConfirmTokenHash = HashStatusPageToken(*subscribers[i].LegacyConfirmToken)
|
||||
_ = DB().Model(&subscribers[i]).Update("confirm_token_hash", subscribers[i].ConfirmTokenHash).Error
|
||||
}
|
||||
}
|
||||
DB().Exec(`UPDATE status_page_subscribers SET confirm_token_hash = '' WHERE confirm_token_hash IS NULL`)
|
||||
DB().Exec(`UPDATE status_page_subscribers SET unsubscribe_token_hash = '' WHERE unsubscribe_token_hash IS NULL`)
|
||||
// A page may expose more than one verified hostname; older schema reserved
|
||||
// only one domain per page.
|
||||
DB().Exec(`DROP INDEX IF EXISTS idx_status_page_domains_status_page_id`)
|
||||
DB().Exec(
|
||||
`CREATE INDEX IF NOT EXISTS status_page_incidents_started
|
||||
ON status_page_incidents (status_page_id, started_at DESC)`,
|
||||
)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS status_page_incidents_event_unique
|
||||
ON status_page_incidents (status_page_id, event_id) WHERE event_id IS NOT NULL`)
|
||||
DB().Exec(
|
||||
`CREATE INDEX IF NOT EXISTS status_page_maintenance_starts
|
||||
ON status_page_maintenance (status_page_id, starts_at DESC)`,
|
||||
)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS maintenances_account_active ON maintenances (account_id) WHERE deleted_at IS NULL`)
|
||||
// Legacy status-page rows predate account-scoped maintenance. Keep their
|
||||
// source IDs so equal title/start rows remain distinct and reruns can safely
|
||||
// preserve every window and each of its joins.
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS maintenances_legacy_status_page_maintenance_unique
|
||||
ON maintenances (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL`)
|
||||
DB().Exec(`INSERT INTO maintenances (account_id, title, description, strategy, duration_sec, start_date, end_date, timezone, active, show_on_all_status_pages, legacy_status_page_maintenance_id, created_at, updated_at)
|
||||
SELECT sp.account_id, old.title, old.description, 'single', EXTRACT(EPOCH FROM (old.ends_at - old.starts_at))::int, old.starts_at, old.ends_at, 'UTC', TRUE, FALSE, old.id, old.created_at, old.updated_at
|
||||
FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id
|
||||
ON CONFLICT (legacy_status_page_maintenance_id) WHERE legacy_status_page_maintenance_id IS NOT NULL DO NOTHING`)
|
||||
DB().Exec(`INSERT INTO maintenance_status_pages (maintenance_id, status_page_id)
|
||||
SELECT m.id, old.status_page_id FROM status_page_maintenance old JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id
|
||||
ON CONFLICT DO NOTHING`)
|
||||
DB().Exec(`INSERT INTO maintenance_monitors (maintenance_id, monitor_id)
|
||||
SELECT m.id, monitor_id FROM status_page_maintenance old JOIN status_pages sp ON sp.id = old.status_page_id JOIN maintenances m ON m.legacy_status_page_maintenance_id = old.id,
|
||||
LATERAL unnest(CASE WHEN cardinality(old.monitor_ids) > 0 THEN old.monitor_ids ELSE sp.monitor_ids END) AS monitor_id
|
||||
ON CONFLICT DO NOTHING`)
|
||||
|
||||
// Worker-driven task queue indexes.
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_queued_due ON tasks (kind, not_before) WHERE state = 'queued'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_leased_expires ON tasks (lease_expires_at) WHERE state = 'leased'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_failed_retry_due ON tasks (not_before) WHERE state = 'failed_retry'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS tasks_dead_kind ON tasks (kind, account_id) WHERE state = 'dead'`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS check_attempts_worker_finished ON check_attempts (worker_node_id, finished_at DESC)`)
|
||||
err = DB().Exec("ALTER TABLE worker_nodes ALTER COLUMN concurrency SET DEFAULT 10").Error
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET DEFAULT '{}'::jsonb").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = DB().Exec("UPDATE checks SET settings = '{}'::jsonb WHERE settings IS NULL").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = DB().Exec("ALTER TABLE checks ALTER COLUMN settings SET NOT NULL").Error
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Inventory models (docs/plans/inventory-management.md §6, §10 M0).
|
||||
// Enum types are created at Step 0 above so they exist before any
|
||||
// AutoMigrate. The numeric labels of each enum value are taken
|
||||
// verbatim from rstuff (`/data/int/rstuff/app/models/*.rb`) so a
|
||||
// future sync layer does not need a value-mapping table — see
|
||||
// docs/parity/rstuff-inventory.md §6.1.
|
||||
|
||||
// Servers — extend with the rstuff-shaped inventory fields. All
|
||||
// statements are IF NOT EXISTS so existing rows keep working
|
||||
// untouched (ext_id/token/price_cents default sensibly; meta gets
|
||||
// an empty jsonb).
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS ext_id VARCHAR(64)`)
|
||||
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_ext_id_unique ON servers (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS kind server_kind NOT NULL DEFAULT 'production'`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS token VARCHAR(64)`)
|
||||
DB().Raw(`CREATE UNIQUE INDEX IF NOT EXISTS servers_token_unique ON servers (token) WHERE token IS NOT NULL`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS price_cents INTEGER NOT NULL DEFAULT 0`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS comment TEXT`)
|
||||
DB().Exec(`ALTER TABLE servers ADD COLUMN IF NOT EXISTS meta JSONB NOT NULL DEFAULT '{}'::jsonb`)
|
||||
|
||||
// WorkerNodes — optional server_id join for inventory correlation.
|
||||
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS server_id BIGINT REFERENCES servers(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_server_idx ON worker_nodes (server_id)`)
|
||||
|
||||
// WorkerNodes — optional account_id FK marking a private (customer-
|
||||
// operated) worker per docs/distributed/private-workers.md. NULL
|
||||
// rows are platform-operated workers eligible to serve any account;
|
||||
// non-NULL rows are pinned to a single account and are removed by
|
||||
// HardDeleteAccount before the account row itself is dropped.
|
||||
DB().Exec(`ALTER TABLE worker_nodes ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS worker_nodes_account_idx ON worker_nodes (account_id)`)
|
||||
|
||||
// Notification credentials are either platform-managed (account_id IS NULL)
|
||||
// or owned by one account. Replace the legacy global name constraint with
|
||||
// scope-aware unique indexes.
|
||||
DB().Exec(`ALTER TABLE notification_credentials ADD COLUMN IF NOT EXISTS account_id BIGINT REFERENCES accounts(id) ON DELETE CASCADE`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS notification_credentials_account_idx ON notification_credentials (account_id)`)
|
||||
DB().Exec(`DROP INDEX IF EXISTS cred_kind_name`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_system_kind_name_unique ON notification_credentials (kind, name) WHERE account_id IS NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS notification_credentials_account_kind_name_unique ON notification_credentials (account_id, kind, name) WHERE account_id IS NOT NULL`)
|
||||
|
||||
// Inventory entities. Order matters: server_ips before sites
|
||||
// (FK), sites before deployments (FK), deployments before
|
||||
// domains (FK). Audit columns (creator_id/updater_id) use the
|
||||
// Audited mixin via concerns.Timestamped + Audited.
|
||||
migrateOrIgnoreTypeExists(
|
||||
&ServerIp{},
|
||||
&Repo{},
|
||||
&Site{},
|
||||
&Deployment{},
|
||||
&SiteRepo{},
|
||||
&Domain{},
|
||||
)
|
||||
// Monitors is migrated earlier for historical FK ordering. Add the optional
|
||||
// site reference only after sites exists on fresh databases.
|
||||
DB().Exec(`ALTER TABLE monitors ADD COLUMN IF NOT EXISTS site_id BIGINT REFERENCES sites(id) ON DELETE SET NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS monitors_site_idx ON monitors (site_id)`)
|
||||
|
||||
// Indexes — kept here so the AutoMigrate path stays the single
|
||||
// source of truth. IF NOT EXISTS guards the rerun case.
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS server_ips_server_idx ON server_ips (server_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS server_ips_address_unique ON server_ips (server_id, address)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_account_idx ON sites (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS sites_server_idx ON sites (server_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_ext_id_unique ON sites (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS sites_account_slug_unique ON sites (account_id, slug)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_ext_id_unique ON repos (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS repos_gitlab_id_unique ON repos (gitlab_id) WHERE gitlab_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_account_idx ON deployments (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_server_idx ON deployments (server_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS deployments_site_idx ON deployments (site_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_ext_id_unique ON deployments (ext_id) WHERE ext_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_account_idx ON domains (account_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_server_idx ON domains (server_id)`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS domains_site_idx ON domains (site_id)`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS domains_name_unique ON domains (name)`)
|
||||
// Dedupe by (server_id, config_path) for nginx sites and by
|
||||
// (server_id, site_id, service_name) for compose services — matches
|
||||
// the upsert keys in app/controllers/api/inventory_deploymentd.go.
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_nginx
|
||||
ON deployments (server_id, config_path) WHERE mode = 'dedicated' AND config_path IS NOT NULL`)
|
||||
DB().Exec(`DROP INDEX IF EXISTS deployments_unique_compose`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS deployments_unique_compose
|
||||
ON deployments (server_id, site_id, service_name) WHERE mode = 'compose' AND site_id IS NOT NULL AND service_name IS NOT NULL`)
|
||||
|
||||
if err := seedCanonicalPlansAndBackfill(false); err != nil {
|
||||
panic(fmt.Sprintf("migrate: billing catalog: %v", err))
|
||||
}
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS plans_active_code_unique ON plans (code) WHERE archived = FALSE`)
|
||||
DB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS subscriptions_provider_external_unique ON subscriptions (provider, provider_subscription_id) WHERE provider_subscription_id IS NOT NULL`)
|
||||
DB().Exec(`CREATE INDEX IF NOT EXISTS subscriptions_account_status_idx ON subscriptions (account_id, status)`)
|
||||
DB().Exec(`DO $$ BEGIN
|
||||
ALTER TABLE plans ADD CONSTRAINT plans_price_nonnegative CHECK (price_monthly_minor >= 0 AND price_annual_minor >= 0);
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
|
||||
DB().Exec(`DO $$ BEGIN
|
||||
ALTER TABLE plans ADD CONSTRAINT plans_limits_valid CHECK (monitor_cap >= 0 AND interval_min_seconds >= 30);
|
||||
EXCEPTION WHEN duplicate_object THEN NULL; END $$`)
|
||||
|
||||
// Seed default region. Use defaultRegionCode (declared in check_jobs.go)
|
||||
// so the literal does not appear three times in the package.
|
||||
localRegion := Region{}
|
||||
DB().Where("code = ?", defaultRegionCode).First(&localRegion)
|
||||
if localRegion.ID == 0 {
|
||||
DB().Create(&Region{Code: defaultRegionCode, Name: "Local (default)", Enabled: true, Priority: 100})
|
||||
}
|
||||
|
||||
DB().Exec(`
|
||||
WITH t AS (
|
||||
select u.id as user_id, i.encrypted_password as encrypted_password
|
||||
from users as u
|
||||
join identities as i on u.id = i.user_id
|
||||
where u.encrypted_password is NULL
|
||||
)
|
||||
UPDATE users
|
||||
SET encrypted_password = t.encrypted_password
|
||||
from t
|
||||
where users.id = t.user_id
|
||||
`)
|
||||
|
||||
log.Println("migrated DB.")
|
||||
}
|
||||
|
||||
func prepareCanonicalPlansTable() error {
|
||||
if !DB().Migrator().HasTable("plans") || DB().Migrator().HasColumn("plans", "code") {
|
||||
return nil
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return tx.Exec(`ALTER TABLE plans RENAME TO plans_legacy`).Error
|
||||
})
|
||||
}
|
||||
|
||||
func planForeignKeyReferences(tx *gorm.DB, table, target string) (bool, error) {
|
||||
var references bool
|
||||
err := tx.Raw(`SELECT EXISTS (
|
||||
SELECT 1 FROM pg_constraint c
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.contype = 'f' AND c.conrelid = to_regclass(?)
|
||||
AND c.confrelid = to_regclass(?) AND a.attname = 'plan_id'
|
||||
)`, table, target).Scan(&references).Error
|
||||
return references, err
|
||||
}
|
||||
|
||||
func seedCanonicalPlansAndBackfill(remapAccounts bool) error {
|
||||
plans := []Plan{
|
||||
{Code: "free", NameRU: "Бесплатный", NameEN: "Free", Currency: "RUB", MonitorCap: 50, IntervalMinSeconds: 300, StatusPagesCap: 1, MaintenanceCap: -1, LoginSeatsIncluded: 3, NotifySeatsIncluded: 0, Integrations: []string{"email", "telegram"}, CheckKinds: []string{"http", "ssl", "dns", "whois", "ping"}, DataRetentionMonths: 3, IsDefault: true},
|
||||
{Code: "solo", NameRU: "Соло", NameEN: "Solo", PriceMonthlyMinor: 74900, PriceAnnualMinor: 64900, Currency: "RUB", MonitorCap: 10, IntervalMinSeconds: 60, StatusPagesCap: 3, MaintenanceCap: 5, LoginSeatsIncluded: 5, NotifySeatsIncluded: 3, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping"}, DataRetentionMonths: 12, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 60},
|
||||
{Code: "team", NameRU: "Команда", NameEN: "Team", PriceMonthlyMinor: 299000, PriceAnnualMinor: 254900, Currency: "RUB", MonitorCap: 100, IntervalMinSeconds: 60, StatusPagesCap: 100, MaintenanceCap: 50, LoginSeatsIncluded: 5, NotifySeatsIncluded: 5, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 24, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 45},
|
||||
{Code: "enterprise", NameRU: "Предприятие", NameEN: "Enterprise", PriceMonthlyMinor: 549000, PriceAnnualMinor: 464900, Currency: "RUB", MonitorCap: 200, IntervalMinSeconds: 30, StatusPagesCap: 0, MaintenanceCap: 0, LoginSeatsIncluded: 0, NotifySeatsIncluded: 0, UnlimitedSeats: true, Integrations: []string{"email", "telegram", "sms", "voice", "mattermost", "webhook", "sso_saml"}, CheckKinds: []string{"http", "ssl", "dns", "ssh", "ftp", "whois", "ping", "rkn_blocklist", "llm"}, DataRetentionMonths: 36, DistributedWorkers: true, Confirmations: true, AllowHardAlerts: true, ConfirmTimeoutSec: 30, SOC2: true, GDPRDPA: true},
|
||||
}
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
for i := range plans {
|
||||
var existing Plan
|
||||
err := tx.Where("code = ?", plans[i].Code).First(&existing).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
if err := tx.Create(&plans[i]).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Catalog rows already existed before maintenance_cap was introduced.
|
||||
// Update this new entitlement only; do not overwrite customer-adjusted
|
||||
// prices or other existing plan attributes during a normal migration.
|
||||
if err := tx.Exec(`UPDATE plans SET maintenance_cap = CASE code
|
||||
WHEN 'free' THEN -1 WHEN 'solo' THEN 5 WHEN 'team' THEN 50 WHEN 'enterprise' THEN 0 ELSE maintenance_cap END
|
||||
WHERE code IN ('free', 'solo', 'team', 'enterprise')`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
hasLegacy := tx.Migrator().HasTable("plans_legacy")
|
||||
hasMigratedLegacy := tx.Migrator().HasTable("plans_legacy_migrated")
|
||||
if !hasLegacy && !hasMigratedLegacy {
|
||||
return nil
|
||||
}
|
||||
hasAccounts := tx.Migrator().HasTable("accounts")
|
||||
accountsNeedRemap := false
|
||||
if remapAccounts && hasLegacy && hasAccounts {
|
||||
referencesCanonical, err := planForeignKeyReferences(tx, "accounts", "plans")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
accountsNeedRemap = !referencesCanonical
|
||||
if accountsNeedRemap {
|
||||
if err := tx.Exec(`UPDATE accounts SET plan_id = CASE
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = accounts.plan_id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
hasSubscriptions := tx.Migrator().HasTable("subscriptions")
|
||||
subscriptionsNeedRemap := false
|
||||
if remapAccounts && hasLegacy && hasSubscriptions {
|
||||
referencesCanonical, err := planForeignKeyReferences(tx, "subscriptions", "plans")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subscriptionsNeedRemap = !referencesCanonical
|
||||
if subscriptionsNeedRemap {
|
||||
if err := tx.Exec(`UPDATE subscriptions SET plan_id = CASE
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND (l."default" = TRUE OR l.price = 0)) THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id AND l.total_monitors > 100) THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
WHERE EXISTS (SELECT 1 FROM plans_legacy l WHERE l.id = subscriptions.plan_id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if remapAccounts && hasLegacy && (accountsNeedRemap || subscriptionsNeedRemap) && tx.Migrator().HasTable("subscription_events") {
|
||||
if err := tx.Exec(`UPDATE subscription_events e SET from_plan_id = CASE
|
||||
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
FROM plans_legacy l WHERE e.from_plan_id = l.id`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`UPDATE subscription_events e SET to_plan_id = CASE
|
||||
WHEN l."default" = TRUE OR l.price = 0 THEN (SELECT id FROM plans WHERE code = 'free')
|
||||
WHEN l.total_monitors > 100 THEN (SELECT id FROM plans WHERE code = 'team')
|
||||
ELSE (SELECT id FROM plans WHERE code = 'solo') END
|
||||
FROM plans_legacy l WHERE e.to_plan_id = l.id`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !remapAccounts && hasAccounts && hasSubscriptions {
|
||||
now := time.Now().UTC()
|
||||
if err := tx.Exec(`INSERT INTO subscriptions (account_id, plan_id, provider, status, billing_cycle, current_period_start, current_period_end, currency, amount_minor, metadata_json, created_at, updated_at)
|
||||
SELECT a.id, a.plan_id, 'manual', 'active', 'monthly', ?, ?, p.currency, p.price_monthly_minor, '{}'::jsonb, ?, ?
|
||||
FROM accounts a JOIN plans p ON p.id = a.plan_id
|
||||
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.account_id = a.id)`, now, now.AddDate(0, 0, 30), now, now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !remapAccounts || !hasLegacy {
|
||||
return nil
|
||||
}
|
||||
if hasAccounts {
|
||||
// AutoMigrate names this association fk_accounts_plan, while an older
|
||||
// migration used accounts_plan_id_fkey. Either name may still point at
|
||||
// plans_legacy after the table rename, so replace both deterministically.
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS accounts_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts DROP CONSTRAINT IF EXISTS fk_accounts_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE accounts ADD CONSTRAINT fk_accounts_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if hasSubscriptions {
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS subscriptions_plan_id_fkey`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions DROP CONSTRAINT IF EXISTS fk_subscriptions_plan`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec(`ALTER TABLE subscriptions ADD CONSTRAINT fk_subscriptions_plan FOREIGN KEY (plan_id) REFERENCES plans(id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Exec("ALTER TABLE plans_legacy RENAME TO plans_legacy_migrated").Error
|
||||
})
|
||||
}
|
||||
162
app/models/migrate_test.go
Обычный файл
162
app/models/migrate_test.go
Обычный файл
@@ -0,0 +1,162 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestMigrateUpgradesLegacyPlansWithoutClosingLockConnection(t *testing.T) {
|
||||
var databaseName string
|
||||
require.NoError(t, DB().Raw("SELECT current_database()").Scan(&databaseName).Error)
|
||||
require.Contains(t, databaseName, "test")
|
||||
|
||||
original := db
|
||||
schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
require.NoError(t, original.Exec("CREATE SCHEMA "+schema).Error)
|
||||
var isolatedSQLDB interface{ Close() error }
|
||||
t.Cleanup(func() {
|
||||
SetDB(original)
|
||||
if isolatedSQLDB != nil {
|
||||
_ = isolatedSQLDB.Close()
|
||||
}
|
||||
original.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE")
|
||||
})
|
||||
|
||||
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable search_path=%s",
|
||||
testDatabaseEnv("DATABASE_HOST", "POSTGRES_HOST", "localhost"),
|
||||
testDatabaseEnv("DATABASE_PORT", "POSTGRES_PORT", "5432"),
|
||||
testDatabaseEnv("DATABASE_USER", "POSTGRES_USER", "rsmon"),
|
||||
testDatabaseEnv("DATABASE_PASSWORD", "POSTGRES_PASSWORD", "rsmon"),
|
||||
databaseName,
|
||||
schema,
|
||||
)
|
||||
isolated, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
sqlDB, err := isolated.DB()
|
||||
require.NoError(t, err)
|
||||
isolatedSQLDB = sqlDB
|
||||
RegisterCallbacks(isolated)
|
||||
SetDB(isolated.Set("gorm:association_autoupdate", false))
|
||||
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE plans (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
price BIGINT NOT NULL DEFAULT 0,
|
||||
total_monitors BIGINT NOT NULL DEFAULT 0,
|
||||
"default" BOOLEAN NOT NULL DEFAULT FALSE
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE accounts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
plan_id BIGINT REFERENCES plans(id)
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE subscriptions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
account_id BIGINT NOT NULL REFERENCES accounts(id),
|
||||
plan_id BIGINT NOT NULL REFERENCES plans(id),
|
||||
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
|
||||
status VARCHAR(24) NOT NULL DEFAULT 'active',
|
||||
billing_cycle VARCHAR(8) NOT NULL DEFAULT 'monthly',
|
||||
currency VARCHAR(3) NOT NULL DEFAULT 'RUB',
|
||||
amount_minor BIGINT NOT NULL DEFAULT 0,
|
||||
metadata_json JSONB NOT NULL DEFAULT '{}'
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`CREATE TABLE subscription_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
subscription_id BIGINT NOT NULL,
|
||||
account_id BIGINT NOT NULL,
|
||||
provider VARCHAR(16) NOT NULL DEFAULT 'manual',
|
||||
kind VARCHAR(32) NOT NULL,
|
||||
from_plan_id BIGINT,
|
||||
to_plan_id BIGINT,
|
||||
payload_json JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)`).Error)
|
||||
require.NoError(t, DB().Exec(`INSERT INTO plans (id, name, price, total_monitors, "default")
|
||||
VALUES (42, 'Legacy free', 0, 10, TRUE)`).Error)
|
||||
require.NoError(t, DB().Exec("INSERT INTO accounts (id, name, plan_id) VALUES (7, 'Legacy account', 42)").Error)
|
||||
require.NoError(t, DB().Exec("INSERT INTO subscriptions (id, account_id, plan_id) VALUES (9, 7, 42)").Error)
|
||||
require.NoError(t, DB().Exec(`INSERT INTO subscription_events
|
||||
(id, subscription_id, account_id, kind, from_plan_id, to_plan_id)
|
||||
VALUES (11, 9, 7, 'legacy_change', 42, 42)`).Error)
|
||||
|
||||
Migrate()
|
||||
|
||||
require.True(t, DB().Migrator().HasColumn("plans", "code"))
|
||||
require.False(t, DB().Migrator().HasTable("plans_legacy"))
|
||||
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
|
||||
var codes []string
|
||||
require.NoError(t, DB().Table("plans").Order("code").Pluck("code", &codes).Error)
|
||||
require.ElementsMatch(t, CanonicalPlanCodes(), codes)
|
||||
var accountPlanCode string
|
||||
require.NoError(t, DB().Table("accounts").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
|
||||
Scan(&accountPlanCode).Error)
|
||||
require.Equal(t, "free", accountPlanCode)
|
||||
var subscriptionPlanCode string
|
||||
require.NoError(t, DB().Table("subscriptions").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = subscriptions.plan_id").Where("subscriptions.id = 9").
|
||||
Scan(&subscriptionPlanCode).Error)
|
||||
require.Equal(t, "free", subscriptionPlanCode)
|
||||
var eventPlanCodes struct {
|
||||
FromCode string
|
||||
ToCode string
|
||||
}
|
||||
require.NoError(t, DB().Table("subscription_events e").
|
||||
Select("fp.code AS from_code, tp.code AS to_code").
|
||||
Joins("JOIN plans fp ON fp.id = e.from_plan_id").
|
||||
Joins("JOIN plans tp ON tp.id = e.to_plan_id").
|
||||
Where("e.id = 11").Scan(&eventPlanCodes).Error)
|
||||
require.Equal(t, "free", eventPlanCodes.FromCode)
|
||||
require.Equal(t, "free", eventPlanCodes.ToCode)
|
||||
require.NoError(t, DB().Exec("SELECT 1").Error)
|
||||
|
||||
// The previous implementation retained this name after remapping. Its
|
||||
// canonical FKs must prevent a retry from remapping the same rows again.
|
||||
require.NoError(t, DB().Exec("ALTER TABLE plans_legacy_migrated RENAME TO plans_legacy").Error)
|
||||
require.NotPanics(t, Migrate)
|
||||
accountPlanCode = ""
|
||||
require.NoError(t, DB().Table("accounts").Select("plans.code").
|
||||
Joins("JOIN plans ON plans.id = accounts.plan_id").Where("accounts.id = 7").
|
||||
Scan(&accountPlanCode).Error)
|
||||
require.Equal(t, "free", accountPlanCode)
|
||||
require.True(t, DB().Migrator().HasTable("plans_legacy_migrated"))
|
||||
}
|
||||
|
||||
func TestMigrationAdvisoryLockIsReleasedAfterPanic(t *testing.T) {
|
||||
sqlDB, err := DB().DB()
|
||||
require.NoError(t, err)
|
||||
otherConn, err := sqlDB.Conn(context.Background())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = otherConn.Close() })
|
||||
|
||||
require.PanicsWithValue(t, "migration failed", func() {
|
||||
withMigrationAdvisoryLock(func() { panic("migration failed") })
|
||||
})
|
||||
|
||||
const migrateAdvisoryLock = int64(1234567890)
|
||||
var acquired bool
|
||||
require.NoError(t, otherConn.QueryRowContext(context.Background(),
|
||||
"SELECT pg_try_advisory_lock($1)", migrateAdvisoryLock).Scan(&acquired))
|
||||
require.True(t, acquired)
|
||||
_, err = otherConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", migrateAdvisoryLock)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func testDatabaseEnv(primary, fallback, defaultValue string) string {
|
||||
if value := os.Getenv(primary); value != "" {
|
||||
return value
|
||||
}
|
||||
if value := os.Getenv(fallback); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
534
app/models/monitor.go
Обычный файл
534
app/models/monitor.go
Обычный файл
@@ -0,0 +1,534 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
stateOK = "OK"
|
||||
stateERR = "ERR"
|
||||
stateWARN = "WARN"
|
||||
stateFail = "FAIL"
|
||||
stateDegraded = "DEGRADED"
|
||||
|
||||
// Event states.
|
||||
stateEnded = "ended"
|
||||
|
||||
// Check column names used in map[string]interface{} GORM updates. Defining
|
||||
// them as constants keeps GORM column references in sync with model fields.
|
||||
colLastStart = "last_start"
|
||||
colLastEnd = "last_end"
|
||||
colState = "state"
|
||||
colWarnings = "warnings"
|
||||
colInfos = "infos"
|
||||
|
||||
// Check kinds. Used to avoid sprinkling magic strings across the codebase.
|
||||
kindHTTP = "http"
|
||||
kindSSL = "ssl"
|
||||
kindSSH = "ssh"
|
||||
kindFTP = "ftp"
|
||||
kindDNS = "dns"
|
||||
kindWhois = "whois"
|
||||
kindRKN = "rkn"
|
||||
kindBSSL = "bssl"
|
||||
kindLLM = "llm"
|
||||
kindLLMHTTP = "llm-http"
|
||||
kindPing = "ping"
|
||||
kindTCP = "tcp"
|
||||
kindUDP = "udp"
|
||||
)
|
||||
|
||||
// Monitor monitor
|
||||
type Monitor struct {
|
||||
concerns.Model
|
||||
// activity status
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
|
||||
// check state, OK - all green, ERR - some checks have failed, UNK - new or not run, FAIL - unable to check
|
||||
State string `gorm:"not null;default:'UNK'" json:"state"`
|
||||
ConfirmState string `gorm:"size:32;not null;default:'none';index" json:"confirm_state"`
|
||||
ConfirmAt *time.Time `json:"confirm_at,omitempty"`
|
||||
ConfirmedByWorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"confirmed_by_worker_id,omitempty"`
|
||||
|
||||
// Group ID
|
||||
GroupID int64 `gorm:"type:bigint REFERENCES groups(id)" json:"group_id,omitempty" validate:"required"`
|
||||
Group *Group `json:"group,omitempty"`
|
||||
|
||||
// Optional inventory Site join (see docs/plans/inventory-management.md §6.3).
|
||||
// Lets the operator navigate monitor → site → deployments → server in one query.
|
||||
SiteID *int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE SET NULL;index" json:"site_id,omitempty"`
|
||||
Site *Site `json:"site,omitempty"`
|
||||
|
||||
// Tags for monitor grouping/searching
|
||||
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
|
||||
|
||||
// PreferredRegions is the optional whitelist of region codes a distributed
|
||||
// worker must be in to receive this monitor's checks. Empty/NULL means
|
||||
// "no preference" — any worker can pick it up. Used by Phase 2 of
|
||||
// docs/plans/worker-notifier-mvp.md (regional job routing); see
|
||||
// app/models/check_jobs.go applyRegionRouting.
|
||||
PreferredRegions pq.StringArray `gorm:"type:varchar(64)[]" json:"preferred_regions,omitempty"`
|
||||
|
||||
// RegionMode controls how PreferredRegions is interpreted by the
|
||||
// distributed-worker job router. Defaults to "any" so monitors without
|
||||
// explicit routing still match every worker — backwards-compatible with
|
||||
// Phase 1 deployments.
|
||||
// "any" — no region filter; legacy behavior (default)
|
||||
// "specific" — only workers whose region_code is in PreferredRegions
|
||||
// "all" — Phase 3 placeholder; today behaves like "any". The
|
||||
// multi-region quorum aggregation is not implemented yet,
|
||||
// see docs/todo.md Phase 3.
|
||||
RegionMode string `gorm:"size:16;not null;default:'any'" json:"region_mode" validate:"omitempty,oneof=any specific all"`
|
||||
|
||||
// Monitor name
|
||||
Name *string `json:"name,omitempty"`
|
||||
|
||||
// Host to monitor
|
||||
Host string `json:"host" validate:"required"`
|
||||
|
||||
// UserID specify user for this monitor (info field)
|
||||
UserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"user_id"`
|
||||
User *User `json:"user,omitempty"`
|
||||
|
||||
// Comment (info field)
|
||||
Comment *string `json:"comment"`
|
||||
|
||||
Checks []Check `json:"checks,omitempty"`
|
||||
DNSRecords []DNSRecord `json:"-"`
|
||||
|
||||
StatsData `gorm:"-:all" sql:"-" json:"stats"`
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// KINDS Check kinds
|
||||
var KINDS = []string{kindHTTP, kindSSL, kindSSH, kindFTP, kindDNS, kindWhois, kindRKN, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
|
||||
|
||||
// ValidCheckKind is the single canonical allow-list for user supplied check
|
||||
// kinds. Keep it beside the kind constants so every transport validates the
|
||||
// same set before a check reaches a worker.
|
||||
func ValidCheckKind(kind string) bool {
|
||||
for _, candidate := range KINDS {
|
||||
if kind == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Region routing mode constants used by Monitor.RegionMode and
|
||||
// app/models/check_jobs.go applyRegionRouting. Centralized so the literal
|
||||
// values are not sprinkled through the codebase.
|
||||
const (
|
||||
// RegionModeAny keeps the legacy behavior: every worker is eligible,
|
||||
// PreferredRegions is ignored. Default for newly-created monitors.
|
||||
RegionModeAny = "any"
|
||||
|
||||
// RegionModeSpecific limits eligible workers to those whose RegionCode
|
||||
// is contained in PreferredRegions. Empty PreferredRegions falls back to
|
||||
// RegionModeAny so the field is safe to leave blank in the UI.
|
||||
RegionModeSpecific = "specific"
|
||||
|
||||
// RegionModeAll is the Phase 3 placeholder: a monitor pinned to all of
|
||||
// its preferred regions for quorum aggregation. Phase 2 treats it as
|
||||
// RegionModeAny and logs a TODO marker so it is easy to grep for.
|
||||
RegionModeAll = "all"
|
||||
)
|
||||
|
||||
// RegionCodesFromSlice is a convenience wrapper so callers (mainly tests
|
||||
// and HTTP handlers) can pass a plain []string and get the pq.StringArray
|
||||
// type the model expects. nil/empty input is preserved as a nil slice so
|
||||
// the GORM column writes a SQL NULL instead of an empty array, matching
|
||||
// the column default.
|
||||
func RegionCodesFromSlice(in []string) pq.StringArray {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(pq.StringArray, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
// Int64ArrayFromSlice mirrors RegionCodesFromSlice for bigint[] columns
|
||||
// such as status_pages.monitor_ids and status_page_maintenance.monitor_ids.
|
||||
// The GORM pq.Int64Array driver expects a non-nil slice for ordered
|
||||
// inserts; callers that always have a non-empty list (the dashboard list
|
||||
// filter, the maintenance form) can rely on this to write a stable shape.
|
||||
func Int64ArrayFromSlice(in []int64) pq.Int64Array {
|
||||
if len(in) == 0 {
|
||||
return pq.Int64Array{}
|
||||
}
|
||||
out := make(pq.Int64Array, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
// ValidateRegionMode returns an error when RegionMode is not one of the
|
||||
// documented values ("any", "specific", "all"). Empty strings are treated as
|
||||
// "any" for backwards compatibility with monitors persisted before the field
|
||||
// existed; the DB column also defaults to "any".
|
||||
func (m *Monitor) ValidateRegionMode() error {
|
||||
switch m.RegionMode {
|
||||
case "", RegionModeAny, RegionModeSpecific, RegionModeAll:
|
||||
return nil
|
||||
default:
|
||||
return errors.Errorf("invalid region_mode %q (expected any|specific|all)", m.RegionMode)
|
||||
}
|
||||
}
|
||||
|
||||
// WantsRegion returns true when the monitor should be routed to a worker
|
||||
// operating in the given region code. Callers use this in
|
||||
// app/models/check_jobs.go to filter the eligible worker pool per check.
|
||||
//
|
||||
// - RegionModeAny: always true (no preference).
|
||||
// - RegionModeAll (Phase 3 placeholder): behaves like Any today; returns
|
||||
// true unconditionally so every region sees the check.
|
||||
// - RegionModeSpecific: true when code is contained in PreferredRegions,
|
||||
// or when PreferredRegions is empty (fall-back to Any).
|
||||
func (m *Monitor) WantsRegion(code string) bool {
|
||||
switch m.RegionMode {
|
||||
case RegionModeSpecific:
|
||||
if len(m.PreferredRegions) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, r := range m.PreferredRegions {
|
||||
if r == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
case RegionModeAll:
|
||||
// TODO(phase3): enumerate PreferredRegions and emit one assignment
|
||||
// per region so the result aggregator can do quorum. Today we
|
||||
// behave like Any so existing workers keep getting checks.
|
||||
return true
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// GetLabel provides functionality.
|
||||
func (m *Monitor) GetLabel() string {
|
||||
if m.Name != nil {
|
||||
return *m.Name
|
||||
}
|
||||
return m.Host
|
||||
}
|
||||
|
||||
// ProcessChecks provides functionality.
|
||||
func (m *Monitor) ProcessChecks(tx *gorm.DB) error {
|
||||
log.Println("process checks")
|
||||
checks := make([]Check, 0)
|
||||
for _, c := range m.Checks { //nolint:gocritic // range copy is acceptable here
|
||||
log.Println("maybe delete check", c.ID, c.Deleted, c.IsNew)
|
||||
if c.Deleted {
|
||||
if !c.IsNew {
|
||||
log.Println("delete check", c.ID)
|
||||
err := tx.Exec("delete from event_checks where check_id = ?", c.ID).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// First, find all message IDs for this check
|
||||
var messageIDs []int64
|
||||
err = tx.Model(&Message{}).Where("check_id = ?", c.ID).Pluck("id", &messageIDs).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete event_messages (join table) first to avoid FK constraint violation
|
||||
if len(messageIDs) > 0 {
|
||||
err = tx.Exec("DELETE FROM event_messages WHERE message_id IN (?)", messageIDs).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Now delete the messages
|
||||
err = tx.Where("check_id = ?", c.ID).Delete(Message{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.Where("id = ? AND monitor_id = ?", c.ID, m.ID).Delete(Check{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if c.IsNew {
|
||||
c.ID = 0
|
||||
}
|
||||
err := c.ValidateSettings()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "check validation error")
|
||||
}
|
||||
checks = append(checks, c)
|
||||
}
|
||||
m.Checks = checks
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveEvent provides functionality.
|
||||
func (m *Monitor) ActiveEvent() Event {
|
||||
evt := Event{}
|
||||
DB().Where("monitor_id = ? AND state != 'old'", m.ID).First(&evt)
|
||||
if evt.ID != 0 {
|
||||
evt.MonitorID = m.ID
|
||||
t := time.Now()
|
||||
evt.StartTime = &t
|
||||
}
|
||||
return evt
|
||||
}
|
||||
|
||||
var mutex sync.Mutex
|
||||
|
||||
// checkSeverityRank assigns an ordinal to each check state so the monitor
|
||||
// aggregator can pick the highest-severity child deterministically.
|
||||
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
|
||||
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
|
||||
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
|
||||
// real check state takes precedence over them.
|
||||
func checkSeverityRank(state string) int {
|
||||
switch state {
|
||||
case stateFail:
|
||||
return 5
|
||||
case stateERR:
|
||||
return 4
|
||||
case stateDegraded:
|
||||
return 3
|
||||
case stateWARN:
|
||||
return 2
|
||||
case stateOK:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateStatusFromChecks updates the monitor status based on its checks.
|
||||
func (m *Monitor) UpdateStatusFromChecks() {
|
||||
mutex.Lock()
|
||||
checks := make([]Check, 0)
|
||||
|
||||
tx := DB().Begin()
|
||||
var locked Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&locked, m.ID).Error; err != nil {
|
||||
tx.Rollback()
|
||||
mutex.Unlock()
|
||||
log.Println("UpdateStatusFromChecks lock monitor", err)
|
||||
return
|
||||
}
|
||||
m.State = locked.State
|
||||
|
||||
_ = tx.Model(m).Association("Checks").Find(&checks)
|
||||
prevState := m.State
|
||||
m.State = stateOK
|
||||
|
||||
// Pick the highest-severity enabled check. The previous implementation
|
||||
// inlined three if-statements with non-obvious precedence (a WARN that
|
||||
// appeared AFTER an ERR in the iteration would never downgrade back,
|
||||
// but a FAIL after ERR would silently get clobbered). Using a single
|
||||
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
|
||||
// independent of slice ordering — the same rule Phase 3 introduces
|
||||
// for DEGRADED, applied uniformly to the existing states too.
|
||||
bestRank := checkSeverityRank(stateOK)
|
||||
bestState := stateOK
|
||||
hasChecks := false
|
||||
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
||||
if check.Enabled == nil || !*check.Enabled {
|
||||
continue
|
||||
}
|
||||
hasChecks = true
|
||||
if r := checkSeverityRank(check.State); r > bestRank {
|
||||
bestRank = r
|
||||
bestState = check.State
|
||||
}
|
||||
}
|
||||
m.State = bestState
|
||||
|
||||
if !hasChecks {
|
||||
m.State = stateWARN
|
||||
}
|
||||
|
||||
if m.State != prevState {
|
||||
err := tx.Model(&m).UpdateColumn("state", m.State).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks fail update state", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
evt := Event{}
|
||||
if err := tx.Where("monitor_id = ? AND state = ?", m.ID, "current").First(&evt).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
tx.Rollback()
|
||||
mutex.Unlock()
|
||||
log.Println("UpdateStatusFromChecks active event", err)
|
||||
return
|
||||
}
|
||||
// log.Println("state", m.State, "active event:", evt.ID)
|
||||
|
||||
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
||||
if check.State == stateERR || check.State == stateFail {
|
||||
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
|
||||
evt.Checks = append(evt.Checks, check)
|
||||
if check.Error != nil {
|
||||
evt.Reason = *check.Error
|
||||
} else {
|
||||
evt.Reason = "unknown error"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch m.State {
|
||||
case stateOK, stateWARN, stateDegraded:
|
||||
// DEGRADED is treated like OK/WARN for the event lifecycle: we do
|
||||
// NOT open a new "current" outage event for a partial regional
|
||||
// failure. Operators see DEGRADED on the check detail page and
|
||||
// the monitor list, but the existing notifier pipeline (down /
|
||||
// restore events) only fires for full ERR/FAIL. A future
|
||||
// improvement can add a separate "degraded" message kind.
|
||||
if evt.ID != 0 {
|
||||
upd := map[string]interface{}{
|
||||
"duration": time.Since(*evt.StartTime).Seconds(),
|
||||
"state": stateEnded,
|
||||
"oks": evt.Oks + 1,
|
||||
}
|
||||
if evt.EndTime == nil {
|
||||
upd["end_time"] = time.Now()
|
||||
}
|
||||
err := tx.Model(&evt).UpdateColumns(upd).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks fail update to ended", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
case stateERR, stateFail:
|
||||
if evt.ID == 0 {
|
||||
tn := time.Now()
|
||||
evt.StartTime = &tn
|
||||
evt.Duration = 0
|
||||
evt.State = "current"
|
||||
evt.MonitorID = m.ID
|
||||
err := tx.Save(&evt).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
spew.Dump(evt)
|
||||
log.Println("UpdateStatusFromChecks fail create", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
} else {
|
||||
upd := map[string]interface{}{
|
||||
"end_time": nil,
|
||||
"state": "current",
|
||||
"errors": evt.Errors + 1,
|
||||
}
|
||||
if evt.StartTime == nil {
|
||||
upd["start_time"] = time.Now()
|
||||
upd["duration"] = 0
|
||||
} else {
|
||||
upd["duration"] = time.Since(*evt.StartTime).Seconds()
|
||||
}
|
||||
|
||||
err := tx.Model(&evt).UpdateColumns(upd).Error
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
spew.Dump(evt)
|
||||
spew.Dump(upd)
|
||||
log.Println("UpdateStatusFromChecks fail update to current", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := m.syncStatusPageIncidentsTx(tx, &evt); err != nil {
|
||||
tx.Rollback()
|
||||
log.Println("UpdateStatusFromChecks status page incident", err)
|
||||
mutex.Unlock()
|
||||
return
|
||||
}
|
||||
err := tx.Commit().Error
|
||||
|
||||
mutex.Unlock()
|
||||
|
||||
if err != nil {
|
||||
log.Println("UpdateStatusFromChecks commit fail", err)
|
||||
return
|
||||
}
|
||||
if m.State != prevState {
|
||||
m.invalidateStatusPages()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Monitor) invalidateStatusPages() {
|
||||
var ids []int64
|
||||
if err := DB().Model(&StatusPage{}).Where("? = ANY(monitor_ids)", m.ID).Pluck("id", &ids).Error; err != nil {
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
InvalidateStatusPageCache(id)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
|
||||
if event == nil || event.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
var pages []StatusPage
|
||||
if err := tx.Where("auto_open_incidents = TRUE AND ? = ANY(monitor_ids)", m.ID).Find(&pages).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range pages {
|
||||
page := &pages[i]
|
||||
var incident StatusPageIncident
|
||||
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
|
||||
if m.State == stateERR || m.State == "FAIL" {
|
||||
if err != nil {
|
||||
// The database uniqueness constraint makes concurrent state updates idempotent.
|
||||
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
|
||||
if err := tx.Create(&incident).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "opened"); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if incident.BodyMD != event.Reason {
|
||||
if err := tx.Model(&incident).Update("body_md", event.Reason).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
incident.BodyMD, incident.UpdatedAt = event.Reason, time.Now()
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "updated"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if err == nil && incident.ResolvedAt == nil {
|
||||
now := time.Now()
|
||||
if err := tx.Model(&incident).Update("resolved_at", now).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
incident.ResolvedAt, incident.UpdatedAt = &now, now
|
||||
if err := EnqueueStatusPageIncidentDeliveriesTx(tx, page, &incident, "resolved"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
215
app/models/monitor_state_test.go
Обычный файл
215
app/models/monitor_state_test.go
Обычный файл
@@ -0,0 +1,215 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// monitorStateTestWorld bundles the account/group/monitor fixture the
|
||||
// monitor-state tests need. It deliberately re-creates the row in each
|
||||
// test rather than sharing, because Monitor.UpdateStatusFromChecks
|
||||
// mutates the row in place and the per-test assertions need a clean
|
||||
// baseline.
|
||||
type monitorStateTestWorld struct {
|
||||
plan models.Plan
|
||||
account models.Account
|
||||
group models.Group
|
||||
monitor models.Monitor
|
||||
}
|
||||
|
||||
// seedMonitorStateWorld provisions one plan/account/group/monitor with
|
||||
// the requested initial state. The monitor is enabled so
|
||||
// UpdateStatusFromChecks treats its checks as live.
|
||||
func seedMonitorStateWorld(t *testing.T, initialState string) monitorStateTestWorld {
|
||||
t.Helper()
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "ms-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
account := models.Account{Name: "ms-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&account).Error)
|
||||
|
||||
group := models.Group{AccountID: account.ID, Name: "ms"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrAgg("mon.test"),
|
||||
Host: "mon.test",
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
State: initialState,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
return monitorStateTestWorld{
|
||||
plan: plan,
|
||||
account: account,
|
||||
group: group,
|
||||
monitor: mon,
|
||||
}
|
||||
}
|
||||
|
||||
// attachCheck creates an enabled check on the monitor with the given
|
||||
// state. Returns the persisted check so the test can re-load it.
|
||||
func attachCheck(t *testing.T, monitorID int64, kind string, state string) models.Check {
|
||||
t.Helper()
|
||||
enTrue := true
|
||||
c := models.Check{
|
||||
MonitorID: monitorID,
|
||||
Kind: kind,
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: state,
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&c).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
// reloadMonitor pulls the latest monitor state from the DB so the test
|
||||
// can compare against the post-UpdateStatusFromChecks row.
|
||||
func reloadMonitor(t *testing.T, id int64) models.Monitor {
|
||||
t.Helper()
|
||||
var m models.Monitor
|
||||
require.NoError(t, models.DB().First(&m, id).Error)
|
||||
return m
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 3 severity rules: FAIL > ERR > DEGRADED > WARN > OK.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestMonitorStateFromChecks_Degraded: monitor with 1 OK + 1 DEGRADED
|
||||
// child → monitor.State = DEGRADED. The new severity rank must
|
||||
// correctly promote DEGRADED above OK.
|
||||
func TestMonitorStateFromChecks_Degraded(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"DEGRADED child must promote monitor above OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DegradedWithError: monitor with 1 ERR +
|
||||
// 1 DEGRADED → monitor.State = ERR. ERR beats DEGRADED in the
|
||||
// severity order.
|
||||
func TestMonitorStateFromChecks_DegradedWithError(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "ERR")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "ERR", got.State,
|
||||
"ERR must beat DEGRADED — the order is FAIL > ERR > DEGRADED > WARN > OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DegradedOnlyOK covers the single-DEGRADED
|
||||
// case explicitly so a regression that treats DEGRADED as "WARN-ish"
|
||||
// would flip this assertion.
|
||||
func TestMonitorStateFromChecks_DegradedOnlyOK(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
attachCheck(t, world.monitor.ID, "http", "DEGRADED")
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "DEGRADED", got.State,
|
||||
"mixed OK+DEGRADED monitor must be DEGRADED, not OK")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_SeverityOrderingTable is a table-driven
|
||||
// sweep of the FAIL > ERR > DEGRADED > WARN > OK ladder. The case set
|
||||
// is intentionally small — every pair that could reveal a wrong
|
||||
// winner under the new severity rank. Keeping it table-driven makes
|
||||
// it trivial to add more cases if a future state is introduced.
|
||||
func TestMonitorStateFromChecks_SeverityOrderingTable(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
checks []string
|
||||
wantMon string
|
||||
}{
|
||||
{"all_ok", []string{"OK", "OK"}, "OK"},
|
||||
{"all_warn", []string{"WARN", "WARN"}, "WARN"},
|
||||
{"ok_with_warn", []string{"OK", "WARN"}, "WARN"},
|
||||
{"warn_with_ok", []string{"WARN", "OK"}, "WARN"}, // order independence
|
||||
{"ok_with_degraded", []string{"OK", "DEGRADED"}, "DEGRADED"},
|
||||
{"warn_with_degraded", []string{"WARN", "DEGRADED"}, "DEGRADED"},
|
||||
{"degraded_with_warn", []string{"DEGRADED", "WARN"}, "DEGRADED"},
|
||||
{"err_with_degraded", []string{"ERR", "DEGRADED"}, "ERR"},
|
||||
{"degraded_with_err", []string{"DEGRADED", "ERR"}, "ERR"},
|
||||
{"fail_with_err", []string{"FAIL", "ERR"}, "FAIL"},
|
||||
{"err_with_fail", []string{"ERR", "FAIL"}, "FAIL"}, // order independence
|
||||
{"fail_alone", []string{"FAIL"}, "FAIL"},
|
||||
{"all_degraded", []string{"DEGRADED", "DEGRADED"}, "DEGRADED"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
for i, st := range c.checks {
|
||||
attachCheck(t, world.monitor.ID,
|
||||
"http-"+string(rune('a'+i)), st)
|
||||
}
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, c.wantMon, got.State,
|
||||
"severity winner for child states %v must be %s", c.checks, c.wantMon)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_NoChecksIsWarn pins the existing "no
|
||||
// enabled checks → WARN" behavior — the new severity rank must not
|
||||
// accidentally produce OK for an empty monitor.
|
||||
func TestMonitorStateFromChecks_NoChecksIsWarn(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "WARN", got.State,
|
||||
"an enabled monitor with zero checks must remain WARN")
|
||||
}
|
||||
|
||||
// TestMonitorStateFromChecks_DisabledCheckIgnored verifies that a
|
||||
// disabled check is not folded into the severity decision. Otherwise
|
||||
// a stuck-in-ERR check that has been disabled would keep tripping the
|
||||
// monitor forever.
|
||||
func TestMonitorStateFromChecks_DisabledCheckIgnored(t *testing.T) {
|
||||
world := seedMonitorStateWorld(t, "UNK")
|
||||
attachCheck(t, world.monitor.ID, "http", "OK")
|
||||
|
||||
enFalse := false
|
||||
disabled := models.Check{
|
||||
MonitorID: world.monitor.ID,
|
||||
Kind: "http-disabled",
|
||||
Interval: 60,
|
||||
Enabled: &enFalse,
|
||||
State: "ERR",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&disabled).Error)
|
||||
|
||||
world.monitor.UpdateStatusFromChecks()
|
||||
|
||||
got := reloadMonitor(t, world.monitor.ID)
|
||||
assert.Equal(t, "OK", got.State,
|
||||
"disabled check must be ignored — only the enabled OK check counts")
|
||||
}
|
||||
|
||||
// guard against time import being pruned by an editor when individual
|
||||
// test bodies stop referencing it directly.
|
||||
var _ = time.Second
|
||||
94
app/models/monitor_transfer_test.go
Обычный файл
94
app/models/monitor_transfer_test.go
Обычный файл
@@ -0,0 +1,94 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// TestMonitorTransferGroupSwap verifies that swapping a monitor's
|
||||
// group_id between two accounts correctly re-homes the monitor without
|
||||
// touching any other monitor data. This is the database primitive that
|
||||
// POST /api/v1/monitors/:id/transfer relies on.
|
||||
func TestMonitorTransferGroupSwap(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
// Two accounts, each with their own default group.
|
||||
accA := models.Account{Name: "A", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&accA).Error)
|
||||
accB := models.Account{Name: "B", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&accB).Error)
|
||||
|
||||
groupA := models.Group{AccountID: accA.ID, Name: "A-default"}
|
||||
require.NoError(t, models.DB().Create(&groupA).Error)
|
||||
groupB := models.Group{AccountID: accB.ID, Name: "B-default"}
|
||||
require.NoError(t, models.DB().Create(&groupB).Error)
|
||||
|
||||
// Monitor lives in account A with one HTTP check.
|
||||
monitor := models.Monitor{
|
||||
GroupID: groupA.ID,
|
||||
Host: "example-a.test",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
check := models.Check{
|
||||
MonitorID: monitor.ID,
|
||||
Kind: "http",
|
||||
URL: ptrString("https://example-a.test/"),
|
||||
Interval: 300,
|
||||
Settings: datatypes.JSON([]byte("{}")),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
|
||||
// Simulate the controller-side update.
|
||||
require.NoError(t, models.DB().
|
||||
Model(&models.Monitor{}).
|
||||
Where("id = ?", monitor.ID).
|
||||
Update("group_id", groupB.ID).Error)
|
||||
|
||||
// Monitor now lives in B; check follows by FK on monitor_id.
|
||||
var reloaded models.Monitor
|
||||
require.NoError(t, models.DB().Preload("Group").First(&reloaded, monitor.ID).Error)
|
||||
assert.Equal(t, groupB.ID, reloaded.GroupID, "monitor group_id should now point at account B's group")
|
||||
assert.Equal(t, accB.ID, reloaded.Group.AccountID, "preloaded group should belong to account B")
|
||||
|
||||
var checkCount int64
|
||||
require.NoError(t, models.DB().Model(&models.Check{}).
|
||||
Where("monitor_id = ?", monitor.ID).Count(&checkCount).Error)
|
||||
assert.Equal(t, int64(1), checkCount, "check rows must follow the monitor across the move")
|
||||
}
|
||||
|
||||
// TestMonitorTransferSameAccountGuard documents the early-return path: the
|
||||
// controller must not silently no-op when the caller picks the current
|
||||
// account, and the test exercises that the DB stays untouched.
|
||||
func TestMonitorTransferSameAccountGuard(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
|
||||
acc := models.Account{Name: "only", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := models.Group{AccountID: acc.ID, Name: "only"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "x.test"}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
|
||||
// No update is issued because the controller rejects same-account moves
|
||||
// before the SQL UPDATE. Verify the row is unchanged.
|
||||
var reloaded models.Monitor
|
||||
require.NoError(t, models.DB().First(&reloaded, monitor.ID).Error)
|
||||
assert.Equal(t, group.ID, reloaded.GroupID)
|
||||
}
|
||||
|
||||
func ptrString(s string) *string { return &s }
|
||||
528
app/models/network_diagnostics.go
Обычный файл
528
app/models/network_diagnostics.go
Обычный файл
@@ -0,0 +1,528 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/influx"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfirmStateNone = "none"
|
||||
ConfirmStatePending = "pending"
|
||||
ConfirmStateConfirmed = "confirmed" // A different worker reproduced the failure.
|
||||
ConfirmStateTimeout = "confirmed_by_timeout"
|
||||
AttemptKindRegular = "regular"
|
||||
AttemptKindConfirm = "confirmation"
|
||||
AttemptStateQueued = "queued"
|
||||
AttemptStateLeased = "leased"
|
||||
AttemptStateFinished = "finished"
|
||||
diagnosticSoft = "soft"
|
||||
diagnosticHard = "hard"
|
||||
diagnosticRecovery = "recovery"
|
||||
)
|
||||
|
||||
type diagnosticSettings struct {
|
||||
confirmTimeout time.Duration
|
||||
healthWindow time.Duration
|
||||
healthRate float64
|
||||
healthMin int64
|
||||
}
|
||||
|
||||
func settingsForAccount(account *Account) diagnosticSettings {
|
||||
settings := diagnosticSettings{confirmTimeout: 90 * time.Second, healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
|
||||
if account == nil || account.Plan == nil {
|
||||
return settings
|
||||
}
|
||||
plan := account.Plan
|
||||
if plan.ConfirmTimeoutSec > 0 {
|
||||
settings.confirmTimeout = time.Duration(plan.ConfirmTimeoutSec) * time.Second
|
||||
}
|
||||
if plan.HealthWindowSec > 0 {
|
||||
settings.healthWindow = time.Duration(plan.HealthWindowSec) * time.Second
|
||||
}
|
||||
if plan.HealthRateThreshold > 0 && plan.HealthRateThreshold <= 1 {
|
||||
settings.healthRate = plan.HealthRateThreshold
|
||||
}
|
||||
if plan.HealthMinAttempts > 0 {
|
||||
settings.healthMin = int64(plan.HealthMinAttempts)
|
||||
}
|
||||
if !plan.Confirmations {
|
||||
return settings
|
||||
}
|
||||
if account.ConfirmTimeoutSec != nil && *account.ConfirmTimeoutSec >= 15 {
|
||||
settings.confirmTimeout = time.Duration(*account.ConfirmTimeoutSec) * time.Second
|
||||
}
|
||||
if account.HealthWindowSec != nil && *account.HealthWindowSec >= 60 {
|
||||
settings.healthWindow = time.Duration(*account.HealthWindowSec) * time.Second
|
||||
}
|
||||
if account.HealthRateThreshold != nil && *account.HealthRateThreshold > 0 && *account.HealthRateThreshold <= 1 {
|
||||
settings.healthRate = *account.HealthRateThreshold
|
||||
}
|
||||
if account.HealthMinAttempts != nil && *account.HealthMinAttempts > 0 {
|
||||
settings.healthMin = int64(*account.HealthMinAttempts)
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
// CheckAttempt is the durable worker-attribution record. Unlike check state,
|
||||
// it is append-only and therefore remains useful after a worker is deweighted.
|
||||
type CheckAttempt struct {
|
||||
concerns.Model
|
||||
JobID string `gorm:"uniqueIndex;size:64;not null" json:"job_id"`
|
||||
CheckID int64 `gorm:"index;not null" json:"check_id"`
|
||||
MonitorID int64 `gorm:"index;not null" json:"monitor_id"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
|
||||
WorkerNode *WorkerNode `json:"worker_node,omitempty"`
|
||||
SourceWorkerNodeID *int64 `gorm:"index" json:"source_worker_node_id,omitempty"`
|
||||
Kind string `gorm:"size:32;not null" json:"kind"`
|
||||
State string `gorm:"size:32;not null" json:"state"`
|
||||
ResultState string `gorm:"size:16" json:"result_state"`
|
||||
Result datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"result"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
LeaseToken string `gorm:"size:64" json:"-"`
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
Deweighted bool `gorm:"not null;default:false" json:"deweighted"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// DiagnosticAuditEvent is a compact, queryable control-plane audit record.
|
||||
type DiagnosticAuditEvent struct {
|
||||
concerns.Model
|
||||
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
|
||||
WorkerNodeID *int64 `gorm:"index" json:"worker_node_id,omitempty"`
|
||||
Kind string `gorm:"size:64;index;not null" json:"kind"`
|
||||
Metadata datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"metadata"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func auditDiagnostic(tx *gorm.DB, kind string, monitorID, workerID *int64, metadata map[string]interface{}) {
|
||||
b, _ := json.Marshal(metadata)
|
||||
_ = tx.Create(&DiagnosticAuditEvent{MonitorID: monitorID, WorkerNodeID: workerID, Kind: kind, Metadata: b}).Error
|
||||
}
|
||||
|
||||
// AuditNetworkRecovery is used by the admin incident-response endpoint.
|
||||
func AuditNetworkRecovery(tx *gorm.DB, workerID int64) {
|
||||
auditDiagnostic(tx, "worker.network_problem_force_recover", nil, &workerID, nil)
|
||||
}
|
||||
|
||||
// enqueueDiagnosticDelivery creates the message and its durable delivery task in
|
||||
// the transition transaction. A transaction advisory lock prevents concurrent
|
||||
// result frames from leaving duplicate messages when the task dedupe wins.
|
||||
func enqueueDiagnosticDelivery(tx *gorm.DB, monitor *Monitor, account *Account, tier string, now time.Time) error {
|
||||
if account == nil || account.Plan == nil {
|
||||
return nil
|
||||
}
|
||||
if tier == diagnosticHard && !account.Plan.AllowHardAlerts {
|
||||
return nil
|
||||
}
|
||||
var notifications []Notification
|
||||
if err := tx.Joins("JOIN notification_groups ON notification_groups.notification_id = notifications.id").
|
||||
Where("notifications.account_id = ? AND notifications.enabled AND notification_groups.group_id = ?", account.ID, monitor.GroupID).
|
||||
Preload("Contacts", "enabled = ?", true).Find(¬ifications).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range notifications {
|
||||
for j := range notifications[i].Contacts {
|
||||
contact := notifications[i].Contacts[j]
|
||||
method := diagnosticContactMethod(contact.Kind)
|
||||
if method == "" || (tier == diagnosticSoft && method != "email" && method != "telegram") {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("diagnostic:%d:%s:%d:%d", monitor.ID, tier, notifications[i].ID, contact.ID)
|
||||
if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtext(?))", key).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var existing Task
|
||||
if err := tx.Where("idempotency_key = ?", key).First(&existing).Error; err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
message := Message{NotificationID: notifications[i].ID, ContactID: contact.ID, Kind: "diagnostic_" + tier, State: TaskStateQueued}
|
||||
if err := tx.Create(&message).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
monitorID, messageID := monitor.ID, message.ID
|
||||
payload, err := json.Marshal(wire.NotificationTask{AccountID: account.ID, MessageID: messageID, NotificationID: notifications[i].ID, MonitorID: &monitorID, Method: method, Contact: wire.NotificationContact{ID: contact.ID, Kind: contact.Kind, Value: contact.Value, Name: contact.Name}, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notifications[i].ID, ContactID: contact.ID, MessageID: &messageID, MonitorID: &monitorID, Method: method, Subject: diagnosticSubject(monitor, tier), BodyText: diagnosticBody(monitor, tier), BodyMarkdown: diagnosticBody(monitor, tier), BodyHTML: diagnosticBody(monitor, tier), Language: "en", MessageKind: message.Kind, NotBefore: now, Payload: payload, IdempotencyKey: key}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func diagnosticContactMethod(kind string) string {
|
||||
switch kind {
|
||||
case "email":
|
||||
return "email"
|
||||
case "telegram_private", "telegram_group":
|
||||
return "telegram"
|
||||
case "webhook", "mattermost", "sms", "voice":
|
||||
return kind
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func diagnosticSubject(monitor *Monitor, tier string) string {
|
||||
return fmt.Sprintf("Monitor %s: %s", monitor.Host, tier)
|
||||
}
|
||||
|
||||
func diagnosticBody(monitor *Monitor, tier string) string {
|
||||
return fmt.Sprintf("Network diagnostic %s for monitor %s.", tier, monitor.Host)
|
||||
}
|
||||
|
||||
// ConfirmationJobsForWorker atomically leases confirmation jobs assigned to this
|
||||
// worker or left unassigned by an expired lease. Unassigned attempts still retain
|
||||
// SourceWorkerNodeID, so the original failing worker can never claim them.
|
||||
func ConfirmationJobsForWorker(worker *WorkerNode, kinds []string, limit int) ([]wire.CheckJob, error) {
|
||||
if worker == nil || worker.AccountID != nil || !worker.SupportsTaskEnvelope() || worker.NetworkProblemActive(time.Now()) || limit < 1 {
|
||||
return nil, nil
|
||||
}
|
||||
var jobs []wire.CheckJob
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var attempts []CheckAttempt
|
||||
if err := tx.Clauses(SkipLockedClause).Where("(worker_node_id = ? OR worker_node_id IS NULL) AND kind = ? AND state = ?", worker.ID, AttemptKindConfirm, AttemptStateQueued).Order("id").Limit(limit).Find(&attempts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range attempts {
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor").First(&check, attempts[i].CheckID).Error; err != nil {
|
||||
continue
|
||||
}
|
||||
if check.Monitor == nil || !check.Monitor.Enabled {
|
||||
continue
|
||||
}
|
||||
if attempts[i].SourceWorkerNodeID != nil && *attempts[i].SourceWorkerNodeID == worker.ID {
|
||||
continue
|
||||
}
|
||||
if !containsString(kinds, check.Kind) || !containsString(worker.CheckTypes(), check.Kind) {
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(DefaultTaskLeaseTTL)
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateLeased, "started_at": now, "lease_token": leaseToken, "lease_expires_at": leaseUntil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
jobs = append(jobs, wire.CheckJob{JobID: attempts[i].JobID, LeaseToken: leaseToken, CheckID: check.ID, MonitorID: check.MonitorID, Kind: check.Kind, Host: check.Monitor.Host, URL: check.URL, Interval: check.Interval, Settings: json.RawMessage(check.Settings)})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return jobs, err
|
||||
}
|
||||
|
||||
// StartConfirmation creates exactly one targeted confirmation for a new outage.
|
||||
func StartConfirmation(checkID, sourceWorkerID int64, now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return StartConfirmationTx(tx, checkID, sourceWorkerID, now)
|
||||
})
|
||||
}
|
||||
|
||||
// StartConfirmationTx is StartConfirmation's transaction-aware form.
|
||||
func StartConfirmationTx(tx *gorm.DB, checkID, sourceWorkerID int64, now time.Time) error {
|
||||
if tx == nil {
|
||||
return errors.New("start confirmation: nil transaction")
|
||||
}
|
||||
{
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
// Confirmations are a paid distributed-check entitlement. Free accounts
|
||||
// retain the legacy direct soft alert path and do not consume worker budget.
|
||||
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil || check.Monitor.Group.Account.Plan == nil || !check.Monitor.Group.Account.Plan.Confirmations {
|
||||
return nil
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor.ConfirmState == ConfirmStatePending || monitor.ConfirmState == ConfirmStateConfirmed {
|
||||
return nil
|
||||
}
|
||||
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, 0, now)
|
||||
if err != nil {
|
||||
monitor.ConfirmState, monitor.ConfirmAt = ConfirmStateTimeout, &now
|
||||
auditDiagnostic(tx, "check.confirm_unavailable", &monitor.ID, &sourceWorkerID, nil)
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticHard, now)
|
||||
}
|
||||
monitor.ConfirmState, monitor.ConfirmAt, monitor.ConfirmedByWorkerID = ConfirmStatePending, &now, &worker.ID
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticSoft, now); err != nil {
|
||||
return err
|
||||
}
|
||||
attempt := CheckAttempt{JobID: uuid.NewString(), CheckID: check.ID, MonitorID: monitor.ID, WorkerNodeID: &worker.ID, SourceWorkerNodeID: &sourceWorkerID, Kind: AttemptKindConfirm, State: AttemptStateQueued}
|
||||
if err := tx.Create(&attempt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_assign", &monitor.ID, &worker.ID, map[string]interface{}{"exclude_worker_id": sourceWorkerID, "job_id": attempt.JobID})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// confirmationWorkerTx selects an independent active platform worker that can
|
||||
// execute this exact kind. Capability filtering is deliberately performed in
|
||||
// Go because the JSON capability format also supports legacy rows safely.
|
||||
func confirmationWorkerTx(tx *gorm.DB, checkKind string, sourceWorkerID, excludeWorkerID int64, now time.Time) (*WorkerNode, error) {
|
||||
var workers []WorkerNode
|
||||
if err := tx.Where("id <> ? AND id <> ? AND account_id IS NULL AND status = 'active' AND (network_problems = FALSE OR network_problems_until <= ? OR network_problems_until IS NULL)", sourceWorkerID, excludeWorkerID, now).Order("id").Find(&workers).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range workers {
|
||||
if workers[i].SupportsTaskEnvelope() && containsString(workers[i].CheckTypes(), checkKind) {
|
||||
return &workers[i], nil
|
||||
}
|
||||
}
|
||||
return nil, gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
// ApplyDiagnosticResult resolves a targeted attempt once. Duplicate reports are ignored.
|
||||
func ApplyDiagnosticResult(report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
|
||||
if report.JobID == "" || worker == nil {
|
||||
return false, nil
|
||||
}
|
||||
handled := true
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
return ApplyDiagnosticResultTx(tx, report, worker, now, &handled)
|
||||
})
|
||||
return handled, err
|
||||
}
|
||||
|
||||
// ApplyDiagnosticResultTx resolves a diagnostic attempt within the caller's
|
||||
// transaction. handled distinguishes a normal check result from a diagnostic.
|
||||
func ApplyDiagnosticResultTx(tx *gorm.DB, report wire.CheckResultReport, worker *WorkerNode, now time.Time, handled *bool) error {
|
||||
if tx == nil {
|
||||
return errors.New("apply diagnostic: nil transaction")
|
||||
}
|
||||
if handled == nil {
|
||||
return errors.New("apply diagnostic: nil handled result")
|
||||
}
|
||||
*handled = true
|
||||
{
|
||||
var attempt CheckAttempt
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("job_id = ?", report.JobID).First(&attempt).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if attempt.Kind != AttemptKindConfirm {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
if attempt.State == AttemptStateFinished {
|
||||
*handled = false
|
||||
return nil
|
||||
}
|
||||
if attempt.State != AttemptStateLeased || attempt.WorkerNodeID == nil || *attempt.WorkerNodeID != worker.ID {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
if attempt.LeaseToken == "" || report.LeaseToken == "" || report.LeaseToken != attempt.LeaseToken || attempt.LeaseExpiresAt == nil || !attempt.LeaseExpiresAt.After(now) {
|
||||
return errors.New("apply diagnostic: lease token is invalid or expired")
|
||||
}
|
||||
payload, _ := json.Marshal(report)
|
||||
deweighted := worker.NetworkProblemActive(now)
|
||||
if err := tx.Model(&attempt).Where("state = ? AND lease_token = ? AND lease_expires_at > ?", AttemptStateLeased, report.LeaseToken, now).Updates(map[string]interface{}{"state": AttemptStateFinished, "result_state": report.State, "result": payload, "finished_at": now, "lease_token": "", "lease_expires_at": nil, "deweighted": deweighted}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, attempt.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if attempt.Kind == AttemptKindConfirm && monitor.ConfirmState == ConfirmStatePending {
|
||||
if report.State == stateERR || report.State == stateFail {
|
||||
monitor.ConfirmState = ConfirmStateConfirmed
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticHard, now); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
monitor.ConfirmState = ConfirmStateNone
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitor, monitorAccount(tx, &monitor), diagnosticRecovery, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_result", &monitor.ID, &worker.ID, map[string]interface{}{"state": report.State, "deweighted": deweighted})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func monitorAccount(tx *gorm.DB, monitor *Monitor) *Account {
|
||||
var group Group
|
||||
if err := tx.Preload("Account.Plan").First(&group, monitor.GroupID).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return group.Account
|
||||
}
|
||||
|
||||
// RecoverDiagnostic clears a completed hard escalation only once and queues the
|
||||
// corresponding recovery tasks in the same transaction.
|
||||
func RecoverDiagnostic(checkID int64, now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
return RecoverDiagnosticTx(tx, checkID, now)
|
||||
})
|
||||
}
|
||||
|
||||
// RecoverDiagnosticTx is RecoverDiagnostic's transaction-aware form.
|
||||
func RecoverDiagnosticTx(tx *gorm.DB, checkID int64, now time.Time) error {
|
||||
if tx == nil {
|
||||
return errors.New("recover diagnostic: nil transaction")
|
||||
}
|
||||
{
|
||||
var check Check
|
||||
if err := tx.Preload("Monitor.Group.Account.Plan").First(&check, checkID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if check.Monitor == nil || check.Monitor.Group == nil || check.Monitor.Group.Account == nil {
|
||||
return nil
|
||||
}
|
||||
var monitor Monitor
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&monitor, check.MonitorID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if monitor.ConfirmState != ConfirmStateConfirmed && monitor.ConfirmState != ConfirmStateTimeout {
|
||||
return nil
|
||||
}
|
||||
monitor.ConfirmState = ConfirmStateNone
|
||||
if err := tx.Save(&monitor).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_recovery", &monitor.ID, nil, nil)
|
||||
return enqueueDiagnosticDelivery(tx, &monitor, check.Monitor.Group.Account, diagnosticRecovery, now)
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkDiagnosticsTick expires confirmations and derives worker health from durable attempts.
|
||||
func NetworkDiagnosticsTick(now time.Time) error {
|
||||
return DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := reapExpiredConfirmationAttemptsTx(tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
var monitors []Monitor
|
||||
if err := tx.Clauses(SkipLockedClause).Preload("Group.Account.Plan").Where("confirm_state = ?", ConfirmStatePending).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range monitors {
|
||||
settings := settingsForAccount(monitors[i].Group.Account)
|
||||
if monitors[i].ConfirmAt == nil || monitors[i].ConfirmAt.After(now.Add(-settings.confirmTimeout)) {
|
||||
continue
|
||||
}
|
||||
if err := tx.Model(&monitors[i]).Update("confirm_state", ConfirmStateTimeout).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
auditDiagnostic(tx, "check.confirm_timeout", &monitors[i].ID, nil, nil)
|
||||
if err := enqueueDiagnosticDelivery(tx, &monitors[i], monitors[i].Group.Account, diagnosticHard, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var workers []WorkerNode
|
||||
if err := tx.Find(&workers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range workers {
|
||||
var total, failed int64
|
||||
// Operated workers serve accounts on different plans. Use the most
|
||||
// sensitive entitled setting among their recent attempts.
|
||||
settings := diagnosticSettings{healthWindow: 5 * time.Minute, healthRate: .5, healthMin: 10}
|
||||
q := tx.Model(&CheckAttempt{}).Where("worker_node_id = ? AND finished_at >= ?", workers[i].ID, now.Add(-settings.healthWindow))
|
||||
q.Count(&total)
|
||||
q.Where("result_state IN ?", []string{stateERR, stateFail}).Count(&failed)
|
||||
flagged := total >= settings.healthMin && float64(failed)/float64(total) >= settings.healthRate
|
||||
updates := map[string]interface{}{"last_total_count": total, "last_failure_count": failed}
|
||||
if flagged {
|
||||
updates["network_problems"] = true
|
||||
updates["network_problems_until"] = now.Add(10 * time.Minute)
|
||||
}
|
||||
if workers[i].NetworkProblems && workers[i].NetworkProblemsUntil != nil && workers[i].NetworkProblemsUntil.Before(now) && !flagged {
|
||||
updates["network_problems"] = false
|
||||
updates["network_problems_until"] = nil
|
||||
auditDiagnostic(tx, "worker.network_problem_unflag", nil, &workers[i].ID, nil)
|
||||
}
|
||||
if flagged && !workers[i].NetworkProblems {
|
||||
auditDiagnostic(tx, "worker.network_problem_flag", nil, &workers[i].ID, map[string]interface{}{"failures": failed, "total": total})
|
||||
}
|
||||
if err := tx.Model(&workers[i]).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
_ = influx.WriteOne("worker_health", map[string]string{"worker_id": workers[i].WorkerID}, map[string]interface{}{"failures": failed, "total": total, "failure_rate": float64(failed) / float64(maxInt64(total, 1))})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func reapExpiredConfirmationAttemptsTx(tx *gorm.DB, now time.Time) error {
|
||||
var attempts []CheckAttempt
|
||||
if err := tx.Clauses(SkipLockedClause).Where("kind = ? AND state = ? AND lease_expires_at <= ?", AttemptKindConfirm, AttemptStateLeased, now).Find(&attempts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range attempts {
|
||||
var check Check
|
||||
if err := tx.First(&check, attempts[i].CheckID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
oldWorkerID := int64(0)
|
||||
if attempts[i].WorkerNodeID != nil {
|
||||
oldWorkerID = *attempts[i].WorkerNodeID
|
||||
}
|
||||
sourceWorkerID := int64(0)
|
||||
if attempts[i].SourceWorkerNodeID != nil {
|
||||
sourceWorkerID = *attempts[i].SourceWorkerNodeID
|
||||
}
|
||||
worker, err := confirmationWorkerTx(tx, check.Kind, sourceWorkerID, oldWorkerID, now)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
// No replacement is available now. Remove the stale assignment so a
|
||||
// later capable independent worker can claim this queued attempt.
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": nil, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", nil).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&attempts[i]).Updates(map[string]interface{}{"worker_node_id": worker.ID, "state": AttemptStateQueued, "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Monitor{}).Where("id = ? AND confirm_state = ?", attempts[i].MonitorID, ConfirmStatePending).Update("confirmed_by_worker_id", worker.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func maxInt64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
193
app/models/notification.go
Обычный файл
193
app/models/notification.go
Обычный файл
@@ -0,0 +1,193 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/workdays"
|
||||
)
|
||||
|
||||
// Notification provides functionality.
|
||||
type Notification struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id)" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
ContactIDs []int64 `gorm:"-:all" json:"contact_ids"`
|
||||
Contacts []Contact `gorm:"many2many:notification_contacts;" json:"contacts,omitempty"`
|
||||
GroupIDs []int64 `gorm:"-:all" json:"group_ids"`
|
||||
Groups []Group `gorm:"many2many:notification_groups;" json:"-"`
|
||||
AlertDelay *int64 `json:"alert_delay,omitempty"`
|
||||
// RepeatAlert *int64 `json:"repeat_alert,omitempty"`
|
||||
BeforeExpiration *int64 `json:"before_expiration,omitempty"`
|
||||
|
||||
NotifyDown bool `gorm:"default:true" json:"notify_down"`
|
||||
NotifyRestore bool `gorm:"default:true" json:"notify_restore"`
|
||||
|
||||
NotifyWHOIS bool `gorm:"default:true" json:"notify_whois"`
|
||||
NotifySSL bool `gorm:"default:true" json:"notify_ssl"`
|
||||
|
||||
NotifyDays *int `json:"notify_days"`
|
||||
NotifyDayStart *int `json:"notify_day_start"`
|
||||
NotifyDayEnd *int `json:"notify_day_end"`
|
||||
NotifyHolidays bool `gorm:"default:true" json:"notify_holidays"`
|
||||
|
||||
Messages []Message `json:"-"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
const notificationDebug = false
|
||||
|
||||
// EnabledNow checks if the notification is enabled at the given time.
|
||||
func (n *Notification) EnabledNow(tn *time.Time) bool {
|
||||
weekday := int(tn.Weekday())
|
||||
// делаем из 0-воскр 1-пн 6-сб вариант 0-пн 6-воскр
|
||||
if weekday == 0 {
|
||||
weekday = 7
|
||||
}
|
||||
weekday--
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, "check enabled now at", tn, "for day", weekday)
|
||||
}
|
||||
|
||||
if !n.NotifyHolidays {
|
||||
c := workdays.GetCalendar()
|
||||
if !c.IsWorkday(*tn) {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, "is not enabled on holiday", tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
minusOneDay := false
|
||||
notifyFromDay := true
|
||||
|
||||
if n.NotifyDayStart != nil && n.NotifyDayEnd != nil {
|
||||
bod := BeginningOfDay(*tn)
|
||||
secondsToday := int(tn.Sub(bod) / time.Second)
|
||||
|
||||
ds := *n.NotifyDayStart
|
||||
de := *n.NotifyDayEnd
|
||||
// вариант 9 утра - 2 часа ночи
|
||||
if ds == de { //nolint:gocritic // complex condition chain
|
||||
notifyFromDay = true
|
||||
} else if ds > de {
|
||||
// с 0 до DayEnd
|
||||
if secondsToday < de { //nolint:gocritic // complex condition chain
|
||||
minusOneDay = true
|
||||
notifyFromDay = true
|
||||
} else if secondsToday < ds {
|
||||
// с DayEnd до DayStart
|
||||
notifyFromDay = false
|
||||
} else {
|
||||
notifyFromDay = true
|
||||
}
|
||||
} else {
|
||||
if secondsToday < ds { //nolint:gocritic // complex condition chain
|
||||
// с 0 до DayStart
|
||||
notifyFromDay = false
|
||||
} else if secondsToday > de {
|
||||
notifyFromDay = false
|
||||
} else {
|
||||
notifyFromDay = true
|
||||
}
|
||||
}
|
||||
// log.Println(secondsToday)
|
||||
}
|
||||
|
||||
if !notifyFromDay {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled as notifyFromDay", tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Если время 0-dayStart считаем что это прошлый день
|
||||
if minusOneDay {
|
||||
weekday--
|
||||
if weekday < 0 {
|
||||
weekday = 6
|
||||
}
|
||||
}
|
||||
|
||||
if n.NotifyDays != nil {
|
||||
if !HasBit(*n.NotifyDays, uint(weekday)) {
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is NOT enabled on weekday", tn.Weekday(), weekday, tn)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is enabled on weekday", tn.Weekday(), weekday, tn)
|
||||
}
|
||||
}
|
||||
|
||||
if notificationDebug {
|
||||
log.Println("notification", n.ID, *n.NotifyDays, "is enabled", tn)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// NotificationLoadIDs provides functionality.
|
||||
func NotificationLoadIDs(notifications *[]Notification) {
|
||||
for i, n := range *notifications { //nolint:gocritic // range copy is acceptable here
|
||||
cids := make([]int64, len(n.Contacts))
|
||||
for i, c := range n.Contacts { //nolint:gocritic // range copy is acceptable here
|
||||
cids[i] = c.ID
|
||||
}
|
||||
(*notifications)[i].ContactIDs = cids
|
||||
|
||||
gids := make([]int64, len(n.Groups))
|
||||
for j, g := range n.Groups { //nolint:gocritic // range copy is acceptable here
|
||||
gids[j] = g.ID
|
||||
}
|
||||
(*notifications)[i].GroupIDs = gids
|
||||
}
|
||||
}
|
||||
|
||||
// PersistRelations provides functionality.
|
||||
func (n *Notification) PersistRelations() error {
|
||||
cts := make([]Contact, len(n.ContactIDs))
|
||||
for i, c := range n.ContactIDs {
|
||||
ct := Contact{}
|
||||
ct.ID = c
|
||||
cts[i] = ct
|
||||
}
|
||||
err := DB().Model(&n).Association("Contacts").Replace(cts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
grp := make([]Group, len(n.GroupIDs))
|
||||
for i, g := range n.GroupIDs {
|
||||
gr := Group{}
|
||||
gr.ID = g
|
||||
grp[i] = gr
|
||||
}
|
||||
err = DB().Model(&n).Association("Groups").Replace(grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetContacts returns the contacts associated with this notification.
|
||||
// Errors are logged and an empty slice is returned instead of panicking so
|
||||
// that a single misconfigured notification cannot kill the scheduler goroutine
|
||||
// that processes expiry alerts (see internal/notifier.RunExp).
|
||||
func (n *Notification) GetContacts() []Contact {
|
||||
contacts := make([]Contact, 0)
|
||||
err := DB().Model(*n).Where("enabled = ?", true).Association("Contacts").Find(&contacts)
|
||||
if err != nil {
|
||||
log.Printf("notification %d: GetContacts failed: %v", n.ID, err)
|
||||
return contacts
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
167
app/models/notification_credential.go
Обычный файл
167
app/models/notification_credential.go
Обычный файл
@@ -0,0 +1,167 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// NotificationCredential kinds stored in the database.
|
||||
const (
|
||||
CredentialKindSMTP = "smtp"
|
||||
CredentialKindTelegram = "telegram"
|
||||
)
|
||||
|
||||
// NotificationCredential stores per-method delivery credentials (SMTP login,
|
||||
// Telegram bot token, etc.) encrypted at rest. Credentials are pushed to
|
||||
// workers through the init/config websocket refresh (see docs/worker-protocol.md
|
||||
// "Credentials Push").
|
||||
type NotificationCredential struct {
|
||||
concerns.Model
|
||||
// AccountID is nil for platform-managed credentials and set for credentials
|
||||
// owned by one customer account.
|
||||
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;index" json:"account_id,omitempty"`
|
||||
Kind string `gorm:"not null" json:"kind"`
|
||||
Name string `gorm:"not null" json:"name"`
|
||||
|
||||
// SMTP-specific
|
||||
Server *string `json:"server,omitempty"`
|
||||
Port *int `json:"port,omitempty"`
|
||||
Login *string `json:"login,omitempty"`
|
||||
FromName *string `json:"from_name,omitempty"`
|
||||
FromAddr *string `json:"from_address,omitempty"`
|
||||
InsecureSkipVerify bool `gorm:"default:false" json:"insecure_skip_verify"`
|
||||
|
||||
// Telegram-specific
|
||||
BotName *string `json:"bot_name,omitempty"`
|
||||
APIURL *string `json:"api_url,omitempty"`
|
||||
WebhookToken string `gorm:"size:128" json:"webhook_token,omitempty"`
|
||||
|
||||
// SecretEnc holds the encrypted (or "plain:"-prefixed fallback) secret
|
||||
// value — SMTP password or Telegram bot token. Decrypt via GetSecret.
|
||||
SecretEnc string `gorm:"column:secret;type:text" json:"-"`
|
||||
SecretMasked string `gorm:"-" json:"secret_masked,omitempty"`
|
||||
|
||||
Enabled *bool `gorm:"not null;default:true" json:"enabled"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;default:'{}'" json:"meta,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// SetSecret encrypts and stores the plaintext secret value.
|
||||
func (c *NotificationCredential) SetSecret(plaintext string) error {
|
||||
enc, err := encryptSecret(plaintext)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.SecretEnc = enc
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSecret decrypts and returns the secret value.
|
||||
func (c *NotificationCredential) GetSecret() (string, error) {
|
||||
return decryptSecret(c.SecretEnc)
|
||||
}
|
||||
|
||||
// FillSecretMasked populates SecretMasked with a display-safe version of the credential secret.
|
||||
func (c *NotificationCredential) FillSecretMasked() {
|
||||
secret, err := c.GetSecret()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.SecretMasked = maskSecret(secret)
|
||||
}
|
||||
|
||||
// EnsureWebhookToken creates the per-bot webhook URL token if it is missing.
|
||||
func (c *NotificationCredential) EnsureWebhookToken() {
|
||||
if c.WebhookToken != "" {
|
||||
return
|
||||
}
|
||||
c.WebhookToken = base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
|
||||
}
|
||||
|
||||
func maskSecret(secret string) string {
|
||||
if secret == "" {
|
||||
return ""
|
||||
}
|
||||
if len(secret) == 1 {
|
||||
return secret
|
||||
}
|
||||
return secret[:1] + "***" + secret[len(secret)-1:]
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (NotificationCredential) TableName() string {
|
||||
return "notification_credentials"
|
||||
}
|
||||
|
||||
// AllNotificationCredentials returns all credentials ordered by kind and name.
|
||||
func AllNotificationCredentials() ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id IS NULL").Order("kind ASC, name ASC").Find(&creds).Error
|
||||
for i := range creds {
|
||||
creds[i].FillSecretMasked()
|
||||
}
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// AccountNotificationCredentials returns credentials owned by accountID.
|
||||
func AccountNotificationCredentials(accountID int64) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id = ?", accountID).Order("kind ASC, name ASC").Find(&creds).Error
|
||||
for i := range creds {
|
||||
creds[i].FillSecretMasked()
|
||||
}
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// EnabledCredentialsByKind returns enabled credentials of the given kind.
|
||||
func EnabledCredentialsByKind(kind string) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id IS NULL AND kind = ? AND enabled = ?", kind, true).Order("name ASC").Find(&creds).Error
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// EnabledCredentialsByAccountAndKind returns only enabled credentials owned by
|
||||
// accountID. It never falls back to platform credentials.
|
||||
func EnabledCredentialsByAccountAndKind(accountID int64, kind string) ([]NotificationCredential, error) {
|
||||
var creds []NotificationCredential
|
||||
err := DB().Where("account_id = ? AND kind = ? AND enabled = ?", accountID, kind, true).Order("name ASC").Find(&creds).Error
|
||||
return creds, err
|
||||
}
|
||||
|
||||
// FindCredential returns a credential by id.
|
||||
func FindCredential(id int64) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().First(&c, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.FillSecretMasked()
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// FindCredentialByName returns a credential by kind and name.
|
||||
func FindCredentialByName(kind, name string) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().Where("kind = ? AND name = ?", kind, name).First(&c).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// FindTelegramCredentialByWebhookToken returns an enabled Telegram credential by webhook token.
|
||||
func FindTelegramCredentialByWebhookToken(token string) (*NotificationCredential, error) {
|
||||
var c NotificationCredential
|
||||
if err := DB().Where("kind = ? AND webhook_token = ? AND enabled = ?", CredentialKindTelegram, token, true).First(&c).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// DeleteCredential removes a credential by id.
|
||||
func DeleteCredential(id int64) error {
|
||||
return DB().Delete(&NotificationCredential{}, id).Error
|
||||
}
|
||||
186
app/models/notification_credential_test.go
Обычный файл
186
app/models/notification_credential_test.go
Обычный файл
@@ -0,0 +1,186 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// TestNotificationCredential_EncryptDecryptRoundTrip verifies that SetSecret
|
||||
// followed by GetSecret returns the original plaintext regardless of which
|
||||
// encryption mode (AES-GCM or plain: fallback) is active.
|
||||
func TestNotificationCredential_EncryptDecryptRoundTrip(t *testing.T) {
|
||||
c := &models.NotificationCredential{}
|
||||
plaintext := "super-secret-smtp-password"
|
||||
|
||||
require.NoError(t, c.SetSecret(plaintext))
|
||||
|
||||
got, err := c.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, plaintext, got)
|
||||
}
|
||||
|
||||
// TestNotificationCredential_EncryptDecryptWithKey verifies that when
|
||||
// RSMON_CRED_KEY is configured the stored value is prefixed with "enc:" and
|
||||
// can still be decrypted back to the original plaintext.
|
||||
func TestNotificationCredential_EncryptDecryptWithKey(t *testing.T) {
|
||||
t.Setenv("RSMON_CRED_KEY", "test-key-rotate-me-1234567890")
|
||||
|
||||
c := &models.NotificationCredential{}
|
||||
plaintext := "bot-token-9876543210:ABCDEFG"
|
||||
|
||||
require.NoError(t, c.SetSecret(plaintext))
|
||||
assert.True(t, strings.HasPrefix(c.SecretEnc, "enc:"),
|
||||
"expected SecretEnc to start with 'enc:' prefix, got %q", c.SecretEnc)
|
||||
assert.NotEqual(t, plaintext, c.SecretEnc, "encrypted value must not equal plaintext")
|
||||
|
||||
got, err := c.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, plaintext, got)
|
||||
}
|
||||
|
||||
// TestNotificationCredential_CRUD exercises create / find-by-id / find-by-name
|
||||
// / delete against the test database.
|
||||
func TestNotificationCredential_CRUD(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
server := "smtp.example.com"
|
||||
port := 587
|
||||
login := "alerts@example.com"
|
||||
enabled := true
|
||||
c := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindSMTP,
|
||||
Name: "primary",
|
||||
Server: &server,
|
||||
Port: &port,
|
||||
Login: &login,
|
||||
Enabled: &enabled,
|
||||
}
|
||||
require.NoError(t, c.SetSecret("smtp-password-xyz"))
|
||||
require.NoError(t, models.DB().Create(c).Error)
|
||||
require.NotZero(t, c.ID, "expected ID to be assigned after Create")
|
||||
|
||||
found, err := models.FindCredential(c.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "primary", found.Name)
|
||||
assert.Equal(t, models.CredentialKindSMTP, found.Kind)
|
||||
require.NotNil(t, found.Server)
|
||||
assert.Equal(t, "smtp.example.com", *found.Server)
|
||||
|
||||
gotSecret, err := found.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "smtp-password-xyz", gotSecret)
|
||||
assert.Equal(t, "s***z", found.SecretMasked)
|
||||
|
||||
byName, err := models.FindCredentialByName(models.CredentialKindSMTP, "primary")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.ID, byName.ID)
|
||||
|
||||
require.NoError(t, models.DeleteCredential(c.ID))
|
||||
|
||||
_, err = models.FindCredential(c.ID)
|
||||
assert.Error(t, err, "FindCredential should fail after delete")
|
||||
}
|
||||
|
||||
// TestNotificationCredential_UniqueKindName verifies that two credentials with
|
||||
// the same (kind, name) pair violate the unique index.
|
||||
func TestNotificationCredential_UniqueKindName(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
enTrue := true
|
||||
first := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: "main-bot",
|
||||
Enabled: &enTrue,
|
||||
}
|
||||
require.NoError(t, first.SetSecret("token-a"))
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
second := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: "main-bot",
|
||||
Enabled: &enTrue,
|
||||
}
|
||||
require.NoError(t, second.SetSecret("token-b"))
|
||||
|
||||
err := models.DB().Create(second).Error
|
||||
require.Error(t, err, "expected unique constraint violation for duplicate (kind, name)")
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"expected error mentioning unique/duplicate, got: %v", err)
|
||||
}
|
||||
|
||||
// TestEnabledCredentialsByKind verifies the kind+enabled filter.
|
||||
func TestEnabledCredentialsByKind(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
enTrue := true
|
||||
enFalse := false
|
||||
enabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "enabled-1", Enabled: &enTrue}
|
||||
disabled := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "disabled-1", Enabled: &enFalse}
|
||||
require.NoError(t, models.DB().Create(enabled).Error)
|
||||
require.NoError(t, models.DB().Create(disabled).Error)
|
||||
|
||||
got, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
|
||||
var names []string
|
||||
for _, c := range got {
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
assert.Contains(t, names, "enabled-1")
|
||||
assert.NotContains(t, names, "disabled-1")
|
||||
}
|
||||
|
||||
func TestNotificationCredentialsAreScopedToSystemOrAccount(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
accountA := &models.Account{Name: "credential-a"}
|
||||
accountB := &models.Account{Name: "credential-b"}
|
||||
require.NoError(t, models.DB().Create(accountA).Error)
|
||||
require.NoError(t, models.DB().Create(accountB).Error)
|
||||
enabled := true
|
||||
system := &models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "system", Enabled: &enabled}
|
||||
ownedA := &models.NotificationCredential{AccountID: &accountA.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
|
||||
ownedB := &models.NotificationCredential{AccountID: &accountB.ID, Kind: models.CredentialKindSMTP, Name: "owned", Enabled: &enabled}
|
||||
require.NoError(t, models.DB().Create(system).Error)
|
||||
require.NoError(t, models.DB().Create(ownedA).Error)
|
||||
require.NoError(t, models.DB().Create(ownedB).Error)
|
||||
|
||||
systemCreds, err := models.EnabledCredentialsByKind(models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, systemCreds, 1)
|
||||
assert.Equal(t, system.ID, systemCreds[0].ID)
|
||||
|
||||
accountCreds, err := models.EnabledCredentialsByAccountAndKind(accountA.ID, models.CredentialKindSMTP)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accountCreds, 1)
|
||||
assert.Equal(t, ownedA.ID, accountCreds[0].ID)
|
||||
}
|
||||
|
||||
func TestNotificationCredentialSecretMasked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
cred := models.NotificationCredential{Kind: models.CredentialKindSMTP, Name: "smtp"}
|
||||
require.NoError(t, cred.SetSecret("password"))
|
||||
require.NoError(t, models.DB().Create(&cred).Error)
|
||||
|
||||
loaded, err := models.FindCredential(cred.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "p***d", loaded.SecretMasked)
|
||||
}
|
||||
146
app/models/notification_get_contacts_test.go
Обычный файл
146
app/models/notification_get_contacts_test.go
Обычный файл
@@ -0,0 +1,146 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/spec/factories"
|
||||
)
|
||||
|
||||
// TestNotificationGetContactsIncludesSystemContact exercises the regression
|
||||
// reported when the production dump was restored into dev: the
|
||||
// contacts.is_system column was missing and the GORM preload panicked on
|
||||
// GetContacts. The fix has two layers:
|
||||
//
|
||||
// 1. AutoMigrate must add is_system (and deletion_requested_at) before the
|
||||
// notifier scheduler starts running.
|
||||
// 2. GetContacts itself must not panic on a query error so a single bad row
|
||||
// cannot tear down the scheduler goroutine.
|
||||
//
|
||||
// This test verifies both layers by:
|
||||
// - asserting that the schema post-Migrate includes is_system, so the
|
||||
// production-like scenario no longer panics; and
|
||||
// - building a notification that contains a contact flagged is_system=true
|
||||
// and checking GetContacts returns it.
|
||||
func TestNotificationGetContactsIncludesSystemContact(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
// Column must exist after Migrate; otherwise GetContacts would fail
|
||||
// with the same panic we saw in production.
|
||||
assertColumnExists(t, "contacts", "is_system")
|
||||
|
||||
account := &models.Account{Name: "acct-get-contacts"}
|
||||
require.NoError(t, models.DB().Create(account).Error)
|
||||
accountID := account.ID
|
||||
|
||||
trueVal := true
|
||||
contact := &models.Contact{
|
||||
AccountID: &accountID,
|
||||
Name: "system-admin",
|
||||
Kind: "email",
|
||||
Value: "ops@example.com",
|
||||
IsSystem: &trueVal,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(contact).Error)
|
||||
|
||||
notification := factories.PersistedNotification(
|
||||
account, []int64{contact.ID}, nil, 300, false,
|
||||
)
|
||||
|
||||
// Reload so the model has its persisted ID; the factory's PersistRelations
|
||||
// may have left ContactIDs empty on the returned value depending on the
|
||||
// GORM version, so fetch fresh.
|
||||
require.NoError(t, models.DB().
|
||||
Preload("Contacts").
|
||||
First(¬ification, notification.ID).Error)
|
||||
|
||||
got := notification.GetContacts()
|
||||
|
||||
ids := make([]int64, 0, len(got))
|
||||
for _, c := range got {
|
||||
ids = append(ids, c.ID)
|
||||
}
|
||||
assert.Contains(t, ids, contact.ID, "GetContacts must include the is_system contact")
|
||||
}
|
||||
|
||||
// TestUserDeletionRequestedAtColumnAndRoundTrip verifies the second missing
|
||||
// column reported by the panic: users.deletion_requested_at. It asserts that
|
||||
// AutoMigrate creates the column and that the field round-trips through the
|
||||
// DB correctly. ProcessPendingDeletions (the consumer of this column) relies
|
||||
// on it being present and queryable.
|
||||
func TestUserDeletionRequestedAtColumnAndRoundTrip(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
assertColumnExists(t, "users", "deletion_requested_at")
|
||||
|
||||
user := factories.PersistedUser("deletion-roundtrip@test.ru", "secret")
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
user.DeletionRequestedAt = &now
|
||||
|
||||
require.NoError(t, models.DB().Save(&user).Error)
|
||||
|
||||
reloaded := models.User{}
|
||||
require.NoError(t, models.DB().First(&reloaded, user.ID).Error)
|
||||
|
||||
require.NotNil(t, reloaded.DeletionRequestedAt, "deletion_requested_at must round-trip via Save/First")
|
||||
assert.True(t, reloaded.DeletionRequestedAt.Equal(now),
|
||||
"deletion_requested_at must preserve the timestamp value (got %v, want %v)",
|
||||
reloaded.DeletionRequestedAt, now)
|
||||
|
||||
// ProcessPendingDeletions should not panic on the populated schema and
|
||||
// must respect the cutoff: a recently-set deletion_requested_at is
|
||||
// still inside the 7-day grace period, so no hard-delete must occur.
|
||||
deleted, err := models.ProcessPendingDeletions()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, deleted, "users within the 7-day grace period must not be hard-deleted")
|
||||
}
|
||||
|
||||
// TestProcessPendingDeletionsQueriesMissingColumnGracefully asserts that even
|
||||
// if the deletion_requested_at column were missing, ProcessPendingDeletions
|
||||
// would not panic (the panic-on-error pattern was historically present in
|
||||
// other notifier helpers). We force the failure by renaming the column back,
|
||||
// calling ProcessPendingDeletions, then restoring the column.
|
||||
func TestProcessPendingDeletionsQueriesMissingColumnGracefully(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
assertColumnExists(t, "users", "deletion_requested_at")
|
||||
|
||||
// Simulate the production-missing-column scenario in a contained way:
|
||||
// rename the column so the SELECT against deletion_requested_at fails.
|
||||
require.NoError(t, models.DB().
|
||||
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at TO deletion_requested_at_hidden").Error)
|
||||
t.Cleanup(func() {
|
||||
// Restore so subsequent tests in this package keep working.
|
||||
_ = models.DB().
|
||||
Exec("ALTER TABLE users RENAME COLUMN deletion_requested_at_hidden TO deletion_requested_at").Error
|
||||
})
|
||||
|
||||
// Must not panic; must return an error.
|
||||
assert.NotPanics(t, func() {
|
||||
_, err := models.ProcessPendingDeletions()
|
||||
assert.Error(t, err, "missing column must surface as an error, not a panic")
|
||||
})
|
||||
}
|
||||
|
||||
// assertColumnExists checks that the given table has the given column by
|
||||
// querying information_schema. It is the canary for the AutoMigrate step
|
||||
// ordering bug: if the column is missing, every test that touches it will
|
||||
// panic with SQLSTATE 42703.
|
||||
func assertColumnExists(t *testing.T, table, column string) {
|
||||
t.Helper()
|
||||
var n int
|
||||
err := models.DB().Raw(
|
||||
`SELECT COUNT(*) FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = ? AND column_name = ?`,
|
||||
table, column,
|
||||
).Scan(&n).Error
|
||||
require.NoError(t, err, "information_schema query must succeed")
|
||||
assert.Equal(t, 1, n, "table %q must have column %q after Migrate()", table, column)
|
||||
}
|
||||
128
app/models/notification_test.go
Обычный файл
128
app/models/notification_test.go
Обычный файл
@@ -0,0 +1,128 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
mondayFriday int
|
||||
allDays int
|
||||
)
|
||||
|
||||
var time1, time3, time8, time9, time10, time23 time.Time
|
||||
|
||||
func init() {
|
||||
// database.ConfigFile = "." + database.ConfigFile
|
||||
// database.Init()
|
||||
// Migrate()
|
||||
// Drop()
|
||||
|
||||
mondayFriday = 0
|
||||
mondayFriday = SetBit(mondayFriday, 0)
|
||||
mondayFriday = SetBit(mondayFriday, 1)
|
||||
mondayFriday = SetBit(mondayFriday, 2)
|
||||
mondayFriday = SetBit(mondayFriday, 3)
|
||||
mondayFriday = SetBit(mondayFriday, 4)
|
||||
|
||||
allDays = 0
|
||||
allDays = SetBit(allDays, 0)
|
||||
allDays = SetBit(allDays, 1)
|
||||
allDays = SetBit(allDays, 2)
|
||||
allDays = SetBit(allDays, 3)
|
||||
allDays = SetBit(allDays, 4)
|
||||
allDays = SetBit(allDays, 5)
|
||||
allDays = SetBit(allDays, 6)
|
||||
|
||||
time1 = time.Date(2019, time.May, 26, 1, 0, 0, 0, time.Local)
|
||||
time3 = time.Date(2019, time.May, 26, 3, 0, 0, 0, time.Local)
|
||||
time8 = time.Date(2019, time.May, 26, 8, 0, 0, 0, time.Local)
|
||||
time9 = time.Date(2019, time.May, 26, 9, 0, 0, 0, time.Local)
|
||||
time10 = time.Date(2019, time.May, 26, 10, 0, 0, 0, time.Local)
|
||||
time23 = time.Date(2019, time.May, 26, 23, 0, 0, 0, time.Local)
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowHolidays(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
|
||||
n.ID = 1
|
||||
n.NotifyHolidays = true
|
||||
holiday := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.Local)
|
||||
assert.Equal(t, true, n.EnabledNow(&holiday), "notification by default should be enabled on holidays")
|
||||
|
||||
n.ID = 2
|
||||
n.NotifyHolidays = false
|
||||
assert.Equal(t, false, n.EnabledNow(&holiday), "notification with NotifyHolidays=fasle should not be enabled on holiday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowMondayFriday(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyHolidays = true
|
||||
|
||||
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
|
||||
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
|
||||
|
||||
n.ID = 3
|
||||
n.NotifyDays = &mondayFriday
|
||||
assert.Equal(t, false, n.EnabledNow(&weekend), "notification mon-fri should not be enabled on sunday")
|
||||
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-fri should be enabled on monday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowAllDays(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyHolidays = true
|
||||
|
||||
weekend := time.Date(2019, time.May, 26, 0, 0, 0, 0, time.Local)
|
||||
weekday := time.Date(2019, time.May, 27, 0, 0, 0, 0, time.Local)
|
||||
|
||||
n.ID = 4
|
||||
n.NotifyDays = &allDays
|
||||
assert.Equal(t, true, n.EnabledNow(&weekend), "notification mon-sat should be enabled on sunday")
|
||||
assert.Equal(t, true, n.EnabledNow(&weekday), "notification mon-sat should be enabled on monday")
|
||||
}
|
||||
|
||||
func TestNotificationEnabledNowNormal(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
n.NotifyHolidays = true
|
||||
|
||||
// 9am - 18pm
|
||||
n.ID = 5
|
||||
start := 9 * 3600
|
||||
end := 18 * 3600
|
||||
n.NotifyDayStart = &start
|
||||
n.NotifyDayEnd = &end
|
||||
|
||||
assert.Equal(t, false, n.EnabledNow(&time1), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time23), "notification should not be enabled outside day")
|
||||
}
|
||||
|
||||
// TestNotificationEnabledNowRollover tests time range that crosses midnight
|
||||
//
|
||||
//nolint:dupl // Test structure similar to TestNotificationEnabledNowNormal but tests different behavior (rollover vs normal time range)
|
||||
func TestNotificationEnabledNowRollover(t *testing.T) {
|
||||
n := &Notification{}
|
||||
n.NotifyDays = &allDays
|
||||
n.NotifyHolidays = true
|
||||
|
||||
// 9am - 2am
|
||||
n.ID = 6
|
||||
start := 9 * 3600
|
||||
end := 2 * 3600
|
||||
n.NotifyDayStart = &start
|
||||
n.NotifyDayEnd = &end
|
||||
|
||||
assert.Equal(t, true, n.EnabledNow(&time1), "notification should be enabled inside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time3), "notification should not be enabled outside day")
|
||||
assert.Equal(t, false, n.EnabledNow(&time8), "notification should not be enabled outside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time9), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time10), "notification should be enabled inside day")
|
||||
assert.Equal(t, true, n.EnabledNow(&time23), "notification should be enabled inside day")
|
||||
}
|
||||
18
app/models/payment.go
Обычный файл
18
app/models/payment.go
Обычный файл
@@ -0,0 +1,18 @@
|
||||
package models
|
||||
|
||||
import "rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
|
||||
// Payment provides functionality.
|
||||
type Payment struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `json:"account_id"`
|
||||
Account User `json:"-"`
|
||||
|
||||
Kind string
|
||||
ExtID string
|
||||
Amount int
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
108
app/models/plan.go
Обычный файл
108
app/models/plan.go
Обычный файл
@@ -0,0 +1,108 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
var canonicalPlanCodes = []string{"free", "solo", "team", "enterprise"}
|
||||
|
||||
// CanonicalPlanCodes returns the supported public catalog codes.
|
||||
func CanonicalPlanCodes() []string {
|
||||
return append([]string(nil), canonicalPlanCodes...)
|
||||
}
|
||||
|
||||
// Plan is a versioned-by-code catalog entry. A zero cap means unlimited.
|
||||
type Plan struct {
|
||||
concerns.Model
|
||||
|
||||
Code string `gorm:"size:32;not null" json:"code"`
|
||||
NameRU string `gorm:"size:64;not null" json:"name_ru"`
|
||||
NameEN string `gorm:"size:64;not null" json:"name_en"`
|
||||
PriceMonthlyMinor int64 `gorm:"not null;default:0" json:"price_monthly_minor"`
|
||||
PriceAnnualMinor int64 `gorm:"not null;default:0" json:"price_annual_minor"`
|
||||
Currency string `gorm:"size:3;not null;default:'RUB'" json:"currency"`
|
||||
MonitorCap int64 `gorm:"not null;default:0" json:"monitor_cap"`
|
||||
IntervalMinSeconds int `gorm:"not null;default:30" json:"interval_min_seconds"`
|
||||
StatusPagesCap int64 `gorm:"not null;default:0" json:"status_pages_cap"`
|
||||
MaintenanceCap int64 `gorm:"not null;default:0" json:"maintenance_cap"`
|
||||
LoginSeatsIncluded int64 `gorm:"not null;default:0" json:"login_seats_included"`
|
||||
NotifySeatsIncluded int64 `gorm:"not null;default:0" json:"notify_seats_included"`
|
||||
UnlimitedSeats bool `gorm:"not null;default:false" json:"unlimited_seats"`
|
||||
Integrations pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"integrations"`
|
||||
CheckKinds pq.StringArray `gorm:"type:text[];not null;default:'{}'" json:"check_kinds"`
|
||||
DataRetentionMonths int `gorm:"not null;default:3" json:"data_retention_months"`
|
||||
DistributedWorkers bool `gorm:"not null;default:false" json:"distributed_workers"`
|
||||
Confirmations bool `gorm:"not null;default:false" json:"confirmations"`
|
||||
AllowHardAlerts bool `gorm:"not null;default:false" json:"allow_hard_alerts"`
|
||||
ConfirmTimeoutSec int `gorm:"not null;default:90" json:"confirm_timeout_sec"`
|
||||
HealthWindowSec int `gorm:"not null;default:300" json:"health_window_sec"`
|
||||
HealthRateThreshold float64 `gorm:"not null;default:0.5" json:"health_rate_threshold"`
|
||||
HealthMinAttempts int `gorm:"not null;default:10" json:"health_min_attempts"`
|
||||
SOC2 bool `gorm:"not null;default:false" json:"soc2"`
|
||||
GDPRDPA bool `gorm:"not null;default:false" json:"gdpr_dpa"`
|
||||
IsDefault bool `gorm:"not null;default:false" json:"is_default"`
|
||||
Archived bool `gorm:"not null;default:false" json:"archived"`
|
||||
|
||||
// Deprecated source-compatibility fields. The legacy plans table is retained
|
||||
// as plans_legacy; these values are never written to the canonical catalog.
|
||||
Default bool `gorm:"-" json:"Default,omitempty"`
|
||||
Name string `gorm:"-" json:"Name,omitempty"`
|
||||
HTTPMonitors *int64 `gorm:"-" json:"HTTPMonitors,omitempty"`
|
||||
DNSMonitors *int64 `gorm:"-" json:"DNSMonitors,omitempty"`
|
||||
WHOISMonitors *int64 `gorm:"-" json:"WHOISMonitors,omitempty"`
|
||||
TotalMonitors *int64 `gorm:"-" json:"TotalMonitors,omitempty"`
|
||||
Price int `gorm:"-" json:"Price,omitempty"`
|
||||
TrialPeriod int `gorm:"-" json:"TrialPeriod,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
func (p *Plan) BeforeCreate(_ *gorm.DB) error {
|
||||
if p.Code == "" {
|
||||
p.Code = "legacy-" + uuid.NewString()[:24]
|
||||
}
|
||||
if p.NameRU == "" {
|
||||
p.NameRU = p.Name
|
||||
}
|
||||
if p.NameEN == "" {
|
||||
p.NameEN = p.NameRU
|
||||
}
|
||||
if p.Currency == "" {
|
||||
p.Currency = "RUB"
|
||||
}
|
||||
if p.IntervalMinSeconds == 0 {
|
||||
p.IntervalMinSeconds = 30
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plan) AfterFind(_ *gorm.DB) error {
|
||||
p.Name = p.NameRU
|
||||
p.Price = int(p.PriceMonthlyMinor / 100)
|
||||
p.Default = p.IsDefault
|
||||
p.TotalMonitors = &p.MonitorCap
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultPlan returns the canonical free plan.
|
||||
func DefaultPlan() Plan {
|
||||
pl := Plan{}
|
||||
if err := DB().Where("code = ? AND archived = FALSE", "free").First(&pl).Error; err != nil {
|
||||
log.Println("unable to find default plan")
|
||||
panic(err)
|
||||
}
|
||||
return pl
|
||||
}
|
||||
|
||||
// AllowsDistributed keeps legacy callers working while using the canonical
|
||||
// entitlement flag for catalog plans.
|
||||
func (p *Plan) AllowsDistributed() bool {
|
||||
return p != nil && (p.DistributedWorkers || p.Price > 0)
|
||||
}
|
||||
20
app/models/region.go
Обычный файл
20
app/models/region.go
Обычный файл
@@ -0,0 +1,20 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// DefaultRegionCode is the historical default region code seeded by
|
||||
// Migrate() (see app/models/migrate.go). Centralized here so admin
|
||||
// endpoints and check_jobs.go agree on the literal.
|
||||
const DefaultRegionCode = "local"
|
||||
|
||||
// Region represents a geographic region where distributed workers can run
|
||||
type Region struct {
|
||||
concerns.Model
|
||||
Code string `gorm:"uniqueIndex;size:20;not null" json:"code"` // e.g. "ru-msk", "us-east", "eu-west"
|
||||
Name string `gorm:"not null" json:"name"` // "Moscow, Russia"
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
Priority int `gorm:"not null;default:0" json:"priority"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
38
app/models/repo.go
Обычный файл
38
app/models/repo.go
Обычный файл
@@ -0,0 +1,38 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Repo is a source repository shared by one or more account-scoped sites.
|
||||
// Repositories themselves are global rstuff mirrors; SiteRepo supplies the
|
||||
// account boundary through its Site.
|
||||
type Repo struct {
|
||||
concerns.Model
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
GitlabID *int64 `json:"gitlab_id,omitempty"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Namespace *string `gorm:"size:120" json:"namespace,omitempty"`
|
||||
Path *string `gorm:"size:255" json:"path,omitempty"`
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the repository table name.
|
||||
func (Repo) TableName() string { return "repos" }
|
||||
|
||||
// SiteRepo is the explicit site/repository join. Role is intentionally data,
|
||||
// rather than an enum, to preserve the rstuff contract as it evolves.
|
||||
type SiteRepo struct {
|
||||
SiteID int64 `gorm:"type:bigint REFERENCES sites(id) ON DELETE CASCADE;primaryKey" json:"site_id"`
|
||||
RepoID int64 `gorm:"type:bigint REFERENCES repos(id) ON DELETE CASCADE;primaryKey" json:"repo_id"`
|
||||
Role string `gorm:"size:32;not null;default:'primary'" json:"role"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the repository assignment table name.
|
||||
func (SiteRepo) TableName() string { return "site_repos" }
|
||||
135
app/models/rkn_domain.go
Обычный файл
135
app/models/rkn_domain.go
Обычный файл
@@ -0,0 +1,135 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// RknDomain is one row of the locally cached ru-blocked domains list. The
|
||||
// `Domain` field carries a lowercased domain string and is the unique key.
|
||||
type RknDomain struct {
|
||||
concerns.Model
|
||||
Domain string `gorm:"size:255;uniqueIndex;not null" json:"domain"`
|
||||
}
|
||||
|
||||
// TableName pins the underlying table name so GORM migrations and raw SQL
|
||||
// (used by IsRknDomainBlocked and the rkn updater) agree.
|
||||
func (RknDomain) TableName() string { return "rkn_domains" }
|
||||
|
||||
// ReplaceRknDomains deletes every existing row and inserts the given domains
|
||||
// in batches. Caller-supplied domains are lower-cased and de-duplicated, and
|
||||
// blank entries are dropped. The whole operation runs in a single
|
||||
// transaction so readers see either the old set or the new set — there is no
|
||||
// in-between state where the table is half-flushed.
|
||||
//
|
||||
// Designed to be called once per parser-updater tick (default every 6h); the
|
||||
// uniqueIndex on Domain guarantees idempotent re-runs even when the caller
|
||||
// forgets to filter.
|
||||
func ReplaceRknDomains(domains []string) error {
|
||||
if domains == nil {
|
||||
domains = []string{}
|
||||
}
|
||||
deduped := make([]string, 0, len(domains))
|
||||
seen := make(map[string]struct{}, len(domains))
|
||||
for _, d := range domains {
|
||||
d = strings.ToLower(strings.TrimSpace(d))
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[d]; ok {
|
||||
continue
|
||||
}
|
||||
seen[d] = struct{}{}
|
||||
deduped = append(deduped, d)
|
||||
}
|
||||
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
// Step 1 — wipe the existing cache. Using a scoped Where("1 = 1") Delete
|
||||
// instead of TRUNCATE so the advisory lock taken by Migrate() does not
|
||||
// become a bottleneck and so any in-flight foreign-key checks against
|
||||
// rkn_domains remain satisfied (the table has no FKs today, but this
|
||||
// matches the convention used by Drop()).
|
||||
if err := tx.Where("1 = 1").Delete(&RknDomain{}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
// Step 2 — bulk-insert in chunks of 1000 rows. CreateInBatches runs N
|
||||
// multi-row INSERT statements, which for the typical ~30k ru-blocked
|
||||
// entries is ~3ms per batch — significantly cheaper than per-row
|
||||
// Create() in tight loops (the previous AddRknDomain implementation).
|
||||
const batchSize = 1000
|
||||
for start := 0; start < len(deduped); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(deduped) {
|
||||
end = len(deduped)
|
||||
}
|
||||
rows := make([]RknDomain, 0, end-start)
|
||||
for _, d := range deduped[start:end] {
|
||||
rows = append(rows, RknDomain{Domain: d})
|
||||
}
|
||||
if err := tx.CreateInBatches(rows, batchSize).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// IsRknDomainBlocked returns true iff `domain` (or its root label, or any
|
||||
// parent suffix already recorded as `*.parent.tld`) is present in the
|
||||
// rkn_domains table.
|
||||
//
|
||||
// Matching rules — see checks/crkn/rkn_init.go for the original logic we
|
||||
// consolidate here:
|
||||
// 1. Exact match against the stored domain string.
|
||||
// 2. Root-domain match (last two labels of the input) — covers the case
|
||||
// where the user passed a subdomain but the upstream only lists the
|
||||
// apex.
|
||||
// 3. Suffix match (`stored LIKE '%' || input || ?`) — covers the case
|
||||
// where the user passed the apex (or a higher-level label) but the
|
||||
// upstream lists a child subdomain.
|
||||
//
|
||||
// All three checks are combined into a single SQL statement via OR so the
|
||||
// table is scanned at most once and the SQL planner can pick a single
|
||||
// index access path.
|
||||
func IsRknDomainBlocked(domain string) (bool, error) {
|
||||
domain = strings.ToLower(strings.TrimSpace(domain))
|
||||
if domain == "" {
|
||||
return false, nil
|
||||
}
|
||||
rootDomain := rootDomainOf(domain)
|
||||
|
||||
// Build the suffix patterns once. Note: every ".X" entry in the table
|
||||
// (i.e. a domain that begins with a dot) matches any subdomain whose
|
||||
// suffix is domain.
|
||||
suffixPattern := "%." + domain
|
||||
|
||||
var count int64
|
||||
err := DB().Raw(
|
||||
"SELECT COUNT(*) FROM rkn_domains WHERE domain = ? OR domain = ? OR domain LIKE ?",
|
||||
domain, rootDomain, suffixPattern,
|
||||
).Scan(&count).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// rootDomainOf returns the last two labels of `domain` (e.g. "a.b.c" → "b.c").
|
||||
// Returns `domain` unchanged when it has fewer than three labels, because a
|
||||
// one- or two-label input IS already the apex/root domain.
|
||||
func rootDomainOf(domain string) string {
|
||||
i := strings.LastIndex(domain, ".")
|
||||
if i < 0 {
|
||||
return domain
|
||||
}
|
||||
j := strings.LastIndex(domain[:i], ".")
|
||||
if j < 0 {
|
||||
return domain
|
||||
}
|
||||
return domain[j+1:]
|
||||
}
|
||||
90
app/models/rkn_domain_test.go
Обычный файл
90
app/models/rkn_domain_test.go
Обычный файл
@@ -0,0 +1,90 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestReplaceRknDomains_BulkAndDedup(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []string{
|
||||
"Foo.example",
|
||||
"foo.example", // dup after lowercasing
|
||||
" bar.example ",
|
||||
"",
|
||||
"baz.example",
|
||||
"qux.example",
|
||||
"qux.example", // dup within input
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknDomains(input))
|
||||
|
||||
got := []string{}
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &got).Error)
|
||||
|
||||
want := []string{"bar.example", "baz.example", "foo.example", "qux.example"}
|
||||
sort.Strings(want)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
func TestReplaceRknDomains_ReplacesExisting(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknDomains([]string{"old1.example", "old2.example"}))
|
||||
|
||||
var n int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(2), n)
|
||||
|
||||
// Second call wipes and replaces — no overlap with old set.
|
||||
require.NoError(t, models.ReplaceRknDomains([]string{"new1.example", "new2.example", "new3.example"}))
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(3), n)
|
||||
|
||||
var domains []string
|
||||
assert.NoError(t, models.DB().Model(&models.RknDomain{}).Order("domain").Pluck("domain", &domains).Error)
|
||||
assert.Equal(t, []string{"new1.example", "new2.example", "new3.example"}, domains)
|
||||
}
|
||||
|
||||
func TestIsRknDomainBlocked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []string{
|
||||
"example.com",
|
||||
"foo.bar",
|
||||
"sub.test",
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknDomains(input))
|
||||
|
||||
cases := []struct {
|
||||
host string
|
||||
want bool
|
||||
}{
|
||||
{"example.com", true}, // exact
|
||||
{"EXAMPLE.com", true}, // case insensitive (caller lowercases)
|
||||
{"www.example.com", true}, // stored has apex; query apex → root-domain match
|
||||
{"deep.nested.example.com", true}, // suffix match via LIKE '%.X'
|
||||
{"foo.bar", true},
|
||||
{"sub.test", true},
|
||||
{"a.sub.test", true},
|
||||
{"unrelated.org", false},
|
||||
{"two.labels", false}, // 2-label input not in list → must not collapse to "labels"
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.host, func(t *testing.T) {
|
||||
got, err := models.IsRknDomainBlocked(c.host)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
193
app/models/rkn_ip.go
Обычный файл
193
app/models/rkn_ip.go
Обычный файл
@@ -0,0 +1,193 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/netaddr"
|
||||
)
|
||||
|
||||
// RknIP stores a single CIDR from the ru-blocked IP list. The cidr column
|
||||
// uses Postgres's native cidr type so the (>>) containment operator works
|
||||
// directly inside IsRknIPBlocked queries — GiST index recommended for any
|
||||
// table large enough to amortize the planner cost; see EnsureRknIndexes.
|
||||
type RknIP struct {
|
||||
concerns.Model
|
||||
Network *netaddr.Cidr `json:"network" gorm:"type:cidr;"`
|
||||
}
|
||||
|
||||
// TableName pins the underlying table name.
|
||||
func (RknIP) TableName() string { return "rkn_ips" }
|
||||
|
||||
// PanicOnErr panics with a spew-formatted error dump if err is non-nil.
|
||||
// Kept for callers that still use the old add-by-row InsertInBatches path
|
||||
// (none after this commit, but kept in case external scripts reference it).
|
||||
func PanicOnErr(err error) {
|
||||
if err != nil {
|
||||
spew.Dump(err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// FindRknIP returns true iff the IP inside `ip` (treated as a /32 or /128
|
||||
// host) falls inside any CIDR already stored in rkn_ips. Kept for callers
|
||||
// that already construct an internal/netaddr.Inet.
|
||||
func FindRknIP(ip netaddr.Inet) (bool, error) {
|
||||
ipstr := ip.Inet.String()
|
||||
if !strings.Contains(ipstr, "/") {
|
||||
if ip.Inet.To4() != nil {
|
||||
ipstr += "/32"
|
||||
} else {
|
||||
ipstr += "/128"
|
||||
}
|
||||
}
|
||||
_, network, err := net.ParseCIDR(ipstr)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
|
||||
var matched []RknIP
|
||||
if err := DB().Raw("SELECT id FROM rkn_ips WHERE network >>= ?", &cidr).Scan(&matched).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return len(matched) > 0, nil
|
||||
}
|
||||
|
||||
// AddRknIP is the legacy per-row insert path. Deprecated: callers should
|
||||
// invoke ReplaceRknIPs from the rkn updater. Kept around so existing cron
|
||||
// scripts that import the symbol keep compiling.
|
||||
func AddRknIP(data []string, count int) {
|
||||
tx := DB().Begin()
|
||||
for k, dataIP := range data {
|
||||
if count > 0 && k > count-1 {
|
||||
break
|
||||
}
|
||||
if !strings.Contains(dataIP, "/") {
|
||||
dataIP += "/32"
|
||||
}
|
||||
_, network, err := net.ParseCIDR(dataIP)
|
||||
PanicOnErr(err)
|
||||
|
||||
cidr := netaddr.Cidr{Cidr: *network, Valid: true}
|
||||
var rknIPs []RknIP
|
||||
err = tx.Raw("SELECT id FROM rkn_ips where network >>= ?", &cidr).Scan(&rknIPs).Error
|
||||
PanicOnErr(err)
|
||||
if len(rknIPs) == 0 {
|
||||
rknIPs = []RknIP{}
|
||||
err = tx.Raw("SELECT id FROM rkn_ips where network << ?", &cidr).Scan(&rknIPs).Error
|
||||
PanicOnErr(err)
|
||||
if len(rknIPs) > 0 {
|
||||
for _, r := range rknIPs {
|
||||
PanicOnErr(tx.Delete(&r).Error)
|
||||
}
|
||||
}
|
||||
PanicOnErr(tx.Create(&RknIP{Network: &cidr}).Error)
|
||||
}
|
||||
log.Println("created:", dataIP)
|
||||
}
|
||||
PanicOnErr(tx.Commit().Error)
|
||||
}
|
||||
|
||||
// ReplaceRknIPs deletes every existing row and bulk-inserts the given CIDRs
|
||||
// in batches of 1000. Whole operation runs in a single transaction so a
|
||||
// partially-applied update can never leave the table in a hybrid state.
|
||||
//
|
||||
// CIDR.parseCIDR-loop uses net.ParseCIDR to canonicalise the address —
|
||||
// upstream .dat files occasionally contain range/mask pairs that aren't
|
||||
// already reduced (e.g. 192.168.0.0/16 spelled as 192.168.5.0/16); the
|
||||
// Postgres cidr type normalises on insert.
|
||||
func ReplaceRknIPs(nets []*net.IPNet) error {
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
|
||||
rows := make([]RknIP, 0, len(nets))
|
||||
seen := make(map[string]struct{}, len(nets))
|
||||
for _, n := range nets {
|
||||
if n == nil || n.IP == nil {
|
||||
continue
|
||||
}
|
||||
// Canonicalise by routing through net.ParseCIDR. This drops the
|
||||
// host bits (a common bug in upstream dumps where a /24 range
|
||||
// is written with the .5 host bit set) and stamps the right
|
||||
// address family flag for Postgres.
|
||||
canonical := n.String()
|
||||
if _, parsed, err := net.ParseCIDR(canonical); err == nil {
|
||||
n = parsed
|
||||
canonical = parsed.String()
|
||||
}
|
||||
if _, ok := seen[canonical]; ok {
|
||||
continue
|
||||
}
|
||||
seen[canonical] = struct{}{}
|
||||
cidr := netaddr.Cidr{Cidr: *n, Valid: true}
|
||||
rows = append(rows, RknIP{Network: &cidr})
|
||||
}
|
||||
|
||||
// Wipe + re-insert in one transaction.
|
||||
if err := tx.Where("1 = 1").Delete(&RknIP{}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
const batchSize = 1000
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
end := start + batchSize
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
if err := tx.CreateInBatches(rows[start:end], batchSize).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// IsRknIPBlocked returns true iff `ip` (any textual form ParseCIDR accepts)
|
||||
// falls inside any CIDR stored in the rkn_ips table. The query uses the
|
||||
// cidr >>= inet containment operator — see EnsureRknIndexes for the GiST
|
||||
// index that makes this fast at scale.
|
||||
func IsRknIPBlocked(ip string) (bool, error) {
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip == "" {
|
||||
return false, nil
|
||||
}
|
||||
var count int64
|
||||
if err := DB().Raw("SELECT COUNT(*) FROM rkn_ips WHERE network >>= ?::inet", ip).Scan(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// EnsureRknIndexes creates the indexes that AutoMigrate cannot express —
|
||||
// the GiST index on rkn_ips.network uses cidr >>= cidr containment (the
|
||||
// expression index `network` already covers equality and prefix ranges,
|
||||
// but the planner benefits from a GiST for `network >>= <other cidr>`
|
||||
// queries against ~30k rows). The unique index on rkn_domains.domain is
|
||||
// also declared in the GORM tag, this function only adds what GORM can't
|
||||
// (GiST) and is idempotent so it's safe to call repeatedly during boot
|
||||
// or migration.
|
||||
func EnsureRknIndexes() error {
|
||||
// GiST on cidr requires the btree_gist contrib — its `cidr_ops`
|
||||
// opclass exposes cidr to GiST. CREATE EXTENSION IF NOT EXISTS is
|
||||
// idempotent.
|
||||
if err := DB().Exec("CREATE EXTENSION IF NOT EXISTS btree_gist").Error; err != nil {
|
||||
return fmt.Errorf("ensure btree_gist: %w", err)
|
||||
}
|
||||
// rkn_ips GiST index on the cidr column supports the >>= containment
|
||||
// operator that IsRknIPBlocked uses. Without it a 30k-row table makes
|
||||
// every IP check a sequential scan; with it each check is an index
|
||||
// probe.
|
||||
if err := DB().Exec(
|
||||
"CREATE INDEX IF NOT EXISTS idx_rkn_ips_network ON rkn_ips USING gist (network)",
|
||||
).Error; err != nil {
|
||||
return fmt.Errorf("ensure idx_rkn_ips_network: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
114
app/models/rkn_ip_test.go
Обычный файл
114
app/models/rkn_ip_test.go
Обычный файл
@@ -0,0 +1,114 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func mustIPNet(t *testing.T, cidr string) *net.IPNet {
|
||||
t.Helper()
|
||||
_, n, err := net.ParseCIDR(cidr)
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
}
|
||||
|
||||
func TestReplaceRknIPs_BulkAndDedup(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
input := []*net.IPNet{
|
||||
mustIPNet(t, "10.0.0.0/8"),
|
||||
mustIPNet(t, "10.5.0.0/8"), // same canonical /8 — dedup
|
||||
mustIPNet(t, "192.168.1.0/24"),
|
||||
mustIPNet(t, "2001:db8::/32"),
|
||||
}
|
||||
require.NoError(t, models.ReplaceRknIPs(input))
|
||||
|
||||
// The uniqueIndex on rkn_ips.network ensures the dedup actually drops
|
||||
// duplicates; ReplaceRknIPs does an in-memory dedup, but the DB-level
|
||||
// constraint is the guarantee.
|
||||
var count int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&count).Error)
|
||||
assert.Equal(t, int64(3), count, "expected dedup to 3 unique CIDRs")
|
||||
}
|
||||
|
||||
func TestReplaceRknIPs_ReplacesExisting(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "8.8.8.0/24"),
|
||||
}))
|
||||
|
||||
var n int64
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(1), n)
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "1.0.0.0/8"),
|
||||
mustIPNet(t, "2.0.0.0/8"),
|
||||
}))
|
||||
|
||||
assert.NoError(t, models.DB().Model(&models.RknIP{}).Count(&n).Error)
|
||||
assert.Equal(t, int64(2), n)
|
||||
}
|
||||
|
||||
func TestIsRknIPBlocked(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.ReplaceRknIPs([]*net.IPNet{
|
||||
mustIPNet(t, "10.0.0.0/8"),
|
||||
mustIPNet(t, "192.168.1.0/24"),
|
||||
mustIPNet(t, "2001:db8::/32"),
|
||||
}))
|
||||
|
||||
cases := []struct {
|
||||
ip string
|
||||
want bool
|
||||
}{
|
||||
{"10.5.7.9", true},
|
||||
{"10.255.255.255", true},
|
||||
{"192.168.1.42", true},
|
||||
{"11.0.0.1", false},
|
||||
{"8.8.8.8", false},
|
||||
{"2001:db8::1", true},
|
||||
{"2001:db9::1", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.ip, func(t *testing.T) {
|
||||
got, err := models.IsRknIPBlocked(c.ip)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, c.want, got, "ip=%s", c.ip)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureRknIndexes_Idempotent(t *testing.T) {
|
||||
// Calling EnsureRknIndexes twice must not error — it's used both by
|
||||
// Migrate() and could be called from boot scripts.
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.NoError(t, models.EnsureRknIndexes())
|
||||
require.NoError(t, models.EnsureRknIndexes())
|
||||
|
||||
// GiST index must exist on rkn_ips.network.
|
||||
var exists bool
|
||||
err := models.DB().Raw(`
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE schemaname='public' AND tablename='rkn_ips'
|
||||
AND indexname='idx_rkn_ips_network'
|
||||
)
|
||||
`).Scan(&exists).Error
|
||||
require.NoError(t, err)
|
||||
assert.True(t, exists, "idx_rkn_ips_network must exist")
|
||||
}
|
||||
55
app/models/selfcheck.go
Обычный файл
55
app/models/selfcheck.go
Обычный файл
@@ -0,0 +1,55 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// SelfCheck provides functionality.
|
||||
type SelfCheck struct {
|
||||
concerns.Model
|
||||
Kind string `gorm:"not null;uniqueIndex:selfchecks" json:"kind"`
|
||||
Server *string `gorm:"uniqueIndex:selfchecks" json:"server"`
|
||||
Info string `json:"info"`
|
||||
LastCheck time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// LogCheck provides functionality.
|
||||
func LogCheck(kind string) error {
|
||||
m := SelfCheck{
|
||||
Kind: kind,
|
||||
Server: nil,
|
||||
}
|
||||
DB().FirstOrInit(&m, m)
|
||||
|
||||
m.LastCheck = time.Now()
|
||||
|
||||
return DB().Save(&m).Error
|
||||
}
|
||||
|
||||
// IsOk checks if the selfcheck for the given kind ran recently.
|
||||
func IsOk(kind string) (bool, string, error) {
|
||||
m := SelfCheck{
|
||||
Kind: kind,
|
||||
Server: nil,
|
||||
}
|
||||
DB().First(&m, m)
|
||||
|
||||
if m.ID == 0 {
|
||||
return false, "", errors.New("not run")
|
||||
}
|
||||
|
||||
var ago time.Time
|
||||
if kind == "exp" {
|
||||
ago = time.Now().Add(-3 * time.Hour)
|
||||
} else {
|
||||
ago = time.Now().Add(-15 * time.Minute)
|
||||
}
|
||||
|
||||
isOk := m.LastCheck.After(ago)
|
||||
|
||||
return isOk, m.LastCheck.Format(time.RFC3339), nil
|
||||
}
|
||||
388
app/models/server.go
Обычный файл
388
app/models/server.go
Обычный файл
@@ -0,0 +1,388 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql/driver"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// AccountMCPToken stores a one-way verifier. MCP tokens are bearer credentials
|
||||
// and must remain valid even when the deployment has no encryption key.
|
||||
type AccountMCPToken struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
TokenEnc string `gorm:"column:token;type:char(64);not null;index" json:"-"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (AccountMCPToken) TableName() string { return "account_mcp_tokens" }
|
||||
|
||||
func (t *AccountMCPToken) SetToken(token string) error {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
t.TokenEnc = hex.EncodeToString(sum[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *AccountMCPToken) TokenMatches(token string) bool {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
encoded := hex.EncodeToString(sum[:])
|
||||
return subtle.ConstantTimeCompare([]byte(t.TokenEnc), []byte(encoded)) == 1
|
||||
}
|
||||
|
||||
func GenerateMCPToken() string {
|
||||
return "mcp_" + base64.RawURLEncoding.EncodeToString(concerns.RandomToken(32))
|
||||
}
|
||||
|
||||
// Server health states — worst-of-monitor-states rollup; see
|
||||
// docs/plans/servers-and-hardware-metrics.md §5.1 for the data model.
|
||||
const (
|
||||
ServerHealthDown = "down"
|
||||
ServerHealthWarn = "warn"
|
||||
ServerHealthUp = "up"
|
||||
ServerHealthPaused = "paused"
|
||||
ServerHealthUnknown = "unknown"
|
||||
)
|
||||
|
||||
// ServerEnvironments is the allow-list for Server.Environment.
|
||||
// New environments require an explicit edit so they are visible in
|
||||
// tests.
|
||||
var ServerEnvironments = []string{"production", "staging", "dev", "test"}
|
||||
|
||||
// ServerKind is the rstuff-mirrored lifecycle label for a Server
|
||||
// (production / staging / old). See
|
||||
// docs/parity/rstuff-inventory.md §6.1 for the byte-stable mapping.
|
||||
type ServerKind string
|
||||
|
||||
// ServerKind values match rstuff's Server.kind enum exactly.
|
||||
// New values require adding a Postgres enum value via
|
||||
// app/models/migrate.go.
|
||||
const (
|
||||
ServerKindProduction ServerKind = "production"
|
||||
ServerKindStaging ServerKind = "staging"
|
||||
ServerKindOld ServerKind = "old"
|
||||
)
|
||||
|
||||
// Scan implements sql.Scanner for ServerKind.
|
||||
func (k *ServerKind) Scan(src any) error {
|
||||
if src == nil {
|
||||
*k = ""
|
||||
return nil
|
||||
}
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
*k = ServerKind(v)
|
||||
case []byte:
|
||||
*k = ServerKind(string(v))
|
||||
default:
|
||||
return fmt.Errorf("server_kind: cannot scan %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for ServerKind.
|
||||
func (k ServerKind) Value() (driver.Value, error) {
|
||||
if k == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return string(k), nil
|
||||
}
|
||||
|
||||
// Server represents a customer-facing logical host (e.g. "prod-web-01").
|
||||
//
|
||||
// One Server can host many WorkerNodes (HA after a VM migration). Each
|
||||
// WorkerNode carries a nullable ServerID so legacy "no server assigned"
|
||||
// rows keep working. The 1:N relation is stored as a nullable FK on
|
||||
// worker_nodes.server_id. The N:M relation to Monitor is the
|
||||
// monitor_servers join table. See
|
||||
// docs/plans/servers-and-hardware-metrics.md §3 for the layer model
|
||||
// and §5.1 for the schema.
|
||||
//
|
||||
// The inventory fields (ExtID, Kind, Token, PriceCents, Comment,
|
||||
// Meta) are added per docs/plans/inventory-management.md §6.2 so
|
||||
// rstuff can push the same row in via Valkey Streams and
|
||||
// deploymentd can authenticate via Token.
|
||||
type Server struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Slug string `gorm:"size:120;not null;index" json:"slug"`
|
||||
Description *string `gorm:"type:text" json:"description"`
|
||||
Region string `gorm:"size:64;not null;index" json:"region"`
|
||||
Environment string `gorm:"size:32;not null;default:'production'" json:"environment"`
|
||||
Tags pq.StringArray `gorm:"type:varchar(255)[]" json:"tags"`
|
||||
Icon *string `gorm:"size:16" json:"icon"`
|
||||
Color *string `gorm:"size:16" json:"color"`
|
||||
Paused bool `gorm:"not null;default:false" json:"paused"`
|
||||
|
||||
// Inventory fields (rstuff mirror; see inventory-management.md §6.2).
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
Kind ServerKind `gorm:"type:server_kind;not null;default:'production'" json:"kind"`
|
||||
// Token is omitted from JSON because it is a bearer credential.
|
||||
// Read it back only via /api/v1/servers/:id/token (operator-only)
|
||||
// and never echoed in list/show responses.
|
||||
Token *string `gorm:"size:64" json:"-"`
|
||||
PriceCents int `gorm:"not null;default:0" json:"price_cents"`
|
||||
Comment *string `gorm:"type:text" json:"comment,omitempty"`
|
||||
// Meta is rstuff-style free-form jsonb; serialized via JSON
|
||||
// encoding (gin renders it as a nested object).
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
HealthState string `gorm:"size:16;not null;default:'unknown';index" json:"health_state"`
|
||||
LastCheckAt *time.Time `json:"last_check_at"`
|
||||
Uptime24h float64 `gorm:"not null;default:1.0" json:"uptime_24h"`
|
||||
Uptime30d float64 `gorm:"not null;default:1.0" json:"uptime_30d"`
|
||||
|
||||
Monitors []Monitor `gorm:"many2many:monitor_servers;joinForeignKey:server_id;joinReferences:monitor_id;" json:"monitors,omitempty"`
|
||||
Workers []WorkerNode `gorm:"foreignKey:ServerID" json:"workers,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (Server) TableName() string { return "servers" }
|
||||
|
||||
// MonitorServer is the join row for the N:M relation between Monitor and
|
||||
// Server. One Monitor can be hosted on many Servers (multi-region
|
||||
// failover); one Server can host many Monitors.
|
||||
type MonitorServer struct {
|
||||
MonitorID int64 `gorm:"type:bigint REFERENCES monitors(id) ON DELETE CASCADE;primaryKey" json:"monitor_id"`
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;primaryKey" json:"server_id"`
|
||||
Position int `gorm:"not null;default:0" json:"position"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (MonitorServer) TableName() string { return "monitor_servers" }
|
||||
|
||||
// ServerMetric is the last-N-point cache written alongside VictoriaMetrics.
|
||||
// The full time-series lives in TSDB; Postgres only keeps the most recent
|
||||
// row per (server, source) for fast health badges and "last seen" cells.
|
||||
// See docs/plans/servers-and-hardware-metrics.md §5.3.
|
||||
type ServerMetric struct {
|
||||
concerns.Model
|
||||
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
WorkerID *int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL" json:"worker_id"`
|
||||
Source string `gorm:"size:32;not null;default:'worker'" json:"source"`
|
||||
CPUPercent *float64 `json:"cpu_percent"`
|
||||
MemUsed *int64 `json:"mem_used"`
|
||||
MemTotal *int64 `json:"mem_total"`
|
||||
DiskUsed *int64 `json:"disk_used"`
|
||||
DiskTotal *int64 `json:"disk_total"`
|
||||
NetRx *int64 `json:"net_rx"`
|
||||
NetTx *int64 `json:"net_tx"`
|
||||
HostUptimeSec *int64 `json:"host_uptime_sec"`
|
||||
Load1 *float64 `json:"load1"`
|
||||
Load5 *float64 `json:"load5"`
|
||||
Load15 *float64 `json:"load15"`
|
||||
ProcessCount *int `json:"process_count"`
|
||||
Processes datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"processes"`
|
||||
Networks datatypes.JSON `gorm:"type:jsonb;not null;default:'[]'::jsonb" json:"networks"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// ServerAlertRule defines one account-owned threshold for a server metric.
|
||||
// ClearThreshold implements hysteresis: a firing rule only recovers after the
|
||||
// value drops below it, avoiding alert flapping around Threshold.
|
||||
type ServerAlertRule struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE CASCADE;not null;index" json:"account_id"`
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
Metric string `gorm:"size:32;not null" json:"metric"`
|
||||
Threshold float64 `gorm:"not null" json:"threshold"`
|
||||
ClearThreshold float64 `gorm:"not null" json:"clear_threshold"`
|
||||
DurationSec int `gorm:"not null;default:300" json:"duration_sec"`
|
||||
NotificationID int64 `gorm:"type:bigint REFERENCES notifications(id) ON DELETE CASCADE;not null" json:"notification_id"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"enabled"`
|
||||
State string `gorm:"size:16;not null;default:'ok'" json:"state"`
|
||||
BreachSince *time.Time `json:"breach_since"`
|
||||
LastValue *float64 `json:"last_value"`
|
||||
LastMetricID *int64 `json:"last_metric_id"`
|
||||
LastFiredAt *time.Time `json:"last_fired_at"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (ServerAlertRule) TableName() string { return "server_alert_rules" }
|
||||
|
||||
// ServerAlertEvent is the durable dedupe/audit record for threshold changes.
|
||||
type ServerAlertEvent struct {
|
||||
concerns.Model
|
||||
RuleID int64 `gorm:"type:bigint REFERENCES server_alert_rules(id) ON DELETE CASCADE;not null;index" json:"rule_id"`
|
||||
State string `gorm:"size:16;not null" json:"state"`
|
||||
Value float64 `gorm:"not null" json:"value"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
func (ServerAlertEvent) TableName() string { return "server_alert_events" }
|
||||
|
||||
// TableName provides functionality.
|
||||
func (ServerMetric) TableName() string { return "server_metrics" }
|
||||
|
||||
// ValidateEnvironment returns nil iff env is in the allow list.
|
||||
func ValidateEnvironment(env string) error {
|
||||
for _, e := range ServerEnvironments {
|
||||
if env == e {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("invalid environment")
|
||||
}
|
||||
|
||||
// Slugify turns a server name into a URL-safe slug.
|
||||
func Slugify(name string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(name))
|
||||
var b strings.Builder
|
||||
for _, r := range slug {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == ' ', r == '_', r == '-', r == '.':
|
||||
b.WriteByte('-')
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "-")
|
||||
if out == "" {
|
||||
out = "server"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AssignMonitors replaces the full set of monitors for a server. Saves
|
||||
// the join table explicitly because gorm:association_autoupdate is
|
||||
// disabled globally (CLAUDE.md GORM Conventions).
|
||||
func (s *Server) AssignMonitors(tx *gorm.DB, monitorIDs []uint) error {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
if err := tx.Exec("DELETE FROM monitor_servers WHERE server_id = ?", s.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(monitorIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[uint]struct{}, len(monitorIDs))
|
||||
for i, mid := range monitorIDs {
|
||||
if _, ok := seen[mid]; ok {
|
||||
continue
|
||||
}
|
||||
seen[mid] = struct{}{}
|
||||
row := MonitorServer{ServerID: s.ID, MonitorID: int64(mid), Position: i}
|
||||
if err := tx.Create(&row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RollupHealthState computes the worst-of-monitor-states. Returns one of
|
||||
// ServerHealth{Down,Warn,Up,Paused,Unknown}.
|
||||
func (s *Server) RollupHealthState(monitors []Monitor) string {
|
||||
if s.Paused {
|
||||
return ServerHealthPaused
|
||||
}
|
||||
if len(monitors) == 0 {
|
||||
return ServerHealthUnknown
|
||||
}
|
||||
allPaused := true
|
||||
worst := ServerHealthUp
|
||||
for i := range monitors {
|
||||
m := &monitors[i]
|
||||
if m.Enabled {
|
||||
allPaused = false
|
||||
}
|
||||
switch m.State {
|
||||
case stateERR, stateFail:
|
||||
return ServerHealthDown
|
||||
case stateWARN:
|
||||
worst = ServerHealthWarn
|
||||
}
|
||||
}
|
||||
if allPaused {
|
||||
return ServerHealthPaused
|
||||
}
|
||||
return worst
|
||||
}
|
||||
|
||||
// SortedTagList returns tags sorted ascending; helper for stable JSON.
|
||||
func (s *Server) SortedTagList() []string {
|
||||
out := make([]string, len(s.Tags))
|
||||
copy(out, s.Tags)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// HealthForServer recomputes and persists health_state + last_check_at
|
||||
// for one server. Called by the distworker health ticker.
|
||||
func HealthForServer(serverID int64) error {
|
||||
server := Server{}
|
||||
if err := DB().First(&server, serverID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var monitors []Monitor
|
||||
if err := DB().Joins("JOIN monitor_servers ms ON ms.monitor_id = monitors.id").
|
||||
Where("ms.server_id = ?", serverID).Find(&monitors).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
state := server.RollupHealthState(monitors)
|
||||
now := time.Now()
|
||||
updates := map[string]interface{}{"health_state": state}
|
||||
if len(monitors) > 0 {
|
||||
updates["last_check_at"] = &now
|
||||
}
|
||||
return DB().Model(&server).Updates(updates).Error
|
||||
}
|
||||
|
||||
// LatestServerMetric returns the newest accepted worker snapshot for a server.
|
||||
func LatestServerMetric(serverID int64) (*ServerMetric, error) {
|
||||
metric := ServerMetric{}
|
||||
err := DB().Where("server_id = ?", serverID).Order("id DESC").First(&metric).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &metric, nil
|
||||
}
|
||||
|
||||
// FindServerByToken returns the Server whose token column equals
|
||||
// the given hex value, or nil with gorm.ErrRecordNotFound when no
|
||||
// row matches. Used by the deploymentd receiver middleware.
|
||||
func FindServerByToken(token string) (*Server, error) {
|
||||
var s Server
|
||||
err := DB().Where("token = ?", token).First(&s).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// GenerateServerToken returns a 32-byte random hex string. Caller
|
||||
// stores the plaintext exactly once (via /servers/:id/rotate-token)
|
||||
// and updates Server.Token; the old value is no longer recoverable.
|
||||
func GenerateServerToken() string {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
// crypto/rand should not fail on Linux; panic keeps the
|
||||
// contract simple for callers in the rare fatal case.
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
43
app/models/server_health_ticker.go
Обычный файл
43
app/models/server_health_ticker.go
Обычный файл
@@ -0,0 +1,43 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HealthTickInterval is exported so focused tests can exercise the same
|
||||
// lifecycle with a short interval.
|
||||
var HealthTickInterval = time.Minute
|
||||
|
||||
var serverHealthTickerOnce sync.Once
|
||||
|
||||
// StartServerHealthTicker periodically refreshes derived server health. The
|
||||
// first production tick is delayed to keep CLI migration paths side-effect
|
||||
// free; assignment/pause paths recompute synchronously.
|
||||
func StartServerHealthTicker(parent context.Context) {
|
||||
serverHealthTickerOnce.Do(func() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(HealthTickInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-parent.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
var ids []int64
|
||||
if err := DB().Model(&Server{}).Pluck("id", &ids).Error; err != nil {
|
||||
log.Printf("server health: list: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, id := range ids {
|
||||
if err := HealthForServer(id); err != nil {
|
||||
log.Printf("server health %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
36
app/models/server_ip.go
Обычный файл
36
app/models/server_ip.go
Обычный файл
@@ -0,0 +1,36 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// ServerIp is a single IP (v4 or v6) bound to a Server. The
|
||||
// canonical source for these rows is the deploymentd server-inventory
|
||||
// receiver (M1) and the network-diagnostics partial plan; today the
|
||||
// only writer is operator-entered via /api/v1/servers/:id/ips.
|
||||
//
|
||||
// `address` is Postgres `inet` so range queries (`<<` / `>>`) work
|
||||
// without parsing text. One row per (server_id, address). The
|
||||
// `is_primary` flag is set when more than one IP exists and the
|
||||
// deploymentd payload signals a primary; otherwise the first row wins.
|
||||
//
|
||||
//nolint:revive // ServerIP rename deferred to M3; rstuff schema uses ServerIp verbatim and parity test depends on it.
|
||||
type ServerIp struct {
|
||||
concerns.Model
|
||||
|
||||
ServerID int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE CASCADE;not null;index" json:"server_id"`
|
||||
Server *Server `json:"-"`
|
||||
// Address is mapped to Postgres inet via raw SQL in Migrate(); the
|
||||
// GORM `type:` tag is not enough because gorm.io/driver/postgres
|
||||
// does not register `inet` in its type map. Field is stored as a
|
||||
// string and validated by the API layer (see
|
||||
// app/controllers/api/server.go ServerIPsAdd).
|
||||
Address string `gorm:"type:inet;not null" json:"address"`
|
||||
IsPrimary bool `gorm:"not null;default:false" json:"is_primary"`
|
||||
RelatedSitesCount int `gorm:"not null;default:0" json:"related_sites_count"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (ServerIp) TableName() string { return "server_ips" }
|
||||
98
app/models/site.go
Обычный файл
98
app/models/site.go
Обычный файл
@@ -0,0 +1,98 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Site represents a customer-facing website or app. One Site can be
|
||||
// hosted on zero or one Server (server_id nullable) and exposes one
|
||||
// or more Deployments (compose services or nginx vhosts) plus zero
|
||||
// or more Repos (via site_repos). The RSMon slice mirrors rstuff's
|
||||
// `sites` table verbatim — see docs/parity/rstuff-inventory.md §2
|
||||
// and docs/plans/inventory-management.md §4.
|
||||
type Site struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Server *Server `json:"-"`
|
||||
ExtID *string `gorm:"size:64" json:"ext_id,omitempty"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
Slug string `gorm:"size:120;not null;index" json:"slug"`
|
||||
URL *string `gorm:"type:text" json:"url,omitempty"`
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
// Kind is a free-text label (not the PG enum) so we can absorb
|
||||
// rstuff additions without a migration. Default "production".
|
||||
Kind string `gorm:"size:32;not null;default:'production'" json:"kind"`
|
||||
IsActive bool `gorm:"not null;default:true" json:"is_active"`
|
||||
Meta datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"meta"`
|
||||
|
||||
Deployments []Deployment `gorm:"foreignKey:SiteID" json:"deployments,omitempty"`
|
||||
Repos []Repo `gorm:"many2many:site_repos;" json:"repos,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
}
|
||||
|
||||
// TableName returns the table name used for Site. Matches rstuff's
|
||||
// `sites` plural exactly so the parity test stays trivial.
|
||||
func (Site) TableName() string { return "sites" }
|
||||
|
||||
// SiteSlugify turns a name into a URL-safe slug. Mirrors the rules
|
||||
// in app/models/server.go:Slugify so /sites/:slug looks the same as
|
||||
// /servers/:slug. Consecutive separators collapse to a single dash;
|
||||
// non-ASCII letters are stripped (the same rule as Slugify — we
|
||||
// don't transliterate in v1, see docs/plans/inventory-management.md
|
||||
// §11 for transliteration as a future hardening item).
|
||||
func SiteSlugify(name string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(name))
|
||||
var b strings.Builder
|
||||
prevDash := false
|
||||
for _, r := range slug {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
prevDash = false
|
||||
case r == ' ', r == '_', r == '-', r == '.':
|
||||
if !prevDash && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
prevDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "-")
|
||||
if out == "" {
|
||||
out = "site"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindOrCreateSiteBySlug returns the site with the given slug for the
|
||||
// given account, creating an empty row (Name=slug, Kind=production,
|
||||
// IsActive=true) when no match exists. The caller's tx wraps the
|
||||
// operation so docker payload ingestion stays atomic.
|
||||
// See docs/plans/inventory-management.md §7.2 — Docker receiver.
|
||||
func FindOrCreateSiteBySlug(tx *gorm.DB, accountID int64, slug string) (*Site, error) {
|
||||
if tx == nil {
|
||||
tx = DB()
|
||||
}
|
||||
var site Site
|
||||
err := tx.Where("account_id = ? AND slug = ?", accountID, slug).First(&site).Error
|
||||
if err == nil {
|
||||
return &site, nil
|
||||
}
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
return nil, err
|
||||
}
|
||||
site = Site{AccountID: accountID, Slug: slug, Name: slug, Kind: "production", IsActive: true}
|
||||
if err := tx.Create(&site).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &site, nil
|
||||
}
|
||||
120
app/models/stats_data.go
Обычный файл
120
app/models/stats_data.go
Обычный файл
@@ -0,0 +1,120 @@
|
||||
package models
|
||||
|
||||
// StatsData provides functionality.
|
||||
type StatsData struct {
|
||||
MonitorID *int64 `json:"monitor_id,omitempty"`
|
||||
Up30d *float64 `json:"up_30d"`
|
||||
Up7d *float64 `json:"up_7d"`
|
||||
Up1d *float64 `json:"up_1d"`
|
||||
}
|
||||
|
||||
// Process provides functionality.
|
||||
func (data *StatsData) Process() {
|
||||
if data.Up1d == nil {
|
||||
dv := 100.0
|
||||
data.Up1d = &dv
|
||||
}
|
||||
if data.Up7d == nil {
|
||||
dv := 100.0
|
||||
data.Up7d = &dv
|
||||
}
|
||||
if data.Up30d == nil {
|
||||
dv := 100.0
|
||||
data.Up30d = &dv
|
||||
}
|
||||
}
|
||||
|
||||
// 86400 seconds / 1d
|
||||
// 604800 seconds / 7d
|
||||
// 2592000 seconds / 30d
|
||||
|
||||
// UptimeSQL provides functionality.
|
||||
const UptimeSQL = `
|
||||
SELECT
|
||||
monitors.id AS monitor_id,
|
||||
round(up1.up::numeric, 3) AS up1d,
|
||||
round(up7.up::numeric, 3) AS up7d,
|
||||
round(up30.up::numeric, 3) AS up30d
|
||||
FROM monitors
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '1' day
|
||||
group by monitor_id
|
||||
) up1 ON monitors.id = up1.monitor_id
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '7' day
|
||||
group by monitor_id
|
||||
) up7 ON monitors.id = up7.monitor_id
|
||||
LEFT JOIN (
|
||||
select monitor_id, CAST(10000 - (10000 * sum(duration) / min(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '30' day
|
||||
group by monitor_id
|
||||
) up30 ON monitors.id = up30.monitor_id
|
||||
`
|
||||
|
||||
// UptimeAllSQL provides functionality.
|
||||
const UptimeAllSQL = `
|
||||
SELECT
|
||||
round(up1.up::numeric, 3) AS up1d,
|
||||
round(up7.up::numeric, 3) AS up7d,
|
||||
round(up30.up::numeric, 3) AS up30d
|
||||
FROM (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 86400) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '1' day AND events.monitor_id IN (?)
|
||||
) up1, (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 604800) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '7' day AND events.monitor_id IN (?)
|
||||
) up7, (
|
||||
select CAST(10000 - (10000 * sum(duration) / sum(lt.least)) as float) / 100 as up from events
|
||||
join (
|
||||
select id, LEAST(extract(epoch from (NOW() - created_at))::int, 2592000) from monitors
|
||||
) lt ON lt.id = events.monitor_id
|
||||
where start_time > NOW() - interval '30' day AND events.monitor_id IN (?)
|
||||
) up30
|
||||
`
|
||||
|
||||
// MonitorStats provides functionality.
|
||||
func MonitorStats(monitors *[]Monitor) error {
|
||||
ids := make([]int64, len(*monitors))
|
||||
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
|
||||
ids[i] = m.ID
|
||||
}
|
||||
|
||||
rows, err := DB().Raw(UptimeSQL+"WHERE monitors.id IN (?)", ids).Rows()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats := make(map[int64]StatsData, 0)
|
||||
|
||||
defer rows.Close() //nolint:errcheck
|
||||
for rows.Next() {
|
||||
data := StatsData{}
|
||||
_ = DB().ScanRows(rows, &data)
|
||||
data.Process()
|
||||
|
||||
stats[*data.MonitorID] = data
|
||||
}
|
||||
|
||||
for i, m := range *monitors { //nolint:gocritic // range copy is acceptable here
|
||||
(*monitors)[i].StatsData = stats[m.ID]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
461
app/models/status_page.go
Обычный файл
461
app/models/status_page.go
Обычный файл
@@ -0,0 +1,461 @@
|
||||
// Package models — status page subsystem (docs/plans/status-pages.md).
|
||||
//
|
||||
// M0 ships the schema for status_pages and its five related tables
|
||||
// (subscribers, incidents, maintenance, domains). The M0 milestone is
|
||||
// read-only at the dashboard level — no editor and no public render yet —
|
||||
// but landing the schema now lets downstream milestones wire public
|
||||
// routes, editor flows, and the notifier→subscriber bridge without
|
||||
// further ALTER TABLE churn. M5 (custom domain) only fills in
|
||||
// status_page_domains rows; the table itself is reserved here so the
|
||||
// M5 migration is just data, not DDL.
|
||||
//
|
||||
// All tables follow the existing RSMon conventions: concerns.Model +
|
||||
// concerns.Timestamped + Audited mixins, gorm.DeletedAt for soft delete
|
||||
// on the top-level status_pages row, pq.Int64Array for the monitor_ids
|
||||
// bigint[] join columns (same shape as sites and the Check.Warnings
|
||||
// slice). Partial-unique indexes (slug, subscriber email) are added via
|
||||
// raw SQL in app/models/migrate.go because the GORM tag language cannot
|
||||
// express a WHERE deleted_at IS NULL predicate.
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"golang.org/x/net/idna"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/config/credis"
|
||||
)
|
||||
|
||||
// Status page color defaults. Matches the existing landing-page primary
|
||||
// green and the accent blue used in the /settings UI, so a freshly
|
||||
// created page already blends in with the rest of the app.
|
||||
const (
|
||||
statusPageDefaultPrimaryColor = "#62c600"
|
||||
statusPageDefaultAccentColor = "#1a73e8"
|
||||
statusPageDefaultHistoryDays = 90
|
||||
statusPageMaxSlugLen = 64
|
||||
statusPageMaxNameLen = 120
|
||||
)
|
||||
|
||||
// Hex color regex — accepts #RGB and #RRGGBB. Centralized so the
|
||||
// controller/model validation agree on the same shape.
|
||||
var hexColorRegex = regexp.MustCompile(`^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$`)
|
||||
|
||||
// StatusPageSlugRegex mirrors the slug format enforced in
|
||||
// StatusPage.NormalizeSlug / ValidateSlug. Lowercase alphanumerics and
|
||||
// dashes, must start and end with an alphanumeric. Length is checked
|
||||
// separately so the regex stays readable.
|
||||
var statusPageSlugRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$`)
|
||||
|
||||
// StatusPage represents a public status page owned by an account. One
|
||||
// account may own many pages (gated by plan in M2+); slugs are globally
|
||||
// unique because public URLs do not contain the account ID. Each page exposes
|
||||
// a curated subset of the account's monitors and a recent-incidents
|
||||
// feed. Soft-deleted rows remain in the table so the partial-unique
|
||||
// index on (account_id, slug) WHERE deleted_at IS NULL still rejects
|
||||
// duplicate slugs against historical records — see the comment on
|
||||
// StatusPagesAccountSlugUnique in migrate.go.
|
||||
type StatusPage struct {
|
||||
concerns.Model
|
||||
|
||||
AccountID int64 `gorm:"type:bigint REFERENCES accounts(id);not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
|
||||
Slug string `gorm:"size:64;not null;index" json:"slug"`
|
||||
Name string `gorm:"size:120;not null" json:"name"`
|
||||
|
||||
Description *string `gorm:"type:text" json:"description,omitempty"`
|
||||
LogoURL *string `gorm:"size:255" json:"logo_url,omitempty"`
|
||||
|
||||
PrimaryColor string `gorm:"size:7;not null;default:'#62c600'" json:"primary_color"`
|
||||
AccentColor string `gorm:"size:7;not null;default:'#1a73e8'" json:"accent_color"`
|
||||
|
||||
// MonitorIDs is the curated subset of account monitors the page
|
||||
// exposes. Order is preserved so the dashboard list and the public
|
||||
// render show the same ordering. Stored as bigint[] to keep
|
||||
// monitor-to-page mapping lookup-free on the read path; M2 will
|
||||
// add a UI to maintain this set.
|
||||
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
|
||||
|
||||
ShowUptimeBars bool `gorm:"not null;default:true" json:"show_uptime_bars"`
|
||||
ShowResponseTimes bool `gorm:"not null;default:true" json:"show_response_times"`
|
||||
ShowHistoryDays int `gorm:"not null;default:90" json:"show_history_days"`
|
||||
|
||||
// PasswordHash is populated in M4 only. Stored at length 255 so
|
||||
// a future bcrypt cost bump does not need a column resize.
|
||||
PasswordHash *string `gorm:"size:255" json:"-"`
|
||||
// GATrackingID — Google Analytics 4 measurement ID; emitted by
|
||||
// the public renderer in M4.
|
||||
GATrackingID *string `gorm:"size:32" json:"ga_tracking_id,omitempty"`
|
||||
|
||||
// NoIndex emits <meta name="robots" content="noindex"> so the
|
||||
// page can be staged without polluting search indexes.
|
||||
NoIndex bool `gorm:"not null;default:false" json:"no_index"`
|
||||
|
||||
// IsPublished gates the public /status/:slug render. Until M2
|
||||
// ships the editor the default value keeps M0 pages invisible.
|
||||
IsPublished bool `gorm:"not null;default:false" json:"is_published"`
|
||||
// AutoOpenIncidents is opt-in so publishing a page does not change
|
||||
// existing alert behavior until an owner explicitly enables it.
|
||||
AutoOpenIncidents bool `gorm:"not null;default:false" json:"auto_open_incidents"`
|
||||
|
||||
concerns.Timestamped
|
||||
Audited
|
||||
|
||||
// DeletedAt is the GORM soft-delete marker. Using gorm.DeletedAt
|
||||
// rather than concerns.SoftDelete because the latter adds a
|
||||
// DeleterID users(id) FK that we do not yet need on status_pages.
|
||||
DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
|
||||
}
|
||||
|
||||
// TableName returns the explicit table name so GORM does not try to
|
||||
// pluralize to "status_pages" via inflection. The plural is already
|
||||
// correct; we declare it anyway for clarity.
|
||||
func (StatusPage) TableName() string { return "status_pages" }
|
||||
|
||||
// StatusPageSubscriberKind — values stored in status_page_subscribers.kind.
|
||||
// "alert" subscribes to incident-driven notifications; "digest_daily"
|
||||
// receives the morning summary (M3). New kinds should be appended so
|
||||
// the JSON serializations stay stable.
|
||||
const (
|
||||
StatusPageSubscriberKindAlert = "alert"
|
||||
StatusPageSubscriberKindDigestDaily = "digest_daily"
|
||||
)
|
||||
|
||||
// StatusPageSubscriber is a row in status_page_subscribers. Email is
|
||||
// stored verbatim (no citext) because the codebase already persists
|
||||
// contact emails as-is; case-insensitive uniqueness is enforced via
|
||||
// the partial unique index in migrate.go using lower(email).
|
||||
type StatusPageSubscriber struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
// ContactID is an internal delivery endpoint. It is never returned by public
|
||||
// subscription APIs; tasks use it to preserve the normal worker email path.
|
||||
ContactID *int64 `gorm:"type:bigint REFERENCES contacts(id) ON DELETE SET NULL;index" json:"-"`
|
||||
|
||||
Email string `gorm:"size:255;not null" json:"email"`
|
||||
Kind string `gorm:"size:16;not null;default:'alert'" json:"kind"`
|
||||
ConfirmTokenHash string `gorm:"size:64" json:"-"`
|
||||
// LegacyConfirmToken is retained only for confirmation links issued before
|
||||
// token hashing shipped. It is cleared on first use or resend.
|
||||
LegacyConfirmToken *string `gorm:"column:confirm_token;size:255" json:"-"`
|
||||
TokenExpiresAt time.Time `json:"-"`
|
||||
UnsubscribeTokenHash string `gorm:"size:64;default:''" json:"-"`
|
||||
ConfirmedAt *time.Time `json:"confirmed_at,omitempty"`
|
||||
UnsubscribedAt *time.Time `json:"unsubscribed_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_subscribers table name.
|
||||
func (StatusPageSubscriber) TableName() string { return "status_page_subscribers" }
|
||||
|
||||
// StatusPageIncident severity values. info = heads-up notices, warn =
|
||||
// degradation, crit = full outage. Used for color-coding in the public
|
||||
// render (M1) and for filtering in the dashboard list (M0).
|
||||
const (
|
||||
StatusPageIncidentSeverityInfo = "info"
|
||||
StatusPageIncidentSeverityWarn = "warn"
|
||||
StatusPageIncidentSeverityCrit = "crit"
|
||||
)
|
||||
|
||||
// StatusPageIncident represents a single incident entry on a status
|
||||
// page. event_id is a soft link back to the existing Event model so
|
||||
// "auto-open on monitor error" can be wired later without a second
|
||||
// migration. posted_by_user_id is nullable so external integrations
|
||||
// can write incidents anonymously.
|
||||
type StatusPageIncident struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
EventID *int64 `gorm:"type:bigint REFERENCES events(id) ON DELETE SET NULL;index" json:"event_id,omitempty"`
|
||||
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
BodyMD string `gorm:"type:text" json:"body_md,omitempty"`
|
||||
Severity string `gorm:"size:16;not null;default:'info'" json:"severity"`
|
||||
|
||||
StartedAt time.Time `gorm:"not null;index" json:"started_at"`
|
||||
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
|
||||
|
||||
PostedByUserID *int64 `gorm:"type:bigint REFERENCES users(id)" json:"posted_by_user_id,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_incidents table name.
|
||||
func (StatusPageIncident) TableName() string { return "status_page_incidents" }
|
||||
|
||||
// StatusPageMaintenance is a scheduled maintenance window. The
|
||||
// monitor_ids column is the set of monitors the window covers; empty
|
||||
// means "all monitors on the page".
|
||||
type StatusPageMaintenance struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
Title string `gorm:"size:200;not null" json:"title"`
|
||||
Description string `gorm:"type:text" json:"description,omitempty"`
|
||||
|
||||
StartsAt time.Time `gorm:"not null;index" json:"starts_at"`
|
||||
EndsAt time.Time `gorm:"not null" json:"ends_at"`
|
||||
|
||||
MonitorIDs pq.Int64Array `gorm:"type:bigint[];not null;default:'{}'" json:"monitor_ids"`
|
||||
|
||||
NotifySubscribers bool `gorm:"not null;default:true" json:"notify_subscribers"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_maintenance table name.
|
||||
func (StatusPageMaintenance) TableName() string { return "status_page_maintenance" }
|
||||
|
||||
// StatusPageDomain is the M5 custom-domain mapping. Reserved in M0 so
|
||||
// the table does not need to be created at M5 — only rows are written
|
||||
// then. domain is unique globally (CNAMEs are hostnames, they cannot
|
||||
// be reused across pages), txt_token is the value the user adds as a
|
||||
// DNS TXT record to prove ownership.
|
||||
type StatusPageDomain struct {
|
||||
concerns.Model
|
||||
|
||||
StatusPageID int64 `gorm:"type:bigint REFERENCES status_pages(id) ON DELETE CASCADE;not null;index" json:"status_page_id"`
|
||||
StatusPage *StatusPage `json:"-"`
|
||||
|
||||
Domain string `gorm:"size:255;not null;uniqueIndex" json:"domain"`
|
||||
VerifiedAt *time.Time `json:"verified_at,omitempty"`
|
||||
TXTToken string `gorm:"size:64;not null" json:"txt_token,omitempty"`
|
||||
VerifyError string `gorm:"type:text" json:"verify_error,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName returns the explicit status_page_domains table name.
|
||||
func (StatusPageDomain) TableName() string { return "status_page_domains" }
|
||||
|
||||
// NormalizeStatusPageDomain accepts a hostname only. URLs, ports, IP literals,
|
||||
// wildcard names, and invalid IDNA are deliberately rejected before DNS work.
|
||||
func NormalizeStatusPageDomain(in string) (string, error) {
|
||||
domain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(in)), ".")
|
||||
if domain == "" || len(domain) > 253 || strings.ContainsAny(domain, "/:@") || net.ParseIP(domain) != nil {
|
||||
return "", errStatusPage("domain must be a hostname")
|
||||
}
|
||||
ascii, err := idna.Lookup.ToASCII(domain)
|
||||
if err != nil || ascii == "" || len(ascii) > 253 || !strings.Contains(ascii, ".") {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
for _, label := range strings.Split(ascii, ".") {
|
||||
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
for _, r := range label {
|
||||
if r != '-' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||
return "", errStatusPage("domain must be a valid hostname")
|
||||
}
|
||||
}
|
||||
}
|
||||
return ascii, nil
|
||||
}
|
||||
|
||||
// HashStatusPageToken keeps bearer-style subscription URLs out of the database.
|
||||
func HashStatusPageToken(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return fmt.Sprintf("%x", sum[:])
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) TokenMatches(token string) bool {
|
||||
return s != nil && s.TokenExpiresAt.After(time.Now()) && s.ConfirmTokenHash == HashStatusPageToken(token)
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) UnsubscribeTokenMatches(token string) bool {
|
||||
return s != nil && s.UnsubscribeTokenHash != "" && s.UnsubscribeTokenHash == HashStatusPageToken(token)
|
||||
}
|
||||
|
||||
// StatusPagePlatformDomain and StatusPagePublicIPs are deployment-owned DNS
|
||||
// targets. Customer domains must point here before they can become routable.
|
||||
func StatusPagePlatformDomain() string {
|
||||
return strings.TrimSuffix(strings.ToLower(os.Getenv("STATUS_PAGE_PLATFORM_DOMAIN")), ".")
|
||||
}
|
||||
|
||||
func StatusPagePublicIPs() []string {
|
||||
return strings.FieldsFunc(os.Getenv("STATUS_PAGE_PUBLIC_IPS"), func(r rune) bool { return r == ',' || r == ' ' })
|
||||
}
|
||||
|
||||
var (
|
||||
statusPageLookupCNAME = net.LookupCNAME
|
||||
statusPageLookupHost = net.LookupHost
|
||||
statusPageLookupTXT = net.LookupTXT
|
||||
)
|
||||
|
||||
// VerifyStatusPageDomain performs the BYO-DNS preflight. A customer must keep
|
||||
// the ownership TXT record and point either a CNAME at the platform hostname or
|
||||
// an A/AAAA record at one of the explicitly configured public addresses.
|
||||
func VerifyStatusPageDomain(domain *StatusPageDomain) error {
|
||||
if domain == nil {
|
||||
return errStatusPage("domain is required")
|
||||
}
|
||||
want, err := NormalizeStatusPageDomain(domain.Domain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
platform := StatusPagePlatformDomain()
|
||||
publicIPs := StatusPagePublicIPs()
|
||||
if platform == "" && len(publicIPs) == 0 {
|
||||
return errStatusPage("custom domain verification is not configured")
|
||||
}
|
||||
matchedTarget := false
|
||||
if platform != "" {
|
||||
if cname, lookupErr := statusPageLookupCNAME(want); lookupErr == nil {
|
||||
matchedTarget = strings.TrimSuffix(strings.ToLower(cname), ".") == platform
|
||||
}
|
||||
}
|
||||
if !matchedTarget && len(publicIPs) > 0 {
|
||||
if hosts, lookupErr := statusPageLookupHost(want); lookupErr == nil {
|
||||
for _, host := range hosts {
|
||||
for _, allowed := range publicIPs {
|
||||
if host == allowed {
|
||||
matchedTarget = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matchedTarget {
|
||||
return errStatusPage("DNS must contain the configured CNAME or public A/AAAA address")
|
||||
}
|
||||
txt, lookupErr := statusPageLookupTXT(want)
|
||||
if lookupErr != nil {
|
||||
return errStatusPage("ownership TXT record was not found")
|
||||
}
|
||||
for _, value := range txt {
|
||||
if value == "rsmon-verify="+domain.TXTToken {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errStatusPage("ownership TXT record does not match")
|
||||
}
|
||||
|
||||
// NormalizeStatusPageSlug lowercases and trims a candidate slug so the
|
||||
// global partial unique index on slug is satisfied
|
||||
// regardless of how the caller capitalizes the input. Returns the
|
||||
// empty string when the result would be unusable as a URL path;
|
||||
// callers should fall back to a name-derived slug in that case.
|
||||
func NormalizeStatusPageSlug(in string) string {
|
||||
slug := strings.ToLower(strings.TrimSpace(in))
|
||||
return slug
|
||||
}
|
||||
|
||||
// ValidateStatusPageSlug enforces the slug format we expose to users:
|
||||
// lowercase alphanumeric plus dash, must start and end with an
|
||||
// alphanumeric, max length 64. Used by the editor before save (M2).
|
||||
// Returns nil when the slug is acceptable.
|
||||
func ValidateStatusPageSlug(slug string) error {
|
||||
if slug == "" {
|
||||
return errStatusPage("slug is required")
|
||||
}
|
||||
if len(slug) > statusPageMaxSlugLen {
|
||||
return errStatusPage("slug is too long")
|
||||
}
|
||||
if !statusPageSlugRegex.MatchString(slug) {
|
||||
return errStatusPage("slug must be lowercase alphanumeric with dashes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateStatusPageColors returns an error if either color is set but
|
||||
// not a valid CSS hex string. Empty strings fall back to the model
|
||||
// defaults when written via BeforeSave hooks.
|
||||
func ValidateStatusPageColors(primary, accent string) error {
|
||||
if primary != "" && !hexColorRegex.MatchString(primary) {
|
||||
return errStatusPage("primary_color must be #RGB or #RRGGBB")
|
||||
}
|
||||
if accent != "" && !hexColorRegex.MatchString(accent) {
|
||||
return errStatusPage("accent_color must be #RGB or #RRGGBB")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BeforeSave is the GORM hook that fills in the canonical defaults
|
||||
// (colors, history days) so callers can pass an empty struct and still
|
||||
// get a usable page. Hook is also the single place where slugs are
|
||||
// normalized, so the unique index never has to chase trailing spaces.
|
||||
// The gorm.DB parameter is required by the hook signature but unused —
|
||||
// the validation here is purely local to the model.
|
||||
func (p *StatusPage) BeforeSave(_ *gorm.DB) error {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
p.Slug = NormalizeStatusPageSlug(p.Slug)
|
||||
if err := ValidateStatusPageSlug(p.Slug); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.PrimaryColor == "" {
|
||||
p.PrimaryColor = statusPageDefaultPrimaryColor
|
||||
}
|
||||
if p.AccentColor == "" {
|
||||
p.AccentColor = statusPageDefaultAccentColor
|
||||
}
|
||||
if p.ShowHistoryDays == 0 {
|
||||
p.ShowHistoryDays = statusPageDefaultHistoryDays
|
||||
}
|
||||
return ValidateStatusPageColors(p.PrimaryColor, p.AccentColor)
|
||||
}
|
||||
|
||||
// IsPublishedNow reports whether the page is publicly visible. M0
|
||||
// always returns false because the editor (M2) is the only thing that
|
||||
// flips IsPublished to true; this helper centralizes that contract.
|
||||
func (p *StatusPage) IsPublishedNow() bool {
|
||||
return p != nil && p.IsPublished && !p.DeletedAt.Valid
|
||||
}
|
||||
|
||||
// errStatusPage builds a validation error carrying the message. The
|
||||
// returned error is a plain error; controllers translate it into a
|
||||
// 422 response.
|
||||
func errStatusPage(msg string) error {
|
||||
if msg == "" {
|
||||
return errors.New("status_page: invalid")
|
||||
}
|
||||
return fmt.Errorf("status_page: %s", msg)
|
||||
}
|
||||
|
||||
// IsActiveSubscriber returns true when the subscriber has confirmed and
|
||||
// has not unsubscribed. Used by the M3 incident-notification loop.
|
||||
func (s *StatusPageSubscriber) IsActiveSubscriber() bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
return s.ConfirmedAt != nil && s.UnsubscribedAt == nil
|
||||
}
|
||||
|
||||
func (s *StatusPageSubscriber) BeforeCreate(_ *gorm.DB) error {
|
||||
if s.TokenExpiresAt.IsZero() {
|
||||
s.TokenExpiresAt = time.Now().Add(24 * time.Hour)
|
||||
}
|
||||
if s.ConfirmTokenHash == "" {
|
||||
s.ConfirmTokenHash = HashStatusPageToken(fmt.Sprintf("legacy-%d-%s", time.Now().UnixNano(), s.Email))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InvalidateStatusPageCache removes the public HTML cache without making
|
||||
// Redis availability part of the monitor or management write path.
|
||||
func InvalidateStatusPageCache(pageID int64) {
|
||||
if credis.Redis != nil {
|
||||
_ = credis.Redis.Del(context.Background(), "statuspage:html:"+strconv.FormatInt(pageID, 10)).Err()
|
||||
}
|
||||
}
|
||||
408
app/models/status_page_delivery.go
Обычный файл
408
app/models/status_page_delivery.go
Обычный файл
@@ -0,0 +1,408 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
const (
|
||||
statusPageDigestHour = 9
|
||||
statusPageOutboxPending = "pending"
|
||||
statusPageOutboxDispatched = "dispatched"
|
||||
statusPageOutboxCanceled = "canceled"
|
||||
)
|
||||
|
||||
// StatusPageDelivery is an outbox row. It does not retain email addresses;
|
||||
// confirmation links are the only transient body data and are redacted on
|
||||
// cancellation. The subscriber/contact data is read while dispatching.
|
||||
type StatusPageDelivery struct {
|
||||
ID int64 `gorm:"primarykey"`
|
||||
StatusPageID int64 `gorm:"not null;index"`
|
||||
SubscriberID int64 `gorm:"not null;index"`
|
||||
IncidentID *int64 `gorm:"index"`
|
||||
Kind string `gorm:"size:32;not null"`
|
||||
Version string `gorm:"size:64;not null"`
|
||||
LocalDate string `gorm:"size:10"`
|
||||
State string `gorm:"size:16;not null;index"`
|
||||
IdempotencyKey string `gorm:"uniqueIndex;size:255;not null"`
|
||||
MessageID *int64 `gorm:"index"`
|
||||
TaskID *int64 `gorm:"index"`
|
||||
LastError string `gorm:"type:text"`
|
||||
Subject string `gorm:"type:text"`
|
||||
BodyText string `gorm:"type:text"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (StatusPageDelivery) TableName() string { return "status_page_deliveries" }
|
||||
|
||||
type StatusPageDigestSchedule struct {
|
||||
ID int64 `gorm:"primarykey"`
|
||||
SubscriberID int64 `gorm:"uniqueIndex:status_page_digest_due;not null"`
|
||||
LocalDate string `gorm:"uniqueIndex:status_page_digest_due;size:10;not null"`
|
||||
DueAt time.Time `gorm:"not null;index"`
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (StatusPageDigestSchedule) TableName() string { return "status_page_digest_schedules" }
|
||||
|
||||
func EnsureStatusPageSubscriberContactTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber) error {
|
||||
if subscriber.ContactID != nil {
|
||||
return nil
|
||||
}
|
||||
accountID := page.AccountID
|
||||
contact := &Contact{AccountID: &accountID, Name: "Status page subscriber", Kind: "email", Value: subscriber.Email, Enabled: true}
|
||||
if err := tx.Create(contact).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subscriber.ContactID = &contact.ID
|
||||
return tx.Model(subscriber).Update("contact_id", contact.ID).Error
|
||||
}
|
||||
|
||||
func enqueueStatusPageDeliveryTx(tx *gorm.DB, pageID, subscriberID int64, incidentID *int64, kind, version, localDate string) error {
|
||||
key := fmt.Sprintf("status-page:%s:subscriber:%d:incident:%d:version:%s:date:%s", kind, subscriberID, valueOrZero(incidentID), version, localDate)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: pageID, SubscriberID: subscriberID, IncidentID: incidentID, Kind: kind, Version: version, LocalDate: localDate, State: statusPageOutboxPending, IdempotencyKey: key}).Error
|
||||
}
|
||||
|
||||
func EnqueueStatusPageConfirmationTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, confirmationURL string) error {
|
||||
key := fmt.Sprintf("status-page:confirm:subscriber:%d:token:%s", subscriber.ID, subscriber.ConfirmTokenHash)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "confirm", Version: subscriber.ConfirmTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Confirm status page subscription", BodyText: "Confirm: " + confirmationURL}).Error
|
||||
}
|
||||
|
||||
func EnqueueStatusPageWelcomeTx(tx *gorm.DB, page *StatusPage, subscriber *StatusPageSubscriber, unsubscribeURL string) error {
|
||||
key := fmt.Sprintf("status-page:welcome:subscriber:%d:token:%s", subscriber.ID, subscriber.UnsubscribeTokenHash)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: page.ID, SubscriberID: subscriber.ID, Kind: "welcome", Version: subscriber.UnsubscribeTokenHash, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "Status page subscription confirmed", BodyText: "Manage subscription: " + unsubscribeURL}).Error
|
||||
}
|
||||
|
||||
func valueOrZero(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
// EnqueueStatusPageIncidentDeliveriesTx is called in the incident transition
|
||||
// transaction, so an incident can never become visible without its delivery
|
||||
// intent being recoverable by the dispatcher.
|
||||
func EnqueueStatusPageIncidentDeliveriesTx(tx *gorm.DB, page *StatusPage, incident *StatusPageIncident, kind string) error {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := tx.Where("status_page_id = ? AND kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", page.ID, StatusPageSubscriberKindAlert).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
version := incident.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
||||
for _, subscriber := range subscribers {
|
||||
if err := enqueueStatusPageDeliveryTx(tx, page.ID, subscriber.ID, &incident.ID, kind, version, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func statusPageNotificationTx(tx *gorm.DB, accountID int64) (*Notification, error) {
|
||||
var notification Notification
|
||||
err := tx.Where("account_id = ? AND name = ?", accountID, "Status page delivery").First(¬ification).Error
|
||||
if err == nil {
|
||||
return ¬ification, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
notification = Notification{Name: "Status page delivery", AccountID: accountID, Enabled: true}
|
||||
return ¬ification, tx.Create(¬ification).Error
|
||||
}
|
||||
|
||||
// DispatchStatusPageDeliveries retries pending outbox rows. It builds ordinary
|
||||
// Message and Task rows in the same transaction, and preserves a pending row on
|
||||
// transient worker/capability failure for the next tick.
|
||||
func DispatchStatusPageDeliveries(now time.Time) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := DB().Where("state = ?", statusPageOutboxPending).Order("id ASC").Limit(200).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, row := range rows {
|
||||
if err := dispatchStatusPageDelivery(row.ID, now); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func dispatchStatusPageDelivery(id int64, now time.Time) error {
|
||||
err := DB().Transaction(func(tx *gorm.DB) error {
|
||||
var outbox StatusPageDelivery
|
||||
if err := tx.First(&outbox, id).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var subscriber StatusPageSubscriber
|
||||
var page StatusPage
|
||||
// Subscriber then outbox is the global lock order shared with
|
||||
// unsubscribe/delete, preventing dispatch-vs-cancel deadlocks.
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, outbox.SubscriberID).Error; err != nil {
|
||||
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber deleted")
|
||||
}
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&outbox, id).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if outbox.State != statusPageOutboxPending {
|
||||
return nil
|
||||
}
|
||||
if subscriber.UnsubscribedAt != nil || (subscriber.ConfirmedAt == nil && outbox.Kind != "confirm") {
|
||||
return cancelStatusPageDeliveryTx(tx, &outbox, "subscriber inactive")
|
||||
}
|
||||
if err := tx.First(&page, outbox.StatusPageID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := EnsureStatusPageSubscriberContactTx(tx, &page, &subscriber); err != nil {
|
||||
return err
|
||||
}
|
||||
notification, err := statusPageNotificationTx(tx, page.AccountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, text := outbox.Subject, outbox.BodyText
|
||||
if subject == "" {
|
||||
subject, text = "["+page.Name+"] status update", "Status update"
|
||||
}
|
||||
if outbox.IncidentID != nil {
|
||||
var incident StatusPageIncident
|
||||
if err := tx.First(&incident, *outbox.IncidentID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
subject = "[" + page.Name + "] " + incident.Title
|
||||
text = strings.ToUpper(outbox.Kind[:1]) + outbox.Kind[1:] + ": " + incident.Title + "\n\n" + incident.BodyMD
|
||||
} else if outbox.BodyText == "" {
|
||||
text = "Open incidents daily digest"
|
||||
}
|
||||
message := &Message{NotificationID: notification.ID, ContactID: *subscriber.ContactID, Kind: "status_page", State: TaskStateQueued}
|
||||
if err := tx.Create(message).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(wire.NotificationTask{AccountID: page.AccountID, MessageID: message.ID, NotificationID: notification.ID, Method: "email", Contact: wire.NotificationContact{ID: *subscriber.ContactID, Kind: "email", Value: subscriber.Email, Name: "Status page subscriber"}, Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := EnqueueNotificationTaskTx(tx, &EnqueueNotificationTaskInput{AccountID: page.AccountID, NotificationID: notification.ID, ContactID: *subscriber.ContactID, MessageID: &message.ID, Method: "email", Subject: subject, BodyText: text, BodyMarkdown: text, BodyHTML: "<p>" + html.EscapeString(text) + "</p>", Language: "en", MessageKind: "status_page", NotBefore: now, Payload: payload, IdempotencyKey: outbox.IdempotencyKey})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Model(&outbox).Updates(map[string]any{"state": statusPageOutboxDispatched, "message_id": message.ID, "task_id": task.ID, "last_error": ""}).Error
|
||||
})
|
||||
if err != nil {
|
||||
// This update intentionally runs after rollback: diagnostic state must not
|
||||
// disappear with the failed message/task transaction.
|
||||
_ = DB().Model(&StatusPageDelivery{}).Where("id = ? AND state = ?", id, statusPageOutboxPending).Update("last_error", err.Error()).Error
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func cancelStatusPageDeliveryTx(tx *gorm.DB, outbox *StatusPageDelivery, reason string) error {
|
||||
updates := map[string]any{"state": statusPageOutboxCanceled, "last_error": reason, "subject": "", "body_text": ""}
|
||||
if err := tx.Model(outbox).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if outbox.TaskID != nil {
|
||||
// Redact every task, including terminal audit rows. State is preserved for
|
||||
// terminal rows, while the payload can no longer disclose the address.
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *outbox.TaskID).Update("payload", []byte(`{}`)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&Task{}).Where("id = ? AND state NOT IN ?", *outbox.TaskID, []string{TaskStateSucceeded, TaskStateFailedPerm, TaskStateDead}).Updates(map[string]any{"state": TaskStateDead, "last_error": "canceled: " + reason, "payload": []byte(`{}`), "lease_owner": "", "lease_token": "", "lease_expires_at": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if outbox.MessageID != nil {
|
||||
return tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *outbox.MessageID, []string{"sent", "error"}).Updates(map[string]any{"state": "error", "error": "canceled", "response": nil}).Error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CancelStatusPageSubscriberDeliveriesTx(tx *gorm.DB, subscriberID int64, reason string) error {
|
||||
var subscriber StatusPageSubscriber
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&subscriber, subscriberID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("subscriber_id = ?", subscriberID).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
if err := cancelStatusPageDeliveryTx(tx, &rows[i], reason); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedactStatusPageSubscriberDeliveryTx removes bearer links and addresses from
|
||||
// retained status-page lineage without touching unrelated account messages.
|
||||
func RedactStatusPageSubscriberDeliveryTx(tx *gorm.DB, subscriberID int64) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Where("subscriber_id = ?", subscriberID).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
row := &rows[i]
|
||||
if err := tx.Model(row).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if row.TaskID != nil {
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *row.TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *row.TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if row.MessageID != nil {
|
||||
if err := tx.Model(&Message{}).Where("id = ?", *row.MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RedactStatusPageConfirmationTx(tx *gorm.DB, subscriberID int64) error {
|
||||
var rows []StatusPageDelivery
|
||||
if err := tx.Where("subscriber_id = ? AND kind = ?", subscriberID, "confirm").Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
if err := tx.Model(&rows[i]).Updates(map[string]any{"subject": "", "body_text": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if rows[i].TaskID != nil {
|
||||
if err := tx.Model(&Task{}).Where("id = ?", *rows[i].TaskID).Updates(map[string]any{"payload": []byte(`{}`), "result": []byte(`{}`), "last_error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&NotificationDelivery{}).Where("task_id = ?", *rows[i].TaskID).Updates(map[string]any{"provider_response": "", "error": ""}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if rows[i].MessageID != nil {
|
||||
if err := tx.Model(&Message{}).Where("id = ?", *rows[i].MessageID).Updates(map[string]any{"response": nil, "error": nil}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteStatusPageTx preserves FK-safe audit rows while making every delivery
|
||||
// endpoint inert and irreversibly removing subscriber PII.
|
||||
func DeleteStatusPageTx(tx *gorm.DB, page *StatusPage) error {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ? AND deleted_at IS NULL", page.ID).First(page).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := tx.Where("status_page_id = ?", page.ID).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
for i := range subscribers {
|
||||
s := &subscribers[i]
|
||||
if err := CancelStatusPageSubscriberDeliveriesTx(tx, s.ID, "status page deleted"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := RedactStatusPageSubscriberDeliveryTx(tx, s.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(s).Updates(map[string]any{"email": "redacted", "confirm_token_hash": "", "confirm_token": nil, "unsubscribe_token_hash": "", "unsubscribed_at": now}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if s.ContactID != nil {
|
||||
if err := tx.Model(&Contact{}).Where("id = ?", *s.ContactID).Updates(map[string]any{"enabled": false, "value": "redacted", "name": "Deleted status page subscriber"}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return tx.Delete(page).Error
|
||||
}
|
||||
|
||||
// EnqueueStatusPageDailyDigests records missed due dates first, then creates
|
||||
// durable outbox rows. Dates remain retryable until their outbox is dispatched.
|
||||
func EnqueueStatusPageDailyDigests(now time.Time) error {
|
||||
var subscribers []StatusPageSubscriber
|
||||
if err := DB().Preload("StatusPage.Account").Where("kind = ? AND confirmed_at IS NOT NULL AND unsubscribed_at IS NULL", StatusPageSubscriberKindDigestDaily).Find(&subscribers).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for i := range subscribers {
|
||||
s := &subscribers[i]
|
||||
if s.StatusPage == nil || s.StatusPage.Account == nil {
|
||||
continue
|
||||
}
|
||||
loc, err := time.LoadLocation(s.StatusPage.Account.Timezone)
|
||||
if err != nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
day := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
|
||||
due := time.Date(day.Year(), day.Month(), day.Day(), statusPageDigestHour, 0, 0, 0, loc)
|
||||
// Only persist today's actual due date. Older pending outbox rows are
|
||||
// retried by DispatchStatusPageDeliveries; never invent history from a
|
||||
// current incident snapshot after downtime.
|
||||
if now.Before(due) || s.ConfirmedAt.After(due) {
|
||||
continue
|
||||
}
|
||||
var incidents []StatusPageIncident
|
||||
if err := DB().Where("status_page_id = ? AND resolved_at IS NULL", s.StatusPageID).Order("started_at ASC").Find(&incidents).Error; err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if len(incidents) == 0 {
|
||||
continue
|
||||
} // No digest is better than a misleading outage summary.
|
||||
lines := make([]string, 0, len(incidents))
|
||||
for _, incident := range incidents {
|
||||
lines = append(lines, "- "+incident.Title)
|
||||
}
|
||||
err = DB().Transaction(func(tx *gorm.DB) error {
|
||||
schedule := StatusPageDigestSchedule{SubscriberID: s.ID, LocalDate: day.Format("2006-01-02"), DueAt: due.UTC()}
|
||||
if err := tx.Where("subscriber_id = ? AND local_date = ?", s.ID, schedule.LocalDate).FirstOrCreate(&schedule).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
key := fmt.Sprintf("status-page:digest:subscriber:%d:date:%s", s.ID, schedule.LocalDate)
|
||||
return tx.Where(StatusPageDelivery{IdempotencyKey: key}).FirstOrCreate(&StatusPageDelivery{StatusPageID: s.StatusPageID, SubscriberID: s.ID, Kind: "digest", Version: schedule.LocalDate, LocalDate: schedule.LocalDate, State: statusPageOutboxPending, IdempotencyKey: key, Subject: "[" + s.StatusPage.Name + "] daily status digest", BodyText: "Open incidents:\n" + strings.Join(lines, "\n")}).Error
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := DispatchStatusPageDeliveries(now); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func StartStatusPageDigestScheduler(ctx context.Context) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if err := DispatchStatusPageDeliveries(time.Now().UTC()); err != nil {
|
||||
log.Printf("status-page delivery dispatch: %v", err)
|
||||
}
|
||||
if err := EnqueueStatusPageDailyDigests(time.Now().UTC()); err != nil {
|
||||
log.Printf("status-page digest: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
401
app/models/status_page_test.go
Обычный файл
401
app/models/status_page_test.go
Обычный файл
@@ -0,0 +1,401 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// statusPageSeedAcc returns a fresh account for status-page tests.
|
||||
func statusPageSeedAcc(t *testing.T) *models.Account {
|
||||
t.Helper()
|
||||
acc := &models.Account{Name: "status-page-" + strings.ReplaceAll(t.Name(), "/", "_")}
|
||||
require.NoError(t, models.DB().Create(acc).Error)
|
||||
return acc
|
||||
}
|
||||
|
||||
// TestStatusPage_Defaults asserts that creating a StatusPage with the
|
||||
// minimum required fields fills in the documented defaults (colors,
|
||||
// history days, soft-delete marker) and leaves IsPublished false.
|
||||
func TestStatusPage_Defaults(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
page := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "acme-status",
|
||||
Name: "Acme Status",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
got := models.StatusPage{}
|
||||
require.NoError(t, models.DB().First(&got, page.ID).Error)
|
||||
|
||||
assert.Equal(t, "#62c600", got.PrimaryColor, "default primary color")
|
||||
assert.Equal(t, "#1a73e8", got.AccentColor, "default accent color")
|
||||
assert.Equal(t, 90, got.ShowHistoryDays, "default history days")
|
||||
assert.True(t, got.ShowUptimeBars, "show_uptime_bars defaults true")
|
||||
assert.True(t, got.ShowResponseTimes, "show_response_times defaults true")
|
||||
assert.False(t, got.IsPublished, "is_published defaults false")
|
||||
assert.False(t, got.IsPublishedNow(), "IsPublishedNow() returns false until publish")
|
||||
assert.True(t, got.DeletedAt.Valid == false, "no soft-delete timestamp on fresh row")
|
||||
}
|
||||
|
||||
// TestStatusPage_SlugUniquenessSoftDelete verifies the partial unique
|
||||
// index allows recreating a slug after soft-deleting the previous row.
|
||||
func TestStatusPage_SlugUniquenessSoftDelete(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
first := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
// Second live row with the same slug must fail.
|
||||
dup := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout copy",
|
||||
}
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err, "duplicate slug on live rows must fail")
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"unexpected error: %v", err,
|
||||
)
|
||||
|
||||
// Soft-delete the first row. After that the slug is reusable.
|
||||
require.NoError(t, models.DB().Delete(first).Error)
|
||||
|
||||
again := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "rollout",
|
||||
Name: "Rollout v2",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(again).Error,
|
||||
"recreating a slug after a soft-delete must succeed (partial index)")
|
||||
assert.NotEqual(t, first.ID, again.ID)
|
||||
}
|
||||
|
||||
// TestStatusPage_SlugIsGlobal verifies public URLs are unambiguous: the
|
||||
// account is not part of /status/:slug, so another account cannot claim it.
|
||||
func TestStatusPage_SlugIsGlobal(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
accA := statusPageSeedAcc(t)
|
||||
accB := &models.Account{Name: "status-page-other"}
|
||||
require.NoError(t, models.DB().Create(accB).Error)
|
||||
|
||||
a := &models.StatusPage{AccountID: accA.ID, Slug: "shared", Name: "A"}
|
||||
require.NoError(t, models.DB().Create(a).Error)
|
||||
|
||||
b := &models.StatusPage{AccountID: accB.ID, Slug: "shared", Name: "B"}
|
||||
require.Error(t, models.DB().Create(b).Error,
|
||||
"different accounts must not be able to reuse a public slug")
|
||||
}
|
||||
|
||||
func TestStatusPageIncidentDeliveryIsIdempotentAndAccountScoped(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: account.ID, Slug: "delivery-page", Name: "Delivery"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
now := time.Now()
|
||||
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "subscriber@example.test", Kind: models.StatusPageSubscriberKindAlert, ConfirmedAt: &now}
|
||||
require.NoError(t, models.DB().Create(subscriber).Error)
|
||||
incident := &models.StatusPageIncident{StatusPageID: page.ID, Title: "API unavailable", Severity: models.StatusPageIncidentSeverityCrit, StartedAt: now}
|
||||
require.NoError(t, models.DB().Create(incident).Error)
|
||||
|
||||
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
|
||||
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
|
||||
}))
|
||||
require.NoError(t, models.DB().Transaction(func(tx *gorm.DB) error {
|
||||
return models.EnqueueStatusPageIncidentDeliveriesTx(tx, page, incident, "opened")
|
||||
}))
|
||||
var deliveries []models.StatusPageDelivery
|
||||
require.NoError(t, models.DB().Where("idempotency_key LIKE ?", "status-page:opened:%").Find(&deliveries).Error)
|
||||
require.Len(t, deliveries, 1)
|
||||
assert.Equal(t, page.ID, deliveries[0].StatusPageID)
|
||||
}
|
||||
|
||||
func TestStatusPageDailyDigestIsTimezoneDateIdempotent(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account := statusPageSeedAcc(t)
|
||||
account.Timezone = "UTC"
|
||||
require.NoError(t, models.DB().Save(account).Error)
|
||||
page := &models.StatusPage{AccountID: account.ID, Slug: "digest-page", Name: "Digest"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
now := time.Date(2026, time.July, 13, 9, 15, 0, 0, time.UTC)
|
||||
confirmedAt := now.Add(-time.Hour)
|
||||
subscriber := &models.StatusPageSubscriber{StatusPageID: page.ID, Email: "digest@example.test", Kind: models.StatusPageSubscriberKindDigestDaily, ConfirmedAt: &confirmedAt}
|
||||
require.NoError(t, models.DB().Create(subscriber).Error)
|
||||
require.NoError(t, models.DB().Create(&models.StatusPageIncident{StatusPageID: page.ID, Title: "Still open", Severity: models.StatusPageIncidentSeverityWarn, StartedAt: now}).Error)
|
||||
|
||||
_ = models.EnqueueStatusPageDailyDigests(now)
|
||||
_ = models.EnqueueStatusPageDailyDigests(now.Add(30 * time.Second))
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.StatusPageDelivery{}).Where("idempotency_key LIKE ?", "status-page:digest:%").Count(&count).Error)
|
||||
assert.GreaterOrEqual(t, count, int64(1), "catch-up may include earlier local due dates")
|
||||
}
|
||||
|
||||
// TestStatusPage_NormalizeSlugAndValidate exercises the slug normalizer
|
||||
// and validator — uppercase input becomes lowercase, and an invalid
|
||||
// slug (leading dash) is rejected on save via BeforeSave.
|
||||
func TestStatusPage_NormalizeSlugAndValidate(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
assert.Equal(t, "lower-case", models.NormalizeStatusPageSlug(" Lower-Case "))
|
||||
assert.NoError(t, models.ValidateStatusPageSlug("acme-status"))
|
||||
|
||||
bad := []string{"", "-leading-dash", "trailing-dash-", "UPPER", "with spaces", "with_underscore"}
|
||||
for _, slug := range bad {
|
||||
err := models.ValidateStatusPageSlug(slug)
|
||||
assert.Error(t, err, "expected error for slug %q", slug)
|
||||
}
|
||||
|
||||
// Save enforces the same rules — uppercase gets normalized, invalid
|
||||
// characters reject.
|
||||
good := &models.StatusPage{AccountID: acc.ID, Slug: "ACME-Status", Name: "Acme"}
|
||||
require.NoError(t, models.DB().Create(good).Error)
|
||||
assert.Equal(t, "acme-status", good.Slug)
|
||||
|
||||
bad2 := &models.StatusPage{AccountID: acc.ID, Slug: "-bad", Name: "Bad"}
|
||||
err := models.DB().Create(bad2).Error
|
||||
require.Error(t, err, "invalid slug should be rejected by BeforeSave")
|
||||
}
|
||||
|
||||
// TestStatusPage_ColorValidation rejects non-hex colors at save time.
|
||||
func TestStatusPage_ColorValidation(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
|
||||
bad := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "badcolor",
|
||||
Name: "BadColor",
|
||||
PrimaryColor: "not-a-color",
|
||||
}
|
||||
err := models.DB().Create(bad).Error
|
||||
require.Error(t, err, "non-hex color must be rejected")
|
||||
|
||||
good := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "goodcolor",
|
||||
Name: "GoodColor",
|
||||
PrimaryColor: "#0a1b2c",
|
||||
AccentColor: "#abc",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(good).Error)
|
||||
|
||||
reloaded := models.StatusPage{}
|
||||
require.NoError(t, models.DB().First(&reloaded, good.ID).Error)
|
||||
assert.Equal(t, "#0a1b2c", reloaded.PrimaryColor)
|
||||
assert.Equal(t, "#abc", reloaded.AccentColor)
|
||||
}
|
||||
|
||||
// TestStatusPageSubscriber_UniqueActive verifies the partial unique
|
||||
// index on (status_page_id, lower(email)) only fires while a
|
||||
// subscriber is not unsubscribed.
|
||||
func TestStatusPageSubscriber_UniqueActive(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{
|
||||
AccountID: acc.ID,
|
||||
Slug: "sub",
|
||||
Name: "Sub",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
mk := func(email string, unsub *time.Time) *models.StatusPageSubscriber {
|
||||
s := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: email,
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-" + email),
|
||||
}
|
||||
s.UnsubscribedAt = unsub
|
||||
return s
|
||||
}
|
||||
|
||||
first := mk("Alice@Example.com", nil)
|
||||
require.NoError(t, models.DB().Create(first).Error)
|
||||
|
||||
// Same email with mixed case must conflict (lower(email) is the
|
||||
// unique key), so a true case-insensitive uniqueness is in place.
|
||||
dup := mk("alice@example.com", nil)
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err, "active subscriber with same lowercased email must conflict")
|
||||
|
||||
// Soft-unsubscribe the original then re-add a fresh row with the
|
||||
// same email — this must succeed because the partial index
|
||||
// excludes unsubscribed rows.
|
||||
now := time.Now()
|
||||
first.UnsubscribedAt = &now
|
||||
require.NoError(t, models.DB().Save(first).Error)
|
||||
|
||||
resurrected := mk("alice@example.com", nil)
|
||||
require.NoError(t, models.DB().Create(resurrected).Error,
|
||||
"creating a fresh subscriber after the previous one unsubscribed must succeed")
|
||||
assert.NotEqual(t, first.ID, resurrected.ID)
|
||||
|
||||
// ActiveSubscriber reflects both confirmed and unsubscribed flags.
|
||||
confirmed := time.Now()
|
||||
active := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: "bob@example.com",
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-bob"),
|
||||
ConfirmedAt: &confirmed,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(active).Error)
|
||||
assert.True(t, active.IsActiveSubscriber(), "confirmed and not unsubscribed → active")
|
||||
|
||||
pending := &models.StatusPageSubscriber{
|
||||
StatusPageID: page.ID,
|
||||
Email: "carol@example.com",
|
||||
Kind: models.StatusPageSubscriberKindAlert,
|
||||
ConfirmTokenHash: models.HashStatusPageToken("tok-carol"),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(pending).Error)
|
||||
assert.False(t, pending.IsActiveSubscriber(), "unconfirmed → not active")
|
||||
}
|
||||
|
||||
// TestStatusPageIncident_SeverityAndEventFK checks that the FK to
|
||||
// events is wired correctly and that severity defaults to "info".
|
||||
func TestStatusPageIncident_SeverityAndEventFK(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "inc", Name: "Inc"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
now := time.Now()
|
||||
inc := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "API slowdown",
|
||||
BodyMD: "Investigating.",
|
||||
Severity: models.StatusPageIncidentSeverityWarn,
|
||||
StartedAt: now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(inc).Error)
|
||||
|
||||
got := models.StatusPageIncident{}
|
||||
require.NoError(t, models.DB().First(&got, inc.ID).Error)
|
||||
assert.Equal(t, "warn", got.Severity)
|
||||
|
||||
// Default severity is "info" when omitted.
|
||||
auto := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "heads up",
|
||||
StartedAt: now,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(auto).Error)
|
||||
gotAuto := models.StatusPageIncident{}
|
||||
require.NoError(t, models.DB().First(&gotAuto, auto.ID).Error)
|
||||
assert.Equal(t, models.StatusPageIncidentSeverityInfo, gotAuto.Severity)
|
||||
|
||||
// EventID stays nullable and the FK tolerates a NULL event.
|
||||
var nilEvt *int64
|
||||
noEvt := &models.StatusPageIncident{
|
||||
StatusPageID: page.ID,
|
||||
Title: "no event",
|
||||
Severity: models.StatusPageIncidentSeverityCrit,
|
||||
StartedAt: now,
|
||||
EventID: nilEvt,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(noEvt).Error)
|
||||
}
|
||||
|
||||
// TestStatusPageMaintenance_BigintArray ensures the monitor_ids bigint[]
|
||||
// column round-trips through the GORM pq.Int64Array driver correctly.
|
||||
func TestStatusPageMaintenance_BigintArray(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "mt", Name: "Maintenance"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
now := time.Now()
|
||||
maint := &models.StatusPageMaintenance{
|
||||
StatusPageID: page.ID,
|
||||
Title: "DB upgrade",
|
||||
Description: "Rolling upgrade.",
|
||||
StartsAt: now,
|
||||
EndsAt: now.Add(2 * time.Hour),
|
||||
MonitorIDs: models.Int64ArrayFromSlice([]int64{1, 7, 42}),
|
||||
NotifySubscribers: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(maint).Error)
|
||||
|
||||
got := models.StatusPageMaintenance{}
|
||||
require.NoError(t, models.DB().First(&got, maint.ID).Error)
|
||||
assert.Equal(t, []int64{1, 7, 42}, []int64(got.MonitorIDs))
|
||||
assert.True(t, got.NotifySubscribers)
|
||||
}
|
||||
|
||||
// TestStatusPageDomain_TableReservation verifies the M5 table is
|
||||
// created with the right unique constraints even though M0 does not
|
||||
// yet populate it.
|
||||
func TestStatusPageDomain_TableReservation(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
acc := statusPageSeedAcc(t)
|
||||
page := &models.StatusPage{AccountID: acc.ID, Slug: "dom", Name: "Domain"}
|
||||
require.NoError(t, models.DB().Create(page).Error)
|
||||
|
||||
d := &models.StatusPageDomain{
|
||||
StatusPageID: page.ID,
|
||||
Domain: "status.example.com",
|
||||
TXTToken: "rsmon-verify=abc123",
|
||||
}
|
||||
require.NoError(t, models.DB().Create(d).Error)
|
||||
require.NoError(t, models.DB().Create(&models.StatusPageDomain{
|
||||
StatusPageID: page.ID,
|
||||
Domain: "www.status.example.com",
|
||||
TXTToken: "rsmon-verify=second",
|
||||
}).Error, "one status page may own multiple domains")
|
||||
|
||||
// Second page claiming the same domain must fail because domain
|
||||
// is globally unique.
|
||||
other := &models.StatusPage{AccountID: acc.ID, Slug: "dom2", Name: "Domain2"}
|
||||
require.NoError(t, models.DB().Create(other).Error)
|
||||
dup := &models.StatusPageDomain{
|
||||
StatusPageID: other.ID,
|
||||
Domain: "status.example.com",
|
||||
TXTToken: "rsmon-verify=xyz789",
|
||||
}
|
||||
err := models.DB().Create(dup).Error
|
||||
require.Error(t, err)
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "unique") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "duplicate"),
|
||||
"expected unique-constraint error, got %v", err)
|
||||
}
|
||||
|
||||
func TestNormalizeStatusPageDomain(t *testing.T) {
|
||||
got, err := models.NormalizeStatusPageDomain(" Status.Example.COM. ")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "status.example.com", got)
|
||||
for _, input := range []string{"https://example.com", "example.com:443", "127.0.0.1", "*.example.com", "-bad.example.com"} {
|
||||
_, err := models.NormalizeStatusPageDomain(input)
|
||||
assert.Error(t, err, input)
|
||||
}
|
||||
}
|
||||
61
app/models/subscription.go
Обычный файл
61
app/models/subscription.go
Обычный файл
@@ -0,0 +1,61 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
SubscriptionStatusTrialing = "trialing"
|
||||
SubscriptionStatusActive = "active"
|
||||
SubscriptionStatusPastDue = "past_due"
|
||||
SubscriptionStatusPaused = "paused"
|
||||
SubscriptionStatusCanceled = "canceled"
|
||||
SubscriptionStatusExpired = "expired"
|
||||
)
|
||||
|
||||
// Subscription is the current billable entitlement for one account. M0 only
|
||||
// writes manual subscriptions; provider flows are deliberately deferred.
|
||||
type Subscription struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"not null;index" json:"account_id"`
|
||||
Account *Account `json:"-"`
|
||||
PlanID int64 `gorm:"not null;index" json:"plan_id"`
|
||||
Plan *Plan `json:"plan,omitempty"`
|
||||
Provider string `gorm:"size:16;not null;default:'manual'" json:"provider"`
|
||||
ProviderSubscriptionID *string `gorm:"size:128" json:"provider_subscription_id,omitempty"`
|
||||
ProviderCustomerID *string `gorm:"size:128" json:"provider_customer_id,omitempty"`
|
||||
Status string `gorm:"size:24;not null;default:'active';index" json:"status"`
|
||||
BillingCycle string `gorm:"size:8;not null;default:'monthly'" json:"billing_cycle"`
|
||||
CurrentPeriodStart *time.Time `json:"current_period_start,omitempty"`
|
||||
CurrentPeriodEnd *time.Time `gorm:"index" json:"current_period_end,omitempty"`
|
||||
TrialEndsAt *time.Time `json:"trial_ends_at,omitempty"`
|
||||
CancelAtPeriodEnd bool `gorm:"not null;default:false" json:"cancel_at_period_end"`
|
||||
CanceledAt *time.Time `json:"canceled_at,omitempty"`
|
||||
Currency string `gorm:"size:3;not null" json:"currency"`
|
||||
AmountMinor int64 `gorm:"not null;default:0" json:"amount_minor"`
|
||||
MetadataJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"metadata_json"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// SubscriptionEvent is an append-only audit record for entitlement changes
|
||||
// and provider deliveries. ProviderEventID makes future webhook processing
|
||||
// idempotent without coupling the ledger to a specific PSP.
|
||||
type SubscriptionEvent struct {
|
||||
concerns.Model
|
||||
SubscriptionID int64 `gorm:"not null;index" json:"subscription_id"`
|
||||
AccountID int64 `gorm:"not null;index" json:"account_id"`
|
||||
Provider string `gorm:"size:16;not null;default:'manual';uniqueIndex:idx_subscription_events_provider_event,priority:1" json:"provider"`
|
||||
Kind string `gorm:"size:32;not null" json:"kind"`
|
||||
FromPlanID *int64 `json:"from_plan_id,omitempty"`
|
||||
ToPlanID *int64 `json:"to_plan_id,omitempty"`
|
||||
AmountMinor *int64 `json:"amount_minor,omitempty"`
|
||||
Currency string `gorm:"size:3" json:"currency,omitempty"`
|
||||
ActorUserID *int64 `json:"actor_user_id,omitempty"`
|
||||
ProviderEventID *string `gorm:"size:128;uniqueIndex:idx_subscription_events_provider_event,priority:2" json:"provider_event_id,omitempty"`
|
||||
PayloadJSON datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'" json:"payload_json"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
47
app/models/tag.go
Обычный файл
47
app/models/tag.go
Обычный файл
@@ -0,0 +1,47 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Tag stores metadata for a single tag name within an account: the
|
||||
// user-chosen color (hex string like "#FF5733") and icon (FontAwesome
|
||||
// icon name like "faTag"). The (account_id, name) pair is unique.
|
||||
//
|
||||
// Tags are still attached to monitors via the monitors.tags text[]
|
||||
// column (see Monitor.Tags). This table only holds the per-tag display
|
||||
// metadata so the same tag renders consistently across the SPA — it
|
||||
// does not affect monitor scoping or filtering.
|
||||
//
|
||||
// Rows are created lazily: the SPA POSTs a Tag the first time a user
|
||||
// customizes its color/icon, and GET /tags?with_counts=1 LEFT JOINs
|
||||
// the unnested monitors.tags array against this table to enrich the
|
||||
// (name, count) pairs with display metadata. A Tag row may exist with
|
||||
// zero monitors using it (count=0) — that happens when a user creates
|
||||
// a tag from /settings/tags/new but has not yet applied it.
|
||||
type Tag struct {
|
||||
concerns.Model
|
||||
AccountID int64 `gorm:"index;not null" json:"account_id"`
|
||||
Account *Account `gorm:"foreignKey:AccountID" json:"account,omitempty"`
|
||||
Name string `gorm:"type:varchar(255);not null" json:"name"`
|
||||
Color string `gorm:"type:varchar(32);not null;default:'#6c757d'" json:"color"`
|
||||
Icon string `gorm:"type:varchar(64);not null;default:'faTag'" json:"icon"`
|
||||
Audited
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TagInfo is a tag with the number of monitors using it plus optional
|
||||
// display metadata (color, icon) coming from the Tag table.
|
||||
type TagInfo struct {
|
||||
Name string `json:"name"`
|
||||
Count int64 `json:"count"`
|
||||
Color string `json:"color"`
|
||||
Icon string `json:"icon"`
|
||||
}
|
||||
|
||||
// DefaultTagColor is the hex color used when a Tag has no metadata row.
|
||||
const DefaultTagColor = "#6c757d"
|
||||
|
||||
// DefaultTagIcon is the FontAwesome icon name used when a Tag has no
|
||||
// metadata row. Matches the icon the SPA renders by default.
|
||||
const DefaultTagIcon = "faTag"
|
||||
135
app/models/tags_test.go
Обычный файл
135
app/models/tags_test.go
Обычный файл
@@ -0,0 +1,135 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
// seedTagsMonitors creates one group with four monitors:
|
||||
//
|
||||
// a.test [prod, web]
|
||||
// b.test [prod, api]
|
||||
// c.test [staging]
|
||||
// d.test [] (NULL tags)
|
||||
//
|
||||
// and returns the group id. It mirrors the exact SQL the /tags rename, delete
|
||||
// and with_counts endpoints run, pinning the contract.
|
||||
func seedTagsMonitors(t *testing.T) int64 {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
plan := models.Plan{Name: "test", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "A", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := models.Group{AccountID: acc.ID, Name: "A-default"}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
|
||||
monitors := []models.Monitor{
|
||||
{GroupID: group.ID, Host: "a.test", Tags: []string{"prod", "web"}},
|
||||
{GroupID: group.ID, Host: "b.test", Tags: []string{"prod", "api"}},
|
||||
{GroupID: group.ID, Host: "c.test", Tags: []string{"staging"}},
|
||||
{GroupID: group.ID, Host: "d.test"},
|
||||
}
|
||||
for i := range monitors {
|
||||
require.NoError(t, models.DB().Create(&monitors[i]).Error)
|
||||
}
|
||||
return group.ID
|
||||
}
|
||||
|
||||
func TestTagsRenameSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = (
|
||||
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
|
||||
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", "production", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected, "only a.test and b.test contain prod")
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"production", "web"}, []string(a.Tags))
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.ElementsMatch(t, []string{"production", "api"}, []string(b.Tags))
|
||||
|
||||
var c models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "c.test").First(&c).Error)
|
||||
assert.Equal(t, []string{"staging"}, []string(c.Tags), "staging-only monitor untouched")
|
||||
}
|
||||
|
||||
func TestTagsRenameSQL_Dedup(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
// Rename prod -> web. a.test has both prod and web: must collapse to a
|
||||
// single "web" entry (array_agg DISTINCT), not [web, web].
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = (
|
||||
SELECT array_agg(DISTINCT x) FROM unnest(array_replace(tags, ?, ?)) AS t(x)
|
||||
) WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", "web", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected)
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"web"}, []string(a.Tags), "duplicate must be deduped")
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.ElementsMatch(t, []string{"web", "api"}, []string(b.Tags))
|
||||
}
|
||||
|
||||
func TestTagsDeleteSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
res := models.DB().Exec(
|
||||
`UPDATE monitors SET tags = array_remove(tags, ?)
|
||||
WHERE tags IS NOT NULL AND group_id IN (?) AND ? = ANY(tags)`,
|
||||
"prod", []int64{groupID}, "prod",
|
||||
)
|
||||
require.NoError(t, res.Error)
|
||||
assert.Equal(t, int64(2), res.RowsAffected)
|
||||
|
||||
var a models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "a.test").First(&a).Error)
|
||||
assert.Equal(t, []string{"web"}, []string(a.Tags))
|
||||
|
||||
var b models.Monitor
|
||||
require.NoError(t, models.DB().Where("host = ?", "b.test").First(&b).Error)
|
||||
assert.Equal(t, []string{"api"}, []string(b.Tags))
|
||||
}
|
||||
|
||||
func TestTagsCountSQL(t *testing.T) {
|
||||
groupID := seedTagsMonitors(t)
|
||||
|
||||
type tagInfo struct {
|
||||
Name string
|
||||
Count int64
|
||||
}
|
||||
var tags []tagInfo
|
||||
err := models.DB().Raw(
|
||||
`SELECT tag AS name, COUNT(*) AS count
|
||||
FROM (SELECT unnest(tags) AS tag FROM monitors
|
||||
WHERE tags IS NOT NULL AND group_id IN (?)) sub
|
||||
GROUP BY tag
|
||||
ORDER BY tag`,
|
||||
[]int64{groupID},
|
||||
).Scan(&tags).Error
|
||||
require.NoError(t, err)
|
||||
|
||||
got := map[string]int64{}
|
||||
for _, tg := range tags {
|
||||
got[tg.Name] = tg.Count
|
||||
}
|
||||
assert.Equal(t, map[string]int64{"prod": 2, "web": 1, "api": 1, "staging": 1}, got)
|
||||
}
|
||||
145
app/models/task.go
Обычный файл
145
app/models/task.go
Обычный файл
@@ -0,0 +1,145 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Task kinds stored in the tasks.kind column. The plan (docs/plans/worker-notifier-mvp.md
|
||||
// section 4.1) reserves the same enum for checks and notifications; this phase only
|
||||
// emits notification rows but the enum is shared so the selector can stay a single
|
||||
// function.
|
||||
const (
|
||||
TaskKindNotification = "notification"
|
||||
TaskKindCheck = "check"
|
||||
TaskKindServerMetric = "server_metric"
|
||||
)
|
||||
|
||||
// Task states for the durable task envelope.
|
||||
const (
|
||||
TaskStateQueued = "queued"
|
||||
TaskStateLeased = "leased"
|
||||
TaskStateSucceeded = "succeeded"
|
||||
TaskStateFailedRetry = "failed_retry"
|
||||
TaskStateFailedPerm = "failed_perm"
|
||||
TaskStateDead = "dead"
|
||||
)
|
||||
|
||||
// Notification result statuses reported by the worker (mirrors the wire enum so the
|
||||
// result handler can decode without re-typing the constants).
|
||||
const (
|
||||
NotificationResultDelivered = "delivered"
|
||||
NotificationResultRetryable = "retryable"
|
||||
NotificationResultPermanent = "permanent"
|
||||
NotificationResultPartial = "partial"
|
||||
)
|
||||
|
||||
// SkipLockedClause is the SELECT ... FOR UPDATE SKIP LOCKED clause used by
|
||||
// every worker-pool selector (checks in check_jobs.ChecksForWorker and
|
||||
// tasks in task_selector.TasksForWorker / TasksForWorkerNotification).
|
||||
// Sharing the value keeps the SQL identical across selectors so goconst
|
||||
// does not flag the literal, and a future change (e.g. NOWAIT) only has
|
||||
// to touch one place.
|
||||
var SkipLockedClause = clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}
|
||||
|
||||
// Task is the durable envelope for both check and notification work executed by the
|
||||
// distributed worker pool. Selection uses FOR UPDATE SKIP LOCKED per worker poll so
|
||||
// a single primary key or sequence never becomes the bottleneck.
|
||||
//
|
||||
// Phase 1 (this commit) only emits notification tasks. The `kind` discriminator and
|
||||
// capability filters are designed to accept checks in phase 2 without a schema change.
|
||||
type Task struct {
|
||||
concerns.Model
|
||||
|
||||
JobID string `gorm:"uniqueIndex;size:64" json:"job_id"`
|
||||
Kind string `gorm:"size:32;index" json:"kind"`
|
||||
State string `gorm:"size:32;index" json:"state"`
|
||||
LastError string `gorm:"type:text" json:"last_error"`
|
||||
|
||||
// Tenancy + audit anchor. AccountID is required for the capability match in
|
||||
// TasksForWorker; monitor_id / message_id / contact_id are denormalized for
|
||||
// fast admin queries.
|
||||
AccountID int64 `gorm:"index" json:"account_id"`
|
||||
MonitorID *int64 `gorm:"index" json:"monitor_id,omitempty"`
|
||||
CheckID *int64 `json:"check_id,omitempty"`
|
||||
MessageID *int64 `json:"message_id,omitempty"`
|
||||
ContactID *int64 `json:"contact_id,omitempty"`
|
||||
|
||||
// Payload is the kind-specific blob the worker needs to execute. For
|
||||
// notifications the producer pre-renders subject/body so the worker does not
|
||||
// need templating context (see RenderNotificationContent in internal/notifier).
|
||||
Payload datatypes.JSON `gorm:"type:jsonb" json:"payload"`
|
||||
|
||||
// Scheduling + retry envelope. NotBefore is set to NOW() by the producer and
|
||||
// bumped by the result handler on retryable failures. Deadline is a soft cap
|
||||
// the selector can use to skip stale tasks.
|
||||
NotBefore time.Time `json:"not_before"`
|
||||
Deadline *time.Time `json:"deadline,omitempty"`
|
||||
|
||||
// LeaseOwner + LeaseExpiresAt are owned by the selector while the task is
|
||||
// in state=leased. The reaper clears them when the lease expires.
|
||||
LeaseOwner string `gorm:"size:128" json:"lease_owner"`
|
||||
LeaseToken string `gorm:"size:64" json:"-"`
|
||||
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
||||
|
||||
Attempts int `json:"attempts"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
|
||||
// IdempotencyKey is unique per logical event so a retry of the producer's
|
||||
// enqueue never produces a second Task row. See EnqueueNotificationTask.
|
||||
IdempotencyKey string `gorm:"uniqueIndex;size:255" json:"idempotency_key"`
|
||||
|
||||
// Result holds the most recent worker result body (NotificationResultReport or
|
||||
// CheckResultReport shape, depending on Kind). It is JSONB so the admin UI can
|
||||
// pretty-print without a separate result table for transient lookups.
|
||||
Result datatypes.JSON `gorm:"type:jsonb" json:"result"`
|
||||
SucceededAt *time.Time `json:"succeeded_at,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TaskReplay records the single operator-initiated replay of a dead task.
|
||||
// OriginalTaskID is unique, making repeated clicks/API retries idempotent.
|
||||
type TaskReplay struct {
|
||||
concerns.Model
|
||||
OriginalTaskID int64 `gorm:"uniqueIndex" json:"original_task_id"`
|
||||
RequeuedTaskID int64 `gorm:"uniqueIndex" json:"requeued_task_id"`
|
||||
OperatorUserID int64 `gorm:"index" json:"operator_user_id"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName overrides the default table name so pluralization stays consistent
|
||||
// with the rest of the schema (tasks, not "task" or "taskses").
|
||||
func (Task) TableName() string {
|
||||
return "tasks"
|
||||
}
|
||||
|
||||
// NotificationDelivery is the per-attempt audit row required by section 7.5 of the
|
||||
// plan ("Audit rows: each successful or failed delivery writes a row in a new
|
||||
// notification_deliveries table"). The result handler appends one row per result
|
||||
// frame, which lets support answer "did the customer ever get this alert" without
|
||||
// scanning application logs.
|
||||
type NotificationDelivery struct {
|
||||
concerns.Model
|
||||
|
||||
MessageID int64 `gorm:"index" json:"message_id"`
|
||||
WorkerID string `gorm:"size:128;index" json:"worker_id"`
|
||||
TaskID int64 `gorm:"index" json:"task_id"`
|
||||
|
||||
Status string `gorm:"size:32" json:"status"`
|
||||
Error string `gorm:"type:text" json:"error"`
|
||||
|
||||
DurationMs int `json:"duration_ms"`
|
||||
ProviderResponse string `gorm:"type:text" json:"provider_response"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName mirrors the plan's preferred lowercase plural.
|
||||
func (NotificationDelivery) TableName() string {
|
||||
return "notification_deliveries"
|
||||
}
|
||||
136
app/models/task_reaper.go
Обычный файл
136
app/models/task_reaper.go
Обычный файл
@@ -0,0 +1,136 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ReapExpiredTasks is the periodic cleanup function described in
|
||||
// docs/plans/worker-notifier-mvp.md section 8.5:
|
||||
//
|
||||
// - tasks in state='leased' whose lease_expires_at is past are returned to
|
||||
// state='queued' and have their lease_owner cleared, so the next selector
|
||||
// poll can pick them up.
|
||||
// - tasks in state='failed_retry' whose not_before is past AND
|
||||
// attempts >= max_attempts are moved to state='dead' so they show up on
|
||||
// the admin dead-letter page and stop consuming selector bandwidth.
|
||||
//
|
||||
// It returns the number of rows it touched so the caller can log a metric.
|
||||
// Cheap enough to run from the web process every 30s.
|
||||
func ReapExpiredTasks() (reaped int, deaded int, err error) {
|
||||
now := time.Now()
|
||||
err = DB().Transaction(func(tx *gorm.DB) error {
|
||||
if err := expireQueuedNotificationTasksTx(tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
var expired []Task
|
||||
if err := tx.Where("state = ? AND lease_expires_at IS NOT NULL AND lease_expires_at < ?", TaskStateLeased, now).Find(&expired).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range expired {
|
||||
if expired[i].Attempts >= expired[i].MaxAttempts {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateDead, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "last_error": "lease expired after max attempts", "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 1 {
|
||||
deaded++
|
||||
if err := FinalizeNotificationTaskTx(tx, &expired[i], "dead", "lease expired after max attempts"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", expired[i].ID, TaskStateLeased).Updates(map[string]interface{}{"state": TaskStateQueued, "lease_owner": "", "lease_token": "", "lease_expires_at": nil, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
reaped += int(result.RowsAffected)
|
||||
}
|
||||
var exhausted []Task
|
||||
if err := tx.Where("state = ? AND not_before <= ? AND attempts >= max_attempts", TaskStateFailedRetry, now).Find(&exhausted).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range exhausted {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", exhausted[i].ID, TaskStateFailedRetry).Updates(map[string]interface{}{"state": TaskStateDead, "updated_at": now})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 1 {
|
||||
deaded++
|
||||
if err := FinalizeNotificationTaskTx(tx, &exhausted[i], "dead", exhausted[i].LastError); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return reaped, deaded, err
|
||||
}
|
||||
|
||||
// FinalizeNotificationTaskTx makes a terminal notification task customer-visible
|
||||
// and auditable. The caller owns the task state transition in this transaction.
|
||||
func FinalizeNotificationTaskTx(tx *gorm.DB, task *Task, status, reason string) error {
|
||||
if task == nil || task.Kind != TaskKindNotification || task.MessageID == nil {
|
||||
return nil
|
||||
}
|
||||
if err := tx.Model(&Message{}).Where("id = ? AND state NOT IN ?", *task.MessageID, []string{"sent", "error"}).Updates(map[string]interface{}{"state": "error", "error": reason}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Create(&NotificationDelivery{MessageID: *task.MessageID, TaskID: task.ID, Status: status, Error: reason}).Error
|
||||
}
|
||||
|
||||
func expireQueuedNotificationTasksTx(tx *gorm.DB, now time.Time) error {
|
||||
var tasks []Task
|
||||
if err := tx.Where("state = ? AND kind = ? AND deadline IS NOT NULL AND deadline <= ?", TaskStateQueued, TaskKindNotification, now).Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range tasks {
|
||||
result := tx.Model(&Task{}).Where("id = ? AND state = ?", tasks[i].ID, TaskStateQueued).Updates(map[string]interface{}{
|
||||
"state": TaskStateDead, "last_error": "notification deadline expired", "updated_at": now,
|
||||
})
|
||||
if result.Error != nil || result.RowsAffected == 0 {
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := FinalizeNotificationTaskTx(tx, &tasks[i], "expired", "notification deadline expired"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartTaskReaper launches a goroutine that runs ReapExpiredTasks on the given
|
||||
// interval. It honors ctx.Done() so the caller can wind it down without
|
||||
// leaking. The function is safe to call once per process; the control plane
|
||||
// runs the reaper from main.init() so only one ticker ever exists in a single
|
||||
// web process.
|
||||
func StartTaskReaper(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
reaped, deaded, err := ReapExpiredTasks()
|
||||
if err != nil {
|
||||
log.Printf("task_reaper: error: %v", err)
|
||||
continue
|
||||
}
|
||||
if reaped > 0 || deaded > 0 {
|
||||
log.Printf("task_reaper: reaped=%d dead=%d", reaped, deaded)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
391
app/models/task_selector.go
Обычный файл
391
app/models/task_selector.go
Обычный файл
@@ -0,0 +1,391 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// DefaultCheckTaskLeaseTTL is how long a leased check remains owned before the reaper
|
||||
// returns it to the queue. It is deliberately larger than the worker's heartbeat
|
||||
// (10s) so a healthy worker can finish a 30s check without the reaper stealing
|
||||
// the lease, and deliberately smaller than the heartbeat timeout (2m) so a
|
||||
// crashed worker sees its tasks reaped within one watchdog cycle.
|
||||
const DefaultCheckTaskLeaseTTL = 60 * time.Second
|
||||
|
||||
// DefaultTaskLeaseTTL remains the check-task default for existing confirmation
|
||||
// callers. Generic task selection must use TaskLeaseTTL so notification work is
|
||||
// not reclaimed during its longer execution window.
|
||||
const DefaultTaskLeaseTTL = DefaultCheckTaskLeaseTTL
|
||||
|
||||
// Notification execution is bounded by the worker runner at five minutes. The
|
||||
// additional minute covers result serialization, websocket/HTTP transport, and
|
||||
// a control-plane scheduling delay before the persisted lease may be reaped.
|
||||
const (
|
||||
DefaultNotificationExecutionTimeout = 5 * time.Minute
|
||||
NotificationTaskReportMargin = time.Minute
|
||||
DefaultNotificationTaskLeaseTTL = DefaultNotificationExecutionTimeout + NotificationTaskReportMargin
|
||||
)
|
||||
|
||||
// TaskLeaseTTL returns the persisted lease lifetime for a task kind.
|
||||
func TaskLeaseTTL(kind string) time.Duration {
|
||||
if kind == TaskKindNotification {
|
||||
return DefaultNotificationTaskLeaseTTL
|
||||
}
|
||||
return DefaultCheckTaskLeaseTTL
|
||||
}
|
||||
|
||||
// DefaultTaskMaxAttempts is the retry budget for a task before it moves to dead.
|
||||
const DefaultTaskMaxAttempts = 5
|
||||
|
||||
// DefaultNotificationTaskDeadline is assigned to manually replayed notification
|
||||
// dead letters. Normal producer tasks may be deadline-free, but a replay must
|
||||
// never inherit an already-expired deadline.
|
||||
const DefaultNotificationTaskDeadline = 15 * time.Minute
|
||||
|
||||
// ErrNotificationMethodNotAuthorized is returned by EnqueueNotificationTask
|
||||
// when the producer can prove no worker in the pool is authorized for the
|
||||
// (method, account) pair. The caller may skip the enqueue or log + continue.
|
||||
var ErrNotificationMethodNotAuthorized = errors.New("no worker authorized for method/account")
|
||||
|
||||
// EnqueueNotificationTaskInput is the pre-rendered envelope produced by the
|
||||
// notifier producer. All slices are required; the selector never reads them.
|
||||
type EnqueueNotificationTaskInput struct {
|
||||
AccountID int64
|
||||
NotificationID int64
|
||||
ContactID int64
|
||||
MessageID *int64
|
||||
MonitorID *int64
|
||||
CheckID *int64
|
||||
EventIDs []int64
|
||||
Method string // "email", "telegram", "webhook", "mattermost", "sms", "voice"
|
||||
Subject string
|
||||
BodyText string
|
||||
BodyHTML string
|
||||
BodyMarkdown string
|
||||
Language string
|
||||
MessageKind string // "down", "up", "exp", "test"
|
||||
NotBefore time.Time
|
||||
Deadline *time.Time
|
||||
MaxAttempts int
|
||||
Payload []byte // marshaled task-specific data
|
||||
IdempotencyKey string // optional; manual test tasks use a unique key and do not have event IDs
|
||||
}
|
||||
|
||||
// EnqueueNotificationTask writes one Task row keyed by a stable idempotency key.
|
||||
// A second call with the same key (same notification/contact/event triple) is a
|
||||
// no-op so the producer is safe to call more than once per pass.
|
||||
//
|
||||
// The capability precheck uses the same NotificationMethods + NotificationAccounts
|
||||
// rule the selector does, so the producer can skip enqueueing work that no
|
||||
// operated worker could ever pick up (sms/voice until phase 4).
|
||||
func EnqueueNotificationTask(input *EnqueueNotificationTaskInput) (*Task, error) {
|
||||
return EnqueueNotificationTaskTx(DB(), input)
|
||||
}
|
||||
|
||||
// EnqueueNotificationTaskTx is the transactional form used by state machines
|
||||
// that must commit their transition, audit event, message, and task together.
|
||||
func EnqueueNotificationTaskTx(tx *gorm.DB, input *EnqueueNotificationTaskInput) (*Task, error) {
|
||||
if tx == nil {
|
||||
return nil, errors.New("enqueue: nil transaction")
|
||||
}
|
||||
if input.AccountID == 0 || input.ContactID == 0 {
|
||||
return nil, errors.New("enqueue: account_id and contact_id are required")
|
||||
}
|
||||
|
||||
idempotencyKey := input.IdempotencyKey
|
||||
if idempotencyKey == "" {
|
||||
if input.NotificationID == 0 || len(input.EventIDs) == 0 {
|
||||
return nil, errors.New("enqueue: notification_id and event_ids are required without explicit idempotency_key")
|
||||
}
|
||||
idempotencyKey = notificationIdempotencyKey(input.NotificationID, input.ContactID, input.EventIDs[0])
|
||||
}
|
||||
|
||||
// Fast path: row already exists from a previous producer tick. Returning
|
||||
// the existing row is the idempotency guarantee — second calls return the
|
||||
// same id, no second INSERT.
|
||||
var existing Task
|
||||
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
|
||||
return &existing, nil
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxAttempts := input.MaxAttempts
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = DefaultTaskMaxAttempts
|
||||
}
|
||||
if !anyWorkerCanDeliver(input.Method, input.AccountID) {
|
||||
return nil, fmt.Errorf("%w: method=%s account=%d", ErrNotificationMethodNotAuthorized, input.Method, input.AccountID)
|
||||
}
|
||||
|
||||
notBefore := input.NotBefore
|
||||
if notBefore.IsZero() {
|
||||
notBefore = time.Now()
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
task := &Task{
|
||||
JobID: uuid.New().String(),
|
||||
Kind: TaskKindNotification,
|
||||
State: TaskStateQueued,
|
||||
AccountID: input.AccountID,
|
||||
MessageID: input.MessageID,
|
||||
ContactID: &input.ContactID,
|
||||
MonitorID: input.MonitorID,
|
||||
CheckID: input.CheckID,
|
||||
NotBefore: notBefore,
|
||||
Deadline: input.Deadline,
|
||||
Attempts: 0,
|
||||
MaxAttempts: maxAttempts,
|
||||
IdempotencyKey: idempotencyKey,
|
||||
}
|
||||
if len(input.Payload) > 0 {
|
||||
task.Payload = input.Payload
|
||||
}
|
||||
task.CreatedAt = now
|
||||
task.UpdatedAt = now
|
||||
|
||||
// ON CONFLICT DO NOTHING so a concurrent producer tick racing with us on
|
||||
// the same idempotency_key loses the race but does not duplicate the row.
|
||||
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(task).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task.ID == 0 {
|
||||
// Lost the race. Re-read and return the winner.
|
||||
if err := tx.Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &existing, nil
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// NotificationIdempotencyKey exposes the producer's idempotency key so the
|
||||
// result handler and admin tooling can match a Task row back to the logical
|
||||
// (notification, contact, event) tuple without re-deriving the format.
|
||||
func NotificationIdempotencyKey(notificationID, contactID, eventID int64) string {
|
||||
return notificationIdempotencyKey(notificationID, contactID, eventID)
|
||||
}
|
||||
|
||||
func notificationIdempotencyKey(notificationID, contactID, eventID int64) string {
|
||||
return fmt.Sprintf("notif:%d:contact:%d:event:%d", notificationID, contactID, eventID)
|
||||
}
|
||||
|
||||
// anyWorkerCanDeliver returns true if at least one active worker in the pool is
|
||||
// authorized to deliver the given (method, account) pair. Used by the producer
|
||||
// to skip enqueues no worker could ever pick up.
|
||||
func anyWorkerCanDeliver(method string, accountID int64) bool {
|
||||
var nodes []WorkerNode
|
||||
if err := DB().Where("status = ? AND last_seen > ?", "active", time.Now().Add(-WorkerHeartbeatFreshness)).Find(&nodes).Error; err != nil {
|
||||
log.Printf("task_selector: cannot list workers: %v", err)
|
||||
// Be permissive on lookup failure: the selector's own filter would still
|
||||
// hold the lease back, so the worst case is a queued task nobody picks
|
||||
// up — which the reaper eventually dead-letters.
|
||||
return true
|
||||
}
|
||||
for i := range nodes {
|
||||
if nodes[i].SupportsTaskEnvelope() && nodes[i].CanDeliverNotification(method, accountID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TasksForWorker leases up to `limit` due tasks for the worker. The selection
|
||||
// is one transaction so the FOR UPDATE SKIP LOCKED + UPDATE that flips state
|
||||
// from queued to leased is atomic. Notification tasks are filtered by the worker's
|
||||
// notification_methods + notification_accounts capability set; check tasks are
|
||||
// filtered by check_types in their payload.
|
||||
//
|
||||
// The function is safe to call from multiple goroutines for different workers.
|
||||
// Two workers that hit the DB at the same time will see disjoint task sets.
|
||||
func TasksForWorker(worker *WorkerNode, limit int) ([]Task, error) {
|
||||
if worker == nil {
|
||||
return nil, errors.New("TasksForWorker: worker is nil")
|
||||
}
|
||||
if !worker.SupportsTaskEnvelope() {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 1
|
||||
}
|
||||
|
||||
notifMethods := worker.NotificationMethods()
|
||||
notifAccounts := worker.AccessibleAccountIDs()
|
||||
hasNotif := len(notifMethods) > 0
|
||||
|
||||
tx := DB().Begin()
|
||||
if tx.Error != nil {
|
||||
return nil, tx.Error
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_ = tx.Rollback().Error
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
now := time.Now()
|
||||
var out []Task
|
||||
|
||||
// First pass: notification tasks the worker is authorized to deliver. We
|
||||
// also bump attempts and flip state to leased in the same row so the
|
||||
// outer selector+lease is atomic. The method filter is a JSONB extract on
|
||||
// payload->>'method' so a single worker query can target one method list.
|
||||
if hasNotif {
|
||||
notifQuery := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'method' IN (?)", notifMethods).
|
||||
Where("payload->>'method' <> ''")
|
||||
if len(notifAccounts) > 0 {
|
||||
notifQuery = notifQuery.Where("account_id IN (?)", notifAccounts)
|
||||
}
|
||||
|
||||
var picked []Task
|
||||
if err := notifQuery.Limit(limit).Find(&picked).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range picked {
|
||||
row := picked[i]
|
||||
newAttempts := row.Attempts + 1
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
|
||||
if err := tx.Model(&row).Updates(map[string]interface{}{
|
||||
colState: TaskStateLeased,
|
||||
"lease_owner": worker.WorkerID,
|
||||
"lease_expires_at": leaseUntil,
|
||||
"attempts": newAttempts,
|
||||
"lease_token": leaseToken,
|
||||
"updated_at": now,
|
||||
}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
row.State = TaskStateLeased
|
||||
row.LeaseOwner = worker.WorkerID
|
||||
row.LeaseExpiresAt = &leaseUntil
|
||||
row.Attempts = newAttempts
|
||||
row.LeaseToken = leaseToken
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
|
||||
remaining := limit - len(out)
|
||||
if checkTypes := worker.CheckTypes(); remaining > 0 && len(checkTypes) > 0 {
|
||||
checkQuery := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindCheck).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'kind' IN (?)", checkTypes)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
checkQuery = checkQuery.Where("account_id IN (?)", accounts)
|
||||
}
|
||||
var picked []Task
|
||||
if err := checkQuery.Limit(remaining).Find(&picked).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
for i := range picked {
|
||||
row := picked[i]
|
||||
newAttempts := row.Attempts + 1
|
||||
leaseToken := uuid.NewString()
|
||||
leaseUntil := now.Add(TaskLeaseTTL(row.Kind))
|
||||
if err := tx.Model(&row).Updates(map[string]interface{}{
|
||||
colState: TaskStateLeased, "lease_owner": worker.WorkerID,
|
||||
"lease_expires_at": leaseUntil, "attempts": newAttempts, "lease_token": leaseToken, "updated_at": now,
|
||||
}).Error; err != nil {
|
||||
_ = tx.Rollback().Error
|
||||
return nil, err
|
||||
}
|
||||
row.State, row.LeaseOwner, row.LeaseExpiresAt, row.Attempts, row.LeaseToken = TaskStateLeased, worker.WorkerID, &leaseUntil, newAttempts, leaseToken
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AvailableWorkerTaskCapacity returns unoccupied local worker slots. Durable
|
||||
// unexpired leases and the worker's heartbeat-reported active/queued workload
|
||||
// describe the same work from different sides, so the larger value is used to
|
||||
// avoid both over-dispatching and double-counting a healthy worker.
|
||||
func AvailableWorkerTaskCapacity(worker *WorkerNode) (int, error) {
|
||||
if worker == nil {
|
||||
return 0, errors.New("worker capacity: worker is nil")
|
||||
}
|
||||
concurrency := worker.Concurrency
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
var leased int64
|
||||
if err := DB().Model(&Task{}).Where("state = ? AND lease_owner = ? AND lease_expires_at > ?", TaskStateLeased, worker.WorkerID, time.Now()).Count(&leased).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var confirmationLeases int64
|
||||
if err := DB().Model(&CheckAttempt{}).Where("worker_node_id = ? AND kind = ? AND state = ? AND lease_expires_at > ?", worker.ID, AttemptKindConfirm, AttemptStateLeased, time.Now()).Count(&confirmationLeases).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
used := int(leased + confirmationLeases)
|
||||
if reported := worker.ReportedWorkload(); reported > used {
|
||||
used = reported
|
||||
}
|
||||
if used >= concurrency {
|
||||
return 0, nil
|
||||
}
|
||||
return concurrency - used, nil
|
||||
}
|
||||
|
||||
// GetTaskByJobID returns one task row keyed by its unique job_id. The result
|
||||
// handler uses this to validate that the incoming JobID exists and matches the
|
||||
// calling worker before it mutates state.
|
||||
func GetTaskByJobID(jobID string) (*Task, error) {
|
||||
if jobID == "" {
|
||||
return nil, errors.New("GetTaskByJobID: empty job_id")
|
||||
}
|
||||
var task Task
|
||||
if err := DB().Where("job_id = ?", jobID).First(&task).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &task, nil
|
||||
}
|
||||
|
||||
// TasksForWorkerTx is the variant exposed for tests so a single SELECT inside a
|
||||
// caller-provided transaction can be inspected without the auto-commit wrapper.
|
||||
// Production code should use TasksForWorker.
|
||||
func TasksForWorkerTx(tx *gorm.DB, worker *WorkerNode, limit int) ([]Task, error) {
|
||||
if tx == nil {
|
||||
return nil, errors.New("TasksForWorkerTx: nil tx")
|
||||
}
|
||||
notifMethods := worker.NotificationMethods()
|
||||
if len(notifMethods) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
now := time.Now()
|
||||
var out []Task
|
||||
// Use SKIP LOCKED to avoid contention between workers (mirror of ChecksForWorker).
|
||||
q := tx.Clauses(SkipLockedClause).
|
||||
Where("state = ? AND kind = ?", TaskStateQueued, TaskKindNotification).
|
||||
Where("not_before <= ?", now).
|
||||
Where("(deadline IS NULL OR deadline > ?)", now).
|
||||
Where("payload->>'method' IN (?)", notifMethods)
|
||||
if accounts := worker.AccessibleAccountIDs(); len(accounts) > 0 {
|
||||
q = q.Where("account_id IN (?)", accounts)
|
||||
}
|
||||
if err := q.Limit(limit).Find(&out).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
681
app/models/task_test.go
Обычный файл
681
app/models/task_test.go
Обычный файл
@@ -0,0 +1,681 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// seedRegion creates a Region row before a worker fixture inserts, so the FK
|
||||
// from worker_nodes -> regions holds. Idempotent: Drop() cleans up.
|
||||
func seedRegion(t *testing.T, code string) models.Region {
|
||||
t.Helper()
|
||||
r := models.Region{}
|
||||
err := models.DB().Where("code = ?", code).First(&r).Error
|
||||
if err == nil {
|
||||
return r
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("seed region lookup: %v", err)
|
||||
}
|
||||
r = models.Region{Code: code, Name: code, Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&r).Error)
|
||||
return r
|
||||
}
|
||||
|
||||
// seedAccountUserPlan returns an account with a default plan and the first user
|
||||
// for the FK chain that contacts/notifications walk. Cleanup is the caller's
|
||||
// responsibility (Drop() at end of test).
|
||||
func seedAccountUserPlan(t *testing.T) (models.Account, models.User) {
|
||||
t.Helper()
|
||||
plan := models.Plan{Name: "test-plan", Default: false}
|
||||
if err := models.DB().Create(&plan).Error; err != nil {
|
||||
t.Fatalf("seed plan: %v", err)
|
||||
}
|
||||
|
||||
user := models.User{Name: "test-user", Email: taskStringPtr("test-" + uuid.NewString() + "@example.com"), Timezone: "UTC"}
|
||||
if err := models.DB().Create(&user).Error; err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
|
||||
account := models.Account{Name: "test-account", Timezone: "UTC", Language: "en", PlanID: &plan.ID}
|
||||
if err := models.DB().Create(&account).Error; err != nil {
|
||||
t.Fatalf("seed account: %v", err)
|
||||
}
|
||||
return account, user
|
||||
}
|
||||
|
||||
func seedNotification(t *testing.T, accountID int64) models.Notification {
|
||||
t.Helper()
|
||||
n := models.Notification{Name: "default", AccountID: accountID, Enabled: true, NotifyDown: true, NotifyRestore: true}
|
||||
require.NoError(t, models.DB().Create(&n).Error)
|
||||
return n
|
||||
}
|
||||
|
||||
func seedEmailContact(t *testing.T, accountID int64) models.Contact {
|
||||
t.Helper()
|
||||
c := models.Contact{Name: "ops", Kind: "email", Value: "ops@example.com", AccountID: &accountID}
|
||||
require.NoError(t, models.DB().Create(&c).Error)
|
||||
return c
|
||||
}
|
||||
|
||||
func taskStringPtr(s string) *string { return &s }
|
||||
|
||||
func TestTaskSchemaMigration(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
require.True(t, models.DB().Migrator().HasTable(&models.Task{}), "tasks table must exist after Migrate()")
|
||||
require.True(t, models.DB().Migrator().HasTable(&models.NotificationDelivery{}), "notification_deliveries table must exist after Migrate()")
|
||||
}
|
||||
|
||||
func TestEnqueueNotificationTask_Idempotency(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
|
||||
// Producer precheck requires at least one worker authorized for the
|
||||
// (method, account) pair. Add an operated-style email worker.
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-idempotency-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
LastSeen: &now,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
payload := []byte(`{"method":"email","subject":"[rsmon] x is down","body_text":"down","body_html":"<p>down</p>","body_markdown":"**down**","language":"en","message_kind":"down"}`)
|
||||
|
||||
input := models.EnqueueNotificationTaskInput{
|
||||
AccountID: account.ID,
|
||||
NotificationID: notification.ID,
|
||||
ContactID: contact.ID,
|
||||
Method: "email",
|
||||
Subject: "[rsmon] x is down",
|
||||
BodyText: "down",
|
||||
BodyHTML: "<p>down</p>",
|
||||
Language: "en",
|
||||
MessageKind: "down",
|
||||
EventIDs: []int64{42},
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
first, err := models.EnqueueNotificationTask(&input)
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, first.ID)
|
||||
|
||||
// Second call with the same logical event must not create a duplicate row.
|
||||
second, err := models.EnqueueNotificationTask(&input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first.ID, second.ID, "idempotency: second enqueue should return the same row")
|
||||
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Task{}).Where("idempotency_key = ?", first.IdempotencyKey).Count(&count).Error)
|
||||
assert.EqualValues(t, 1, count, "exactly one Task row per idempotency key")
|
||||
}
|
||||
|
||||
// TestEnqueueNotificationTask_AuthorizationSkip makes sure the producer can
|
||||
// observe ErrNotificationMethodNotAuthorized when no worker is eligible for
|
||||
// the (method, account) pair. The producer uses this to avoid enqueueing work
|
||||
// no operated worker could ever pick up.
|
||||
func TestEnqueueNotificationTask_AuthorizationSkip(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
// Register a worker that only knows telegram. An email task must fail the
|
||||
// precheck.
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-tg-only-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"telegram"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
_, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
||||
AccountID: account.ID,
|
||||
NotificationID: notification.ID,
|
||||
ContactID: contact.ID,
|
||||
Method: "email",
|
||||
EventIDs: []int64{1},
|
||||
Payload: []byte(`{"method":"email"}`),
|
||||
})
|
||||
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
||||
}
|
||||
|
||||
func TestEnqueueNotificationTask_RequiresTaskEnvelopeWorker(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
now := time.Now()
|
||||
worker := &models.WorkerNode{WorkerID: "legacy-notify-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"}, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
||||
}))}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
require.NoError(t, models.DB().Model(&models.WorkerNode{}).Where("id <> ?", worker.ID).Update("status", "dead").Error)
|
||||
input := &models.EnqueueNotificationTaskInput{AccountID: account.ID, NotificationID: notification.ID, ContactID: contact.ID, Method: "email", EventIDs: []int64{77}, Payload: []byte(`{"method":"email"}`)}
|
||||
_, err := models.EnqueueNotificationTask(input)
|
||||
require.ErrorIs(t, err, models.ErrNotificationMethodNotAuthorized)
|
||||
var count int64
|
||||
require.NoError(t, models.DB().Model(&models.Task{}).Count(&count).Error)
|
||||
assert.Zero(t, count)
|
||||
worker.Capabilities = datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"}, "task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{},
|
||||
}))
|
||||
require.NoError(t, models.DB().Save(worker).Error)
|
||||
task, err := models.EnqueueNotificationTask(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, models.TaskStateQueued, task.State)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-email-only-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
LastSeen: &now,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
// Enqueue 3 tasks of different methods; only the email ones should be
|
||||
// picked up by the worker. We use the raw helper because the producer's
|
||||
// precheck would refuse the telegram row when no worker handles telegram —
|
||||
// the selector test must exercise the SELECT-side filter, not the
|
||||
// producer-side authorization.
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{1})
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "telegram", []int64{2})
|
||||
mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{3})
|
||||
|
||||
picked, err := models.TasksForWorker(w, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 2, "only email tasks should be leased")
|
||||
|
||||
for _, p := range picked {
|
||||
assert.Equal(t, models.TaskStateLeased, p.State)
|
||||
assert.Equal(t, w.WorkerID, p.LeaseOwner)
|
||||
assert.NotNil(t, p.LeaseExpiresAt)
|
||||
assert.Equal(t, 1, p.Attempts)
|
||||
}
|
||||
|
||||
// A second call must not return the same rows.
|
||||
picked2, err := models.TasksForWorker(w, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked2, "second selector poll should see an empty queue while leased")
|
||||
}
|
||||
|
||||
func TestEnqueueDueCheckTasks_UsesGenericEnvelope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
group := models.Group{Name: "checks", AccountID: account.ID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: "example.com", Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
enabled := true
|
||||
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "generic-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
|
||||
require.NoError(t, models.EnqueueDueCheckTasks(worker, []string{"http"}, 1))
|
||||
picked, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 1)
|
||||
assert.Equal(t, models.TaskKindCheck, picked[0].Kind)
|
||||
assert.Equal(t, models.TaskStateLeased, picked[0].State)
|
||||
assert.Equal(t, check.ID, *picked[0].CheckID)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_ChecksRespectPrivateAccountScope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
accountA, _ := seedAccountUserPlan(t)
|
||||
accountB, _ := seedAccountUserPlan(t)
|
||||
accountID := accountA.ID
|
||||
private := &models.WorkerNode{
|
||||
WorkerID: "private-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
platform := &models.WorkerNode{
|
||||
WorkerID: "platform-check-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(),
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"check_types": []string{"http"}, "task_envelope": true})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(private).Error)
|
||||
require.NoError(t, models.DB().Create(platform).Error)
|
||||
task := models.Task{JobID: uuid.NewString(), Kind: models.TaskKindCheck, State: models.TaskStateQueued, AccountID: accountB.ID, Payload: datatypes.JSON([]byte(`{"kind":"http"}`)), NotBefore: time.Now().Add(-time.Second), MaxAttempts: 5, IdempotencyKey: "cross-account-" + uuid.NewString()}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
picked, err := models.TasksForWorker(private, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked)
|
||||
picked, err = models.TasksForWorker(platform, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, picked, 1)
|
||||
assert.Equal(t, task.ID, picked[0].ID)
|
||||
}
|
||||
|
||||
func TestChecksForWorker_RespectsPrivateAccountScope(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
accountA, _ := seedAccountUserPlan(t)
|
||||
accountB, _ := seedAccountUserPlan(t)
|
||||
makeCheck := func(accountID int64, host string) models.Check {
|
||||
group := models.Group{Name: host, AccountID: accountID}
|
||||
require.NoError(t, models.DB().Create(&group).Error)
|
||||
monitor := models.Monitor{GroupID: group.ID, Host: host, Enabled: true}
|
||||
require.NoError(t, models.DB().Create(&monitor).Error)
|
||||
enabled := true
|
||||
check := models.Check{MonitorID: monitor.ID, Enabled: &enabled, Kind: "http", Interval: 60, Settings: datatypes.JSON([]byte(`{}`))}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return check
|
||||
}
|
||||
owned := makeCheck(accountA.ID, "owned.example")
|
||||
_ = makeCheck(accountB.ID, "other.example")
|
||||
accountID := accountA.ID
|
||||
private := &models.WorkerNode{
|
||||
WorkerID: "private-legacy-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), AccountID: &accountID,
|
||||
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(private).Error)
|
||||
checks := models.ChecksForWorker(private, []string{"http"}, 10)
|
||||
require.Len(t, checks, 1)
|
||||
assert.Equal(t, owned.ID, checks[0].ID)
|
||||
}
|
||||
|
||||
func TestTasksForWorker_SkipsExpiredDeadline(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
now := time.Now()
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "worker-deadline-" + uuid.NewString(), RegionCode: "test", Status: "active", LastSeen: &now, AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
|
||||
expired := time.Now().Add(-time.Second)
|
||||
task := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{91})
|
||||
require.NoError(t, models.DB().Model(&task).Update("deadline", expired).Error)
|
||||
picked, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, picked)
|
||||
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, stored.State)
|
||||
assert.Equal(t, 0, stored.Attempts)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasksTerminatesExpiredQueuedNotification(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
deadline := time.Now().Add(-time.Minute)
|
||||
// This reaper test intentionally has no eligible worker; insert directly
|
||||
// so it tests deadline handling rather than producer capability validation.
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{92})
|
||||
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{"message_id": message.ID, "deadline": deadline}).Error)
|
||||
|
||||
_, _, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, stored.State)
|
||||
assert.Equal(t, "notification deadline expired", stored.LastError)
|
||||
var storedMessage models.Message
|
||||
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
||||
assert.Equal(t, "error", storedMessage.State)
|
||||
var auditCount int64
|
||||
require.NoError(t, models.DB().Model(&models.NotificationDelivery{}).Where("task_id = ? AND status = ?", task.ID, "expired").Count(&auditCount).Error)
|
||||
assert.EqualValues(t, 1, auditCount)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasks_RecyclesLeasesAndDeadsExhaustedRetries(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
|
||||
now := time.Now()
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "worker-reap-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
LastSeen: &now,
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"task_envelope": true,
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
// 1) A leased task whose lease expired — should go back to queued.
|
||||
expiredLease := time.Now().Add(-time.Minute)
|
||||
leased := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{10})
|
||||
require.NoError(t, models.DB().Model(&leased).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateLeased,
|
||||
"lease_owner": w.WorkerID,
|
||||
"lease_expires_at": expiredLease,
|
||||
"attempts": 1,
|
||||
}).Error)
|
||||
|
||||
// 2) A failed_retry task past its not_before and at max_attempts — should move to dead.
|
||||
failedRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{11})
|
||||
require.NoError(t, models.DB().Model(&failedRetry).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateFailedRetry,
|
||||
"attempts": 5,
|
||||
"max_attempts": 5,
|
||||
"not_before": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
|
||||
// 3) A failed_retry task past not_before but attempts < max_attempts — must stay failed_retry.
|
||||
pendingRetry := mustEnqueue(t, account.ID, notification.ID, contact.ID, "email", []int64{12})
|
||||
require.NoError(t, models.DB().Model(&pendingRetry).Updates(map[string]interface{}{
|
||||
"state": models.TaskStateFailedRetry,
|
||||
"attempts": 2,
|
||||
"max_attempts": 5,
|
||||
"not_before": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
|
||||
reaped, deaded, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped, "one expired lease should be returned to queue")
|
||||
assert.Equal(t, 1, deaded, "one exhausted retry should move to dead")
|
||||
|
||||
var leasedRow models.Task
|
||||
require.NoError(t, models.DB().First(&leasedRow, leased.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, leasedRow.State)
|
||||
assert.Empty(t, leasedRow.LeaseOwner)
|
||||
assert.Nil(t, leasedRow.LeaseExpiresAt)
|
||||
|
||||
var deadRow models.Task
|
||||
require.NoError(t, models.DB().First(&deadRow, failedRetry.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, deadRow.State)
|
||||
|
||||
var pendingRow models.Task
|
||||
require.NoError(t, models.DB().First(&pendingRow, pendingRetry.ID).Error)
|
||||
assert.Equal(t, models.TaskStateFailedRetry, pendingRow.State)
|
||||
}
|
||||
|
||||
func TestReapExpiredTasks_ExpiredNotificationLeaseExhaustionFinalizesMessage(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
message := models.Message{NotificationID: notification.ID, ContactID: contact.ID, Kind: "down", State: models.TaskStateQueued}
|
||||
require.NoError(t, models.DB().Create(&message).Error)
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{99})
|
||||
require.NoError(t, models.DB().Model(&task).Updates(map[string]interface{}{
|
||||
"message_id": message.ID, "state": models.TaskStateLeased, "attempts": 5, "max_attempts": 5,
|
||||
"lease_expires_at": time.Now().Add(-time.Minute),
|
||||
}).Error)
|
||||
_, deaded, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, deaded)
|
||||
var storedTask models.Task
|
||||
require.NoError(t, models.DB().First(&storedTask, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateDead, storedTask.State)
|
||||
assert.Equal(t, "lease expired after max attempts", storedTask.LastError)
|
||||
var storedMessage models.Message
|
||||
require.NoError(t, models.DB().First(&storedMessage, message.ID).Error)
|
||||
assert.Equal(t, "error", storedMessage.State)
|
||||
require.NotNil(t, storedMessage.Error)
|
||||
assert.Equal(t, "lease expired after max attempts", *storedMessage.Error)
|
||||
var delivery models.NotificationDelivery
|
||||
require.NoError(t, models.DB().Where("task_id = ?", task.ID).First(&delivery).Error)
|
||||
assert.Equal(t, "dead", delivery.Status)
|
||||
assert.Equal(t, "lease expired after max attempts", delivery.Error)
|
||||
}
|
||||
|
||||
func TestNotificationTaskLeaseOutlivesExecutionTimeoutAndReapsAfterExpiry(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
account, _ := seedAccountUserPlan(t)
|
||||
notification := seedNotification(t, account.ID)
|
||||
contact := seedEmailContact(t, account.ID)
|
||||
seedRegion(t, "test")
|
||||
worker := &models.WorkerNode{
|
||||
WorkerID: "notification-lease-" + uuid.NewString(), RegionCode: "test", Status: "active", AuthToken: uuid.NewString(), Concurrency: 1,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{"task_envelope": true, "notification_methods": []string{"email"}, "notification_accounts": []int64{}})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(worker).Error)
|
||||
task := mustEnqueueRaw(t, account.ID, notification.ID, contact.ID, "email", []int64{101})
|
||||
leased, err := models.TasksForWorker(worker, 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, leased, 1)
|
||||
require.NotNil(t, leased[0].LeaseExpiresAt)
|
||||
assert.GreaterOrEqual(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationTaskLeaseTTL-time.Second)
|
||||
assert.Greater(t, leased[0].LeaseExpiresAt.Sub(time.Now()), models.DefaultNotificationExecutionTimeout)
|
||||
|
||||
withinExecution := time.Now().Add(models.DefaultNotificationExecutionTimeout)
|
||||
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", withinExecution).Error)
|
||||
reaped, _, err := models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reaped)
|
||||
var stored models.Task
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateLeased, stored.State)
|
||||
|
||||
require.NoError(t, models.DB().Model(&task).Update("lease_expires_at", time.Now().Add(-time.Second)).Error)
|
||||
reaped, _, err = models.ReapExpiredTasks()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
require.NoError(t, models.DB().First(&stored, task.ID).Error)
|
||||
assert.Equal(t, models.TaskStateQueued, stored.State)
|
||||
}
|
||||
|
||||
func mustEnqueue(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
||||
t.Helper()
|
||||
payload := []byte(`{"method":"` + method + `"}`)
|
||||
task, err := models.EnqueueNotificationTask(&models.EnqueueNotificationTaskInput{
|
||||
AccountID: accountID,
|
||||
NotificationID: notificationID,
|
||||
ContactID: contactID,
|
||||
Method: method,
|
||||
EventIDs: eventIDs,
|
||||
Payload: payload,
|
||||
NotBefore: time.Now().Add(-time.Second),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, task.ID)
|
||||
return *task
|
||||
}
|
||||
|
||||
// mustEnqueueRaw inserts a Task row directly without going through the producer
|
||||
// precheck. The selector test deliberately mixes methods (email + telegram) on a
|
||||
// worker that only handles email; the producer would refuse the telegram row,
|
||||
// which is the wrong thing to assert about in a selector test.
|
||||
func mustEnqueueRaw(t *testing.T, accountID, notificationID, contactID int64, method string, eventIDs []int64) models.Task {
|
||||
t.Helper()
|
||||
payload := datatypes.JSON([]byte(`{"method":"` + method + `"}`))
|
||||
contact := contactID
|
||||
task := models.Task{
|
||||
JobID: uuid.New().String(),
|
||||
Kind: models.TaskKindNotification,
|
||||
State: models.TaskStateQueued,
|
||||
AccountID: accountID,
|
||||
ContactID: &contact,
|
||||
Payload: payload,
|
||||
NotBefore: time.Now().Add(-time.Second),
|
||||
Attempts: 0,
|
||||
MaxAttempts: 5,
|
||||
IdempotencyKey: models.NotificationIdempotencyKey(notificationID, contactID, eventIDs[0]),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&task).Error)
|
||||
require.NotZero(t, task.ID)
|
||||
return task
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v interface{}) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestWorkerNodeCapabilities_NotificationFlags confirms that the JSON-backed
|
||||
// capabilities blob correctly exposes the notification_methods and
|
||||
// notification_accounts arrays that the selector and credential push depend on.
|
||||
func TestWorkerNodeCapabilities_NotificationFlags(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
seedRegion(t, "test")
|
||||
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: "caps-worker-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"email", "telegram"},
|
||||
"notification_accounts": []int64{7, 8},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
|
||||
got := models.WorkerNode{}
|
||||
require.NoError(t, models.DB().First(&got, w.ID).Error)
|
||||
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
||||
assert.Equal(t, []int64{7, 8}, got.NotificationAccounts())
|
||||
assert.True(t, got.CanDeliverNotification("email", 7))
|
||||
assert.False(t, got.CanDeliverNotification("email", 9), "account 9 is not in the allowed list")
|
||||
assert.False(t, got.CanDeliverNotification("mattermost", 7), "method not authorized")
|
||||
|
||||
// Operated-style worker: empty accounts list means "all accounts".
|
||||
w2 := &models.WorkerNode{
|
||||
WorkerID: "ops-worker-" + uuid.NewString(),
|
||||
RegionCode: "test",
|
||||
Status: "active",
|
||||
AuthToken: uuid.NewString(),
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON(mustJSON(t, map[string]interface{}{
|
||||
"check_types": []string{"http"},
|
||||
"notification_methods": []string{"email"},
|
||||
"notification_accounts": []int64{},
|
||||
})),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w2).Error)
|
||||
|
||||
got2 := models.WorkerNode{}
|
||||
require.NoError(t, models.DB().First(&got2, w2.ID).Error)
|
||||
assert.True(t, got2.CanDeliverNotification("email", 9999), "empty accounts list means all accounts")
|
||||
}
|
||||
|
||||
func TestWorkerNodeReportedWorkloadUsesDisjointHeartbeatFields(t *testing.T) {
|
||||
w := &models.WorkerNode{Capabilities: datatypes.JSON([]byte(`{"active_checks":2,"queue_depth":3,"active_notifications":5,"notification_queue_depth":7}`))}
|
||||
assert.Equal(t, 17, w.ReportedWorkload())
|
||||
}
|
||||
|
||||
// TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults verifies that
|
||||
// EnsureConfiguredWorkerNode (the in-cluster worker provisioner) populates
|
||||
// notification_methods / notification_accounts on the JSON blob so the new
|
||||
// selector and the new credential push work out of the box for the bundled
|
||||
// worker.
|
||||
func TestEnsureConfiguredWorkerNode_NotificationCapabilitiesDefaults(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
t.Setenv("WORKER_AUTH_TOKEN", "secret-token-for-test-xyz")
|
||||
t.Setenv("DEPLOY_ENV", "test-env")
|
||||
t.Setenv("RSMON_WORKER_ID", "worker-test")
|
||||
t.Setenv("NOTIFICATION_METHODS", "email,telegram")
|
||||
t.Setenv("NOTIFICATION_ACCOUNTS", "11,22")
|
||||
|
||||
models.EnsureConfiguredWorkerNode()
|
||||
|
||||
var got models.WorkerNode
|
||||
require.NoError(t, models.DB().Where("worker_id = ?", "worker-test").First(&got).Error)
|
||||
assert.Equal(t, []string{"email", "telegram"}, got.NotificationMethods())
|
||||
assert.Equal(t, []int64{11, 22}, got.NotificationAccounts())
|
||||
}
|
||||
|
||||
// TestNotificationIdempotencyKeyFormat pins the producer-side key shape so the
|
||||
// result handler can re-derive it for matching without depending on internal
|
||||
// package state.
|
||||
func TestNotificationIdempotencyKeyFormat(t *testing.T) {
|
||||
key := models.NotificationIdempotencyKey(7, 9, 13)
|
||||
assert.Equal(t, "notif:7:contact:9:event:13", key)
|
||||
}
|
||||
|
||||
// guard against uuid being accidentally dropped from the imports.
|
||||
var _ = uuid.New
|
||||
77
app/models/telegram_bot.go
Обычный файл
77
app/models/telegram_bot.go
Обычный файл
@@ -0,0 +1,77 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
const (
|
||||
// TelegramBotMessageReceived marks inbound bot messages.
|
||||
TelegramBotMessageReceived = "received"
|
||||
// TelegramBotMessageSent marks outbound bot replies.
|
||||
TelegramBotMessageSent = "sent"
|
||||
// TelegramBotStatusMain is the singleton status row name for the bot.
|
||||
TelegramBotStatusMain = "main"
|
||||
)
|
||||
|
||||
// TelegramBotMessage stores a Telegram bot chat message for admin history.
|
||||
type TelegramBotMessage struct {
|
||||
concerns.Model
|
||||
|
||||
Direction string `gorm:"size:16;index" json:"direction"`
|
||||
ChatID int64 `gorm:"index" json:"chat_id"`
|
||||
ChatType string `gorm:"size:32" json:"chat_type"`
|
||||
Username string `gorm:"size:255" json:"username"`
|
||||
Text string `gorm:"type:text" json:"text"`
|
||||
Command string `gorm:"size:64" json:"command"`
|
||||
ContactID *int64 `gorm:"index" json:"contact_id,omitempty"`
|
||||
Error string `gorm:"type:text" json:"error,omitempty"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (TelegramBotMessage) TableName() string {
|
||||
return "telegram_bot_messages"
|
||||
}
|
||||
|
||||
// TelegramBotStatus stores the current Telegram bot heartbeat/status.
|
||||
type TelegramBotStatus struct {
|
||||
concerns.Model
|
||||
|
||||
Name string `gorm:"uniqueIndex;size:64;not null" json:"name"`
|
||||
Username string `gorm:"size:255" json:"username"`
|
||||
Online bool `gorm:"not null;default:false" json:"online"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty"`
|
||||
LastError string `gorm:"type:text" json:"last_error,omitempty"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName overrides the default table name.
|
||||
func (TelegramBotStatus) TableName() string {
|
||||
return "telegram_bot_statuses"
|
||||
}
|
||||
|
||||
// RecentTelegramBotMessages returns the latest Telegram bot messages, capped to a safe limit.
|
||||
func RecentTelegramBotMessages(limit int) ([]TelegramBotMessage, error) {
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
messages := []TelegramBotMessage{}
|
||||
err := DB().Order("id DESC").Limit(limit).Find(&messages).Error
|
||||
return messages, err
|
||||
}
|
||||
|
||||
// TelegramBotCurrentStatus returns the singleton Telegram bot status row.
|
||||
func TelegramBotCurrentStatus() (*TelegramBotStatus, error) {
|
||||
status := TelegramBotStatus{}
|
||||
if err := DB().Where("name = ?", TelegramBotStatusMain).First(&status).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status.LastSeen == nil || time.Since(*status.LastSeen) > 2*time.Minute {
|
||||
status.Online = false
|
||||
}
|
||||
return &status, nil
|
||||
}
|
||||
24
app/models/telegram_bot_test.go
Обычный файл
24
app/models/telegram_bot_test.go
Обычный файл
@@ -0,0 +1,24 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
)
|
||||
|
||||
func TestTelegramBotCurrentStatusMarksStaleOffline(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seen := time.Now().Add(-3 * time.Minute)
|
||||
status := models.TelegramBotStatus{Name: models.TelegramBotStatusMain, Online: true, LastSeen: &seen}
|
||||
require.NoError(t, models.DB().Create(&status).Error)
|
||||
|
||||
got, err := models.TelegramBotCurrentStatus()
|
||||
require.NoError(t, err)
|
||||
assert.False(t, got.Online)
|
||||
}
|
||||
233
app/models/user.go
Обычный файл
233
app/models/user.go
Обычный файл
@@ -0,0 +1,233 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/authidentity"
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// User represents a platform user.
|
||||
//
|
||||
// A User is a global identity that may belong to many tenants via the
|
||||
// Access join table (see Access). Contacts created by or assigned to a
|
||||
// User carry that UserID; admins reach them via the cross-tenant
|
||||
// /admin/users page, while per-account management is via
|
||||
// /settings/users (which shows Accesses preloaded with User + Invite).
|
||||
//
|
||||
// Authentication state (password, confirmation, lock, recover) lives
|
||||
// here, not on Access, because those attributes are account-independent.
|
||||
type User struct {
|
||||
// concerns.Model
|
||||
ID int64 `gorm:"primarykey" json:"id"`
|
||||
|
||||
Email *string `gorm:"uniqueIndex;size:255" json:"email" validate:"required"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `gorm:"not null;default:true" json:"-"`
|
||||
// Operator grants platform-wide operational access. Account ownership alone
|
||||
// must never grant cross-tenant task inspection or replay.
|
||||
Operator bool `gorm:"not null;default:false" json:"operator"`
|
||||
Timezone string `json:"timezone"`
|
||||
Language string `gorm:"default:ru" json:"language"`
|
||||
// Settings holds small per-user UI preferences. It intentionally stays
|
||||
// separate from account settings because the sidebar is a personal view.
|
||||
Settings datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"settings"`
|
||||
Phone string `gorm:"index,size:255" json:"phone"`
|
||||
TelegramID *int64 `gorm:"uniqueIndex" json:"telegram_id"`
|
||||
TelegramUsername string `gorm:"size:255" json:"telegram_username"`
|
||||
|
||||
Accesses []Access `json:"-"`
|
||||
Contacts []Contact `json:"-"`
|
||||
|
||||
LastActiveAt *time.Time `json:"last_active_at"`
|
||||
LastActiveIP *string `json:"last_active_ip"`
|
||||
|
||||
EncryptedPassword *string `json:"-"`
|
||||
PasswordSetAt *time.Time `json:"-"`
|
||||
|
||||
// Confirm
|
||||
ConfirmationToken *string `json:"-"`
|
||||
Confirmed bool `json:"confirmed"`
|
||||
|
||||
// Lock
|
||||
AttemptCount int `json:"-"`
|
||||
LastAttempt *time.Time `json:"-"`
|
||||
LockedAt *time.Time `json:"-"`
|
||||
|
||||
// Recover
|
||||
RecoverToken *string `json:"-"`
|
||||
RecoverTokenAt *time.Time `json:"-"`
|
||||
|
||||
RememberTokens pq.StringArray `gorm:"index;type:varchar(100)[]" json:"-"`
|
||||
|
||||
// DeletionRequestedAt is set when the user requests account deletion.
|
||||
// During the 7-day grace period the user can cancel the deletion
|
||||
// (clearing this field). After 7 days the user and all related data
|
||||
// are hard-deleted by a scheduled job.
|
||||
DeletionRequestedAt *time.Time `json:"deletion_requested_at"`
|
||||
|
||||
concerns.Timestamped `json:"-"`
|
||||
}
|
||||
|
||||
// DeletionPending returns true if the user has requested deletion and is
|
||||
// still within the 7-day grace period.
|
||||
func (u *User) DeletionPending() bool {
|
||||
return u.DeletionRequestedAt != nil
|
||||
}
|
||||
|
||||
// GetLabel returns info label for user.
|
||||
func (u *User) GetLabel() string {
|
||||
return u.DisplayName()
|
||||
}
|
||||
|
||||
// DisplayName implements qor.CurrentUser for admin.
|
||||
func (u *User) DisplayName() string {
|
||||
if u.Email != nil {
|
||||
return u.Name + " " + *u.Email
|
||||
}
|
||||
return u.Name
|
||||
}
|
||||
|
||||
// AfterSocialLogin is a callback after social login.
|
||||
func (u *User) AfterSocialLogin(inviteID int64) (*User, error) {
|
||||
oldUser := User{}
|
||||
DB().Where("email = ?", u.Email).Where("id != ?", u.ID).First(&oldUser)
|
||||
if oldUser.ID > 0 {
|
||||
log.Println("new user", u.ID, "has same email", u.Email, "as old user", oldUser.ID, "so replacing")
|
||||
err := DB().Model(&authidentity.AuthIdentity{}).Where("user_id = ?", u.ID).Updates(
|
||||
authidentity.Basic{
|
||||
UserID: &oldUser.ID,
|
||||
},
|
||||
).Error
|
||||
|
||||
return &oldUser, err
|
||||
}
|
||||
err := u.AfterRegister(inviteID)
|
||||
return u, err
|
||||
}
|
||||
|
||||
// AfterInvite is a callback after invite acceptance.
|
||||
func (u *User) AfterInvite(invite *Invite) error {
|
||||
invite.InviteeID = &u.ID
|
||||
|
||||
err := DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to add accesses to invited user")
|
||||
}
|
||||
|
||||
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{
|
||||
"confirmed_at": time.Now(),
|
||||
}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed set user as confirmed")
|
||||
}
|
||||
|
||||
eml := invite.Email
|
||||
u.Email = &eml
|
||||
if invite.Name != "" {
|
||||
u.Name = invite.Name
|
||||
}
|
||||
err = DB().Save(&u).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to save user")
|
||||
}
|
||||
|
||||
if invite.Name == "" {
|
||||
log.Println("set name", u.Name)
|
||||
invite.Name = u.Name
|
||||
}
|
||||
|
||||
invite.State = stateOK
|
||||
|
||||
err = DB().Save(&invite).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to save invite")
|
||||
}
|
||||
|
||||
invite.Invitee = u
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AfterRegister is a callback after registration.
|
||||
func (u *User) AfterRegister(inviteID int64) error {
|
||||
var err error
|
||||
|
||||
if inviteID > 0 {
|
||||
invite := Invite{}
|
||||
err := DB().First(&invite, inviteID).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invite: failed to find invite")
|
||||
}
|
||||
|
||||
err = u.AfterInvite(&invite)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = CreateAccountForUser("", u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AfterLogin is a callback after login.
|
||||
func (u *User) AfterLogin(inviteID int64) error {
|
||||
var err error
|
||||
if inviteID > 0 {
|
||||
invite := Invite{}
|
||||
err = DB().First(&invite, inviteID).Error
|
||||
if err == nil {
|
||||
invite.InviteeID = &u.ID
|
||||
invite.State = stateOK
|
||||
err = DB().Save(&invite).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to save invite")
|
||||
}
|
||||
err = DB().Table("accesses").Where("invite_id = ?", invite.ID).Updates(map[string]interface{}{"user_id": u.ID}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to add accesses to invited user")
|
||||
}
|
||||
|
||||
err = DB().Model(&authidentity.AuthIdentity{}).Where("provider = ? AND user_id = ?", "password", u.ID).Updates(map[string]interface{}{ //nolint:lll
|
||||
"confirmed_at": time.Now(),
|
||||
}).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed set user as confirmed")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Gravatar returns the Gravatar URL for the user.
|
||||
func (u *User) Gravatar(size int) string {
|
||||
if u.Email == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
hash := md5.Sum([]byte(*u.Email))
|
||||
return fmt.Sprintf("https://www.gravatar.com/avatar/%x?s=%d&d=blank", hash, size)
|
||||
}
|
||||
|
||||
// AsJSON returns a JSON representation of user.
|
||||
func (u User) AsJSON() map[string]interface{} { //nolint:gocritic // hugeParam: accepted for interface compatibility
|
||||
r := map[string]interface{}{
|
||||
"id": u.ID,
|
||||
"email": u.Email,
|
||||
"avatar": u.Gravatar(32),
|
||||
"deletion_requested_at": u.DeletionRequestedAt,
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
23
app/models/whois.go
Обычный файл
23
app/models/whois.go
Обычный файл
@@ -0,0 +1,23 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// Whois provides functionality.
|
||||
type Whois struct {
|
||||
concerns.Model
|
||||
|
||||
MonitorID *int64
|
||||
Monitor *Monitor
|
||||
|
||||
Tld string
|
||||
Nameservers pq.StringArray `gorm:"type:varchar(255)[]"`
|
||||
Expires *time.Time
|
||||
UpdatedAt *time.Time
|
||||
Data string
|
||||
}
|
||||
31
app/models/worker_log_event.go
Обычный файл
31
app/models/worker_log_event.go
Обычный файл
@@ -0,0 +1,31 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// WorkerLogEvent stores critical log events sent by worker nodes. The same
|
||||
// payload may also be forwarded to VictoriaLogs; Postgres keeps a compact audit
|
||||
// copy so the control plane can show recent critical events even if external log
|
||||
// storage is temporarily unavailable.
|
||||
type WorkerLogEvent struct {
|
||||
concerns.Model
|
||||
|
||||
WorkerID int64 `gorm:"type:bigint REFERENCES worker_nodes(id) ON DELETE SET NULL;index" json:"worker_id"`
|
||||
WorkerNodeID string `gorm:"size:100;not null;index" json:"worker_node_id"`
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
Level string `gorm:"size:16;not null;index" json:"level"`
|
||||
Message string `gorm:"type:text;not null" json:"message"`
|
||||
Source string `gorm:"size:64" json:"source"`
|
||||
Payload datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'::jsonb" json:"payload"`
|
||||
OccurredAt time.Time `gorm:"not null;index" json:"occurred_at"`
|
||||
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
// TableName provides functionality.
|
||||
func (WorkerLogEvent) TableName() string { return "worker_log_events" }
|
||||
452
app/models/worker_node.go
Обычный файл
452
app/models/worker_node.go
Обычный файл
@@ -0,0 +1,452 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
)
|
||||
|
||||
// WorkerNode represents a distributed monitoring worker
|
||||
type WorkerNode struct {
|
||||
concerns.Model
|
||||
WorkerID string `gorm:"uniqueIndex;size:100;not null" json:"worker_id"` // UUID or configured ID
|
||||
RegionCode string `gorm:"size:20;not null;index" json:"region_code"`
|
||||
Region *Region `gorm:"foreignKey:RegionCode;references:Code" json:"region,omitempty"`
|
||||
Status string `gorm:"not null;default:'registered'" json:"status"` // registered, active, inactive, dead
|
||||
AuthToken string `gorm:"uniqueIndex;size:64;not null" json:"-"`
|
||||
LastSeen *time.Time `json:"last_seen"`
|
||||
Version string `gorm:"size:50" json:"version"`
|
||||
URL string `gorm:"size:500" json:"url"` //nolint:lll // publicly-advertised URL; may differ from bind host:port when behind Traefik
|
||||
Capabilities datatypes.JSON `gorm:"not null;default:'{}'" json:"capabilities"` // {"check_types": ["http","ssl","dns",...]}
|
||||
Concurrency int `gorm:"not null;default:20" json:"concurrency"`
|
||||
NetworkProblems bool `gorm:"not null;default:false;index" json:"network_problems"`
|
||||
NetworkProblemsUntil *time.Time `json:"network_problems_until,omitempty"`
|
||||
LastFailureCount int `gorm:"not null;default:0" json:"last_failure_count"`
|
||||
LastTotalCount int `gorm:"not null;default:0" json:"last_total_count"`
|
||||
// Capability flags (see docs/plans/inventory-management.md §3).
|
||||
// All default to true so an existing worker row that pre-dates
|
||||
// this migration keeps running checks. Toggle from the admin UI
|
||||
// or POST a boolean to /api/v1/workers to disable any one of
|
||||
// them; the distworker client re-reads these flags on every
|
||||
// task poll.
|
||||
//
|
||||
// Pointer types so GORM can distinguish "client didn't supply
|
||||
// the key, fall back to the DB default" from "client explicitly
|
||||
// set false". A plain `bool` would be silently re-overwritten
|
||||
// by the column default on Save (default:true kicks in when
|
||||
// GORM sees the zero value, regardless of whether the handler
|
||||
// asked for false). See the controller tests for the
|
||||
// partial-update case.
|
||||
RunChecks *bool `gorm:"not null;default:true" json:"run_checks"`
|
||||
CollectMetrics *bool `gorm:"not null;default:true" json:"collect_metrics"`
|
||||
DetectProjects *bool `gorm:"not null;default:true" json:"detect_projects"`
|
||||
// ServerID is the optional inventory Server this worker daemon
|
||||
// is running on (see docs/plans/servers-and-hardware-metrics.md
|
||||
// §3 and docs/plans/inventory-management.md §1). Nullable so
|
||||
// legacy "no server assigned" rows keep working. Indexed because
|
||||
// the distworker health ticker joins servers→workers frequently.
|
||||
ServerID *int64 `gorm:"type:bigint REFERENCES servers(id) ON DELETE SET NULL;index" json:"server_id,omitempty"`
|
||||
// AccountID scopes a worker to a single customer account (private
|
||||
// worker per docs/distributed/private-workers.md). NULL means a
|
||||
// platform-operated worker eligible to serve any account; non-NULL
|
||||
// is a customer-operated worker pinned to one account. ON DELETE
|
||||
// SET NULL keeps an operated worker valid if its account row is
|
||||
// ever removed without an explicit private-worker cleanup.
|
||||
AccountID *int64 `gorm:"type:bigint REFERENCES accounts(id) ON DELETE SET NULL;index" json:"account_id,omitempty"`
|
||||
LLMs []LLM `json:"llms,omitempty" gorm:"many2many:worker_llms;"`
|
||||
concerns.Timestamped
|
||||
}
|
||||
|
||||
const WorkerHeartbeatFreshness = 2 * time.Minute
|
||||
|
||||
func (w *WorkerNode) NetworkProblemActive(now time.Time) bool {
|
||||
return w != nil && w.NetworkProblems && (w.NetworkProblemsUntil == nil || w.NetworkProblemsUntil.After(now))
|
||||
}
|
||||
|
||||
// WorkerStatuses provides functionality.
|
||||
// WorkerStatus represents the possible worker statuses
|
||||
var WorkerStatuses = []string{"registered", "active", "inactive", "dead"}
|
||||
|
||||
// AllWorkerCheckKinds is the complete distributed-worker capability set.
|
||||
func AllWorkerCheckKinds() []string {
|
||||
return []string{kindHTTP, kindSSL, kindDNS, kindSSH, kindFTP, kindWhois, kindBSSL, kindLLM, kindLLMHTTP, kindPing, kindTCP, kindUDP}
|
||||
}
|
||||
|
||||
// NormalizeWorkerCapabilities expands aliases and removes unknown or duplicate capabilities.
|
||||
func NormalizeWorkerCapabilities(capabilities []string) []string {
|
||||
allowed := make(map[string]bool)
|
||||
for _, kind := range AllWorkerCheckKinds() {
|
||||
allowed[kind] = true
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
normalized := make([]string, 0, len(capabilities))
|
||||
for _, capability := range capabilities {
|
||||
capability = strings.TrimSpace(strings.ToLower(capability))
|
||||
if capability == "all" || capability == "*" {
|
||||
return AllWorkerCheckKinds()
|
||||
}
|
||||
if !allowed[capability] || seen[capability] {
|
||||
continue
|
||||
}
|
||||
seen[capability] = true
|
||||
normalized = append(normalized, capability)
|
||||
}
|
||||
if len(normalized) == 0 {
|
||||
return AllWorkerCheckKinds()
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// IsAlive returns true if the worker is considered alive based on last_seen
|
||||
func (w *WorkerNode) IsAlive() bool {
|
||||
if w.LastSeen == nil {
|
||||
return false
|
||||
}
|
||||
// Worker is considered dead if no heartbeat for 2 minutes
|
||||
return w.LastSeen.After(time.Now().Add(-WorkerHeartbeatFreshness))
|
||||
}
|
||||
|
||||
// NotificationMethods returns the notification methods the worker is authorized
|
||||
// to deliver (e.g. ["email", "telegram"]). Empty slice means "no notification
|
||||
// delivery authorized". See docs/plans/worker-notifier-mvp.md section 4.3.
|
||||
func (w *WorkerNode) NotificationMethods() []string {
|
||||
caps := w.capabilitiesMap()
|
||||
if caps == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := caps["notification_methods"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return parseStringList(raw)
|
||||
}
|
||||
|
||||
// NotificationAccounts returns the account IDs the worker is authorized to
|
||||
// serve for notifications. Empty slice means "owned by RSMon, all accounts".
|
||||
// Customer-hosted workers (phase 4) ship a non-empty slice to scope credentials.
|
||||
func (w *WorkerNode) NotificationAccounts() []int64 {
|
||||
caps := w.capabilitiesMap()
|
||||
if caps == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := caps["notification_accounts"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return parseInt64List(raw)
|
||||
}
|
||||
|
||||
// AccessibleAccountIDs returns the accounts this worker may access. Empty means
|
||||
// RSMon-operated/global worker. It combines the legacy single AccountID field
|
||||
// with the newer notification_accounts capability list.
|
||||
func (w *WorkerNode) AccessibleAccountIDs() []int64 {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if w.AccountID != nil && *w.AccountID > 0 {
|
||||
// A private worker cannot widen its account scope through a mutable
|
||||
// capability JSON blob.
|
||||
return []int64{*w.AccountID}
|
||||
}
|
||||
seen := map[int64]bool{}
|
||||
out := []int64{}
|
||||
for _, id := range w.NotificationAccounts() {
|
||||
if id > 0 && !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CanDeliverNotification returns true when the worker is allowed to deliver
|
||||
// the given method for the given account. An empty NotificationAccounts slice
|
||||
// means the worker is RSMon-operated and may serve any account.
|
||||
func (w *WorkerNode) CanDeliverNotification(method string, accountID int64) bool {
|
||||
methods := w.NotificationMethods()
|
||||
if len(methods) == 0 {
|
||||
return false
|
||||
}
|
||||
if !containsString(methods, method) {
|
||||
return false
|
||||
}
|
||||
accounts := w.AccessibleAccountIDs()
|
||||
if len(accounts) == 0 {
|
||||
return true
|
||||
}
|
||||
return containsInt64(accounts, accountID)
|
||||
}
|
||||
|
||||
// CheckTypes returns the check kinds this worker may execute.
|
||||
func (w *WorkerNode) CheckTypes() []string {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return nil
|
||||
}
|
||||
return parseStringList(capabilities["check_types"])
|
||||
}
|
||||
|
||||
// SupportsTaskEnvelope is an explicit protocol capability. Version labels are
|
||||
// build metadata (and may be "latest" or a commit SHA), not a wire contract.
|
||||
// Rows created before this capability existed intentionally remain v1.
|
||||
func (w *WorkerNode) SupportsTaskEnvelope() bool {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return false
|
||||
}
|
||||
supported, _ := capabilities["task_envelope"].(bool)
|
||||
return supported
|
||||
}
|
||||
|
||||
// ReportedWorkload is the worker's local active and queued work across both
|
||||
// checks and notifications. It is advisory; durable leases remain authoritative.
|
||||
func (w *WorkerNode) ReportedWorkload() int {
|
||||
capabilities := w.capabilitiesMap()
|
||||
if capabilities == nil {
|
||||
return 0
|
||||
}
|
||||
keys := []string{"active_checks", "queue_depth", "active_notifications", "notification_queue_depth"}
|
||||
total := 0
|
||||
for _, key := range keys {
|
||||
switch value := capabilities[key].(type) {
|
||||
case float64:
|
||||
if value > 0 {
|
||||
total += int(value)
|
||||
}
|
||||
case int:
|
||||
if value > 0 {
|
||||
total += value
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (w *WorkerNode) capabilitiesMap() map[string]interface{} {
|
||||
if w == nil || len(w.Capabilities) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out map[string]interface{}
|
||||
if err := json.Unmarshal(w.Capabilities, &out); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseStringList(raw interface{}) []string {
|
||||
switch v := raw.(type) {
|
||||
case []interface{}:
|
||||
out := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
out := make([]string, 0, len(v))
|
||||
for _, s := range v {
|
||||
if s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInt64List(raw interface{}) []int64 {
|
||||
switch v := raw.(type) {
|
||||
case []interface{}:
|
||||
out := make([]int64, 0, len(v))
|
||||
for _, item := range v {
|
||||
switch n := item.(type) {
|
||||
case float64:
|
||||
out = append(out, int64(n))
|
||||
case int64:
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []int64:
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsString(haystack []string, needle string) bool {
|
||||
for _, s := range haystack {
|
||||
if s == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsInt64(haystack []int64, needle int64) bool {
|
||||
for _, n := range haystack {
|
||||
if n == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EnsureConfiguredWorkerNode creates or updates the bundled Docker Compose worker from environment variables.
|
||||
func EnsureConfiguredWorkerNode() {
|
||||
token := envFirst("WORKER_AUTH_TOKEN", "RSMON_AUTH_TOKEN")
|
||||
if token == "" {
|
||||
return
|
||||
}
|
||||
|
||||
deployEnv := envDefault("DEPLOY_ENV", "production")
|
||||
workerID := envDefault("RSMON_WORKER_ID", "worker-"+deployEnv+"-01")
|
||||
regionCode := envDefault("RSMON_REGION_CODE", deployEnv)
|
||||
version := envDefault("RSMON_WORKER_VERSION", envDefault("IMAGE_TAG", "latest"))
|
||||
concurrency := envInt("WORKER_CONCURRENCY", 20)
|
||||
workerURL := strings.TrimSpace(os.Getenv("WORKER_URL"))
|
||||
capabilities := NormalizeWorkerCapabilities(splitEnvList(envDefault(
|
||||
"RSMON_CAPABILITIES",
|
||||
"http,ssl,dns,ssh,ftp,whois,bssl,llm,llm-http,ping,tcp,udp",
|
||||
)))
|
||||
|
||||
region := Region{}
|
||||
if err := DB().Where("code = ?", regionCode).First(®ion).Error; err != nil {
|
||||
region = Region{Code: regionCode, Name: regionCode, Enabled: true}
|
||||
if err := DB().Create(®ion).Error; err != nil {
|
||||
log.Printf("worker: failed to create configured worker region %s: %v", regionCode, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
capJSON, err := json.Marshal(map[string]interface{}{
|
||||
"check_types": capabilities,
|
||||
"task_envelope": true,
|
||||
"notification_methods": parseStringList(notificationMethodsFromEnv()),
|
||||
"notification_accounts": parseInt64List(notificationAccountsFromEnv()),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("worker: failed to marshal configured worker capabilities: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
worker := WorkerNode{}
|
||||
DB().Where("worker_id = ? OR auth_token = ?", workerID, token).First(&worker)
|
||||
created := worker.ID == 0
|
||||
worker.WorkerID = workerID
|
||||
worker.RegionCode = regionCode
|
||||
worker.Status = "registered"
|
||||
worker.AuthToken = token
|
||||
worker.Version = version
|
||||
worker.URL = workerURL
|
||||
worker.Capabilities = datatypes.JSON(capJSON)
|
||||
worker.Concurrency = concurrency
|
||||
|
||||
if err := DB().Save(&worker).Error; err != nil {
|
||||
log.Printf("worker: failed to provision configured worker %s: %v", workerID, err)
|
||||
return
|
||||
}
|
||||
if created {
|
||||
log.Printf("worker: provisioned configured worker %s in region %s", workerID, regionCode)
|
||||
} else {
|
||||
log.Printf("worker: updated configured worker %s in region %s", workerID, regionCode)
|
||||
}
|
||||
}
|
||||
|
||||
func splitEnvList(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
items := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
items = append(items, part)
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func envFirst(keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func envDefault(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func envInt(key string, defaultValue int) int {
|
||||
value, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || value <= 0 {
|
||||
return defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// Notification method constants used across the package. Centralized here so
|
||||
// the literal does not appear three or more times (goconst).
|
||||
const (
|
||||
methodEmail = "email"
|
||||
methodTelegram = "telegram"
|
||||
methodWebhook = "webhook"
|
||||
methodMattermost = "mattermost"
|
||||
)
|
||||
|
||||
// defaultNotificationMethods is the operated-worker default notification method
|
||||
// list. Lives at package scope so goconst does not flag the literal across
|
||||
// the package (account.go and user.go already reference "email").
|
||||
var defaultNotificationMethods = []string{methodEmail, methodTelegram, methodWebhook, methodMattermost}
|
||||
|
||||
// notificationMethodsFromEnv reads the optional NOTIFICATION_METHODS env var.
|
||||
// Empty result yields the default "all four" list so the operated worker can
|
||||
// deliver email / telegram / webhook / mattermost out of the box.
|
||||
func notificationMethodsFromEnv() []string {
|
||||
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_METHODS"))
|
||||
if raw == "" {
|
||||
return append([]string{}, defaultNotificationMethods...)
|
||||
}
|
||||
out := make([]string, 0, 4)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notificationAccountsFromEnv reads the optional NOTIFICATION_ACCOUNTS env var.
|
||||
// Empty result means "all accounts allowed" (the RSMon-operated default).
|
||||
func notificationAccountsFromEnv() []int64 {
|
||||
raw := strings.TrimSpace(os.Getenv("NOTIFICATION_ACCOUNTS"))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
out := make([]int64, 0, 4)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.ParseInt(part, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
687
checks/calls/init.go
Обычный файл
687
checks/calls/init.go
Обычный файл
@@ -0,0 +1,687 @@
|
||||
// Package calls provides functionality.
|
||||
package calls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/openai/openai-go/v3"
|
||||
"github.com/openai/openai-go/v3/option"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/internal/checkresult"
|
||||
"rsgit.ru/rsmon/rsmon/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
verdictError = "error"
|
||||
verdictWarning = "warning"
|
||||
verdictNormal = "normal"
|
||||
)
|
||||
|
||||
// Result is a result of an AI health check
|
||||
type Result struct {
|
||||
checkresult.CheckResult
|
||||
|
||||
// Captured data
|
||||
Screenshot string `json:"screenshot"` // base64 encoded screenshot
|
||||
DOM string `json:"dom"` // HTML content
|
||||
NetworkLogs string `json:"network_logs"` // JSON array of network logs
|
||||
ConsoleLogs string `json:"console_logs"` // JSON array of console logs
|
||||
URL string `json:"url"` // Final URL after redirects
|
||||
StatusCode int `json:"status_code"` // HTTP status code
|
||||
Title string `json:"title"` // Page title
|
||||
|
||||
// LLM analysis
|
||||
LLMResponse string `json:"llm_response"` // Full LLM response
|
||||
LLMVerdict string `json:"llm_verdict"` // "normal", "error", "warning"
|
||||
LLMReasoning string `json:"llm_reasoning"` // LLM's explanation
|
||||
HasChanges bool `json:"has_changes"` // Whether changes were detected from previous
|
||||
|
||||
// Previous screenshot comparison
|
||||
PreviousURL string `json:"previous_url"` // URL of previous screenshot (if any)
|
||||
PreviousTime string `json:"previous_time"` // Timestamp of previous screenshot
|
||||
|
||||
// Metadata
|
||||
CheckID uint `json:"check_id"` // For database reference
|
||||
ScreenshotID uint `json:"screenshot_id"` // ID of stored screenshot
|
||||
}
|
||||
|
||||
// NetworkLog represents a network request log
|
||||
type NetworkLog struct {
|
||||
URL string `json:"url"`
|
||||
Method string `json:"method"`
|
||||
StatusCode int `json:"status_code"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size"`
|
||||
Time float64 `json:"time"`
|
||||
}
|
||||
|
||||
// ConsoleLog represents a console log entry
|
||||
type ConsoleLog struct {
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// ScreenshotRecord represents a stored screenshot in the database
|
||||
type ScreenshotRecord struct {
|
||||
ID uint `gorm:"primarykey"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CheckID uint `json:"check_id"`
|
||||
MonitorID uint `json:"monitor_id"`
|
||||
URL string `json:"url"`
|
||||
// ScreenshotPath stores the S3 object path (e.g., "screenshots/monitor_123/2025-01-15_123456.jpg")
|
||||
|
||||
ScreenshotPath string `json:"screenshot_path" gorm:"type:varchar(500)"`
|
||||
DOM string `gorm:"type:longtext" json:"dom"`
|
||||
NetworkLogs string `gorm:"type:longtext" json:"network_logs"`
|
||||
ConsoleLogs string `gorm:"type:longtext" json:"console_logs"`
|
||||
StatusCode int `json:"status_code"`
|
||||
Title string `json:"title"`
|
||||
LLMVerdict string `json:"llm_verdict"`
|
||||
LLMReasoning string `json:"llm_reasoning"`
|
||||
}
|
||||
|
||||
// Perform executes the AI health check
|
||||
func Perform(c *models.Check) *Result {
|
||||
result := &Result{
|
||||
CheckID: uint(c.ID),
|
||||
}
|
||||
result.Warnings = []string{}
|
||||
result.Infos = []string{}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
// Get the target URL
|
||||
targetURL := getTargetURL(c)
|
||||
result.URL = targetURL
|
||||
|
||||
// Step 1: Capture screenshot, DOM, and logs
|
||||
screenshotData, domData, networkLogs, consoleLogs, statusCode, title, err := captureWebData(targetURL)
|
||||
if err != nil {
|
||||
result.Error = fmt.Errorf("failed to capture web data: %w", err)
|
||||
result.Duration = time.Since(start)
|
||||
result.State = "FAIL"
|
||||
return result
|
||||
}
|
||||
|
||||
result.Screenshot = screenshotData
|
||||
result.DOM = domData
|
||||
result.NetworkLogs = networkLogs
|
||||
result.ConsoleLogs = consoleLogs
|
||||
result.StatusCode = statusCode
|
||||
result.Title = title
|
||||
|
||||
// Step 2: Store current screenshot in database (if available)
|
||||
var screenshotID uint
|
||||
if isDBAvailable() {
|
||||
screenshotID, err = storeScreenshot(uint(c.MonitorID), uint(c.ID), result)
|
||||
if err != nil {
|
||||
result.Warnings = append(result.Warnings, "Failed to store screenshot: "+err.Error())
|
||||
log.Printf("[ai-check] Failed to store screenshot: %v", err)
|
||||
} else {
|
||||
result.ScreenshotID = screenshotID
|
||||
result.Infos = append(result.Infos, "Screenshot stored for reference")
|
||||
}
|
||||
} else {
|
||||
result.Infos = append(result.Infos, "Database not available - running in standalone mode")
|
||||
}
|
||||
|
||||
// Step 3: Get previous screenshots for comparison (if DB available)
|
||||
var previousScreenshots []ScreenshotRecord
|
||||
if isDBAvailable() {
|
||||
previousScreenshots, err = getPreviousScreenshots(uint(c.MonitorID), 3)
|
||||
if err != nil {
|
||||
result.Warnings = append(result.Warnings, "Could not retrieve previous screenshots: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if len(previousScreenshots) > 0 {
|
||||
result.PreviousURL = previousScreenshots[0].URL
|
||||
result.PreviousTime = previousScreenshots[0].CreatedAt.Format(time.RFC3339)
|
||||
result.Infos = append(result.Infos, fmt.Sprintf("Comparing with %d previous screenshot(s)", len(previousScreenshots)))
|
||||
}
|
||||
|
||||
// Step 4: Analyze with LLM
|
||||
llmVerdict, llmReasoning, hasChanges, err := analyzeWithLLM(targetURL, result, previousScreenshots)
|
||||
if err != nil {
|
||||
result.Error = fmt.Errorf("LLM analysis failed: %w", err)
|
||||
result.LLMResponse = err.Error()
|
||||
result.Duration = time.Since(start)
|
||||
result.State = "FAIL"
|
||||
return result
|
||||
}
|
||||
|
||||
result.LLMVerdict = llmVerdict
|
||||
result.LLMReasoning = llmReasoning
|
||||
result.HasChanges = hasChanges
|
||||
|
||||
// Update the screenshot record with LLM analysis
|
||||
if screenshotID > 0 {
|
||||
_ = updateScreenshotWithLLM(screenshotID, llmVerdict, llmReasoning)
|
||||
}
|
||||
|
||||
// Determine final state based on LLM verdict
|
||||
switch llmVerdict {
|
||||
case verdictError:
|
||||
result.State = "ERR"
|
||||
result.Error = errors.New(llmReasoning)
|
||||
case verdictWarning:
|
||||
result.State = "WARN"
|
||||
result.Warnings = append(result.Warnings, llmReasoning)
|
||||
default:
|
||||
result.State = "OK"
|
||||
if hasChanges {
|
||||
result.Infos = append(result.Infos, "Visual changes detected from previous screenshot")
|
||||
}
|
||||
}
|
||||
|
||||
result.Duration = time.Since(start)
|
||||
return result
|
||||
}
|
||||
|
||||
// getTargetURL constructs the target URL from the check
|
||||
func getTargetURL(c *models.Check) string {
|
||||
if c.URL != nil && *c.URL != "" {
|
||||
return *c.URL
|
||||
}
|
||||
|
||||
// Construct from monitor host
|
||||
host := c.Monitor.Host
|
||||
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
|
||||
return "https://" + host
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// captureWebData captures screenshot, DOM, network and console logs using chromedp
|
||||
func captureWebData(targetURL string) (screenshot, dom, networkLogsJSON, consoleLogsJSON string, statusCode int, title string, err error) {
|
||||
// Create a context with timeout
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Create chromedp context with options for headless chrome
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(ctx,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.NoFirstRun,
|
||||
chromedp.DisableGPU,
|
||||
chromedp.IgnoreCertErrors,
|
||||
chromedp.Flag("headless", "new"),
|
||||
chromedp.Flag("no-sandbox", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
chromedp.Flag("hide-scrollbars", true),
|
||||
chromedp.Flag("mute-audio", true),
|
||||
chromedp.WindowSize(1920, 1080),
|
||||
)
|
||||
defer allocCancel()
|
||||
|
||||
taskCtx, taskCancel := chromedp.NewContext(allocCtx)
|
||||
defer taskCancel()
|
||||
|
||||
// Collect network logs
|
||||
networkLogs := []NetworkLog{}
|
||||
consoleLogs := []ConsoleLog{}
|
||||
|
||||
var finalStatusCode int
|
||||
|
||||
// Listen for network events
|
||||
chromedp.ListenTarget(taskCtx, func(ev interface{}) {
|
||||
switch e := ev.(type) {
|
||||
case *network.EventRequestWillBeSent:
|
||||
// Track the main request
|
||||
if e.Type == network.ResourceTypeDocument {
|
||||
networkLogs = append(networkLogs, NetworkLog{
|
||||
URL: e.Request.URL,
|
||||
Method: e.Request.Method,
|
||||
Type: string(e.Type),
|
||||
})
|
||||
}
|
||||
case *network.EventResponseReceived:
|
||||
// Update the main request with status code
|
||||
for i, log := range networkLogs {
|
||||
if log.URL == e.Response.URL && log.Method == "" { // Not yet updated
|
||||
networkLogs[i].StatusCode = int(e.Response.Status)
|
||||
networkLogs[i].Size = int64(e.Response.EncodedDataLength)
|
||||
|
||||
// Capture final status code for main document
|
||||
if e.Type == network.ResourceTypeDocument && finalStatusCode == 0 {
|
||||
finalStatusCode = int(e.Response.Status)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
case *runtime.EventConsoleAPICalled:
|
||||
// Capture console logs
|
||||
value := ""
|
||||
for _, arg := range e.Args {
|
||||
// Get the string representation of the argument
|
||||
value += arg.Value.String()
|
||||
}
|
||||
if value != "" && value != "undefined" {
|
||||
consoleLogs = append(consoleLogs, ConsoleLog{
|
||||
Type: "console." + string(e.Type),
|
||||
Value: strings.TrimSpace(value),
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Run the browser tasks
|
||||
var buf []byte
|
||||
var html string
|
||||
var pageTitle string
|
||||
|
||||
tasks := chromedp.Tasks{
|
||||
network.Enable(),
|
||||
chromedp.Navigate(targetURL),
|
||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||
chromedp.Sleep(2 * time.Second), // Wait for page to fully render
|
||||
chromedp.Title(&pageTitle),
|
||||
chromedp.OuterHTML(":root", &html, chromedp.ByQueryAll),
|
||||
chromedp.FullScreenshot(&buf, 90), // 90% quality JPEG
|
||||
}
|
||||
|
||||
if err := chromedp.Run(taskCtx, tasks); err != nil {
|
||||
// Check if it's a timeout error
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "", "", "", "", 0, "", fmt.Errorf("page load timeout after 30s")
|
||||
}
|
||||
return "", "", "", "", 0, "", fmt.Errorf("chromedp error: %w", err)
|
||||
}
|
||||
|
||||
// Encode screenshot as base64
|
||||
screenshot = base64.StdEncoding.EncodeToString(buf)
|
||||
dom = html
|
||||
|
||||
// Convert logs to JSON
|
||||
networkBytes, _ := json.Marshal(networkLogs)
|
||||
networkLogsJSON = string(networkBytes)
|
||||
|
||||
consoleBytes, _ := json.Marshal(consoleLogs)
|
||||
consoleLogsJSON = string(consoleBytes)
|
||||
|
||||
// Set status code if we got it from network events
|
||||
if finalStatusCode == 0 {
|
||||
finalStatusCode = 200 // Assume OK if we got this far
|
||||
}
|
||||
|
||||
return screenshot, dom, networkLogsJSON, consoleLogsJSON, finalStatusCode, pageTitle, nil
|
||||
}
|
||||
|
||||
// storeScreenshot stores the screenshot data in S3 and metadata in the database
|
||||
func storeScreenshot(monitorID, checkID uint, result *Result) (uint, error) {
|
||||
// Initialize storage if not already initialized
|
||||
if !storage.IsAvailable() {
|
||||
if err := storage.Init(); err != nil {
|
||||
return 0, fmt.Errorf("failed to initialize storage: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a unique filename for the screenshot
|
||||
// Format: screenshots/monitor_<monitor_id>/YYYY-MM-DD/<uuid>.jpg
|
||||
timestamp := time.Now().Format("2006-01-02")
|
||||
screenshotID := uuid.New().String()
|
||||
objectPath := fmt.Sprintf("screenshots/monitor_%d/%s/%s.jpg", monitorID, timestamp, screenshotID)
|
||||
|
||||
// Decode base64 screenshot data
|
||||
screenshotBytes, err := base64.StdEncoding.DecodeString(result.Screenshot)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to decode screenshot: %w", err)
|
||||
}
|
||||
|
||||
// Upload screenshot to S3
|
||||
ctx := context.Background()
|
||||
reader := bytes.NewReader(screenshotBytes)
|
||||
objectSize := int64(len(screenshotBytes))
|
||||
|
||||
_, err = storage.PutObject(ctx, objectPath, reader, objectSize, minio.PutObjectOptions{
|
||||
ContentType: "image/jpeg",
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to upload screenshot to S3: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("[ai-check] Stored screenshot at S3 path: %s (size: %d bytes)", objectPath, objectSize)
|
||||
|
||||
// Store metadata in database
|
||||
record := &ScreenshotRecord{
|
||||
MonitorID: monitorID,
|
||||
CheckID: checkID,
|
||||
URL: result.URL,
|
||||
ScreenshotPath: objectPath,
|
||||
DOM: truncateString(result.DOM, 50000), // Limit DOM size
|
||||
NetworkLogs: result.NetworkLogs,
|
||||
ConsoleLogs: result.ConsoleLogs,
|
||||
StatusCode: result.StatusCode,
|
||||
Title: result.Title,
|
||||
}
|
||||
|
||||
// Auto migrate the table
|
||||
if err := models.DB().AutoMigrate(&ScreenshotRecord{}); err != nil {
|
||||
return 0, fmt.Errorf("failed to auto-migrate: %w", err)
|
||||
}
|
||||
|
||||
if err := models.DB().Create(record).Error; err != nil {
|
||||
return 0, fmt.Errorf("failed to insert screenshot metadata: %w", err)
|
||||
}
|
||||
|
||||
return record.ID, nil
|
||||
}
|
||||
|
||||
// loadScreenshotData loads screenshot data from S3 for a given record
|
||||
func loadScreenshotData(record *ScreenshotRecord) (string, error) {
|
||||
if record.ScreenshotPath == "" {
|
||||
return "", errors.New("no screenshot path in record")
|
||||
}
|
||||
|
||||
if !storage.IsAvailable() {
|
||||
if err := storage.Init(); err != nil {
|
||||
return "", fmt.Errorf("failed to initialize storage: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
object, err := storage.GetObject(ctx, record.ScreenshotPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get screenshot from S3: %w", err)
|
||||
}
|
||||
defer object.Close() //nolint:errcheck // accepted lint exception
|
||||
|
||||
// Read the object data
|
||||
var buf bytes.Buffer
|
||||
_, err = io.Copy(&buf, object)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read screenshot data: %w", err)
|
||||
}
|
||||
|
||||
// Encode as base64 for LLM analysis
|
||||
return base64.StdEncoding.EncodeToString(buf.Bytes()), nil
|
||||
}
|
||||
|
||||
// updateScreenshotWithLLM updates a screenshot record with LLM analysis
|
||||
func updateScreenshotWithLLM(screenshotID uint, verdict, reasoning string) error {
|
||||
return models.DB().Model(&ScreenshotRecord{}).
|
||||
Where("id = ?", screenshotID).
|
||||
Updates(map[string]interface{}{
|
||||
"llm_verdict": verdict,
|
||||
"llm_reasoning": reasoning,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// isDBAvailable checks if the database is available
|
||||
func isDBAvailable() bool {
|
||||
return models.IsDBAvailable()
|
||||
}
|
||||
|
||||
// getPreviousScreenshots retrieves previous screenshots for comparison
|
||||
func getPreviousScreenshots(monitorID uint, limit int) ([]ScreenshotRecord, error) {
|
||||
var records []ScreenshotRecord
|
||||
|
||||
err := models.DB().Model(&ScreenshotRecord{}).
|
||||
Where("monitor_id = ? AND llm_verdict != ''", monitorID).
|
||||
Order("created_at DESC").
|
||||
Limit(limit).
|
||||
Find(&records).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// truncateString truncates a string to a maximum length
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "... (truncated)"
|
||||
}
|
||||
|
||||
// analyzeWithLLM sends the screenshot and data to the LLM for analysis
|
||||
func analyzeWithLLM(targetURL string, current *Result, previous []ScreenshotRecord) (verdict, reasoning string, hasChanges bool, err error) { //nolint:lll
|
||||
// Get LLM configuration from environment
|
||||
llmKey := firstEnv("LLM_APIKEY", "LLAMA_KEY")
|
||||
llmURL := firstEnv("LLM_URL", "LLAMA_URL")
|
||||
llmModel := firstEnv("LLM_MODEL", "LLAMA_MODEL")
|
||||
|
||||
if llmKey == "" || llmURL == "" {
|
||||
return "", "", false, errors.New("LLM credentials not configured (LLAMA_KEY, LLAMA_URL)")
|
||||
}
|
||||
|
||||
if llmModel == "" {
|
||||
llmModel = "llama3.2-vision" // Default model
|
||||
}
|
||||
|
||||
// Create OpenAI client with custom base URL
|
||||
client := openai.NewClient(
|
||||
option.WithBaseURL(llmURL),
|
||||
option.WithAPIKey(llmKey),
|
||||
)
|
||||
|
||||
// Build the system prompt
|
||||
systemPrompt := buildSystemPrompt(len(previous) > 0)
|
||||
|
||||
// Build user message with current screenshot
|
||||
userContent := buildUserMessage(targetURL, current, previous)
|
||||
|
||||
// Prepare messages
|
||||
messages := []openai.ChatCompletionMessageParamUnion{
|
||||
openai.SystemMessage(systemPrompt),
|
||||
openai.UserMessage(userContent),
|
||||
}
|
||||
|
||||
// Call LLM with timeout
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
params := openai.ChatCompletionNewParams{
|
||||
Messages: messages,
|
||||
Model: llmModel,
|
||||
MaxTokens: openai.Int(2000),
|
||||
Temperature: openai.Float(0.3), // Lower temperature for more consistent analysis
|
||||
}
|
||||
|
||||
completion, err := client.Chat.Completions.New(ctx, params)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("LLM request failed: %w", err)
|
||||
}
|
||||
|
||||
if len(completion.Choices) == 0 {
|
||||
return "", "", false, errors.New("LLM returned no choices")
|
||||
}
|
||||
|
||||
response := completion.Choices[0].Message.Content
|
||||
|
||||
// Parse the LLM response to extract verdict and reasoning
|
||||
return parseLLMResponse(response)
|
||||
}
|
||||
|
||||
func firstEnv(keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildSystemPrompt creates the system prompt for the LLM
|
||||
func buildSystemPrompt(hasPrevious bool) string {
|
||||
basePrompt := `
|
||||
You are a web application health monitoring assistant. Your task is to analyze website screenshots
|
||||
and determine if the website appears to be functioning normally or if there are issues.
|
||||
|
||||
|
||||
Consider the following aspects:
|
||||
1. Visual layout - Is the page rendered correctly? Are elements properly aligned?
|
||||
2. Error messages - Are there any visible error messages, 404s, 500s, or similar?
|
||||
3. Loading states - Is the page stuck loading or showing spinners?
|
||||
4. Broken elements - Are there broken images, missing styles, or layout shifts?
|
||||
5. Content availability - Is the expected content visible?
|
||||
|
||||
Respond in the following JSON format:
|
||||
{
|
||||
"verdict": "normal" | "warning" | "error",
|
||||
"reasoning": "Brief explanation of your assessment",
|
||||
"has_changes": true | false
|
||||
}
|
||||
|
||||
Verdict guidelines:
|
||||
- "error": Page is clearly broken (error messages, completely failed to load, blank page)
|
||||
- "warning": Page loads but has issues (partial content, broken elements, degraded appearance)
|
||||
- "normal": Page appears to be functioning correctly`
|
||||
|
||||
if hasPrevious {
|
||||
basePrompt += `
|
||||
|
||||
You will also be shown previous screenshots of the same website. Compare the current screenshot with previous ones and report any significant visual changes that might indicate problems (layout shifts, missing elements, color changes, etc.).` //nolint:lll
|
||||
}
|
||||
|
||||
return basePrompt
|
||||
}
|
||||
|
||||
// buildUserMessage creates the user message with screenshot data
|
||||
func buildUserMessage(targetURL string, current *Result, previous []ScreenshotRecord) string {
|
||||
var sb strings.Builder
|
||||
|
||||
_, _ = fmt.Fprintf(&sb, "Analyze the screenshot of: %s\n\n", targetURL)
|
||||
_, _ = fmt.Fprintf(&sb, "HTTP Status Code: %d\n", current.StatusCode)
|
||||
_, _ = fmt.Fprintf(&sb, "Page Title: %s\n\n", current.Title)
|
||||
|
||||
// Add console errors if any
|
||||
var consoleErrors []ConsoleLog
|
||||
json.Unmarshal([]byte(current.ConsoleLogs), &consoleErrors) //nolint:errcheck // accepted lint exception
|
||||
errorCount := 0
|
||||
for _, log := range consoleErrors {
|
||||
if strings.Contains(log.Type, "error") || strings.Contains(log.Type, "warn") {
|
||||
errorCount++
|
||||
}
|
||||
}
|
||||
if errorCount > 0 {
|
||||
_, _ = fmt.Fprintf(&sb, "Console Errors/Warnings: %d\n\n", errorCount)
|
||||
}
|
||||
|
||||
// Add network errors
|
||||
var networkLogs []NetworkLog
|
||||
json.Unmarshal([]byte(current.NetworkLogs), &networkLogs) //nolint:errcheck // accepted lint exception
|
||||
failedRequests := 0
|
||||
for _, req := range networkLogs {
|
||||
if req.StatusCode >= 400 {
|
||||
failedRequests++
|
||||
}
|
||||
}
|
||||
if failedRequests > 0 {
|
||||
_, _ = fmt.Fprintf(&sb, "Failed Network Requests: %d\n\n", failedRequests)
|
||||
}
|
||||
|
||||
sb.WriteString("Current screenshot (base64):\n")
|
||||
sb.WriteString(current.Screenshot)
|
||||
sb.WriteString("\n\n")
|
||||
|
||||
// Add previous screenshots if available (load from S3)
|
||||
if len(previous) > 0 {
|
||||
_, _ = fmt.Fprintf(&sb, "For comparison, here are %d previous screenshot(s):\n\n", len(previous))
|
||||
for i, prev := range previous { //nolint:gocritic // range copy is acceptable here
|
||||
// Load screenshot data from S3
|
||||
screenshotData, err := loadScreenshotData(&prev)
|
||||
if err != nil {
|
||||
log.Printf("[ai-check] Failed to load previous screenshot %d: %v", i+1, err)
|
||||
_, _ = fmt.Fprintf(&sb, "Previous screenshot #%d (from %s): [error loading screenshot]\n\n",
|
||||
i+1, prev.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||
continue
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(&sb, "Previous screenshot #%d (from %s):\n", i+1, prev.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||
sb.WriteString(screenshotData)
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// parseLLMResponse parses the LLM response to extract structured data
|
||||
func parseLLMResponse(response string) (verdict, reasoning string, hasChanges bool, err error) {
|
||||
// Try to extract JSON from the response
|
||||
response = strings.TrimSpace(response)
|
||||
|
||||
// Look for JSON block
|
||||
jsonStart := strings.Index(response, "{")
|
||||
jsonEnd := strings.LastIndex(response, "}")
|
||||
|
||||
if jsonStart == -1 || jsonEnd == -1 {
|
||||
// No JSON found, try to parse text response
|
||||
return parseTextResponse(response)
|
||||
}
|
||||
|
||||
jsonStr := response[jsonStart : jsonEnd+1]
|
||||
|
||||
var parsed struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Reasoning string `json:"reasoning"`
|
||||
HasChanges bool `json:"has_changes"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(jsonStr), &parsed); err != nil {
|
||||
// JSON parse failed, fall back to text parsing
|
||||
return parseTextResponse(response)
|
||||
}
|
||||
|
||||
// Validate verdict
|
||||
switch parsed.Verdict {
|
||||
case "normal", "ok", "healthy":
|
||||
return verdictNormal, parsed.Reasoning, parsed.HasChanges, nil
|
||||
case "warning", "warn":
|
||||
return verdictWarning, parsed.Reasoning, parsed.HasChanges, nil
|
||||
case "error", "fail", "unhealthy":
|
||||
return verdictError, parsed.Reasoning, parsed.HasChanges, nil
|
||||
default:
|
||||
// Unknown verdict, default to normal with warning
|
||||
if parsed.Verdict != "" {
|
||||
return verdictWarning, parsed.Reasoning, parsed.HasChanges, nil
|
||||
}
|
||||
return verdictNormal, "Unable to determine specific issues from analysis", false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// parseTextResponse parses a non-JSON response
|
||||
func parseTextResponse(response string) (verdict, reasoning string, hasChanges bool, err error) {
|
||||
responseLower := strings.ToLower(response)
|
||||
|
||||
// Look for keywords
|
||||
hasError := strings.Contains(responseLower, "error") ||
|
||||
strings.Contains(responseLower, "broken") ||
|
||||
strings.Contains(responseLower, "failed") ||
|
||||
strings.Contains(responseLower, "not working")
|
||||
|
||||
hasWarning := strings.Contains(responseLower, "warning") ||
|
||||
strings.Contains(responseLower, "issue") ||
|
||||
strings.Contains(responseLower, "problem") ||
|
||||
strings.Contains(responseLower, "degraded")
|
||||
|
||||
hasChange := strings.Contains(responseLower, "change") ||
|
||||
strings.Contains(responseLower, "different") ||
|
||||
strings.Contains(responseLower, "modified")
|
||||
|
||||
if hasError {
|
||||
return verdictError, response, hasChange, nil
|
||||
} else if hasWarning {
|
||||
return verdictWarning, response, hasChange, nil
|
||||
}
|
||||
|
||||
return verdictNormal, response, hasChange, nil
|
||||
}
|
||||
11
checks/calls/init_test.go
Обычный файл
11
checks/calls/init_test.go
Обычный файл
@@ -0,0 +1,11 @@
|
||||
package calls
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsDBAvailableWithoutInitializedDatabase(t *testing.T) {
|
||||
assert.False(t, isDBAvailable())
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Ссылка в новой задаче
Block a user