feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
451
app/models/check_jobs_test.go
Обычный файл
451
app/models/check_jobs_test.go
Обычный файл
@@ -0,0 +1,451 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/datatypes"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/config/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Init()
|
||||
}
|
||||
|
||||
// seedRegionRoutingWorld builds two regions and three groups (one per
|
||||
// monitor) plus three workers. The monitors and their PreferredRegions /
|
||||
// RegionMode are configured by the caller via a callback so each test can
|
||||
// express the exact routing scenario it wants to assert on.
|
||||
//
|
||||
// Returns a routerTestWorld that knows how to look up each fixture by name
|
||||
// for readable assertions.
|
||||
type routerTestWorld struct {
|
||||
groupAny *models.Group
|
||||
groupSpec *models.Group
|
||||
groupAll *models.Group
|
||||
|
||||
workerMsk *models.WorkerNode
|
||||
workerUSEast *models.WorkerNode
|
||||
workerLocal *models.WorkerNode
|
||||
}
|
||||
|
||||
// seedWorker creates a worker bound to regionCode. AuthToken is unique per
|
||||
// worker so the FOR UPDATE SKIP LOCKED path can exercise two distinct
|
||||
// concurrent callers.
|
||||
func seedWorker(t *testing.T, id, regionCode string) *models.WorkerNode {
|
||||
t.Helper()
|
||||
seedRegion(t, regionCode)
|
||||
w := &models.WorkerNode{
|
||||
WorkerID: id,
|
||||
RegionCode: regionCode,
|
||||
Status: "active",
|
||||
AuthToken: "tok-" + id,
|
||||
Concurrency: 4,
|
||||
Capabilities: datatypes.JSON([]byte(`{"check_types":["http"]}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(w).Error)
|
||||
return w
|
||||
}
|
||||
|
||||
// seedRouterMonitor creates a Monitor with the given region routing
|
||||
// attributes and one ready-to-run http Check. The check has Interval=60
|
||||
// (minimum allowed) and LastStart=nil so ChecksForWorker picks it up
|
||||
// immediately on the next poll.
|
||||
func seedRouterMonitor(t *testing.T, group *models.Group, host, regionMode string, preferred []string) (models.Monitor, models.Check) {
|
||||
t.Helper()
|
||||
enTrue := true
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
if regionMode != "" {
|
||||
mon.RegionMode = regionMode
|
||||
}
|
||||
if preferred != nil {
|
||||
mon.PreferredRegions = models.RegionCodesFromSlice(preferred)
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
|
||||
check := models.Check{
|
||||
MonitorID: mon.ID,
|
||||
Kind: "http",
|
||||
Interval: 60,
|
||||
Enabled: &enTrue,
|
||||
State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&check).Error)
|
||||
return mon, check
|
||||
}
|
||||
|
||||
func stringPtrRouter(s string) *string { return &s }
|
||||
|
||||
// seedRouterWorld is the common fixture for the TestRegionRouting_* table.
|
||||
// It provisions two regions (ru-msk, us-east) and three monitors pinned to
|
||||
// different routing modes; the workers are created lazily by the caller.
|
||||
func seedRouterWorld(t *testing.T) routerTestWorld {
|
||||
t.Helper()
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "ru-msk")
|
||||
seedRegion(t, "us-east")
|
||||
|
||||
plan := models.Plan{Name: "router", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "router-acc", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
|
||||
groupAny := &models.Group{AccountID: acc.ID, Name: "any"}
|
||||
groupSpec := &models.Group{AccountID: acc.ID, Name: "spec"}
|
||||
groupAll := &models.Group{AccountID: acc.ID, Name: "all"}
|
||||
require.NoError(t, models.DB().Create(groupAny).Error)
|
||||
require.NoError(t, models.DB().Create(groupSpec).Error)
|
||||
require.NoError(t, models.DB().Create(groupAll).Error)
|
||||
|
||||
seedRouterMonitor(t, groupAny, "any.test", models.RegionModeAny, nil)
|
||||
seedRouterMonitor(t, groupSpec, "spec-msk.test", models.RegionModeSpecific, []string{"ru-msk"})
|
||||
seedRouterMonitor(t, groupSpec, "spec-us.test", models.RegionModeSpecific, []string{"us-east"})
|
||||
seedRouterMonitor(t, groupAll, "all.test", models.RegionModeAll, []string{"ru-msk", "us-east"})
|
||||
|
||||
return routerTestWorld{
|
||||
groupAny: groupAny,
|
||||
groupSpec: groupSpec,
|
||||
groupAll: groupAll,
|
||||
workerMsk: seedWorker(t, "w-msk", "ru-msk"),
|
||||
workerUSEast: seedWorker(t, "w-us", "us-east"),
|
||||
workerLocal: seedWorker(t, "w-local", "local"),
|
||||
}
|
||||
}
|
||||
|
||||
// idsOf returns a sorted list of monitor IDs assigned to the worker for
|
||||
// easier assertions across checks.
|
||||
func idsOf(checks []*models.Check) []int64 {
|
||||
out := make([]int64, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestRegionRouting_AnyReturnsAll confirms the SQL filter preserves the
|
||||
// legacy behavior for region_mode='any' monitors and the Phase 3
|
||||
// placeholder 'all' monitors: a worker in a region nobody explicitly
|
||||
// whitelisted must still see them, while monitors pinned to specific
|
||||
// regions stay filtered out.
|
||||
//
|
||||
// seedRouterWorld configures four monitors:
|
||||
// - any.test → region_mode='any', no PreferredRegions
|
||||
// - spec-msk.test → region_mode='specific', preferred=[ru-msk]
|
||||
// - spec-us.test → region_mode='specific', preferred=[us-east]
|
||||
// - all.test → region_mode='all', preferred=[ru-msk, us-east]
|
||||
//
|
||||
// A worker in region "remote" (whitelisted by nobody) must see exactly
|
||||
// {any.test, all.test} — the two monitors whose region_mode bypasses the
|
||||
// whitelist — and nothing else.
|
||||
func TestRegionRouting_AnyReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
seedRegion(t, "remote")
|
||||
w := seedWorker(t, "w-remote", "remote")
|
||||
|
||||
checks := models.ChecksForWorker(w, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
|
||||
assert.ElementsMatch(t, []string{"any.test", "all.test"}, hosts,
|
||||
"region_mode='any' and the Phase 3 'all' placeholder must bypass the whitelist")
|
||||
assert.NotContains(t, hosts, "spec-msk.test",
|
||||
"specific-mode monitor with whitelisted ru-msk must NOT reach a remote worker")
|
||||
assert.NotContains(t, hosts, "spec-us.test",
|
||||
"specific-mode monitor with whitelisted us-east must NOT reach a remote worker")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificFiltersByRegion proves the core Phase 2 promise:
|
||||
// workers in different regions never receive a monitor whose PreferredRegions
|
||||
// does not include their region code. The check is run with parallel
|
||||
// goroutines because ChecksForWorker stamps `last_start` on every row it
|
||||
// leases — a sequential second poll would always see an already-claimed
|
||||
// queue and the filter would have nothing to test against.
|
||||
//
|
||||
// Under the FOR UPDATE SKIP LOCKED race, whichever SELECT fires first grabs
|
||||
// every matching row, so the *exact* per-worker host list is non-deterministic.
|
||||
// The deterministic invariant the test asserts is the no-leak property: a
|
||||
// worker in ru-msk must never see spec-us.test, and vice versa.
|
||||
func TestRegionRouting_SpecificFiltersByRegion(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mskChecks []*models.Check
|
||||
usChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
mskChecks = models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
usChecks = models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
mskHosts := hostsOf(mskChecks)
|
||||
usHosts := hostsOf(usChecks)
|
||||
|
||||
// Aggregate coverage: together the two workers must see every check
|
||||
// the routing layer would ever allow them — the four seeded monitors.
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
},
|
||||
append(append([]string{}, mskHosts...), usHosts...),
|
||||
"union of both workers' slices must cover every seeded monitor (any/specific/all × region)")
|
||||
|
||||
// Core Phase 2 invariant: regional filtering never leaks across
|
||||
// PreferredRegions boundaries. This is the only assertion a
|
||||
// concurrent SKIP LOCKED race lets us pin deterministically.
|
||||
assert.NotContains(t, mskHosts, "spec-us.test",
|
||||
"ru-msk worker must never see a monitor whitelisted for us-east only")
|
||||
assert.NotContains(t, usHosts, "spec-msk.test",
|
||||
"us-east worker must never see a monitor whitelisted for ru-msk only")
|
||||
}
|
||||
|
||||
// TestRegionRouting_SpecificEmptyPreferredFallsBackToAny confirms the
|
||||
// documented fall-back: a monitor in RegionModeSpecific with no
|
||||
// PreferredRegions behaves like RegionModeAny so the field is safe to
|
||||
// leave blank. We poll from the us-east worker — without the fall-back it
|
||||
// would only see any.test + all.test + spec-us.test.
|
||||
func TestRegionRouting_SpecificEmptyPreferredFallsBackToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
// Reset the spec-msk monitor to have an empty PreferredRegions list
|
||||
// (the seed above gave it one). The Monitor row's RegionMode stays
|
||||
// 'specific'.
|
||||
require.NoError(t, models.DB().Model(&models.Monitor{}).
|
||||
Where("host = ?", "spec-msk.test").
|
||||
Update("preferred_regions", models.RegionCodesFromSlice(nil)).Error)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerUSEast, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "spec-msk.test",
|
||||
"empty PreferredRegions with region_mode=specific must fall back to 'any'")
|
||||
}
|
||||
|
||||
// TestRegionRouting_AllDeferredToAny pins the Phase 3 placeholder behavior:
|
||||
// region_mode='all' is logged and treated as 'any' today. The test asserts
|
||||
// the monitor flows to a worker in any region (the TODO log marker is
|
||||
// emitted from applyRegionRouting — pinned here as a code-grep contract).
|
||||
func TestRegionRouting_AllDeferredToAny(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerMsk, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.Contains(t, hosts, "all.test",
|
||||
"region_mode='all' must currently behave like 'any' so existing checks keep flowing")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LocalWorkerBypass ensures the historic "local" region
|
||||
// still routes everything: the in-process scheduler handles those monitors
|
||||
// and we don't want the Phase 2 filter to leak platform workers through it.
|
||||
func TestRegionRouting_LocalWorkerBypass(t *testing.T) {
|
||||
world := seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(world.workerLocal, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "worker in region 'local' must receive every check (bypass)")
|
||||
}
|
||||
|
||||
// TestRegionRouting_NilWorkerReturnsAll asserts the diagnostic-friendly
|
||||
// escape hatch: passing nil for the worker skips the routing filter and
|
||||
// returns every check the kinds/limit envelope allows.
|
||||
func TestRegionRouting_NilWorkerReturnsAll(t *testing.T) {
|
||||
seedRouterWorld(t)
|
||||
|
||||
checks := models.ChecksForWorker(nil, []string{"http"}, 50)
|
||||
hosts := hostsOf(checks)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"any.test",
|
||||
"spec-msk.test",
|
||||
"spec-us.test",
|
||||
"all.test",
|
||||
}, hosts, "nil worker must bypass the routing filter")
|
||||
}
|
||||
|
||||
// TestRegionRouting_LoadBalanceImplicit confirms the SKIP LOCKED implicit
|
||||
// load-balancing story: when two workers in the same region race for a pool
|
||||
// of pending checks, each of them receives a non-empty disjoint slice. The
|
||||
// two polls run in parallel goroutines so the FOR UPDATE SKIP LOCKED race
|
||||
// window is actually exercised.
|
||||
//
|
||||
// IMPORTANT: SKIP LOCKED with a large LIMIT is unfair — whichever
|
||||
// transaction's SELECT fires first grabs everything. The test therefore
|
||||
// uses LIMIT=4 with 10 pending rows so each worker is forced to leave some
|
||||
// rows unlocked for the other worker to pick up. Together they must cover
|
||||
// at most 8 rows (LIMIT × workers) without overlap; the remaining rows are
|
||||
// intentionally left for a future poll cycle, which mirrors production
|
||||
// behavior where workers continually drain a backlog.
|
||||
func TestRegionRouting_LoadBalanceImplicit(t *testing.T) {
|
||||
models.Drop()
|
||||
models.Migrate()
|
||||
|
||||
seedRegion(t, "shared")
|
||||
plan := models.Plan{Name: "lb-plan", Default: true}
|
||||
require.NoError(t, models.DB().Create(&plan).Error)
|
||||
acc := models.Account{Name: "lb", PlanID: &plan.ID}
|
||||
require.NoError(t, models.DB().Create(&acc).Error)
|
||||
group := &models.Group{AccountID: acc.ID, Name: "lb-g"}
|
||||
require.NoError(t, models.DB().Create(group).Error)
|
||||
|
||||
enTrue := true
|
||||
for i := 0; i < 10; i++ {
|
||||
host := "lb-" + string(rune('a'+i)) + ".test"
|
||||
mon := models.Monitor{
|
||||
Name: stringPtrRouter(host),
|
||||
Host: host,
|
||||
GroupID: group.ID,
|
||||
Enabled: true,
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&mon).Error)
|
||||
ck := models.Check{
|
||||
MonitorID: mon.ID, Kind: "http", Interval: 60,
|
||||
Enabled: &enTrue, State: "UNK",
|
||||
Settings: datatypes.JSON([]byte(`{}`)),
|
||||
}
|
||||
require.NoError(t, models.DB().Create(&ck).Error)
|
||||
}
|
||||
|
||||
w1 := seedWorker(t, "lb-w1", "shared")
|
||||
w2 := seedWorker(t, "lb-w2", "shared")
|
||||
|
||||
const limitPerWorker = 4
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
aChecks, bChecks []*models.Check
|
||||
)
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
aChecks = models.ChecksForWorker(w1, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
bChecks = models.ChecksForWorker(w2, []string{"http"}, limitPerWorker)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
assert.Greater(t, len(aChecks), 0, "worker 1 must receive at least one check")
|
||||
assert.Greater(t, len(bChecks), 0, "worker 2 must receive at least one check")
|
||||
assert.LessOrEqual(t, len(aChecks)+len(bChecks), 2*limitPerWorker,
|
||||
"two concurrent workers with LIMIT each can lease at most LIMIT*2 rows per cycle")
|
||||
assert.Empty(t, intersectHosts(aChecks, bChecks),
|
||||
"the two slices must be disjoint (FOR UPDATE SKIP LOCKED must not double-lease)")
|
||||
}
|
||||
|
||||
// TestMonitorValidateRegionMode exercises the documented enum on the
|
||||
// Monitor type so the validator surface does not regress.
|
||||
func TestMonitorValidateRegionMode(t *testing.T) {
|
||||
cases := []struct {
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", false},
|
||||
{"any", false},
|
||||
{"specific", false},
|
||||
{"all", false},
|
||||
{"round-robin", true},
|
||||
{"RANDOM", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run("mode="+c.mode, func(t *testing.T) {
|
||||
m := models.Monitor{RegionMode: c.mode}
|
||||
err := m.ValidateRegionMode()
|
||||
if c.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMonitorWantsRegion pins the public WantsRegion helper that powers the
|
||||
// non-SQL callers (UI preview, plan validation). Phase 3 will swap the
|
||||
// 'all' branch's behavior, so the table covers all three modes today.
|
||||
func TestMonitorWantsRegion(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mode string
|
||||
regions []string
|
||||
workerRC string
|
||||
want bool
|
||||
}{
|
||||
{"any_always_true", "any", []string{"ru-msk"}, "us-east", true},
|
||||
{"any_empty_pref_still_true", "any", nil, "us-east", true},
|
||||
{"specific_match", "specific", []string{"ru-msk", "eu-west"}, "ru-msk", true},
|
||||
{"specific_no_match", "specific", []string{"ru-msk", "eu-west"}, "us-east", false},
|
||||
{"specific_empty_pref_fallback", "specific", nil, "us-east", true},
|
||||
{"all_placeholder_true", "all", []string{"ru-msk", "us-east"}, "ru-msk", true},
|
||||
{"all_placeholder_foreign_region", "all", []string{"ru-msk", "us-east"}, "eu-west", true},
|
||||
{"empty_mode_defaults_to_any", "", nil, "us-east", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
m := models.Monitor{
|
||||
RegionMode: c.mode,
|
||||
PreferredRegions: models.RegionCodesFromSlice(c.regions),
|
||||
}
|
||||
assert.Equal(t, c.want, m.WantsRegion(c.workerRC))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hostsOf extracts the hostnames from the assigned checks for readable
|
||||
// assertions in table-driven tests.
|
||||
func hostsOf(checks []*models.Check) []string {
|
||||
out := make([]string, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// intersectHosts returns the hostnames present in both slices — used to
|
||||
// prove two concurrent workers did not lease the same check twice.
|
||||
func intersectHosts(a, b []*models.Check) []string {
|
||||
set := make(map[string]struct{}, len(a))
|
||||
for _, c := range a {
|
||||
if c.Monitor != nil {
|
||||
set[c.Monitor.Host] = struct{}{}
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, c := range b {
|
||||
if c.Monitor == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[c.Monitor.Host]; ok {
|
||||
out = append(out, c.Monitor.Host)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Ссылка в новой задаче
Block a user