feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
185
internal/distworker/client.go
Обычный файл
185
internal/distworker/client.go
Обычный файл
@@ -0,0 +1,185 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// HTTPStatusError reports an HTTP response returned while dialing the websocket.
|
||||
type HTTPStatusError struct {
|
||||
StatusCode int
|
||||
Status string
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *HTTPStatusError) Error() string {
|
||||
if e.Body == "" {
|
||||
return fmt.Sprintf("websocket dial failed: %s", e.Status)
|
||||
}
|
||||
return fmt.Sprintf("websocket dial failed: %s: %s", e.Status, e.Body)
|
||||
}
|
||||
|
||||
// Client is the HTTP client for communicating with the control plane
|
||||
type Client struct {
|
||||
endpoint string
|
||||
authToken string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new API client
|
||||
func NewClient(endpoint, authToken string) *Client {
|
||||
return &Client{
|
||||
endpoint: endpoint,
|
||||
authToken: authToken,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// postJSON is a helper for sending JSON POST requests
|
||||
func (c *Client) postJSON(path string, payload interface{}) (*http.Response, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequest("POST", c.endpoint+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if c.authToken != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.authToken)
|
||||
}
|
||||
|
||||
return c.httpClient.Do(httpReq)
|
||||
}
|
||||
|
||||
// checkStatusCode checks if the response status is OK, returns error otherwise
|
||||
func checkStatusCode(resp *http.Response, errorMsg string) error {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("%s failed: %s: %s", errorMsg, resp.Status, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Heartbeat sends a heartbeat to the control plane
|
||||
func (c *Client) Heartbeat(req wire.HeartbeatRequest) error {
|
||||
resp, err := c.postJSON("/api/internal/workers/heartbeat", req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
return checkStatusCode(resp, "heartbeat")
|
||||
}
|
||||
|
||||
// RotateToken asks the main app's internal API to mint a new
|
||||
// bearer token for this worker. The current bearer is used for
|
||||
// authentication; the response carries the freshly issued token.
|
||||
//
|
||||
// Returns the new token string. The main app invalidates the old
|
||||
// token immediately.
|
||||
func (c *Client) RotateToken() (string, error) {
|
||||
resp, err := c.postJSON("/api/internal/workers/rotate-token", struct{}{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("rotate-token failed: %s: %s", resp.Status, string(body))
|
||||
}
|
||||
var out struct {
|
||||
AuthToken string `json:"auth_token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return "", fmt.Errorf("decode rotate-token response: %w", err)
|
||||
}
|
||||
if out.AuthToken == "" {
|
||||
return "", fmt.Errorf("rotate-token response empty")
|
||||
}
|
||||
return out.AuthToken, nil
|
||||
}
|
||||
|
||||
// GetJobs fetches available check jobs from the control plane
|
||||
func (c *Client) GetJobs() (*wire.JobsResponse, error) {
|
||||
httpReq, err := http.NewRequest("GET", c.endpoint+"/api/internal/workers/jobs", http.NoBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.authToken)
|
||||
|
||||
resp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("get jobs failed: %s: %s", resp.Status, string(body))
|
||||
}
|
||||
|
||||
var result wire.JobsResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// ReportResults sends check results to the control plane
|
||||
func (c *Client) ReportResults(req wire.ResultsRequest) error {
|
||||
resp, err := c.postJSON("/api/internal/workers/results", req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
return checkStatusCode(resp, "report results")
|
||||
}
|
||||
|
||||
// WorkerSocket connects to the websocket task channel.
|
||||
func (c *Client) WorkerSocket() (*websocket.Conn, error) {
|
||||
endpoint := strings.TrimRight(c.endpoint, "/")
|
||||
wsURL := endpoint
|
||||
if !strings.HasSuffix(wsURL, "/worker") && !strings.HasSuffix(wsURL, "/api/worker") {
|
||||
wsURL += "/worker"
|
||||
}
|
||||
if strings.HasPrefix(wsURL, "https://") {
|
||||
wsURL = "wss://" + strings.TrimPrefix(wsURL, "https://")
|
||||
} else if strings.HasPrefix(wsURL, "http://") {
|
||||
wsURL = "ws://" + strings.TrimPrefix(wsURL, "http://")
|
||||
}
|
||||
|
||||
u, err := url.Parse(wsURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("token", c.authToken)
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
conn, resp, err := websocket.DefaultDialer.Dial(u.String(), nil)
|
||||
if err != nil && resp != nil {
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, &HTTPStatusError{StatusCode: resp.StatusCode, Status: resp.Status, Body: string(body)}
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
203
internal/distworker/config.go
Обычный файл
203
internal/distworker/config.go
Обычный файл
@@ -0,0 +1,203 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMaxConcurrency is the default upper bound for the worker pool size
|
||||
// and the number of dispatch goroutines that drain the job queue.
|
||||
const DefaultMaxConcurrency = 32
|
||||
|
||||
const (
|
||||
// DefaultHTTPHost is the bind address used when WORKER_HOST is unset.
|
||||
DefaultHTTPHost = "0.0.0.0"
|
||||
// DefaultHTTPPort is the bind port used when WORKER_PORT is unset.
|
||||
// Picked >20000 to avoid colliding with the main RSMon app (which
|
||||
// binds 7401 by default) when the worker is co-located on the
|
||||
// same host. Operators are still free to override via WORKER_PORT.
|
||||
DefaultHTTPPort = 27401
|
||||
|
||||
// schemeHTTP / schemeHTTPS are the only schemes accepted on
|
||||
// WORKER_URL. Peer workers and the main app need an http(s) origin
|
||||
// they can dial with Go's net/http stack.
|
||||
schemeHTTP = "http"
|
||||
schemeHTTPS = "https"
|
||||
|
||||
// loopbackHostnames lists hostnames treated as loopback for the
|
||||
// http-on-non-loopback warning emitted by ValidateHTTPConfig.
|
||||
loopbackHostnames = "localhost,127.0.0.1,::1,0.0.0.0"
|
||||
)
|
||||
|
||||
// HTTPConfig holds the settings that govern the worker's local HTTP
|
||||
// listener (web app MVP in Task 3 and Raft peer connections in Task 4).
|
||||
// Host/Port are the bind interface. URL is the publicly-advertised
|
||||
// location peers and the main app use to reach the worker; it is NOT
|
||||
// derived from Host:Port because workers commonly sit behind a reverse
|
||||
// proxy / Traefik with HTTPS while listening on plain HTTP internally.
|
||||
//
|
||||
// Login/Password are basic auth credentials for the worker web app API
|
||||
// (Task 3). They are kept in memory only; Task 3 may hash them before
|
||||
// any persistent store.
|
||||
type HTTPConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
URL string
|
||||
Login string
|
||||
Password string
|
||||
}
|
||||
|
||||
// IsAuthConfigured reports whether both WORKER_LOGIN and WORKER_PASSWORD
|
||||
// are set. The HTTP listener refuses to start otherwise.
|
||||
func (c HTTPConfig) IsAuthConfigured() bool {
|
||||
return c.Login != "" && c.Password != ""
|
||||
}
|
||||
|
||||
// IsListenConfigured reports whether the listener should bind at all.
|
||||
// Currently always true while WORKER_PORT > 0; placeholder for Task 3
|
||||
// to wire "skip listen" semantics.
|
||||
func (c HTTPConfig) IsListenConfigured() bool {
|
||||
return c.Port > 0
|
||||
}
|
||||
|
||||
// Config holds the local worker connection settings.
|
||||
// Runtime settings are delivered by the control plane over websocket.
|
||||
type Config struct {
|
||||
URL string
|
||||
Token string
|
||||
|
||||
// MaxConcurrency caps the worker pool size and the number of dispatcher
|
||||
// goroutines. A value <= 0 falls back to DefaultMaxConcurrency.
|
||||
// Runtime configuration delivered by the control plane via the websocket
|
||||
// "init" or "config" message is clamped to this value when resizing.
|
||||
MaxConcurrency int
|
||||
|
||||
// HTTP holds the local web app / Raft listener settings (Task 2).
|
||||
HTTP HTTPConfig
|
||||
}
|
||||
|
||||
// ConfigFromEnv creates a Config from environment variables.
|
||||
func ConfigFromEnv() Config {
|
||||
return Config{
|
||||
URL: normalizeURL(os.Getenv("RSMON_URL")),
|
||||
Token: os.Getenv("RSMON_TOKEN"),
|
||||
HTTP: HTTPConfigFromEnv(),
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPConfigFromEnv reads HTTP listener settings from the environment.
|
||||
// Empty WORKER_HOST defaults to DefaultHTTPHost; empty WORKER_PORT defaults
|
||||
// to DefaultHTTPPort. A malformed WORKER_PORT falls back to the default.
|
||||
// URL is parsed loosely here; ValidateHTTPConfig does the real check.
|
||||
func HTTPConfigFromEnv() HTTPConfig {
|
||||
port := DefaultHTTPPort
|
||||
if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" {
|
||||
if v, err := strconv.Atoi(raw); err == nil && v > 0 && v <= 65535 {
|
||||
port = v
|
||||
}
|
||||
}
|
||||
host := strings.TrimSpace(os.Getenv("WORKER_HOST"))
|
||||
if host == "" {
|
||||
host = DefaultHTTPHost
|
||||
}
|
||||
return HTTPConfig{
|
||||
Host: host,
|
||||
Port: port,
|
||||
URL: strings.TrimSpace(os.Getenv("WORKER_URL")),
|
||||
Login: os.Getenv("WORKER_LOGIN"),
|
||||
Password: os.Getenv("WORKER_PASSWORD"),
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateHTTPConfig enforces the Task 2 invariants:
|
||||
//
|
||||
// - WORKER_LOGIN and WORKER_PASSWORD must be both set or both empty.
|
||||
// A mixed state (XOR) is a config bug and must fail fast so an
|
||||
// operator notices immediately. Both empty is allowed while the HTTP
|
||||
// listener is not started yet.
|
||||
// - When the listener would actually start (WORKER_PORT > 0), both
|
||||
// must be set; otherwise we are going to expose an unauthenticated
|
||||
// endpoint.
|
||||
// - WORKER_URL, if set, must be a parseable absolute URL. Relative
|
||||
// URLs are rejected because peer workers and the main app need a
|
||||
// concrete origin to dial.
|
||||
// - A WORKER_URL with scheme=http on a non-loopback host logs a
|
||||
// warning: production deployments normally terminate TLS at a
|
||||
// reverse proxy (Traefik, nginx).
|
||||
func ValidateHTTPConfig(c HTTPConfig, willListen bool) error {
|
||||
loginSet := c.Login != ""
|
||||
passSet := c.Password != ""
|
||||
if loginSet != passSet {
|
||||
return fmt.Errorf(
|
||||
"WORKER_LOGIN and WORKER_PASSWORD must be both set or both empty (got login=%s password=%s)",
|
||||
boolStr(loginSet), boolStr(passSet))
|
||||
}
|
||||
if willListen && !c.IsAuthConfigured() {
|
||||
return fmt.Errorf("HTTP listener refused to start: WORKER_LOGIN and WORKER_PASSWORD must be set when WORKER_PORT > 0")
|
||||
}
|
||||
if c.URL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(c.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("WORKER_URL is not a valid URL: %v", err)
|
||||
}
|
||||
if u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("WORKER_URL must be an absolute URL with scheme and host (got %q)", c.URL)
|
||||
}
|
||||
if u.Scheme != schemeHTTP && u.Scheme != schemeHTTPS {
|
||||
return fmt.Errorf("WORKER_URL scheme must be http or https (got %q)", u.Scheme)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WarnInsecurePublicURL logs a warning when WORKER_URL uses plain http
|
||||
// for a non-loopback host. Returns the host so the caller can log it.
|
||||
// A no-op for the loopback case (typical local dev) and for https URLs.
|
||||
func WarnInsecurePublicURL(rawURL string) (host string, shouldWarn bool) {
|
||||
if rawURL == "" {
|
||||
return "", false
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if u.Scheme != schemeHTTP {
|
||||
return u.Host, false
|
||||
}
|
||||
if isLoopbackHost(u.Hostname()) {
|
||||
return u.Host, false
|
||||
}
|
||||
return u.Host, true
|
||||
}
|
||||
|
||||
func isLoopbackHost(host string) bool {
|
||||
host = strings.ToLower(host)
|
||||
for _, h := range strings.Split(loopbackHostnames, ",") {
|
||||
if host == strings.TrimSpace(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func boolStr(b bool) string {
|
||||
if b {
|
||||
return "set"
|
||||
}
|
||||
return "empty"
|
||||
}
|
||||
|
||||
func normalizeURL(endpoint string) string {
|
||||
endpoint = strings.TrimRight(endpoint, "/")
|
||||
if strings.HasSuffix(endpoint, "/api/worker") {
|
||||
return strings.TrimSuffix(endpoint, "/api/worker")
|
||||
}
|
||||
if strings.HasSuffix(endpoint, "/worker") {
|
||||
return strings.TrimSuffix(endpoint, "/worker")
|
||||
}
|
||||
return endpoint
|
||||
}
|
||||
197
internal/distworker/config_test.go
Обычный файл
197
internal/distworker/config_test.go
Обычный файл
@@ -0,0 +1,197 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestHTTPConfigFromEnv_Defaults verifies the documented defaults when no
|
||||
// HTTP-related env vars are set: 0.0.0.0:27401 and empty URL / login /
|
||||
// password. The default port was bumped from 7401 to 27401 to avoid
|
||||
// colliding with the main RSMon app when the worker is co-located.
|
||||
func TestHTTPConfigFromEnv_Defaults(t *testing.T) {
|
||||
t.Setenv("WORKER_HOST", "")
|
||||
t.Setenv("WORKER_PORT", "")
|
||||
t.Setenv("WORKER_URL", "")
|
||||
t.Setenv("WORKER_LOGIN", "")
|
||||
t.Setenv("WORKER_PASSWORD", "")
|
||||
|
||||
cfg := HTTPConfigFromEnv()
|
||||
assert.Equal(t, DefaultHTTPHost, cfg.Host, "host should default to 0.0.0.0")
|
||||
assert.Equal(t, DefaultHTTPPort, cfg.Port, "port should default to 27401")
|
||||
assert.Equal(t, "", cfg.URL)
|
||||
assert.Equal(t, "", cfg.Login)
|
||||
assert.Equal(t, "", cfg.Password)
|
||||
assert.False(t, cfg.IsAuthConfigured())
|
||||
}
|
||||
|
||||
// TestHTTPConfigFromEnv_Overrides verifies the env vars win over the
|
||||
// defaults when explicitly set, including a non-default port.
|
||||
func TestHTTPConfigFromEnv_Overrides(t *testing.T) {
|
||||
t.Setenv("WORKER_HOST", "127.0.0.1")
|
||||
t.Setenv("WORKER_PORT", "9100")
|
||||
t.Setenv("WORKER_URL", "https://worker.example.com")
|
||||
t.Setenv("WORKER_LOGIN", "ops")
|
||||
t.Setenv("WORKER_PASSWORD", "s3cret")
|
||||
|
||||
cfg := HTTPConfigFromEnv()
|
||||
assert.Equal(t, "127.0.0.1", cfg.Host)
|
||||
assert.Equal(t, 9100, cfg.Port)
|
||||
assert.Equal(t, "https://worker.example.com", cfg.URL)
|
||||
assert.Equal(t, "ops", cfg.Login)
|
||||
assert.Equal(t, "s3cret", cfg.Password)
|
||||
assert.True(t, cfg.IsAuthConfigured())
|
||||
}
|
||||
|
||||
// TestHTTPConfigFromEnv_PortGarbageFallsBackToDefault covers the "operator
|
||||
// fat-fingered WORKER_PORT" case: the helper must not panic and must
|
||||
// fall back to DefaultHTTPPort instead of starting on a zero port.
|
||||
func TestHTTPConfigFromEnv_PortGarbageFallsBackToDefault(t *testing.T) {
|
||||
t.Setenv("WORKER_PORT", "not-a-port")
|
||||
|
||||
cfg := HTTPConfigFromEnv()
|
||||
assert.Equal(t, DefaultHTTPPort, cfg.Port,
|
||||
"unparseable WORKER_PORT should fall back to the default")
|
||||
}
|
||||
|
||||
// TestHTTPConfigFromEnv_PortOutOfRangeFallsBackToDefault covers zero and
|
||||
// out-of-range port values which are rejected by url/strconv semantics.
|
||||
func TestHTTPConfigFromEnv_PortOutOfRangeFallsBackToDefault(t *testing.T) {
|
||||
for _, v := range []string{"0", "-1", "70000"} {
|
||||
t.Run("port="+v, func(t *testing.T) {
|
||||
t.Setenv("WORKER_PORT", v)
|
||||
cfg := HTTPConfigFromEnv()
|
||||
assert.Equal(t, DefaultHTTPPort, cfg.Port,
|
||||
"WORKER_PORT=%q should fall back to the default", v)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_LoginXORPasswordRejected covers the central
|
||||
// invariant: a mixed login/password state is a config bug and must
|
||||
// fail fast.
|
||||
func TestValidateHTTPConfig_LoginXORPasswordRejected(t *testing.T) {
|
||||
t.Run("only login set", func(t *testing.T) {
|
||||
err := ValidateHTTPConfig(HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
Login: "ops", Password: "",
|
||||
}, false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "WORKER_LOGIN and WORKER_PASSWORD")
|
||||
})
|
||||
t.Run("only password set", func(t *testing.T) {
|
||||
err := ValidateHTTPConfig(HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
Login: "", Password: "s3cret",
|
||||
}, false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "WORKER_LOGIN and WORKER_PASSWORD")
|
||||
})
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_BothEmptyAllowedWhenNotListening reflects the
|
||||
// Task 2 deferral: while the HTTP listener is not started (Task 3),
|
||||
// leaving both unset is permitted so a worker that does not yet need
|
||||
// the web app can still start.
|
||||
func TestValidateHTTPConfig_BothEmptyAllowedWhenNotListening(t *testing.T) {
|
||||
err := ValidateHTTPConfig(HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
Login: "", Password: "",
|
||||
}, false)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_BothEmptyRejectedWhenListening confirms that
|
||||
// when Task 3 actually starts the listener, an unauthenticated listener
|
||||
// is refused.
|
||||
func TestValidateHTTPConfig_BothEmptyRejectedWhenListening(t *testing.T) {
|
||||
err := ValidateHTTPConfig(HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
Login: "", Password: "",
|
||||
}, true)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "refused to start")
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_URLAbsoluteRequired covers the relative-URL
|
||||
// rejection: peer workers and the main app need a concrete origin.
|
||||
func TestValidateHTTPConfig_URLAbsoluteRequired(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
}{
|
||||
{"relative path", "/worker"},
|
||||
{"missing scheme", "example.com"},
|
||||
{"missing host", "https://"},
|
||||
{"empty after trim", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, URL: tc.url}
|
||||
err := ValidateHTTPConfig(cfg, false)
|
||||
// empty url is allowed
|
||||
if tc.url == "" {
|
||||
assert.NoError(t, err)
|
||||
return
|
||||
}
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_URLSchemeAllowed covers the two accepted schemes.
|
||||
func TestValidateHTTPConfig_URLSchemeAllowed(t *testing.T) {
|
||||
for _, scheme := range []string{"http", "https"} {
|
||||
t.Run(scheme, func(t *testing.T) {
|
||||
cfg := HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
URL: scheme + "://localhost:27401",
|
||||
}
|
||||
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateHTTPConfig_URLSchemeRejected covers non-http(s) schemes.
|
||||
func TestValidateHTTPConfig_URLSchemeRejected(t *testing.T) {
|
||||
for _, scheme := range []string{"ftp", "file", "ws", "wss"} {
|
||||
t.Run(scheme, func(t *testing.T) {
|
||||
cfg := HTTPConfig{
|
||||
Host: "0.0.0.0", Port: 27401,
|
||||
URL: scheme + "://localhost:27401",
|
||||
}
|
||||
err := ValidateHTTPConfig(cfg, false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, strings.ToLower(err.Error()), "scheme must be http or https")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarnInsecurePublicURL exercises the warning helper: only http on
|
||||
// non-loopback hosts should warn. The returned host is the parsed host
|
||||
// (host:port when present) so the caller can log a useful target.
|
||||
func TestWarnInsecurePublicURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
wantWarn bool
|
||||
wantHost string
|
||||
}{
|
||||
{"empty", "", false, ""},
|
||||
{"https production", "https://worker.example.com", false, "worker.example.com"},
|
||||
{"http loopback", "http://localhost:27401", false, "localhost:27401"},
|
||||
{"http 127.0.0.1", "http://127.0.0.1:27401", false, "127.0.0.1:27401"},
|
||||
{"http 0.0.0.0", "http://0.0.0.0:27401", false, "0.0.0.0:27401"},
|
||||
{"http production", "http://worker.example.com", true, "worker.example.com"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
host, warn := WarnInsecurePublicURL(tc.url)
|
||||
assert.Equal(t, tc.wantWarn, warn)
|
||||
assert.Equal(t, tc.wantHost, host)
|
||||
})
|
||||
}
|
||||
}
|
||||
237
internal/distworker/consensus.go
Обычный файл
237
internal/distworker/consensus.go
Обычный файл
@@ -0,0 +1,237 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// consensusDecision is the outcome of one consensus round. The
|
||||
// selfcheck module drives its alert state machine off Down/Up; NoQuorum
|
||||
// is the "we don't know yet" verdict returned when too few voters
|
||||
// reported for either side to be a majority.
|
||||
type consensusDecision int
|
||||
|
||||
const (
|
||||
// consensusNoQuorum means we have fewer than the minimum voters
|
||||
// required to reach a majority. The selfcheck holds the
|
||||
// previous state instead of flipping.
|
||||
consensusNoQuorum consensusDecision = iota
|
||||
// consensusUp means a majority of (self + known peers) reported
|
||||
// the master API as up.
|
||||
consensusUp
|
||||
// consensusDown means a majority reported the master API as
|
||||
// down. The selfcheck starts (or continues) the down timer.
|
||||
consensusDown
|
||||
)
|
||||
|
||||
// String makes the verdict easy to log without bespoke formatting.
|
||||
func (d consensusDecision) String() string {
|
||||
switch d {
|
||||
case consensusUp:
|
||||
return "up"
|
||||
case consensusDown:
|
||||
return "down"
|
||||
default:
|
||||
return "no-quorum"
|
||||
}
|
||||
}
|
||||
|
||||
// tallyConsensus counts how many of self+peers are up vs down. The
|
||||
// returned sizes are useful for logging/debugging and let the
|
||||
// selfcheck report "2 of 3 voters say down" in its log line.
|
||||
//
|
||||
// selfUp == nil means "self vote is unknown" (e.g. the first tick
|
||||
// has not completed). The cluster size is the number of known votes
|
||||
// (self, if known, plus each peer observation).
|
||||
func tallyConsensus(selfUp *bool, peers []peerObservation) (up, down, total int) {
|
||||
if selfUp != nil {
|
||||
total++
|
||||
if *selfUp {
|
||||
up++
|
||||
} else {
|
||||
down++
|
||||
}
|
||||
}
|
||||
for i := range peers {
|
||||
total++
|
||||
if peers[i].Up {
|
||||
up++
|
||||
} else {
|
||||
down++
|
||||
}
|
||||
}
|
||||
return up, down, total
|
||||
}
|
||||
|
||||
// decideConsensus applies the simple-majority rule: whichever side
|
||||
// (up or down) has at least floor(total/2)+1 votes wins. When total
|
||||
// is 0 (no votes at all) or neither side reaches that threshold the
|
||||
// result is consensusNoQuorum.
|
||||
//
|
||||
// minVotes is the minimum number of fresh votes (self + peers) that
|
||||
// must be present before any verdict is reported. It is the gate
|
||||
// that turns simple-majority into a peer-backed cluster quorum: a
|
||||
// multi-worker deployment (WorkerInit.Peers non-empty) passes
|
||||
// minVotes=2 so a lone self vote — no fresh peer observations yet —
|
||||
// is consensusNoQuorum, both for up and down. A single-worker /
|
||||
// no-peer deployment passes minVotes=1 to preserve the prior
|
||||
// single-node behavior. Values below 1 are clamped to 1.
|
||||
//
|
||||
// The "self" vote is required to reach quorum: a worker that has not
|
||||
// produced its own first probe cannot make a down consensus call
|
||||
// (its own vote would be missing). Pass nil for selfUp to model the
|
||||
// pre-first-probe window.
|
||||
func decideConsensus(selfUp *bool, peers []peerObservation, minVotes int) consensusDecision {
|
||||
if selfUp == nil {
|
||||
return consensusNoQuorum
|
||||
}
|
||||
if minVotes < 1 {
|
||||
minVotes = 1
|
||||
}
|
||||
up, down, total := tallyConsensus(selfUp, peers)
|
||||
if total < minVotes {
|
||||
return consensusNoQuorum
|
||||
}
|
||||
// Standard majority for a non-empty set: floor(total/2)+1.
|
||||
// For 1 voter (self only, no peers yet) this collapses to 1
|
||||
// which still requires unanimous agreement with self.
|
||||
majority := total/2 + 1
|
||||
if up >= majority {
|
||||
return consensusUp
|
||||
}
|
||||
if down >= majority {
|
||||
return consensusDown
|
||||
}
|
||||
return consensusNoQuorum
|
||||
}
|
||||
|
||||
// consensusState tracks the cluster-level up/down verdict over time
|
||||
// so the selfcheck module can fire alerts only after the verdict has
|
||||
// held for selfcheckConsensusWait. This mirrors the "incident
|
||||
// state machine" idea from
|
||||
// docs/distributed/worker-to-worker-raft.md §9.1 in miniature: we
|
||||
// only have two states (down / clear) and one wait threshold, but
|
||||
// the structure is the same so the next slice can swap the rule for
|
||||
// the full FSM without changing the alert site.
|
||||
type consensusState struct {
|
||||
mu sync.Mutex
|
||||
// downSince records the wall-clock time the cluster verdict
|
||||
// first flipped to consensusDown. Cleared when the verdict
|
||||
// flips back to consensusUp. nil means "not currently down".
|
||||
downSince *time.Time
|
||||
// alertActive mirrors the prior selfcheckState.sent* flags. It
|
||||
// is true between the down-alert firing and the recovery alert
|
||||
// firing so a duplicate probe does not re-send the same
|
||||
// "master is down" notification.
|
||||
alertActive bool
|
||||
// lastVerdict keeps the most recent decision for the next tick
|
||||
// to compare against without recomputing from scratch.
|
||||
lastVerdict consensusDecision
|
||||
// notificationLeader is the last deterministic worker elected to
|
||||
// send system-contact notifications for this local view of the
|
||||
// cluster. The first non-empty leader only initializes the field;
|
||||
// later changes are alert-worthy.
|
||||
notificationLeader string
|
||||
}
|
||||
|
||||
func (s *consensusState) notificationLeaderChanged(leader string) (old string, changed bool) {
|
||||
if s == nil || leader == "" {
|
||||
return "", false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.notificationLeader == "" {
|
||||
s.notificationLeader = leader
|
||||
return "", false
|
||||
}
|
||||
if s.notificationLeader == leader {
|
||||
return "", false
|
||||
}
|
||||
old = s.notificationLeader
|
||||
s.notificationLeader = leader
|
||||
return old, true
|
||||
}
|
||||
|
||||
// isDownConsensusHeld reports whether the supplied verdict means
|
||||
// "down for at least selfcheckConsensusWait" given the current
|
||||
// state. A fresh down verdict sets downSince; a subsequent down
|
||||
// verdict keeps the original timestamp so the wait is measured from
|
||||
// the first observation, not the most recent.
|
||||
func (s *consensusState) isDownConsensusHeld(verdict consensusDecision, now time.Time) bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
switch verdict {
|
||||
case consensusDown:
|
||||
if s.downSince == nil {
|
||||
t := now
|
||||
s.downSince = &t
|
||||
}
|
||||
s.lastVerdict = verdict
|
||||
return now.Sub(*s.downSince) >= selfcheckConsensusWait
|
||||
case consensusUp:
|
||||
s.downSince = nil
|
||||
s.alertActive = false
|
||||
s.lastVerdict = verdict
|
||||
return false
|
||||
default:
|
||||
// NoQuorum: hold the existing state. Do not reset
|
||||
// downSince (a transient blip should not extend the
|
||||
// timer, but it should not erase progress either).
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// markDownAlertFired records that the down alert has been emitted so
|
||||
// the next tick does not fire it again. Idempotent.
|
||||
func (s *consensusState) markDownAlertFired() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.alertActive = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// shouldFireRecovery reports whether the cluster has been up long
|
||||
// enough to fire a recovery alert. Recovery uses the same wait
|
||||
// window as the down alert so a flapping verdict does not spam
|
||||
// recovery notifications.
|
||||
func (s *consensusState) shouldFireRecovery(verdict consensusDecision, now time.Time) bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if verdict != consensusUp {
|
||||
return false
|
||||
}
|
||||
if !s.alertActive {
|
||||
return false
|
||||
}
|
||||
if s.lastVerdict != consensusUp {
|
||||
// Just flipped from down to up; stamp the recovery timer.
|
||||
s.lastVerdict = verdict
|
||||
t := now
|
||||
s.downSince = &t
|
||||
return false
|
||||
}
|
||||
if s.downSince == nil {
|
||||
return false
|
||||
}
|
||||
return now.Sub(*s.downSince) >= selfcheckConsensusWait
|
||||
}
|
||||
|
||||
// markRecoveryFired clears the alert-active flag so a future
|
||||
// down verdict can fire the down alert again.
|
||||
func (s *consensusState) markRecoveryFired() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.alertActive = false
|
||||
s.downSince = nil
|
||||
s.mu.Unlock()
|
||||
}
|
||||
280
internal/distworker/notification.go
Обычный файл
280
internal/distworker/notification.go
Обычный файл
@@ -0,0 +1,280 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/internal/sender"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// ExecuteNotification runs one NotificationTask end-to-end. It looks up the
|
||||
// matching credential from the worker's in-memory cache (pushed via the init
|
||||
// / config websocket message) and dispatches to the per-method executor.
|
||||
// Returns a NotificationResultReport ready to send back to the control plane.
|
||||
//
|
||||
// Phase 1 of docs/plans/worker-notifier-mvp.md ships four supported methods
|
||||
// (email / telegram / webhook / mattermost). sms and voice are explicit
|
||||
// permanent failures with status=unsupported_method so the operator knows
|
||||
// the worker received the task and chose not to deliver it (rather than
|
||||
// silently dropping it as the production plan forbids).
|
||||
//
|
||||
//nolint:gocritic // task model is shared with the rest of the dispatcher; keep by-value
|
||||
func (r *Runner) ExecuteNotification(ctx context.Context, task models.Task) wire.NotificationResultReport {
|
||||
start := time.Now()
|
||||
|
||||
report := wire.NotificationResultReport{
|
||||
JobID: task.JobID,
|
||||
Status: wire.NotificationResultPermanent,
|
||||
DurationMs: 0,
|
||||
}
|
||||
|
||||
if len(task.Payload) == 0 {
|
||||
report.Status = wire.NotificationResultPermanent
|
||||
report.Error = stringPtr("notification task payload is empty")
|
||||
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||
return report
|
||||
}
|
||||
|
||||
var nt wire.NotificationTask
|
||||
if err := json.Unmarshal(task.Payload, &nt); err != nil {
|
||||
report.Error = stringPtr("invalid notification payload: " + err.Error())
|
||||
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||
return report
|
||||
}
|
||||
if nt.JobID == "" {
|
||||
nt.JobID = task.JobID
|
||||
}
|
||||
if nt.MessageID == 0 && task.MessageID != nil {
|
||||
nt.MessageID = *task.MessageID
|
||||
}
|
||||
report.JobID = nt.JobID
|
||||
report.LeaseToken = nt.LeaseToken
|
||||
report.MessageID = nt.MessageID
|
||||
if task.Deadline != nil && !task.Deadline.After(time.Now()) {
|
||||
report.Error = stringPtr("notification deadline expired")
|
||||
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||
return report
|
||||
}
|
||||
|
||||
creds := r.Credentials()
|
||||
if creds == nil {
|
||||
report.Error = stringPtr("worker has no notification credentials pushed yet")
|
||||
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||
return report
|
||||
}
|
||||
|
||||
status, providerResp, errStr, retry, execErr := r.deliverNotification(ctx, nt, creds)
|
||||
if task.Deadline != nil && !task.Deadline.After(time.Now()) {
|
||||
status, providerResp, errStr, retry, execErr = wire.NotificationResultPermanent, "", "notification deadline expired", nil, context.DeadlineExceeded
|
||||
}
|
||||
report.Status = status
|
||||
report.ProviderResponse = stringPtrOrNil(providerResp)
|
||||
report.Error = stringPtrOrNil(errStr)
|
||||
if retry != nil {
|
||||
report.RetryAfterSeconds = retry
|
||||
}
|
||||
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||
if execErr != nil {
|
||||
log.Printf("worker: notification delivery job=%s message=%d method=%s status=%s error_class=delivery_failed duration_ms=%d",
|
||||
nt.JobID, nt.MessageID, nt.Method, status, report.DurationMs)
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
// deliverNotification dispatches one notification task to the per-method
|
||||
// sub-executor. Returns (status, providerResponse, error, retryAfterSeconds,
|
||||
// internalError). The internalError is non-nil only for unexpected panics or
|
||||
// credential-resolution failures the result report should log.
|
||||
//
|
||||
//nolint:gocritic // wire payload is shared; keep by-value
|
||||
func (r *Runner) deliverNotification(
|
||||
ctx context.Context,
|
||||
nt wire.NotificationTask,
|
||||
creds *wire.NotificationCredentials,
|
||||
) (string, string, string, *int, error) {
|
||||
switch nt.Method {
|
||||
case systemContactKindEmail:
|
||||
if len(creds.SMTP) == 0 {
|
||||
return wire.NotificationResultPermanent,
|
||||
"", "no SMTP credential authorized for this worker", nil,
|
||||
errors.New("smtp: no credential")
|
||||
}
|
||||
cred, ok := selectSMTPCredential(creds.SMTP, nt.CredentialID)
|
||||
if !ok {
|
||||
return wire.NotificationResultPermanent, "", "requested SMTP credential is not authorized for this worker",
|
||||
nil, errors.New("smtp: credential not found")
|
||||
}
|
||||
err := sender.SendEmailWithCredentialContext(ctx,
|
||||
nt.Contact.Value, nt.Subject, nt.BodyText, nt.BodyHTML, &cred,
|
||||
)
|
||||
return classifyResult(err, "")
|
||||
|
||||
case "telegram":
|
||||
if len(creds.Telegram) == 0 {
|
||||
return wire.NotificationResultPermanent, "", "no Telegram credential authorized for this worker",
|
||||
nil, errors.New("telegram: no credential")
|
||||
}
|
||||
// Convert wire.TelegramCredential -> models.NotificationCredential
|
||||
// for the existing tg.SendMessageWithToken entry point. We do not
|
||||
// persist this; it lives only on the goroutine stack.
|
||||
cred, ok := selectTelegramCredential(creds.Telegram, nt.CredentialID)
|
||||
if !ok {
|
||||
return wire.NotificationResultPermanent, "", "requested Telegram credential is not authorized for this worker",
|
||||
nil, errors.New("telegram: credential not found")
|
||||
}
|
||||
nc := wireTelegramToModel(&cred)
|
||||
err := sender.SendTelegramWithCredentialContext(ctx,
|
||||
nt.Contact.Value, nt.Subject, nt.BodyMarkdown, nc,
|
||||
)
|
||||
return classifyResult(err, "")
|
||||
|
||||
case "webhook":
|
||||
signingSecret := ""
|
||||
if creds.Webhook != nil {
|
||||
signingSecret = creds.Webhook.SigningSecret
|
||||
}
|
||||
body, mErr := json.Marshal(nt)
|
||||
if mErr != nil {
|
||||
return wire.NotificationResultPermanent, "", "marshal webhook payload: " + mErr.Error(), nil, mErr
|
||||
}
|
||||
resp, err := sender.SendWebhookWithCredentialContext(ctx, body, nt.Contact.Value, signingSecret)
|
||||
return classifyResult(err, stringOrEmpty(resp))
|
||||
|
||||
case "mattermost":
|
||||
var mc *wire.MattermostCredential
|
||||
if creds.Mattermost != nil {
|
||||
mc = creds.Mattermost
|
||||
}
|
||||
resp, err := sender.SendMattermostWithCredentialContext(ctx,
|
||||
nt.Contact.Value, nt.MessageKind, nt.Subject, nt.BodyMarkdown, mc,
|
||||
)
|
||||
return classifyResult(err, stringOrEmpty(resp))
|
||||
|
||||
case "sms", "voice":
|
||||
return wire.NotificationResultPermanent, "", "unsupported_method: " + nt.Method,
|
||||
nil, errors.New("unsupported method: " + nt.Method)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return wire.NotificationResultRetryable, "", "context canceled: " + err.Error(), intPtr(5), err
|
||||
}
|
||||
return wire.NotificationResultPermanent, "", "unknown notification method: " + nt.Method,
|
||||
nil, errors.New("unknown method: " + nt.Method)
|
||||
}
|
||||
|
||||
func selectSMTPCredential(creds []wire.SMTPCredential, id *int64) (wire.SMTPCredential, bool) {
|
||||
if id != nil {
|
||||
for _, cred := range creds {
|
||||
if cred.ID == *id {
|
||||
return cred, true
|
||||
}
|
||||
}
|
||||
return wire.SMTPCredential{}, false
|
||||
}
|
||||
return creds[0], true
|
||||
}
|
||||
|
||||
func selectTelegramCredential(creds []wire.TelegramCredential, id *int64) (wire.TelegramCredential, bool) {
|
||||
if id != nil {
|
||||
for _, cred := range creds {
|
||||
if cred.ID == *id {
|
||||
return cred, true
|
||||
}
|
||||
}
|
||||
return wire.TelegramCredential{}, false
|
||||
}
|
||||
return creds[0], true
|
||||
}
|
||||
|
||||
// classifyResult translates a delivery error into the wire status enum.
|
||||
// SMTP 421 / 4xx with a Retry-After hint becomes retryable + suggested
|
||||
// delay. Network errors become retryable with a fixed 30s backoff. Anything
|
||||
// else is permanent.
|
||||
func classifyResult(err error, providerResp string) (string, string, string, *int, error) {
|
||||
if err == nil {
|
||||
return wire.NotificationResultDelivered, providerResp, "", nil, nil
|
||||
}
|
||||
errStr := err.Error()
|
||||
|
||||
// Heuristic: any 4xx SMTP response is retryable. The legacy sender does
|
||||
// not parse this, so the worker does a substring match on the error.
|
||||
if isRetryableSMTP(errStr) {
|
||||
retry := 30
|
||||
return wire.NotificationResultRetryable, providerResp, errStr, &retry, err
|
||||
}
|
||||
// Network / DNS / TLS / context errors are typically transient.
|
||||
if isTransientTransport(errStr) {
|
||||
retry := 15
|
||||
return wire.NotificationResultRetryable, providerResp, errStr, &retry, err
|
||||
}
|
||||
// Telegram "chat not found", webhook 4xx response (non-5xx) -> permanent.
|
||||
return wire.NotificationResultPermanent, providerResp, errStr, nil, err
|
||||
}
|
||||
|
||||
func isRetryableSMTP(s string) bool {
|
||||
prefixes := []string{"smtp: 4", "421", "450", "451", "452"}
|
||||
for _, p := range prefixes {
|
||||
if len(s) >= len(p) && s[:len(p)] == p {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isTransientTransport(s string) bool {
|
||||
markers := []string{"timeout", "tempor", "connection refused", "no such host", "i/o timeout", "tls"}
|
||||
for _, m := range markers {
|
||||
if bytes.Contains([]byte(s), []byte(m)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stringOrEmpty(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func stringPtr(s string) *string { return &s }
|
||||
|
||||
func stringPtrOrNil(s string) *string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return &s
|
||||
}
|
||||
|
||||
func intPtr(i int) *int { return &i }
|
||||
|
||||
// wireTelegramToModel translates a wire.TelegramCredential (init push shape)
|
||||
// into a models.NotificationCredential (DB shape) so it can be passed to the
|
||||
// existing tg.SendMessageWithToken entry point. The BotName/APIURL fields
|
||||
// are preserved; the secret is the bot token.
|
||||
func wireTelegramToModel(wt *wire.TelegramCredential) *models.NotificationCredential {
|
||||
nc := &models.NotificationCredential{
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: wt.Name,
|
||||
}
|
||||
if wt.BotName != "" {
|
||||
nc.BotName = &wt.BotName
|
||||
}
|
||||
if wt.APIURL != "" {
|
||||
nc.APIURL = &wt.APIURL
|
||||
}
|
||||
// Store the token in SecretEnc without encryption; SendMessageWithToken
|
||||
// does not read SecretEnc directly — it expects the helper to decrypt
|
||||
// via GetSecret. We go around that by writing the plaintext into the
|
||||
// struct field via the SetSecret path. Plain prefix lets the legacy
|
||||
// decrypt path return the raw value.
|
||||
nc.SetSecret(wt.Token) //nolint:errcheck // best-effort; failure becomes runtime error in SendMessageWithToken
|
||||
return nc
|
||||
}
|
||||
233
internal/distworker/notification_test.go
Обычный файл
233
internal/distworker/notification_test.go
Обычный файл
@@ -0,0 +1,233 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// runnerWithCreds is the smallest fixture that yields a Runner with a
|
||||
// credentials block already applied (no DB, no init websocket).
|
||||
func runnerWithCreds(creds *wire.NotificationCredentials) *Runner {
|
||||
r := NewRunner(&Config{MaxConcurrency: 4})
|
||||
r.credentialsMu.Lock()
|
||||
r.credentials = creds
|
||||
r.credentialsMu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// TestExecuteNotification_Email_NoCredentials verifies that the executor
|
||||
// returns a permanent failure when no SMTP credential is pushed.
|
||||
func TestExecuteNotification_Email_NoCredentials(t *testing.T) {
|
||||
r := runnerWithCreds(nil)
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{
|
||||
JobID: "job-1",
|
||||
Payload: []byte(`{
|
||||
"job_id":"job-1","message_id":1,"notification_id":1,
|
||||
"method":"email","subject":"x","body_text":"x","body_html":"<p>x</p>",
|
||||
"contact":{"id":1,"kind":"email","value":"ops@example.com","name":"ops"},
|
||||
"message_kind":"down"
|
||||
}`),
|
||||
})
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status)
|
||||
assert.NotNil(t, report.Error)
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status, "must be permanent so the producer does not loop")
|
||||
}
|
||||
|
||||
func TestClassifyResult_NilEmailErrorDoesNotPanic(t *testing.T) {
|
||||
status, _, text, retry, err := classifyResult(nil, "")
|
||||
assert.Equal(t, wire.NotificationResultDelivered, status)
|
||||
assert.Empty(t, text)
|
||||
assert.Nil(t, retry)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestExecuteNotification_UnsupportedMethod covers sms/voice returning
|
||||
// permanent + unsupported_method.
|
||||
func TestExecuteNotification_UnsupportedMethod(t *testing.T) {
|
||||
r := runnerWithCreds(&wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{Server: "smtp.example.com", Port: 587, Login: "u", Password: "p"}},
|
||||
Telegram: []wire.TelegramCredential{{Name: "bot", Token: "123:abc"}},
|
||||
})
|
||||
for _, method := range []string{"sms", "voice"} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{
|
||||
JobID: "job-" + method,
|
||||
Payload: []byte(`{
|
||||
"job_id":"job-` + method + `","message_id":1,"notification_id":1,
|
||||
"method":"` + method + `","subject":"x","body_text":"x","body_html":"<p>x</p>",
|
||||
"contact":{"id":1,"kind":"` + method + `","value":"x","name":"ops"},
|
||||
"message_kind":"down"
|
||||
}`),
|
||||
})
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status)
|
||||
require.NotNil(t, report.Error)
|
||||
assert.Contains(t, *report.Error, "unsupported_method")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecuteNotification_UnknownMethod classifies unknown methods as
|
||||
// permanent so we never loop on bad producer output.
|
||||
func TestExecuteNotification_UnknownMethod(t *testing.T) {
|
||||
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{
|
||||
JobID: "job-x",
|
||||
Payload: []byte(`{
|
||||
"job_id":"job-x","message_id":1,"notification_id":1,
|
||||
"method":"pigeon","subject":"x","body_text":"x","body_html":"<p>x</p>",
|
||||
"contact":{"id":1,"kind":"pigeon","value":"ops@example.com","name":"ops"},
|
||||
"message_kind":"down"
|
||||
}`),
|
||||
})
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status)
|
||||
}
|
||||
|
||||
// TestExecuteNotification_EmptyPayloadPermanent: a malformed task must be
|
||||
// rejected permanently so the operator can spot it on the admin page.
|
||||
func TestExecuteNotification_EmptyPayloadPermanent(t *testing.T) {
|
||||
r := runnerWithCreds(&wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{Server: "smtp.example.com", Port: 587, Login: "u", Password: "p"}},
|
||||
})
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{JobID: "job-empty"})
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status)
|
||||
require.NotNil(t, report.Error)
|
||||
}
|
||||
|
||||
func TestExecuteNotification_ExpiredDeadlineDoesNotDeliver(t *testing.T) {
|
||||
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||
expired := time.Now().Add(-time.Second)
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{JobID: "expired", Deadline: &expired, Payload: []byte(`{"job_id":"expired","message_id":1,"method":"email","contact":{"id":1,"kind":"email"}}`)})
|
||||
require.NotNil(t, report.Error)
|
||||
assert.Contains(t, *report.Error, "deadline expired")
|
||||
assert.Equal(t, wire.NotificationResultPermanent, report.Status)
|
||||
}
|
||||
|
||||
// TestExecuteNotification_DurationPositive: every report carries a non-zero
|
||||
// duration_ms even when the work happens instantly. This is what the
|
||||
// notification_deliveries audit row expects.
|
||||
func TestExecuteNotification_DurationPositive(t *testing.T) {
|
||||
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{
|
||||
JobID: "job-d",
|
||||
Payload: []byte(`{
|
||||
"job_id":"job-d","message_id":1,"notification_id":1,
|
||||
"method":"sms","subject":"x","body_text":"x","body_html":"<p>x</p>",
|
||||
"contact":{"id":1,"kind":"sms","value":"x","name":"ops"},
|
||||
"message_kind":"down"
|
||||
}`),
|
||||
})
|
||||
assert.GreaterOrEqual(t, report.DurationMs, 0)
|
||||
}
|
||||
|
||||
// TestClassifyResult verifies the error -> status mapping.
|
||||
func TestClassifyResult(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
wantStatus string
|
||||
wantRetryAfter *int
|
||||
}{
|
||||
{"nil error", nil, wire.NotificationResultDelivered, nil},
|
||||
{"permanent provider error", errors.New("550 mailbox not found"), wire.NotificationResultPermanent, nil},
|
||||
{"smtp 421 retryable", errors.New("smtp: 421 try again later"), wire.NotificationResultRetryable, intPtr(30)},
|
||||
{"smtp 452 retryable", errors.New("452 insufficient storage"), wire.NotificationResultRetryable, intPtr(30)},
|
||||
{"network timeout retryable", errors.New("dial tcp: i/o timeout"), wire.NotificationResultRetryable, intPtr(15)},
|
||||
{"tls handshake retryable", errors.New("tls: handshake failure"), wire.NotificationResultRetryable, intPtr(15)},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
status, _, errStr, retry, _ := classifyResult(tc.err, "")
|
||||
assert.Equal(t, tc.wantStatus, status)
|
||||
if tc.err != nil {
|
||||
assert.NotEmpty(t, errStr)
|
||||
}
|
||||
if tc.wantRetryAfter == nil {
|
||||
assert.Nil(t, retry)
|
||||
} else {
|
||||
require.NotNil(t, retry)
|
||||
assert.Equal(t, *tc.wantRetryAfter, *retry)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWireTelegramToModel_RoundTrip sanity-checks the wire->DB credential
|
||||
// translation used by the telegram branch.
|
||||
func TestWireTelegramToModel_RoundTrip(t *testing.T) {
|
||||
wire := &wire.TelegramCredential{
|
||||
ID: 1,
|
||||
Name: "main-bot",
|
||||
BotName: "rsmon_bot",
|
||||
Token: "123456:ABCDEFG",
|
||||
APIURL: "https://api.telegram.org",
|
||||
}
|
||||
nc := wireTelegramToModel(wire)
|
||||
require.NotNil(t, nc)
|
||||
assert.Equal(t, models.CredentialKindTelegram, nc.Kind)
|
||||
assert.Equal(t, "main-bot", nc.Name)
|
||||
require.NotNil(t, nc.BotName)
|
||||
assert.Equal(t, "rsmon_bot", *nc.BotName)
|
||||
require.NotNil(t, nc.APIURL)
|
||||
assert.Equal(t, "https://api.telegram.org", *nc.APIURL)
|
||||
got, err := nc.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "123456:ABCDEFG", got)
|
||||
}
|
||||
|
||||
func TestSelectNotificationCredentialByID(t *testing.T) {
|
||||
id := int64(2)
|
||||
smtp, ok := selectSMTPCredential([]wire.SMTPCredential{{ID: 1, Name: "first"}, {ID: 2, Name: "second"}}, &id)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "second", smtp.Name)
|
||||
|
||||
tg, ok := selectTelegramCredential([]wire.TelegramCredential{{ID: 1, Name: "first"}, {ID: 2, Name: "second"}}, &id)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "second", tg.Name)
|
||||
|
||||
missingID := int64(3)
|
||||
_, ok = selectSMTPCredential([]wire.SMTPCredential{{ID: 1, Name: "first"}}, &missingID)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
// TestEnqueueNotification_BoundedQueue ensures the notifyQueue provides
|
||||
// backpressure: the channel capacity is queueCapacity() and EnqueueNotification
|
||||
// blocks once it is full.
|
||||
func TestEnqueueNotification_BoundedQueue(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 4})
|
||||
r.notifyQueue = make(chan wire.NotificationTask, 4)
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
require.True(t, r.EnqueueNotification(wire.NotificationTask{JobID: "x"}))
|
||||
}
|
||||
// Channel is full; a non-blocking send must fail. We cannot truly verify
|
||||
// the blocking case in unit tests, so we just assert the depth counter.
|
||||
assert.Equal(t, int64(4), r.notifyDepth)
|
||||
}
|
||||
|
||||
// TestExecuteNotification_ReportsJobIDFromPayload verifies the executor
|
||||
// falls back to the task's JobID when the payload has none.
|
||||
func TestExecuteNotification_ReportsJobIDFromPayload(t *testing.T) {
|
||||
r := runnerWithCreds(nil)
|
||||
report := r.ExecuteNotification(context.Background(), models.Task{
|
||||
JobID: "outer-job",
|
||||
Payload: []byte(`{
|
||||
"job_id":"","message_id":1,"notification_id":1,
|
||||
"method":"sms","subject":"x","body_text":"x","body_html":"<p>x</p>",
|
||||
"contact":{"id":1,"kind":"sms","value":"x","name":"ops"},
|
||||
"message_kind":"down"
|
||||
}`),
|
||||
})
|
||||
assert.Equal(t, "outer-job", report.JobID)
|
||||
}
|
||||
|
||||
// guard against time import being unused if the above compile-time helpers
|
||||
// are dropped in a future refactor.
|
||||
var _ = time.Second
|
||||
241
internal/distworker/peer.go
Обычный файл
241
internal/distworker/peer.go
Обычный файл
@@ -0,0 +1,241 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// peerStatusPath is the HTTP path the worker webapp serves at
|
||||
// GET /api/peer/status. Kept on a single constant so the polling
|
||||
// client and the handler can never drift.
|
||||
const peerStatusPath = "/api/peer/status"
|
||||
|
||||
// peerStatusTimeout bounds how long a single peer probe is allowed
|
||||
// to take. Short enough that a sluggish peer does not stall the
|
||||
// selfcheck loop; long enough to absorb a single TCP retransmit.
|
||||
const peerStatusTimeout = 5 * time.Second
|
||||
|
||||
// peerStatusMaxAge is how stale a peer's last observation is allowed
|
||||
// to get before the consensus treats it as "unknown" (excluded from
|
||||
// the vote). Equal to two poll intervals so a single dropped probe
|
||||
// does not immediately disqualify a peer.
|
||||
const peerStatusMaxAge = 2 * peerPollInterval
|
||||
|
||||
// peerPollInterval drives the peer-poller cadence. Kept equal to the
|
||||
// local selfcheck interval so observations stay aligned.
|
||||
const peerPollInterval = 30 * time.Second
|
||||
|
||||
// PeerStatus is the JSON a worker returns at GET /api/peer/status.
|
||||
// The shape is stable so peer workers can pin against it without
|
||||
// coordinating a schema bump. ObservedAt is the wall-clock time the
|
||||
// local selfcheck produced this status; peers ignore entries older
|
||||
// than peerStatusMaxAge.
|
||||
type PeerStatus struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
Up bool `json:"up"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
|
||||
// peerObservation is the per-peer entry the consensus uses. The
|
||||
// selfcheck module does not reach into the wire type directly; it
|
||||
// always works with the cached peerObservation so the JSON shape can
|
||||
// evolve without forcing a selfcheck refactor.
|
||||
type peerObservation struct {
|
||||
WorkerID string
|
||||
Up bool
|
||||
ObservedAt time.Time
|
||||
Err error // transient fetch error; nil = observation is valid
|
||||
}
|
||||
|
||||
// peerCache holds the latest observation per peer worker_id. The map
|
||||
// is guarded by a single RWMutex because reads (consensus) vastly
|
||||
// outnumber writes (one per peer per poll tick). Entries survive
|
||||
// across applyInit refreshes; the peer poller overwrites the
|
||||
// per-worker slot every tick.
|
||||
type peerCache struct {
|
||||
mu sync.RWMutex
|
||||
items map[string]peerObservation
|
||||
}
|
||||
|
||||
func newPeerCache() *peerCache {
|
||||
return &peerCache{items: map[string]peerObservation{}}
|
||||
}
|
||||
|
||||
// snapshot returns a defensive copy of the current observations in
|
||||
// no particular order. Callers must not mutate the returned slice.
|
||||
func (c *peerCache) snapshot() []peerObservation {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
out := make([]peerObservation, 0, len(c.items))
|
||||
for _, v := range c.items {
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// put stores a fresh observation. The poller calls this every tick;
|
||||
// the consensus reads via snapshot().
|
||||
func (c *peerCache) put(obs peerObservation) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.items[obs.WorkerID] = obs
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// resetFor replaces the cache contents with one fresh observation per
|
||||
// peer in the supplied list. Used when applyInit refreshes the peer
|
||||
// set so a removed peer's stale observation is dropped immediately
|
||||
// rather than lingering until peerStatusMaxAge expires.
|
||||
func (c *peerCache) resetFor(peers []wire.PeerInfo) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.items = make(map[string]peerObservation, len(peers))
|
||||
for i := range peers {
|
||||
p := peers[i]
|
||||
if p.WorkerID == "" {
|
||||
continue
|
||||
}
|
||||
c.items[p.WorkerID] = peerObservation{WorkerID: p.WorkerID}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchPeerStatus issues a single GET /api/peer/status to the given
|
||||
// peer and decodes the response. Network errors and non-2xx codes
|
||||
// are returned as a populated peerObservation with Err set so the
|
||||
// caller can decide whether to update the cache (we do, to mark the
|
||||
// peer as "we tried"). Caller is responsible for not calling this
|
||||
// concurrently for the same peer in a way that would race the
|
||||
// peerCache lock; the per-peer update is serialized via the cache.
|
||||
//
|
||||
// peer is taken by pointer to keep the wire.PeerInfo copy off the
|
||||
// hot path; the function is called once per peer per poll interval.
|
||||
func fetchPeerStatus(ctx context.Context, peer *wire.PeerInfo) peerObservation {
|
||||
obs := peerObservation{WorkerID: peer.WorkerID}
|
||||
if peer.URL == "" {
|
||||
obs.Err = fmt.Errorf("peer %s: empty url", peer.WorkerID)
|
||||
return obs
|
||||
}
|
||||
u, err := url.Parse(peer.URL)
|
||||
if err != nil || (u.Scheme != schemeHTTP && u.Scheme != schemeHTTPS) {
|
||||
obs.Err = fmt.Errorf("peer %s: bad url %q", peer.WorkerID, peer.URL)
|
||||
return obs
|
||||
}
|
||||
endpoint := strings.TrimRight(peer.URL, "/") + peerStatusPath
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
|
||||
if err != nil {
|
||||
obs.Err = fmt.Errorf("peer %s: build request: %w", peer.WorkerID, err)
|
||||
return obs
|
||||
}
|
||||
if peer.Login != "" {
|
||||
req.SetBasicAuth(peer.Login, peer.Password)
|
||||
}
|
||||
client := &http.Client{Timeout: peerStatusTimeout}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
obs.Err = fmt.Errorf("peer %s: get: %w", peer.WorkerID, err)
|
||||
return obs
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
obs.Err = fmt.Errorf("peer %s: status %d", peer.WorkerID, resp.StatusCode)
|
||||
return obs
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8*1024))
|
||||
if err != nil {
|
||||
obs.Err = fmt.Errorf("peer %s: read body: %w", peer.WorkerID, err)
|
||||
return obs
|
||||
}
|
||||
var status PeerStatus
|
||||
if err := json.Unmarshal(body, &status); err != nil {
|
||||
obs.Err = fmt.Errorf("peer %s: decode: %w", peer.WorkerID, err)
|
||||
return obs
|
||||
}
|
||||
obs.Up = status.Up
|
||||
obs.ObservedAt = status.ObservedAt
|
||||
return obs
|
||||
}
|
||||
|
||||
// peerPollerLoop runs once per peerPollInterval and refreshes the
|
||||
// cache. It exits when ctx is canceled. The loop runs each peer
|
||||
// sequentially so a slow / unreachable peer does not spawn N
|
||||
// concurrent goroutines that would overwhelm the local listener.
|
||||
func (r *Runner) peerPollerLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(peerPollInterval)
|
||||
defer ticker.Stop()
|
||||
// Run once shortly after startup so a worker that boots while
|
||||
// the master is already down does not wait a full interval
|
||||
// before the first peer observation lands.
|
||||
first := time.NewTimer(2 * time.Second)
|
||||
defer first.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
case <-ticker.C:
|
||||
}
|
||||
r.pollPeersOnce(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// pollPeersOnce fetches every cached peer's status and stores the
|
||||
// results. Safe to call directly from tests to drive a deterministic
|
||||
// poll cycle.
|
||||
func (r *Runner) pollPeersOnce(ctx context.Context) {
|
||||
peers := r.Peers()
|
||||
if len(peers) == 0 {
|
||||
return
|
||||
}
|
||||
for i := range peers {
|
||||
p := peers[i]
|
||||
obs := fetchPeerStatus(ctx, &p)
|
||||
r.peerCache.put(obs)
|
||||
if obs.Err != nil {
|
||||
log.Printf("worker peer poll: %s: %v", p.WorkerID, obs.Err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// peerObservationsForConsensus returns the peer observations the
|
||||
// consensus should consider. A peer whose last observation is older
|
||||
// than peerStatusMaxAge is dropped from the vote (treated as
|
||||
// "unknown") so a worker that lost connectivity to one peer cannot
|
||||
// single-handedly decide the cluster is healthy.
|
||||
func (r *Runner) peerObservationsForConsensus(now time.Time) []peerObservation {
|
||||
all := r.peerCache.snapshot()
|
||||
out := make([]peerObservation, 0, len(all))
|
||||
for _, obs := range all {
|
||||
if obs.WorkerID == "" {
|
||||
continue
|
||||
}
|
||||
if obs.Err != nil {
|
||||
continue
|
||||
}
|
||||
if obs.ObservedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if now.Sub(obs.ObservedAt) > peerStatusMaxAge {
|
||||
continue
|
||||
}
|
||||
out = append(out, obs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
424
internal/distworker/peer_test.go
Обычный файл
424
internal/distworker/peer_test.go
Обычный файл
@@ -0,0 +1,424 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
func TestDecideConsensus(t *testing.T) {
|
||||
up := true
|
||||
down := false
|
||||
peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
|
||||
peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
self *bool
|
||||
peers []peerObservation
|
||||
minVotes int
|
||||
want consensusDecision
|
||||
}{
|
||||
{
|
||||
name: "self only up (single-node, minVotes=1)",
|
||||
self: &up,
|
||||
peers: nil,
|
||||
minVotes: 1,
|
||||
want: consensusUp,
|
||||
},
|
||||
{
|
||||
name: "self only down (single-node, minVotes=1)",
|
||||
self: &down,
|
||||
peers: nil,
|
||||
minVotes: 1,
|
||||
want: consensusDown,
|
||||
},
|
||||
{
|
||||
name: "self down + 2 peers down = majority down",
|
||||
self: &down,
|
||||
peers: []peerObservation{
|
||||
{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
|
||||
{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
|
||||
},
|
||||
minVotes: 2,
|
||||
want: consensusDown,
|
||||
},
|
||||
{
|
||||
name: "self up + 2 peers up = majority up",
|
||||
self: &up,
|
||||
peers: []peerObservation{
|
||||
{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
|
||||
{WorkerID: "p2", Up: true, ObservedAt: time.Now()},
|
||||
},
|
||||
minVotes: 2,
|
||||
want: consensusUp,
|
||||
},
|
||||
{
|
||||
name: "self up + 1 up + 1 down = majority up (2 of 3)",
|
||||
self: &up,
|
||||
peers: []peerObservation{
|
||||
{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
|
||||
{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
|
||||
},
|
||||
minVotes: 2,
|
||||
want: consensusUp,
|
||||
},
|
||||
{
|
||||
name: "self up + 1 down (1 up, 1 down) = tie (no quorum)",
|
||||
self: &up,
|
||||
peers: []peerObservation{
|
||||
{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
|
||||
},
|
||||
minVotes: 2,
|
||||
want: consensusNoQuorum,
|
||||
},
|
||||
{
|
||||
name: "self down + 1 up + 1 down = majority down (2 of 3)",
|
||||
self: &down,
|
||||
peers: []peerObservation{
|
||||
{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
|
||||
{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
|
||||
},
|
||||
minVotes: 2,
|
||||
want: consensusDown,
|
||||
},
|
||||
{
|
||||
name: "self vote missing = no quorum",
|
||||
self: nil,
|
||||
peers: peersDown,
|
||||
minVotes: 2,
|
||||
want: consensusNoQuorum,
|
||||
},
|
||||
{
|
||||
name: "self vote missing + 1 up peer = still no quorum",
|
||||
self: nil,
|
||||
peers: peersUp,
|
||||
minVotes: 2,
|
||||
want: consensusNoQuorum,
|
||||
},
|
||||
{
|
||||
name: "minVotes=0 clamped to 1 (single-node self up)",
|
||||
self: &up,
|
||||
peers: nil,
|
||||
minVotes: 0,
|
||||
want: consensusUp,
|
||||
},
|
||||
{
|
||||
name: "minVotes<0 clamped to 1 (single-node self down)",
|
||||
self: &down,
|
||||
peers: nil,
|
||||
minVotes: -3,
|
||||
want: consensusDown,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, decideConsensus(tc.self, tc.peers, tc.minVotes))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsensusState_FiresAfterWait(t *testing.T) {
|
||||
state := &consensusState{}
|
||||
now := time.Now()
|
||||
|
||||
// Stamp the initial down verdict at the baseline so subsequent
|
||||
// ticks measure against a known start time.
|
||||
held := state.isDownConsensusHeld(consensusDown, now)
|
||||
assert.False(t, held, "must not fire on the first down tick")
|
||||
assert.NotNil(t, state.downSince, "downSince should be stamped on first down verdict")
|
||||
|
||||
// Not yet fired even after 4m59s.
|
||||
held = state.isDownConsensusHeld(consensusDown, now.Add(4*time.Minute+59*time.Second))
|
||||
assert.False(t, held, "must not fire before the wait elapses")
|
||||
|
||||
// Fire after the wait elapses.
|
||||
held = state.isDownConsensusHeld(consensusDown, now.Add(5*time.Minute+time.Second))
|
||||
assert.True(t, held, "must fire once the wait has elapsed")
|
||||
state.markDownAlertFired()
|
||||
assert.True(t, state.alertActive)
|
||||
|
||||
// Subsequent down verdicts do not re-stamp downSince.
|
||||
stampBefore := *state.downSince
|
||||
_ = state.isDownConsensusHeld(consensusDown, now.Add(10*time.Minute))
|
||||
assert.Equal(t, stampBefore, *state.downSince,
|
||||
"downSince must be set on the first observation, not re-stamped")
|
||||
|
||||
// Recovery clears the down timer and the alert flag.
|
||||
_ = state.isDownConsensusHeld(consensusUp, now.Add(11*time.Minute))
|
||||
assert.Nil(t, state.downSince)
|
||||
assert.False(t, state.alertActive)
|
||||
}
|
||||
|
||||
func TestConsensusState_RecoveryRequiresHeld(t *testing.T) {
|
||||
state := &consensusState{}
|
||||
now := time.Now()
|
||||
|
||||
// Pretend we already fired a down alert.
|
||||
state.alertActive = true
|
||||
t0 := now
|
||||
state.downSince = &t0
|
||||
|
||||
// First up verdict stamps a fresh recovery timer; the alert
|
||||
// must NOT fire yet.
|
||||
assert.False(t, state.shouldFireRecovery(consensusUp, now.Add(time.Second)))
|
||||
assert.Equal(t, consensusUp, state.lastVerdict)
|
||||
|
||||
// Stale up verdict (verdict != up) does not advance the
|
||||
// recovery timer.
|
||||
assert.False(t, state.shouldFireRecovery(consensusDown, now.Add(time.Minute)))
|
||||
|
||||
// Held long enough, recovery fires.
|
||||
assert.True(t, state.shouldFireRecovery(consensusUp, now.Add(selfcheckConsensusWait+time.Second)))
|
||||
state.markRecoveryFired()
|
||||
assert.False(t, state.alertActive)
|
||||
assert.Nil(t, state.downSince)
|
||||
}
|
||||
|
||||
func TestConsensusState_NoQuorumHoldsState(t *testing.T) {
|
||||
state := &consensusState{}
|
||||
now := time.Now()
|
||||
|
||||
// First establish a down verdict + timer.
|
||||
_ = state.isDownConsensusHeld(consensusDown, now)
|
||||
stampBefore := *state.downSince
|
||||
state.markDownAlertFired()
|
||||
|
||||
// NoQuorum must not reset the timer; the alert stays fired
|
||||
// so the next down verdict does not double-fire.
|
||||
_ = state.isDownConsensusHeld(consensusNoQuorum, now.Add(2*time.Minute))
|
||||
assert.Equal(t, stampBefore, *state.downSince)
|
||||
assert.True(t, state.alertActive)
|
||||
}
|
||||
|
||||
func TestTallyConsensus(t *testing.T) {
|
||||
up, down := true, false
|
||||
peers := []peerObservation{
|
||||
{Up: true}, {Up: false}, {Up: true},
|
||||
}
|
||||
gotUp, gotDown, gotTotal := tallyConsensus(&up, peers)
|
||||
assert.Equal(t, 3, gotUp)
|
||||
assert.Equal(t, 1, gotDown)
|
||||
assert.Equal(t, 4, gotTotal)
|
||||
|
||||
gotUp, gotDown, gotTotal = tallyConsensus(&down, nil)
|
||||
assert.Equal(t, 0, gotUp)
|
||||
assert.Equal(t, 1, gotDown)
|
||||
assert.Equal(t, 1, gotTotal)
|
||||
|
||||
gotUp, gotDown, gotTotal = tallyConsensus(nil, peers)
|
||||
assert.Equal(t, 2, gotUp)
|
||||
assert.Equal(t, 1, gotDown)
|
||||
assert.Equal(t, 3, gotTotal)
|
||||
}
|
||||
|
||||
func TestPeerCacheSnapshotAndPut(t *testing.T) {
|
||||
c := newPeerCache()
|
||||
c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
|
||||
c.put(peerObservation{WorkerID: "b", Up: false, ObservedAt: time.Now()})
|
||||
|
||||
snap := c.snapshot()
|
||||
assert.Len(t, snap, 2)
|
||||
}
|
||||
|
||||
func TestPeerCacheResetForDropsRemovedPeers(t *testing.T) {
|
||||
c := newPeerCache()
|
||||
c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
|
||||
c.put(peerObservation{WorkerID: "b", Up: true, ObservedAt: time.Now()})
|
||||
|
||||
c.resetFor([]wire.PeerInfo{{WorkerID: "a"}, {WorkerID: "c"}})
|
||||
snap := c.snapshot()
|
||||
ids := map[string]peerObservation{}
|
||||
for _, s := range snap {
|
||||
ids[s.WorkerID] = s
|
||||
}
|
||||
_, hasA := ids["a"]
|
||||
_, hasB := ids["b"]
|
||||
_, hasC := ids["c"]
|
||||
assert.True(t, hasA, "peer 'a' retained")
|
||||
assert.False(t, hasB, "removed peer 'b' dropped")
|
||||
assert.True(t, hasC, "new peer 'c' added")
|
||||
}
|
||||
|
||||
func TestApplyInitStoresPeersAndResetsCache(t *testing.T) {
|
||||
executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
require.NotNil(t, r.peerCache, "runner must own a peer cache after NewRunner")
|
||||
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 2,
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
{WorkerID: "w-3", URL: "http://127.0.0.1:27403"},
|
||||
},
|
||||
})
|
||||
got := r.Peers()
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, "w-2", got[0].WorkerID)
|
||||
assert.Equal(t, "w-3", got[1].WorkerID)
|
||||
|
||||
// Cache should be primed with empty observations for the new
|
||||
// peers; the per-peer entries are visible via snapshot().
|
||||
snap := r.peerCache.snapshot()
|
||||
assert.Len(t, snap, 2)
|
||||
for _, s := range snap {
|
||||
assert.True(t, s.ObservedAt.IsZero(),
|
||||
"freshly-reset peer %s should have a zero observed_at", s.WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchPeerStatus_DecodesUpResponse(t *testing.T) {
|
||||
up := true
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(PeerStatus{
|
||||
WorkerID: "remote",
|
||||
Up: up,
|
||||
ObservedAt: now,
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
|
||||
WorkerID: "remote",
|
||||
URL: srv.URL,
|
||||
})
|
||||
assert.True(t, obs.Up)
|
||||
assert.NoError(t, obs.Err)
|
||||
assert.Equal(t, "remote", obs.WorkerID)
|
||||
assert.Equal(t, now, obs.ObservedAt)
|
||||
}
|
||||
|
||||
func TestFetchPeerStatus_NetworkErrorIsRecorded(t *testing.T) {
|
||||
obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
|
||||
WorkerID: "unreachable",
|
||||
URL: "http://127.0.0.1:1",
|
||||
})
|
||||
assert.Error(t, obs.Err)
|
||||
assert.False(t, obs.Up)
|
||||
}
|
||||
|
||||
func TestFetchPeerStatus_BadURLRejected(t *testing.T) {
|
||||
obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
|
||||
WorkerID: "weird",
|
||||
URL: "ftp://example.com",
|
||||
})
|
||||
assert.Error(t, obs.Err)
|
||||
}
|
||||
|
||||
func TestDecideConsensus_MultiWorkerQuorumGate(t *testing.T) {
|
||||
// When the control plane has configured peers (a multi-worker
|
||||
// install), a lone self vote must NOT be treated as a
|
||||
// cluster-wide verdict in either direction. A second,
|
||||
// independent vote is required to reach any verdict.
|
||||
up := true
|
||||
down := false
|
||||
peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}
|
||||
peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
|
||||
|
||||
t.Run("self down + configured peers but no fresh peer = no quorum", func(t *testing.T) {
|
||||
assert.Equal(t, consensusNoQuorum, decideConsensus(&down, nil, 2),
|
||||
"a lone down vote must not page when peers are configured but not reporting")
|
||||
assert.Equal(t, consensusNoQuorum, decideConsensus(&up, nil, 2),
|
||||
"a lone up vote must not clear an active alert when peers are configured but not reporting")
|
||||
})
|
||||
|
||||
t.Run("self down + one fresh down peer = down consensus", func(t *testing.T) {
|
||||
assert.Equal(t, consensusDown, decideConsensus(&down, peersDown, 2))
|
||||
})
|
||||
|
||||
t.Run("self up + one fresh up peer = up consensus", func(t *testing.T) {
|
||||
assert.Equal(t, consensusUp, decideConsensus(&up, peersUp, 2))
|
||||
})
|
||||
|
||||
t.Run("self down + one fresh up peer = no quorum (1/1 split)", func(t *testing.T) {
|
||||
// With two voters and one of each, neither side has a
|
||||
// majority, so the verdict stays NoQuorum. This is the
|
||||
// intended conservative behavior — we don't want a
|
||||
// single dissenting peer to override self.
|
||||
assert.Equal(t, consensusNoQuorum, decideConsensus(&down, peersUp, 2))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDecideConsensus_SingleNodePreserved(t *testing.T) {
|
||||
// When no peers are configured (single-worker / no-peer
|
||||
// deployment) the caller passes minVotes=1, which preserves
|
||||
// the prior single-node behavior: a self-only vote is enough
|
||||
// to drive a verdict in either direction.
|
||||
up := true
|
||||
down := false
|
||||
|
||||
assert.Equal(t, consensusUp, decideConsensus(&up, nil, 1),
|
||||
"single-node self up must still reach up consensus")
|
||||
assert.Equal(t, consensusDown, decideConsensus(&down, nil, 1),
|
||||
"single-node self down must still reach down consensus")
|
||||
}
|
||||
|
||||
func TestPeerObservationsForConsensus_DropsStale(t *testing.T) {
|
||||
executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
|
||||
now := time.Now()
|
||||
fresh := peerObservation{WorkerID: "p1", Up: true, ObservedAt: now.Add(-30 * time.Second)}
|
||||
stale := peerObservation{WorkerID: "p2", Up: true, ObservedAt: now.Add(-5 * time.Minute)}
|
||||
errored := peerObservation{WorkerID: "p3", Up: true, Err: assert.AnError}
|
||||
unknown := peerObservation{WorkerID: "p4"}
|
||||
r.peerCache.put(fresh)
|
||||
r.peerCache.put(stale)
|
||||
r.peerCache.put(errored)
|
||||
r.peerCache.put(unknown)
|
||||
|
||||
got := r.peerObservationsForConsensus(now)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "p1", got[0].WorkerID)
|
||||
}
|
||||
|
||||
func TestPollPeersOnceIsConcurrencySafe(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(PeerStatus{
|
||||
WorkerID: "remote",
|
||||
Up: true,
|
||||
ObservedAt: time.Now().UTC(),
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
URL: "http://127.0.0.1:1", // local self probe will fail, no effect on peer poll
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "p1", URL: srv.URL},
|
||||
},
|
||||
})
|
||||
|
||||
// Run multiple concurrent poll cycles. The cache uses a single
|
||||
// RWMutex; this verifies there is no data race under that
|
||||
// access pattern.
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
r.pollPeersOnce(context.Background())
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
assert.GreaterOrEqual(t, atomic.LoadInt32(&hits), int32(1))
|
||||
}
|
||||
156
internal/distworker/results.go
Обычный файл
156
internal/distworker/results.go
Обычный файл
@@ -0,0 +1,156 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// recentResultsSize is the in-memory ring buffer capacity for the
|
||||
// last N check results produced by the worker. The webapp reads
|
||||
// from this buffer for the /checks page and the recent-results
|
||||
// counters on /overview.
|
||||
const recentResultsSize = 200
|
||||
|
||||
// recentNotificationsSize mirrors recentResultsSize for emitted
|
||||
// notifications. Phase 1 only writes selfcheck alerts to this
|
||||
// buffer (the main app's notification flow still lives in the main
|
||||
// app); the buffer is shape-stable so future phases can append
|
||||
// without changing the page contract.
|
||||
const recentNotificationsSize = 100
|
||||
|
||||
// ResultRow is one row from the worker's in-memory result ring
|
||||
// buffer. Kept in the distworker package so the webapp can read it
|
||||
// without going through wire (which is a payload envelope, not a
|
||||
// stable render type).
|
||||
type ResultRow struct {
|
||||
MonitorID int64
|
||||
CheckID int64
|
||||
Kind string
|
||||
Host string
|
||||
State string
|
||||
DurationMs int64
|
||||
Error string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// NotificationRow is one row from the worker's notification ring
|
||||
// buffer. Phase 1 only fills this from selfcheck alerts; the row
|
||||
// shape is forward-compatible with main-app-issued notifications.
|
||||
type NotificationRow struct {
|
||||
Kind string // "email", "telegram_private", "telegram_group"
|
||||
Channel string
|
||||
Subject string
|
||||
Body string
|
||||
OK bool
|
||||
Error string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// resultBuffer is a thread-safe FIFO ring buffer of ResultRow. The
|
||||
// Runner owns one; both the dispatcher goroutine and the webapp
|
||||
// readers can touch it concurrently.
|
||||
type resultBuffer struct {
|
||||
mu sync.RWMutex
|
||||
buf []ResultRow
|
||||
head int
|
||||
size int
|
||||
}
|
||||
|
||||
func newResultBuffer() *resultBuffer {
|
||||
return &resultBuffer{buf: make([]ResultRow, recentResultsSize)}
|
||||
}
|
||||
|
||||
// add appends one entry, evicting the oldest when full.
|
||||
func (b *resultBuffer) add(r *ResultRow) {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.buf[b.head] = *r
|
||||
b.head = (b.head + 1) % len(b.buf)
|
||||
if b.size < len(b.buf) {
|
||||
b.size++
|
||||
}
|
||||
}
|
||||
|
||||
// snapshot returns the most recent n rows in chronological order.
|
||||
// n <= 0 returns an empty slice.
|
||||
func (b *resultBuffer) snapshot(n int) []ResultRow {
|
||||
return ringSnapshot(&b.mu, &b.head, &b.size, &b.buf, n)
|
||||
}
|
||||
|
||||
// notificationBuffer mirrors resultBuffer for NotificationRow.
|
||||
type notificationBuffer struct {
|
||||
mu sync.RWMutex
|
||||
buf []NotificationRow
|
||||
head int
|
||||
size int
|
||||
}
|
||||
|
||||
func newNotificationBuffer() *notificationBuffer {
|
||||
return ¬ificationBuffer{buf: make([]NotificationRow, recentNotificationsSize)}
|
||||
}
|
||||
|
||||
func (b *notificationBuffer) add(r *NotificationRow) {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.buf[b.head] = *r
|
||||
b.head = (b.head + 1) % len(b.buf)
|
||||
if b.size < len(b.buf) {
|
||||
b.size++
|
||||
}
|
||||
}
|
||||
|
||||
func (b *notificationBuffer) snapshot(n int) []NotificationRow {
|
||||
return ringSnapshot(&b.mu, &b.head, &b.size, &b.buf, n)
|
||||
}
|
||||
|
||||
// ringSnapshot is a generic FIFO ring-buffer snapshot. The caller
|
||||
// passes the mutex/head/size/buf by pointer; the lock is taken
|
||||
// while reading. n <= 0 returns nil. The returned slice is a copy
|
||||
// so callers can hand it to non-locking code paths (e.g. the
|
||||
// webapp's templates) without holding the buffer lock.
|
||||
func ringSnapshot[T any](mu *sync.RWMutex, head, size *int, buf *[]T, n int) []T {
|
||||
if mu == nil || n <= 0 {
|
||||
return nil
|
||||
}
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
if *size == 0 {
|
||||
return nil
|
||||
}
|
||||
if n > *size {
|
||||
n = *size
|
||||
}
|
||||
out := make([]T, 0, n)
|
||||
start := (*head - n + len(*buf)) % len(*buf)
|
||||
for i := 0; i < n; i++ {
|
||||
idx := (start + i) % len(*buf)
|
||||
out = append(out, (*buf)[idx])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resultRowFromReport converts a wire.CheckResultReport into the
|
||||
// internal ResultRow type the webapp renders.
|
||||
func resultRowFromReport(env *resultEnvelope, report *wire.CheckResultReport, at time.Time) *ResultRow {
|
||||
row := &ResultRow{
|
||||
MonitorID: report.MonitorID,
|
||||
CheckID: report.CheckID,
|
||||
Kind: env.job.Kind,
|
||||
Host: env.job.Host,
|
||||
State: report.State,
|
||||
DurationMs: report.DurationMs,
|
||||
At: at,
|
||||
}
|
||||
if report.Error != nil {
|
||||
row.Error = *report.Error
|
||||
}
|
||||
return row
|
||||
}
|
||||
141
internal/distworker/results_test.go
Обычный файл
141
internal/distworker/results_test.go
Обычный файл
@@ -0,0 +1,141 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
func TestResultBufferSnapshotEmpty(t *testing.T) {
|
||||
b := newResultBuffer()
|
||||
assert.Nil(t, b.snapshot(0))
|
||||
assert.Nil(t, b.snapshot(10))
|
||||
}
|
||||
|
||||
func TestResultBufferAppendAndSnapshot(t *testing.T) {
|
||||
b := newResultBuffer()
|
||||
for i := 0; i < 12; i++ {
|
||||
b.add(&ResultRow{MonitorID: int64(i), State: "OK"})
|
||||
}
|
||||
got := b.snapshot(20)
|
||||
// Buffer is fixed-size so we cap at recentResultsSize; here we
|
||||
// added only 12 entries so the buffer still holds all of them.
|
||||
require.Len(t, got, 12)
|
||||
assert.Equal(t, int64(0), got[0].MonitorID)
|
||||
assert.Equal(t, int64(11), got[11].MonitorID)
|
||||
}
|
||||
|
||||
func TestResultBufferEvictsOldest(t *testing.T) {
|
||||
b := newResultBuffer()
|
||||
total := recentResultsSize + 5
|
||||
for i := 0; i < total; i++ {
|
||||
b.add(&ResultRow{MonitorID: int64(i)})
|
||||
}
|
||||
got := b.snapshot(recentResultsSize)
|
||||
require.Len(t, got, recentResultsSize)
|
||||
// The first surviving entry is i=5 (the (N+1)-th add).
|
||||
assert.Equal(t, int64(5), got[0].MonitorID)
|
||||
assert.Equal(t, int64(total-1), got[recentResultsSize-1].MonitorID)
|
||||
}
|
||||
|
||||
func TestResultBufferConcurrentAccess(t *testing.T) {
|
||||
b := newResultBuffer()
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < 4; w++ {
|
||||
wg.Add(1)
|
||||
go func(off int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 100; i++ {
|
||||
b.add(&ResultRow{MonitorID: int64(off*100 + i)})
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
for w := 0; w < 4; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 50; i++ {
|
||||
_ = b.snapshot(10)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
got := b.snapshot(recentResultsSize)
|
||||
assert.NotNil(t, got)
|
||||
}
|
||||
|
||||
func TestNotificationBufferAppendAndSnapshot(t *testing.T) {
|
||||
b := newNotificationBuffer()
|
||||
for i := 0; i < 3; i++ {
|
||||
b.add(&NotificationRow{Subject: "s" + itoaForTest(i), At: time.Now()})
|
||||
}
|
||||
got := b.snapshot(10)
|
||||
require.Len(t, got, 3)
|
||||
assert.Equal(t, "s0", got[0].Subject)
|
||||
}
|
||||
|
||||
func TestRunnerRecentResultsBeforeStart(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 2})
|
||||
assert.Nil(t, r.RecentResults(5),
|
||||
"RecentResults must be safe before Start")
|
||||
}
|
||||
|
||||
func TestRunnerRecentNotificationsBeforeStart(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 2})
|
||||
assert.Nil(t, r.RecentNotifications(5))
|
||||
}
|
||||
|
||||
func TestRunnerRecordNotification(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 2})
|
||||
r.RecordNotification(&NotificationRow{Subject: "x", OK: true})
|
||||
r.RecordNotification(&NotificationRow{Subject: "y", OK: false, Error: "boom"})
|
||||
got := r.RecentNotifications(10)
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, "x", got[0].Subject)
|
||||
assert.True(t, got[0].OK)
|
||||
assert.False(t, got[1].OK)
|
||||
assert.Equal(t, "boom", got[1].Error)
|
||||
}
|
||||
|
||||
func TestRunnerIdentityFieldsFromInit(t *testing.T) {
|
||||
executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
RegionCode: "eu",
|
||||
Version: "v1.2.3",
|
||||
Capabilities: []string{"http", "ssl"},
|
||||
})
|
||||
assert.Equal(t, "w-1", r.WorkerID())
|
||||
assert.Equal(t, "eu", r.RegionCode())
|
||||
assert.Equal(t, "v1.2.3", r.WorkerVersion())
|
||||
assert.Equal(t, []string{"http", "ssl"}, r.WorkerCapabilities())
|
||||
}
|
||||
|
||||
func TestRunnerTokenAccessors(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 1, Token: "tok-1"})
|
||||
assert.Equal(t, "tok-1", r.Token())
|
||||
assert.True(t, r.TokenRotatedAt().IsZero(), "no rotation yet")
|
||||
r.tokenRotatedMu.Lock()
|
||||
r.tokenRotatedAt = time.Now().UTC()
|
||||
r.tokenRotatedMu.Unlock()
|
||||
assert.False(t, r.TokenRotatedAt().IsZero())
|
||||
}
|
||||
|
||||
func itoaForTest(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
const d = "0123456789"
|
||||
out := ""
|
||||
for n > 0 {
|
||||
out = string(d[n%10]) + out
|
||||
n /= 10
|
||||
}
|
||||
return out
|
||||
}
|
||||
1030
internal/distworker/runner.go
Обычный файл
1030
internal/distworker/runner.go
Обычный файл
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
19
internal/distworker/runner_protocol_test.go
Обычный файл
19
internal/distworker/runner_protocol_test.go
Обычный файл
@@ -0,0 +1,19 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
func TestEnqueueTaskMessagePrefersEnvelopeOverLegacyCheck(t *testing.T) {
|
||||
r := &Runner{jobQueue: make(chan wire.CheckJob, 2), stopCh: make(chan struct{})}
|
||||
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{Type: wire.TaskTypeCheck, Job: &wire.CheckJob{JobID: "v2"}}, Task: &wire.CheckJob{JobID: "v1"}}
|
||||
require.True(t, r.enqueueTaskMessage(message))
|
||||
job := <-r.jobQueue
|
||||
assert.Equal(t, "v2", job.JobID)
|
||||
assert.Empty(t, r.jobQueue, "the sibling legacy field must not run a second check")
|
||||
}
|
||||
318
internal/distworker/runner_test.go
Обычный файл
318
internal/distworker/runner_test.go
Обычный файл
@@ -0,0 +1,318 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Jeffail/tunny"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// newTestRunner builds a runner with a deterministic executor and the
|
||||
// fixed dispatcher/pool layout. It registers a cleanup hook that drains
|
||||
// the runner so tests can leak-free.
|
||||
func newTestRunner(t *testing.T, maxConc, poolSize int, fn func(payload interface{}) interface{}) *Runner {
|
||||
t.Helper()
|
||||
r := NewRunner(&Config{MaxConcurrency: maxConc})
|
||||
r.executor = fn
|
||||
require.NotNil(t, r.executor)
|
||||
atomic.StoreInt64(&r.concurrency, int64(poolSize))
|
||||
r.jobQueue = make(chan wire.CheckJob, r.queueCapacity())
|
||||
r.results = make(chan resultEnvelope, r.queueCapacity())
|
||||
r.pool = tunny.NewFunc(poolSize, r.executor)
|
||||
for i := 0; i < maxConc; i++ {
|
||||
r.wg.Add(1)
|
||||
go r.dispatcher()
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
r.Stop()
|
||||
r.wg.Wait()
|
||||
r.pool.Close()
|
||||
close(r.results)
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
func TestQueueCapacityScalesWithMaxConcurrency(t *testing.T) {
|
||||
cases := []struct {
|
||||
maxConc int
|
||||
wantCapacity int
|
||||
}{
|
||||
// 2*maxConc, floored at minQueueCapacity so a small pool still
|
||||
// has backpressure headroom.
|
||||
{maxConc: 1, wantCapacity: minQueueCapacity},
|
||||
{maxConc: 4, wantCapacity: minQueueCapacity},
|
||||
{maxConc: 16, wantCapacity: 2 * 16},
|
||||
{maxConc: 64, wantCapacity: 2 * 64},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run("max="+strconv.Itoa(tc.maxConc), func(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: tc.maxConc})
|
||||
assert.Equal(t, tc.wantCapacity, r.QueueCapacity(),
|
||||
"queue capacity should scale with max concurrency")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatcherRunsJobsConcurrently(t *testing.T) {
|
||||
const (
|
||||
maxConc = 8
|
||||
poolSize = 4
|
||||
jobCount = 12
|
||||
hold = 80 * time.Millisecond
|
||||
)
|
||||
var (
|
||||
inFlight atomic.Int64
|
||||
peak atomic.Int64
|
||||
)
|
||||
|
||||
executor := func(payload interface{}) interface{} {
|
||||
cur := inFlight.Add(1)
|
||||
for {
|
||||
p := peak.Load()
|
||||
if cur <= p || peak.CompareAndSwap(p, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(hold)
|
||||
inFlight.Add(-1)
|
||||
job := payload.(wire.CheckJob)
|
||||
return []wire.CheckResultReport{{
|
||||
JobID: job.JobID,
|
||||
CheckID: job.CheckID,
|
||||
State: "OK",
|
||||
}}
|
||||
}
|
||||
|
||||
r := newTestRunner(t, maxConc, poolSize, executor)
|
||||
|
||||
// Drain the results channel so dispatchers do not block.
|
||||
var drainWG sync.WaitGroup
|
||||
drainWG.Add(1)
|
||||
go func() {
|
||||
defer drainWG.Done()
|
||||
for i := 0; i < jobCount; i++ {
|
||||
select {
|
||||
case <-r.results:
|
||||
case <-r.stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < jobCount; i++ {
|
||||
job := wire.CheckJob{
|
||||
JobID: "job-" + strconv.Itoa(i),
|
||||
CheckID: int64(i + 1),
|
||||
Kind: "http",
|
||||
Host: "example.com",
|
||||
}
|
||||
require.True(t, r.Enqueue(job))
|
||||
}
|
||||
|
||||
drainWG.Wait()
|
||||
// The tunny.Pool size (poolSize) limits how many jobs run in
|
||||
// parallel, so the peak should be at most poolSize and at least 2
|
||||
// (otherwise the test would pass on a serial pool).
|
||||
observed := peak.Load()
|
||||
assert.GreaterOrEqual(t, observed, int64(2),
|
||||
"expected concurrent execution, observed peak=%d", observed)
|
||||
assert.LessOrEqual(t, observed, int64(poolSize),
|
||||
"peak should be bounded by pool size, observed peak=%d", observed)
|
||||
}
|
||||
|
||||
func TestEnqueueRespectsBackpressure(t *testing.T) {
|
||||
// Use a small queue with no dispatchers consuming it, so the bounded
|
||||
// channel is the only source of backpressure. This is the cleanest
|
||||
// way to assert that Enqueue parks when the buffer is full.
|
||||
const queueCap = 4
|
||||
r := NewRunner(&Config{MaxConcurrency: 2})
|
||||
r.jobQueue = make(chan wire.CheckJob, queueCap)
|
||||
r.results = make(chan resultEnvelope, queueCap)
|
||||
t.Cleanup(func() {
|
||||
r.Stop()
|
||||
close(r.results)
|
||||
})
|
||||
|
||||
// Fill the bounded queue.
|
||||
for i := 0; i < queueCap; i++ {
|
||||
require.True(t, r.Enqueue(wire.CheckJob{JobID: "prefill-" + strconv.Itoa(i)}))
|
||||
}
|
||||
assert.Equal(t, queueCap, r.QueueDepth(),
|
||||
"queue should be full after %d enqueues", queueCap)
|
||||
|
||||
// The next Enqueue must block because the queue is full.
|
||||
enqueueDone := make(chan bool, 1)
|
||||
go func() {
|
||||
enqueueDone <- r.Enqueue(wire.CheckJob{JobID: "blocking"})
|
||||
}()
|
||||
|
||||
select {
|
||||
case got := <-enqueueDone:
|
||||
t.Fatalf("Enqueue returned %v while the queue was full; expected backpressure", got)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
// expected: still parked
|
||||
}
|
||||
|
||||
// Free a slot and confirm the parked Enqueue unblocks.
|
||||
select {
|
||||
case <-r.jobQueue:
|
||||
case <-r.stopCh:
|
||||
t.Fatal("runner stopped unexpectedly")
|
||||
}
|
||||
select {
|
||||
case ok := <-enqueueDone:
|
||||
assert.True(t, ok, "Enqueue should succeed once a slot is free")
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Enqueue did not unblock after slot was freed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyInitResizesPool(t *testing.T) {
|
||||
executor := func(payload interface{}) interface{} {
|
||||
return []wire.CheckResultReport{}
|
||||
}
|
||||
|
||||
r := newTestRunner(t, 8, 1, executor)
|
||||
// Replace the pool with a proxy that records SetSize calls. The
|
||||
// proxy delegates Close back to the underlying pool so the test
|
||||
// runner cleanup only closes the pool once.
|
||||
original := r.pool
|
||||
proxy := newSizeProbe(original)
|
||||
r.pool = proxy
|
||||
|
||||
r.applyInit(&wire.WorkerInit{Concurrency: 5, WorkerID: "w-1"})
|
||||
assert.Equal(t, 5, r.Concurrency(), "applyInit should update pool size")
|
||||
require.NotEmpty(t, proxy.sizes, "expected pool.SetSize to be called")
|
||||
assert.Equal(t, 5, proxy.sizes[len(proxy.sizes)-1])
|
||||
|
||||
// Concurrency above maxConcurrency should be clamped.
|
||||
r.applyInit(&wire.WorkerInit{Concurrency: 999, WorkerID: "w-1"})
|
||||
assert.Equal(t, r.MaxConcurrency(), r.Concurrency(),
|
||||
"applyInit should clamp concurrency to maxConcurrency")
|
||||
}
|
||||
|
||||
// sizeProbePool wraps a jobPool and records SetSize calls. Close is
|
||||
// forwarded to the wrapped pool so cleanup happens exactly once.
|
||||
type sizeProbePool struct {
|
||||
jobPool
|
||||
sizes []int
|
||||
}
|
||||
|
||||
func newSizeProbe(p jobPool) *sizeProbePool {
|
||||
return &sizeProbePool{jobPool: p}
|
||||
}
|
||||
|
||||
func (s *sizeProbePool) SetSize(n int) {
|
||||
s.sizes = append(s.sizes, n)
|
||||
s.jobPool.SetSize(n)
|
||||
}
|
||||
|
||||
func TestApplyInitIgnoresNonPositive(t *testing.T) {
|
||||
executor := func(payload interface{}) interface{} {
|
||||
return []wire.CheckResultReport{}
|
||||
}
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
// newTestRunner mirrors Start()'s initial concurrency of 1.
|
||||
require.Equal(t, 1, r.Concurrency())
|
||||
|
||||
// Concurrency = 0 must not break the runner or change its size.
|
||||
r.applyInit(&wire.WorkerInit{Concurrency: 0})
|
||||
assert.Equal(t, 1, r.Concurrency(),
|
||||
"non-positive concurrency should be ignored")
|
||||
|
||||
// Concurrency = -5 should also be ignored.
|
||||
r.applyInit(&wire.WorkerInit{Concurrency: -5})
|
||||
assert.Equal(t, 1, r.Concurrency())
|
||||
}
|
||||
|
||||
// TestApplyInitStoresCredentialsInMemory verifies that WorkerInit.Credentials
|
||||
// is stored on the Runner via applyInit and is retrievable through
|
||||
// Credentials(). It also confirms that a subsequent applyInit with nil
|
||||
// credentials replaces the previous value.
|
||||
func TestApplyInitStoresCredentialsInMemory(t *testing.T) {
|
||||
executor := func(payload interface{}) interface{} {
|
||||
return []wire.CheckResultReport{}
|
||||
}
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
|
||||
// Before any applyInit, Credentials() returns nil.
|
||||
assert.Nil(t, r.Credentials(), "Credentials() must return nil before any init")
|
||||
|
||||
want := &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{
|
||||
ID: 11,
|
||||
Name: "primary",
|
||||
Server: "smtp.example.com",
|
||||
Port: 587,
|
||||
Login: "alerts@example.com",
|
||||
Password: "smtp-password-xyz",
|
||||
}},
|
||||
Telegram: []wire.TelegramCredential{{
|
||||
ID: 22,
|
||||
Name: "main-bot",
|
||||
Token: "bot-token-9876543210:ABCDEFG",
|
||||
}},
|
||||
}
|
||||
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 2,
|
||||
Credentials: want,
|
||||
})
|
||||
|
||||
got := r.Credentials()
|
||||
require.NotNil(t, got, "Credentials() must return non-nil after applyInit")
|
||||
require.Len(t, got.SMTP, 1)
|
||||
require.Len(t, got.Telegram, 1)
|
||||
assert.Equal(t, "primary", got.SMTP[0].Name)
|
||||
assert.Equal(t, "smtp-password-xyz", got.SMTP[0].Password)
|
||||
assert.Equal(t, "main-bot", got.Telegram[0].Name)
|
||||
assert.Equal(t, "bot-token-9876543210:ABCDEFG", got.Telegram[0].Token)
|
||||
|
||||
// A subsequent applyInit with nil Credentials replaces the value.
|
||||
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2})
|
||||
assert.Nil(t, r.Credentials(),
|
||||
"Credentials() must return nil after applyInit with nil Credentials")
|
||||
}
|
||||
|
||||
func TestNotificationHeartbeatCounters(t *testing.T) {
|
||||
r := NewRunner(&Config{MaxConcurrency: 1})
|
||||
atomic.StoreInt64(&r.notifyDepth, 2)
|
||||
atomic.StoreInt64(&r.notifyActive, 3)
|
||||
assert.Equal(t, 5, r.ActiveNotifications())
|
||||
assert.Equal(t, 2, r.NotificationQueueDepth())
|
||||
}
|
||||
|
||||
// TestApplyInitStoresURLInMemory verifies that the URL field added in
|
||||
// Task 2 is stored on the Runner via applyInit and exposed through
|
||||
// URL(). A subsequent applyInit with empty URL replaces the previous
|
||||
// value (consistent with the Credentials contract).
|
||||
func TestApplyInitStoresURLInMemory(t *testing.T) {
|
||||
executor := func(payload interface{}) interface{} {
|
||||
return []wire.CheckResultReport{}
|
||||
}
|
||||
r := newTestRunner(t, 4, 1, executor)
|
||||
|
||||
// Before any applyInit, URL() returns empty.
|
||||
assert.Equal(t, "", r.URL(), "URL() must return empty before any init")
|
||||
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 2,
|
||||
URL: "https://worker-eu.example.com",
|
||||
})
|
||||
assert.Equal(t, "https://worker-eu.example.com", r.URL(),
|
||||
"URL() must return the value pushed by applyInit")
|
||||
|
||||
// Empty URL in a subsequent applyInit must clear the stored value.
|
||||
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2})
|
||||
assert.Equal(t, "", r.URL(),
|
||||
"URL() must return empty after applyInit with empty URL")
|
||||
}
|
||||
373
internal/distworker/selfcheck.go
Обычный файл
373
internal/distworker/selfcheck.go
Обычный файл
@@ -0,0 +1,373 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
mathrand "math/rand/v2"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/app/models"
|
||||
"rsgit.ru/rsmon/rsmon/app/models/concerns"
|
||||
"rsgit.ru/rsmon/rsmon/internal/checkexec"
|
||||
"rsgit.ru/rsmon/rsmon/internal/notify"
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
const (
|
||||
selfcheckInterval = 30 * time.Second
|
||||
selfcheckTimeoutMillis = 10_000
|
||||
selfcheckJitterMax = 30 * time.Second
|
||||
selfcheckProbePath = "/up"
|
||||
// selfcheckConsensusWait is the time the cluster-level down
|
||||
// verdict must hold before the selfcheck fires a system alert.
|
||||
// 5 minutes matches the user-facing requirement: a single
|
||||
// blip should not page, but a sustained outage must.
|
||||
selfcheckConsensusWait = 5 * time.Minute
|
||||
selfcheckDownMessage = "Нет связи с основным api"
|
||||
selfcheckRecoveryMessage = "Связь с основным api восстановлена"
|
||||
selfcheckLeaderMessage = "Изменился мастер воркер оповещений"
|
||||
selfcheckStateOK = "OK"
|
||||
selfcheckStateWarn = "WARN"
|
||||
// systemContactKindEmail is the wire-level identifier for email
|
||||
// system contacts, matching Contact.Kind values from the control plane.
|
||||
systemContactKindEmail = "email"
|
||||
|
||||
// notificationChannel labels are short tags the webapp renders
|
||||
// alongside each row on /notifications. Kept distinct from the
|
||||
// systemContactKind* constants so a single contact can be
|
||||
// reached over multiple channels without an extra column.
|
||||
notificationChannelSMTP = "smtp"
|
||||
notificationChannelTelegram = "telegram"
|
||||
)
|
||||
|
||||
// selfcheckState tracks the cluster-level up/down verdict for the
|
||||
// master API as decided by simple-majority consensus over self +
|
||||
// reachable peers. Alert fires only after the down verdict has held
|
||||
// for selfcheckConsensusWait; recovery fires only after a matching
|
||||
// up verdict has held for the same window.
|
||||
type selfcheckState struct {
|
||||
consensus *consensusState
|
||||
}
|
||||
|
||||
// startSelfcheck launches the periodic selfcheck loop. The first tick is
|
||||
// delayed by a random 0..selfcheckJitterMax so a fleet restart does not
|
||||
// stampede the main API simultaneously. Returns when ctx is canceled.
|
||||
func (r *Runner) startSelfcheck(ctx context.Context) {
|
||||
jitter := time.Duration(mathrand.Int64N(int64(selfcheckJitterMax))) //nolint:gosec // non-cryptographic jitter to stagger fleet probes
|
||||
select {
|
||||
case <-time.After(jitter):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
ticker := time.NewTicker(selfcheckInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
r.runSelfcheckOnce(ctx, state)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.runSelfcheckOnce(ctx, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) runSelfcheckOnce(ctx context.Context, state *selfcheckState) {
|
||||
target := selfcheckTarget(r.config.URL)
|
||||
if target == "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
ok := runMainAPIHTTPCheck(target)
|
||||
upFlag := ok
|
||||
r.SetMasterStatus(&upFlag, now)
|
||||
|
||||
peers := r.peerObservationsForConsensus(now)
|
||||
configuredPeers := len(r.Peers())
|
||||
// In a multi-worker deployment we need a peer-backed quorum
|
||||
// (self + at least one fresh peer) before any verdict is
|
||||
// reported. A lone self vote — peers configured but no fresh
|
||||
// peer observations yet — is consensusNoQuorum so the
|
||||
// selfcheck state machine does not start its 5-minute down
|
||||
// timer on what might be a transient fleet-bootstrap blip.
|
||||
// The configured peer list itself is the signal that the
|
||||
// operator wants cross-worker consensus; peerObservations
|
||||
// above already drops stale/error observations so "configured
|
||||
// peers" maps 1:1 to "peer observations can be fresh".
|
||||
minVotes := 1
|
||||
if configuredPeers > 0 {
|
||||
minVotes = 2
|
||||
}
|
||||
verdict := decideConsensus(&upFlag, peers, minVotes)
|
||||
up, down, total := tallyConsensus(&upFlag, peers)
|
||||
voters := consensusVoters(r.WorkerID(), peers)
|
||||
notificationLeader := consensusNotificationLeader(voters)
|
||||
log.Printf("worker: selfcheck verdict=%s up=%d down=%d total=%d min_votes=%d target=%s",
|
||||
verdict, up, down, total, minVotes, target)
|
||||
if oldLeader, changed := state.consensus.notificationLeaderChanged(notificationLeader); changed && r.isConsensusNotificationLeader(notificationLeader) {
|
||||
log.Printf("worker: selfcheck notification leader changed old=%s new=%s; firing system alert", oldLeader, notificationLeader)
|
||||
r.sendSystemAlert(ctx, true, r.formatSelfcheckLeaderMessage(oldLeader, target, verdict, up, down, total, minVotes, configuredPeers, len(peers), notificationLeader, voters))
|
||||
}
|
||||
|
||||
if verdict == consensusDown {
|
||||
if state.consensus.isDownConsensusHeld(verdict, now) &&
|
||||
!state.consensus.alertActive {
|
||||
state.consensus.markDownAlertFired()
|
||||
if !r.isConsensusNotificationLeader(notificationLeader) {
|
||||
log.Printf("worker: selfcheck down consensus held %s; notification leader=%s; skipping send", selfcheckConsensusWait, notificationLeader)
|
||||
return
|
||||
}
|
||||
log.Printf("worker: selfcheck down consensus held %s; firing system alert", selfcheckConsensusWait)
|
||||
r.sendSystemAlert(ctx, true, r.formatSelfcheckMessage(selfcheckDownMessage, target, verdict, up, down, total, minVotes, configuredPeers, len(peers), notificationLeader, voters))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if verdict == consensusUp && state.consensus.shouldFireRecovery(verdict, now) {
|
||||
state.consensus.markRecoveryFired()
|
||||
if !r.isConsensusNotificationLeader(notificationLeader) {
|
||||
log.Printf("worker: selfcheck up consensus held %s; notification leader=%s; skipping recovery", selfcheckConsensusWait, notificationLeader)
|
||||
return
|
||||
}
|
||||
log.Printf("worker: selfcheck up consensus held %s; firing recovery", selfcheckConsensusWait)
|
||||
r.sendSystemAlert(ctx, false, r.formatSelfcheckMessage(selfcheckRecoveryMessage, target, verdict, up, down, total, minVotes, configuredPeers, len(peers), notificationLeader, voters))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) formatSelfcheckLeaderMessage(
|
||||
oldLeader, target string,
|
||||
verdict consensusDecision,
|
||||
up, down, total, minVotes, configuredPeers, freshPeers int,
|
||||
notificationLeader string,
|
||||
voters []string,
|
||||
) string {
|
||||
return r.formatSelfcheckMessage(
|
||||
fmt.Sprintf("%s: %s -> %s", selfcheckLeaderMessage, oldLeader, notificationLeader),
|
||||
target, verdict, up, down, total, minVotes, configuredPeers, freshPeers, notificationLeader, voters,
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Runner) formatSelfcheckMessage(
|
||||
message, target string,
|
||||
verdict consensusDecision,
|
||||
up, down, total, minVotes, configuredPeers, freshPeers int,
|
||||
notificationLeader string,
|
||||
voters []string,
|
||||
) string {
|
||||
workerID := r.WorkerID()
|
||||
if workerID == "" {
|
||||
workerID = "unknown"
|
||||
}
|
||||
region := r.RegionCode()
|
||||
if region == "" {
|
||||
region = "unknown"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"%s\n\nWorker: %s\nRegion: %s\nTarget: %s\nRaft status: lightweight peer quorum\nNotification leader: %s\nConsensus: %s\nVotes: up=%d down=%d total=%d min_votes=%d\nWorkers seen: %d current votes, %d fresh peers of %d configured peers\nVoters: %s\nHold time: %s",
|
||||
message, workerID, region, target, notificationLeader, verdict, up, down, total, minVotes, total, freshPeers, configuredPeers, strings.Join(voters, ","), selfcheckConsensusWait,
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Runner) isConsensusNotificationLeader(leader string) bool {
|
||||
return leader != "" && r.WorkerID() == leader
|
||||
}
|
||||
|
||||
func consensusVoters(self string, peers []peerObservation) []string {
|
||||
voters := make([]string, 0, len(peers)+1)
|
||||
if self != "" {
|
||||
voters = append(voters, self)
|
||||
}
|
||||
for _, peer := range peers {
|
||||
if peer.WorkerID != "" {
|
||||
voters = append(voters, peer.WorkerID)
|
||||
}
|
||||
}
|
||||
sort.Strings(voters)
|
||||
return voters
|
||||
}
|
||||
|
||||
func consensusNotificationLeader(voters []string) string {
|
||||
if len(voters) == 0 {
|
||||
return ""
|
||||
}
|
||||
return voters[0]
|
||||
}
|
||||
|
||||
func selfcheckTarget(baseURL string) string {
|
||||
baseURL = strings.TrimRight(baseURL, "/")
|
||||
if baseURL == "" {
|
||||
return ""
|
||||
}
|
||||
return baseURL + selfcheckProbePath
|
||||
}
|
||||
|
||||
func runMainAPIHTTPCheck(target string) bool {
|
||||
settings, _ := json.Marshal(models.CheckSettings{
|
||||
ExpectedAnswer: "default",
|
||||
RequestMethod: "GET",
|
||||
Timeout: selfcheckTimeoutMillis,
|
||||
SlowTime: selfcheckTimeoutMillis,
|
||||
})
|
||||
name := "main api selfcheck"
|
||||
monitor := &models.Monitor{Host: selfcheckHost(target)}
|
||||
check := models.Check{
|
||||
ID: -1,
|
||||
Name: &name,
|
||||
Kind: "http", //nolint:goconst // check kind is a wire string, not the http package identifier
|
||||
Interval: int(selfcheckInterval.Seconds()),
|
||||
URL: &target,
|
||||
Settings: settings,
|
||||
}
|
||||
results := checkexec.Execute(monitor, []models.Check{check})
|
||||
if len(results) == 0 {
|
||||
return false
|
||||
}
|
||||
state := results[0].Result.State
|
||||
return state == selfcheckStateOK || state == selfcheckStateWarn
|
||||
}
|
||||
|
||||
func selfcheckHost(target string) string {
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return target
|
||||
}
|
||||
return parsed.Host
|
||||
}
|
||||
|
||||
// sendSystemAlert notifies all cached system contacts using cached credentials.
|
||||
// failure=true means "down" message, false means "recovered".
|
||||
func (r *Runner) sendSystemAlert(_ context.Context, failure bool, message string) {
|
||||
creds := r.Credentials()
|
||||
if creds == nil {
|
||||
log.Printf("worker: selfcheck alert skipped: no credentials")
|
||||
return
|
||||
}
|
||||
|
||||
contacts := r.SystemContacts()
|
||||
if len(contacts) == 0 {
|
||||
log.Printf("worker: selfcheck alert skipped: no system contacts")
|
||||
return
|
||||
}
|
||||
|
||||
subject := "RSMon worker alert"
|
||||
if !failure {
|
||||
subject = "RSMon worker recovery"
|
||||
}
|
||||
|
||||
for i := range contacts {
|
||||
c := contacts[i]
|
||||
switch c.Kind {
|
||||
case systemContactKindEmail:
|
||||
if len(creds.SMTP) == 0 {
|
||||
continue
|
||||
}
|
||||
cred := smtpCredToModel(&creds.SMTP[0])
|
||||
now := time.Now().UTC()
|
||||
row := &NotificationRow{
|
||||
Kind: c.Kind,
|
||||
Channel: notificationChannelSMTP,
|
||||
Subject: subject,
|
||||
Body: message,
|
||||
At: now,
|
||||
}
|
||||
if err := notify.Email(cred, c.Value, subject, message, ""); err != nil {
|
||||
log.Printf("worker: selfcheck email to %s failed: %v", c.Value, err)
|
||||
row.OK = false
|
||||
row.Error = err.Error()
|
||||
} else {
|
||||
row.OK = true
|
||||
}
|
||||
r.RecordNotification(row)
|
||||
case "telegram_private", "telegram_group":
|
||||
if len(creds.Telegram) == 0 {
|
||||
continue
|
||||
}
|
||||
chatID, err := parseTelegramChatID(c.Value)
|
||||
if err != nil {
|
||||
log.Printf("worker: selfcheck telegram chat_id parse failed for %s: %v", c.Value, err)
|
||||
continue
|
||||
}
|
||||
cred := telegramCredToModel(creds.Telegram[0])
|
||||
body := fmt.Sprintf("%s\n\n%s", subject, message)
|
||||
now := time.Now().UTC()
|
||||
row := &NotificationRow{
|
||||
Kind: c.Kind,
|
||||
Channel: notificationChannelTelegram,
|
||||
Subject: subject,
|
||||
Body: message,
|
||||
At: now,
|
||||
}
|
||||
if err := notify.Telegram(cred, chatID, body); err != nil {
|
||||
log.Printf("worker: selfcheck telegram to %s failed: %v", c.Value, err)
|
||||
row.OK = false
|
||||
row.Error = err.Error()
|
||||
} else {
|
||||
row.OK = true
|
||||
}
|
||||
r.RecordNotification(row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseTelegramChatID converts a numeric telegram chat id stored as a
|
||||
// string into an int64.
|
||||
func parseTelegramChatID(raw string) (int64, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0, fmt.Errorf("empty chat id")
|
||||
}
|
||||
id, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse chat id: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// smtpCredToModel wraps a wire SMTP credential in a models.NotificationCredential.
|
||||
// SecretEnc carries the plaintext with the "plain:" prefix so models.GetSecret
|
||||
// returns it unchanged — workers do not have the encryption key configured.
|
||||
func smtpCredToModel(c *wire.SMTPCredential) *models.NotificationCredential {
|
||||
port := c.Port
|
||||
enabled := true
|
||||
return &models.NotificationCredential{
|
||||
Model: concerns.Model{ID: c.ID},
|
||||
Kind: models.CredentialKindSMTP,
|
||||
Name: c.Name,
|
||||
Server: &c.Server,
|
||||
Port: &port,
|
||||
Login: &c.Login,
|
||||
FromName: &c.FromName,
|
||||
FromAddr: &c.FromAddress,
|
||||
InsecureSkipVerify: c.InsecureSkipVerify,
|
||||
Enabled: &enabled,
|
||||
SecretEnc: "plain:" + c.Password,
|
||||
}
|
||||
}
|
||||
|
||||
// telegramCredToModel wraps a wire Telegram credential in a
|
||||
// models.NotificationCredential. SecretEnc carries the plaintext with the
|
||||
// "plain:" prefix so models.GetSecret returns it unchanged.
|
||||
func telegramCredToModel(c wire.TelegramCredential) *models.NotificationCredential {
|
||||
enabled := true
|
||||
cred := &models.NotificationCredential{
|
||||
Model: concerns.Model{ID: c.ID},
|
||||
Kind: models.CredentialKindTelegram,
|
||||
Name: c.Name,
|
||||
BotName: &c.BotName,
|
||||
APIURL: &c.APIURL,
|
||||
Enabled: &enabled,
|
||||
SecretEnc: "plain:" + c.Token,
|
||||
}
|
||||
return cred
|
||||
}
|
||||
625
internal/distworker/selfcheck_test.go
Обычный файл
625
internal/distworker/selfcheck_test.go
Обычный файл
@@ -0,0 +1,625 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
func TestRunMainAPIHTTPCheck_OK(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ok := runMainAPIHTTPCheck(srv.URL)
|
||||
assert.True(t, ok, "200 OK should be a successful probe")
|
||||
}
|
||||
|
||||
func TestRunMainAPIHTTPCheck_404IsDown(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ok := runMainAPIHTTPCheck(srv.URL)
|
||||
assert.False(t, ok, "normal HTTP check flow treats non-200 as down")
|
||||
}
|
||||
|
||||
func TestRunMainAPIHTTPCheck_5xxIsDown(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ok := runMainAPIHTTPCheck(srv.URL)
|
||||
assert.False(t, ok, "5xx should be treated as down")
|
||||
}
|
||||
|
||||
func TestRunMainAPIHTTPCheck_NetworkErrorIsDown(t *testing.T) {
|
||||
ok := runMainAPIHTTPCheck("http://127.0.0.1:1")
|
||||
assert.False(t, ok, "connection refused should be treated as down")
|
||||
}
|
||||
|
||||
func TestSelfcheckTarget(t *testing.T) {
|
||||
assert.Equal(t, "https://rsmon.ru/up", selfcheckTarget("https://rsmon.ru"))
|
||||
assert.Equal(t, "https://rsmon.ru/up", selfcheckTarget("https://rsmon.ru/"))
|
||||
assert.Equal(t, "https://api.example.com/api/v1/up", selfcheckTarget("https://api.example.com/api/v1"))
|
||||
assert.Equal(t, "", selfcheckTarget(""), "empty base URL yields empty target")
|
||||
}
|
||||
|
||||
func TestParseTelegramChatID(t *testing.T) {
|
||||
id, err := parseTelegramChatID("200318758")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(200318758), id)
|
||||
|
||||
_, err = parseTelegramChatID("")
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = parseTelegramChatID("not-a-number")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSMTPCredToModelCarriesPlaintextSecret(t *testing.T) {
|
||||
c := wire.SMTPCredential{
|
||||
ID: 11,
|
||||
Name: "primary",
|
||||
Server: "smtp.example.com",
|
||||
Port: 587,
|
||||
Login: "alerts@example.com",
|
||||
Password: "smtp-password-xyz",
|
||||
FromName: "RSMon",
|
||||
FromAddress: "alerts@example.com",
|
||||
}
|
||||
cred := smtpCredToModel(&c)
|
||||
require.NotNil(t, cred)
|
||||
assert.Equal(t, "plain:smtp-password-xyz", cred.SecretEnc)
|
||||
assert.Equal(t, int64(11), cred.ID)
|
||||
assert.Equal(t, models_credentialKindSMTP(), cred.Kind)
|
||||
|
||||
got, err := cred.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "smtp-password-xyz", got,
|
||||
"worker-side credential must yield plaintext via models.GetSecret")
|
||||
}
|
||||
|
||||
func TestTelegramCredToModelCarriesPlaintextToken(t *testing.T) {
|
||||
c := wire.TelegramCredential{
|
||||
ID: 22,
|
||||
Name: "main-bot",
|
||||
BotName: "rsmon_alerts_bot",
|
||||
Token: "bot-token-9876543210:ABCDEFG",
|
||||
APIURL: "https://api.telegram.org",
|
||||
}
|
||||
cred := telegramCredToModel(c)
|
||||
require.NotNil(t, cred)
|
||||
assert.Equal(t, "plain:bot-token-9876543210:ABCDEFG", cred.SecretEnc)
|
||||
assert.Equal(t, int64(22), cred.ID)
|
||||
assert.Equal(t, models_credentialKindTelegram(), cred.Kind)
|
||||
|
||||
got, err := cred.GetSecret()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "bot-token-9876543210:ABCDEFG", got)
|
||||
}
|
||||
|
||||
// models_credentialKindSMTP/Telegram are local shims to avoid an import-cycle
|
||||
// in this isolated test file. The real constants live in app/models and are
|
||||
// asserted at runtime through the wire translation functions.
|
||||
func models_credentialKindSMTP() string { return "smtp" }
|
||||
func models_credentialKindTelegram() string { return "telegram" }
|
||||
|
||||
func TestApplyInitStoresSystemContacts(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
|
||||
assert.Empty(t, r.SystemContacts(), "no system contacts before init")
|
||||
|
||||
want := []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com", Name: "ops"},
|
||||
{ID: 2, Kind: "telegram_group", Value: "200318758", Name: "alerts"},
|
||||
}
|
||||
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 2,
|
||||
SystemContacts: want,
|
||||
})
|
||||
|
||||
got := r.SystemContacts()
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, want[0].ID, got[0].ID)
|
||||
assert.Equal(t, "email", got[0].Kind)
|
||||
assert.Equal(t, "ops@example.com", got[0].Value)
|
||||
assert.Equal(t, "telegram_group", got[1].Kind)
|
||||
assert.Equal(t, "200318758", got[1].Value)
|
||||
|
||||
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2})
|
||||
assert.Empty(t, r.SystemContacts(),
|
||||
"subsequent init with empty SystemContacts should replace cache")
|
||||
}
|
||||
|
||||
func TestRunSelfcheckOnce_FailureStartsDownStateNoAlert(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.config.URL = srv.URL
|
||||
creds := &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp.example.com", Port: 587, Login: "a@a", FromAddress: "a@a", Password: "x"}},
|
||||
}
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: creds,
|
||||
SystemContacts: []wire.SystemContact{{ID: 1, Kind: "email", Value: "ops@example.com"}},
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
require.NotNil(t, state.consensus.downSince,
|
||||
"first down probe must stamp downSince")
|
||||
assert.False(t, state.consensus.alertActive,
|
||||
"no alert expected before selfcheckConsensusWait elapses")
|
||||
}
|
||||
|
||||
func TestRunSelfcheckOnce_AlertFiredAfterConsensusWait(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.config.URL = "http://127.0.0.1:1"
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
downSince := time.Now().Add(-selfcheckConsensusWait - time.Second)
|
||||
state.consensus.downSince = &downSince
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.True(t, state.consensus.alertActive,
|
||||
"alert must be active after selfcheckConsensusWait elapses")
|
||||
}
|
||||
|
||||
func TestRunSelfcheckOnce_RecoveryClearsAlert(t *testing.T) {
|
||||
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srvOK.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
state.consensus.alertActive = true
|
||||
downSince := time.Now().Add(-20 * time.Minute)
|
||||
state.consensus.downSince = &downSince
|
||||
|
||||
r.config.URL = srvOK.URL
|
||||
// First up tick: marks the up consensus start; recovery alert
|
||||
// waits for selfcheckConsensusWait. Assert the alert is still
|
||||
// active so we can distinguish "up just arrived" from "up held".
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.True(t, state.consensus.alertActive,
|
||||
"recovery alert should not fire on the first up tick")
|
||||
assert.Equal(t, consensusUp, state.consensus.lastVerdict)
|
||||
|
||||
// Tick again with the up verdict now held past
|
||||
// selfcheckConsensusWait so the recovery alert fires.
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-selfcheckConsensusWait - time.Second))
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.False(t, state.consensus.alertActive,
|
||||
"recovery alert must clear alertActive after selfcheckConsensusWait up")
|
||||
assert.Nil(t, state.consensus.downSince)
|
||||
}
|
||||
|
||||
func TestRunSelfcheckOnce_RequiresSelfVote(t *testing.T) {
|
||||
// When self has not yet produced a verdict (nil), the
|
||||
// consensus must refuse to call the cluster down regardless of
|
||||
// what peers report.
|
||||
up := true
|
||||
peers := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
|
||||
verdict := decideConsensus(nil, peers, 2)
|
||||
assert.Equal(t, consensusNoQuorum, verdict,
|
||||
"missing self vote must prevent a down consensus")
|
||||
|
||||
verdict = decideConsensus(&up, peers, 2)
|
||||
assert.Equal(t, consensusUp, verdict,
|
||||
"self up + peer up must reach consensus up")
|
||||
}
|
||||
|
||||
func TestSendSystemAlert_NoCredentials(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
SystemContacts: []wire.SystemContact{{ID: 1, Kind: "email", Value: "ops@example.com"}},
|
||||
})
|
||||
|
||||
r.sendSystemAlert(t.Context(), true, "test message")
|
||||
}
|
||||
|
||||
func TestSendSystemAlert_NoContacts(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
})
|
||||
|
||||
r.sendSystemAlert(t.Context(), true, "test message")
|
||||
}
|
||||
|
||||
func TestWorkerInit_SystemContactsJSONRoundtrip(t *testing.T) {
|
||||
init := wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 4,
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com", Name: "ops"},
|
||||
{ID: 2, Kind: "telegram_private", Value: "200318758", Name: "alerts"},
|
||||
},
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(init)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(raw), `"system_contacts":`)
|
||||
|
||||
var decoded wire.WorkerInit
|
||||
require.NoError(t, json.Unmarshal(raw, &decoded))
|
||||
require.Len(t, decoded.SystemContacts, 2)
|
||||
assert.Equal(t, init.SystemContacts[0], decoded.SystemContacts[0])
|
||||
assert.Equal(t, init.SystemContacts[1], decoded.SystemContacts[1])
|
||||
}
|
||||
|
||||
func TestWorkerInit_PeersJSONRoundtrip(t *testing.T) {
|
||||
init := wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Concurrency: 4,
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402", RegionCode: "local"},
|
||||
{WorkerID: "w-3", URL: "http://127.0.0.1:27403", RegionCode: "local", Login: "ops", Password: "x"},
|
||||
},
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(init)
|
||||
require.NoError(t, err)
|
||||
body := string(raw)
|
||||
assert.Contains(t, body, `"peers":[`)
|
||||
assert.Contains(t, body, `"worker_id":"w-2"`)
|
||||
assert.Contains(t, body, `"url":"http://127.0.0.1:27402"`)
|
||||
assert.Contains(t, body, `"login":"ops"`)
|
||||
assert.Contains(t, body, `"password":"x"`)
|
||||
|
||||
var decoded wire.WorkerInit
|
||||
require.NoError(t, json.Unmarshal(raw, &decoded))
|
||||
require.Len(t, decoded.Peers, 2)
|
||||
assert.Equal(t, init.Peers[0], decoded.Peers[0])
|
||||
assert.Equal(t, init.Peers[1], decoded.Peers[1])
|
||||
}
|
||||
|
||||
func TestWorkerInit_PeersOmittedWhenEmpty(t *testing.T) {
|
||||
init := wire.WorkerInit{WorkerID: "w-1", Concurrency: 1}
|
||||
raw, err := json.Marshal(init)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(raw), `"peers"`,
|
||||
"empty peers slice must be omitted so legacy workers stay wire-compatible")
|
||||
}
|
||||
|
||||
// TestRunSelfcheckOnce_ConfiguredPeersNoFreshPeer_NoAlert verifies that
|
||||
// in a multi-worker install a transient local probe failure (or
|
||||
// success) does NOT fire an alert or reset the state machine when
|
||||
// peers are configured but have not produced a fresh observation yet.
|
||||
// The first peer poll lands 2s after startup, and after that on the
|
||||
// peer-poll interval, so the early ticks are a real "lone self vote"
|
||||
// window in production.
|
||||
func TestRunSelfcheckOnce_ConfiguredPeersNoFreshPeer_NoAlert(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.config.URL = srv.URL
|
||||
creds := &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
}
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: creds,
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com"},
|
||||
},
|
||||
// Two peers configured but neither has produced a
|
||||
// fresh observation yet — the selfcheck is the lone
|
||||
// voter.
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
{WorkerID: "w-3", URL: "http://127.0.0.1:27403"},
|
||||
},
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
|
||||
assert.Nil(t, state.consensus.downSince,
|
||||
"with configured peers and no fresh peer observations, the down timer must not start")
|
||||
assert.False(t, state.consensus.alertActive,
|
||||
"no alert can fire without a peer-backed down verdict")
|
||||
}
|
||||
|
||||
// TestRunSelfcheckOnce_ConfiguredPeersWithFreshDownPeer_FiresAfterWait
|
||||
// exercises the happy path: self and one peer both see the master as
|
||||
// down, the consensus verdict is down, and after selfcheckConsensusWait
|
||||
// the alert fires exactly once.
|
||||
func TestRunSelfcheckOnce_ConfiguredPeersWithFreshDownPeer_FiresAfterWait(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.config.URL = srv.URL
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com"},
|
||||
},
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
},
|
||||
})
|
||||
|
||||
// Seed a fresh down observation from the peer as if its
|
||||
// selfcheck tick has just completed.
|
||||
r.peerCache.put(peerObservation{
|
||||
WorkerID: "w-2",
|
||||
Up: false,
|
||||
ObservedAt: time.Now(),
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
|
||||
// First tick: stamps downSince, does not yet fire.
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
require.NotNil(t, state.consensus.downSince,
|
||||
"peer-backed down verdict must stamp downSince on first tick")
|
||||
assert.False(t, state.consensus.alertActive,
|
||||
"alert must not fire before selfcheckConsensusWait elapses")
|
||||
|
||||
// Second tick, simulate the 5-minute wait having elapsed
|
||||
// by rewinding downSince past the wait threshold. This
|
||||
// avoids waiting wall-clock time in a unit test.
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-selfcheckConsensusWait - time.Second))
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.True(t, state.consensus.alertActive,
|
||||
"down consensus held for selfcheckConsensusWait must fire the alert exactly once")
|
||||
}
|
||||
|
||||
func TestRunSelfcheckOnce_NonLeaderSkipsDuplicateAlert(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.config.URL = srv.URL
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "worker-local-2",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
SystemContacts: []wire.SystemContact{{ID: 1, Kind: "email", Value: "ops@example.com"}},
|
||||
Peers: []wire.PeerInfo{{WorkerID: "worker-local-1", URL: "http://127.0.0.1:27401"}},
|
||||
})
|
||||
r.peerCache.put(peerObservation{WorkerID: "worker-local-1", Up: false, ObservedAt: time.Now()})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-selfcheckConsensusWait - time.Second))
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
|
||||
assert.True(t, state.consensus.alertActive, "non-leader still marks the incident handled locally")
|
||||
assert.Empty(t, r.RecentNotifications(10), "non-leader must not deliver duplicate system-contact notifications")
|
||||
}
|
||||
|
||||
func TestFormatSelfcheckLeaderMessage_IncludesOldAndNewLeader(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{WorkerID: "worker-local-2", RegionCode: "local", Concurrency: 1})
|
||||
|
||||
out := r.formatSelfcheckLeaderMessage("worker-local-1", "http://localhost:7401/up", consensusDown, 0, 2, 2, 2, 2, 1,
|
||||
"worker-local-2", []string{"worker-local-2", "worker-local-3"})
|
||||
|
||||
assert.Contains(t, out, "Изменился мастер воркер оповещений: worker-local-1 -> worker-local-2")
|
||||
assert.Contains(t, out, "Notification leader: worker-local-2")
|
||||
assert.Contains(t, out, "Voters: worker-local-2,worker-local-3")
|
||||
}
|
||||
|
||||
// TestRunSelfcheckOnce_RecoveryRequiresHeldUpConsensus verifies that
|
||||
// after a peer-backed down alert, recovery only fires after the up
|
||||
// verdict has itself been held for selfcheckConsensusWait, and that
|
||||
// the recovery flow respects the multi-worker gate (an isolated up
|
||||
// tick with no fresh peer must NOT clear the alert).
|
||||
func TestRunSelfcheckOnce_RecoveryRequiresHeldUpConsensus(t *testing.T) {
|
||||
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srvOK.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com"},
|
||||
},
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
},
|
||||
})
|
||||
|
||||
// Pretend the peer also sees the master as up — the only
|
||||
// way the up verdict is allowed past the multi-worker gate.
|
||||
r.peerCache.put(peerObservation{
|
||||
WorkerID: "w-2",
|
||||
Up: true,
|
||||
ObservedAt: time.Now(),
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
state.consensus.alertActive = true
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-20 * time.Minute))
|
||||
r.config.URL = srvOK.URL
|
||||
|
||||
// First up tick: stamps the recovery timer; alert stays
|
||||
// active because the up verdict is not yet held for
|
||||
// selfcheckConsensusWait.
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.True(t, state.consensus.alertActive,
|
||||
"recovery alert must not fire on the first up tick")
|
||||
assert.Equal(t, consensusUp, state.consensus.lastVerdict)
|
||||
|
||||
// Simulate the 5-minute recovery wait having elapsed.
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-selfcheckConsensusWait - time.Second))
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.False(t, state.consensus.alertActive,
|
||||
"recovery alert must clear alertActive after selfcheckConsensusWait up consensus")
|
||||
assert.Nil(t, state.consensus.downSince)
|
||||
}
|
||||
|
||||
// TestRunSelfcheckOnce_RecoveryIsolatedUpDoesNotClearAlert verifies
|
||||
// that an isolated up tick (configured peers, no fresh peer
|
||||
// observation) cannot unilaterally clear an active alert. The
|
||||
// alert must remain active until a peer-backed up verdict is held.
|
||||
func TestRunSelfcheckOnce_RecoveryIsolatedUpDoesNotClearAlert(t *testing.T) {
|
||||
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srvOK.Close()
|
||||
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com"},
|
||||
},
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
},
|
||||
})
|
||||
// Note: no fresh peer observation is seeded — the selfcheck
|
||||
// is the lone voter in this scenario.
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
state.consensus.alertActive = true
|
||||
state.consensus.downSince = ptrTime(time.Now().Add(-20 * time.Minute))
|
||||
r.config.URL = srvOK.URL
|
||||
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
|
||||
assert.True(t, state.consensus.alertActive,
|
||||
"isolated up tick must not clear an active alert when peers are configured")
|
||||
}
|
||||
|
||||
// TestRunSelfcheckOnce_NoQuorumDoesNotResetDownTimer verifies that a
|
||||
// NoQuorum verdict neither stamps nor advances the 5-minute down
|
||||
// timer. A brief blip in peer reachability must not page or reset
|
||||
// progress toward the consensus wait.
|
||||
func TestRunSelfcheckOnce_NoQuorumDoesNotResetDownTimer(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{
|
||||
WorkerID: "w-1",
|
||||
Credentials: &wire.NotificationCredentials{
|
||||
SMTP: []wire.SMTPCredential{{ID: 1, Name: "primary", Server: "smtp", Port: 25, Login: "a", FromAddress: "a", Password: "x"}},
|
||||
},
|
||||
SystemContacts: []wire.SystemContact{
|
||||
{ID: 1, Kind: "email", Value: "ops@example.com"},
|
||||
},
|
||||
Peers: []wire.PeerInfo{
|
||||
{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
|
||||
},
|
||||
})
|
||||
// Peer observation is older than peerStatusMaxAge so the
|
||||
// selfcheck is alone in the vote.
|
||||
r.peerCache.put(peerObservation{
|
||||
WorkerID: "w-2",
|
||||
Up: false,
|
||||
ObservedAt: time.Now().Add(-10 * time.Minute),
|
||||
})
|
||||
|
||||
state := &selfcheckState{consensus: &consensusState{}}
|
||||
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
assert.Nil(t, state.consensus.downSince,
|
||||
"first NoQuorum must not stamp the down timer")
|
||||
assert.False(t, state.consensus.alertActive)
|
||||
|
||||
// A subsequent NoQuorum must also not advance the timer;
|
||||
// the existing stamp (from a previous verdict) is held.
|
||||
previous := ptrTime(time.Now().Add(-3 * time.Minute))
|
||||
state.consensus.downSince = previous
|
||||
r.runSelfcheckOnce(t.Context(), state)
|
||||
require.NotNil(t, state.consensus.downSince)
|
||||
assert.Equal(t, *previous, *state.consensus.downSince,
|
||||
"NoQuorum must not advance the down timer")
|
||||
}
|
||||
|
||||
func TestFormatSelfcheckMessage_IncludesWorkerAndConsensus(t *testing.T) {
|
||||
r := newTestRunner(t, 4, 1, func(interface{}) interface{} { return []wire.CheckResultReport{} })
|
||||
r.applyInit(&wire.WorkerInit{WorkerID: "worker-local-1", RegionCode: "local", Concurrency: 1})
|
||||
|
||||
out := r.formatSelfcheckMessage(selfcheckDownMessage, "http://localhost:7401/up", consensusDown, 0, 3, 3, 2, 2, 2,
|
||||
"worker-local-1", []string{"worker-local-1", "worker-local-2", "worker-local-3"})
|
||||
|
||||
assert.Contains(t, out, "Нет связи с основным api")
|
||||
assert.Contains(t, out, "Worker: worker-local-1")
|
||||
assert.Contains(t, out, "Region: local")
|
||||
assert.Contains(t, out, "Target: http://localhost:7401/up")
|
||||
assert.Contains(t, out, "Raft status: lightweight peer quorum")
|
||||
assert.Contains(t, out, "Notification leader: worker-local-1")
|
||||
assert.Contains(t, out, "Consensus: down")
|
||||
assert.Contains(t, out, "Votes: up=0 down=3 total=3 min_votes=2")
|
||||
assert.Contains(t, out, "Workers seen: 3 current votes, 2 fresh peers of 2 configured peers")
|
||||
assert.Contains(t, out, "Voters: worker-local-1,worker-local-2,worker-local-3")
|
||||
assert.Contains(t, out, "Hold time: 5m0s")
|
||||
}
|
||||
|
||||
func TestConsensusNotificationLeader(t *testing.T) {
|
||||
voters := consensusVoters("worker-local-2", []peerObservation{{WorkerID: "worker-local-3"}, {WorkerID: "worker-local-1"}})
|
||||
assert.Equal(t, []string{"worker-local-1", "worker-local-2", "worker-local-3"}, voters)
|
||||
assert.Equal(t, "worker-local-1", consensusNotificationLeader(voters))
|
||||
}
|
||||
|
||||
func TestConsensusState_NotificationLeaderChanged(t *testing.T) {
|
||||
state := &consensusState{}
|
||||
old, changed := state.notificationLeaderChanged("worker-local-1")
|
||||
assert.False(t, changed)
|
||||
assert.Empty(t, old)
|
||||
|
||||
old, changed = state.notificationLeaderChanged("worker-local-1")
|
||||
assert.False(t, changed)
|
||||
assert.Empty(t, old)
|
||||
|
||||
old, changed = state.notificationLeaderChanged("worker-local-2")
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "worker-local-1", old)
|
||||
}
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
271
internal/distworker/server_metrics.go
Обычный файл
271
internal/distworker/server_metrics.go
Обычный файл
@@ -0,0 +1,271 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
type serverMetricSample struct {
|
||||
at time.Time
|
||||
cpuTotal, cpuIdle uint64
|
||||
rx, tx int64
|
||||
}
|
||||
|
||||
var serverMetricSamples = struct {
|
||||
sync.Mutex
|
||||
byServer map[int64]serverMetricSample
|
||||
}{byServer: make(map[int64]serverMetricSample)}
|
||||
|
||||
const serverMetricInterval = 5 * time.Second
|
||||
|
||||
const (
|
||||
maxServerMetricProcesses = 20
|
||||
maxServerMetricNetworks = 16
|
||||
)
|
||||
|
||||
// serverMetricLoop collects only on a worker assigned to a Server. It uses
|
||||
// the real Linux /proc and statfs collector; unsupported platforms return no
|
||||
// report rather than fabricated values. The worker has no control-plane DB
|
||||
// access and forwards snapshots on its authenticated websocket.
|
||||
func (r *Runner) serverMetricLoop() {
|
||||
ticker := time.NewTicker(serverMetricInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
serverID := r.serverID.Load()
|
||||
if serverID == 0 || r.metricResults == nil {
|
||||
continue
|
||||
}
|
||||
report, ok := collectServerMetric(serverID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case r.metricResults <- report:
|
||||
case <-r.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) {
|
||||
memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc")
|
||||
if !ok {
|
||||
return wire.ServerMetricReport{}, false
|
||||
}
|
||||
memUsed := memTotal - memAvailable
|
||||
var rxTotal, txTotal int64
|
||||
networks := make([]wire.NetworkMetric, 0)
|
||||
if data, err := os.ReadFile("/proc/net/dev"); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 10 {
|
||||
name := strings.TrimSuffix(fields[0], ":")
|
||||
if name == "lo" {
|
||||
continue
|
||||
}
|
||||
rx, tx := parseI64(fields[1]), parseI64(fields[9])
|
||||
rxTotal += rx
|
||||
txTotal += tx
|
||||
networks = append(networks, wire.NetworkMetric{Interface: name, RxBytes: rx, TxBytes: tx})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(networks, func(i, j int) bool {
|
||||
return networks[i].RxBytes+networks[i].TxBytes > networks[j].RxBytes+networks[j].TxBytes
|
||||
})
|
||||
if len(networks) > maxServerMetricNetworks {
|
||||
networks = networks[:maxServerMetricNetworks]
|
||||
}
|
||||
processCount := countProcesses("/proc")
|
||||
processes := collectProcesses("/proc")
|
||||
now := time.Now()
|
||||
cpuPercent, netRx, netTx := sampledRates(serverID, now, rxTotal, txTotal)
|
||||
diskUsed, diskTotal, diskOK := rootDiskUsage()
|
||||
var diskUsedPtr, diskTotalPtr *int64
|
||||
if diskOK {
|
||||
diskUsedPtr, diskTotalPtr = &diskUsed, &diskTotal
|
||||
}
|
||||
return wire.ServerMetricReport{
|
||||
ServerID: serverID, CPUPercent: cpuPercent, MemUsed: &memUsed, MemTotal: &memTotal,
|
||||
DiskUsed: diskUsedPtr, DiskTotal: diskTotalPtr, NetRx: netRx, NetTx: netTx, HostUptimeSec: &uptime,
|
||||
Load1: &load1, Load5: &load5, Load15: &load15, ProcessCount: &processCount,
|
||||
Processes: processes, Networks: networks,
|
||||
}, true
|
||||
}
|
||||
|
||||
// sampledRates turns monotonic /proc counters into percent and bytes/second.
|
||||
// A first sample, a clock anomaly, or a counter reset intentionally has no rate.
|
||||
func sampledRates(serverID int64, now time.Time, rx, tx int64) (*float64, *int64, *int64) {
|
||||
total, idle, ok := readCPUCounters("/proc/stat")
|
||||
if !ok {
|
||||
return nil, nil, nil
|
||||
}
|
||||
serverMetricSamples.Lock()
|
||||
defer serverMetricSamples.Unlock()
|
||||
previous, hasPrevious := serverMetricSamples.byServer[serverID]
|
||||
serverMetricSamples.byServer[serverID] = serverMetricSample{at: now, cpuTotal: total, cpuIdle: idle, rx: rx, tx: tx}
|
||||
if !hasPrevious || !now.After(previous.at) || total <= previous.cpuTotal || idle < previous.cpuIdle || rx < previous.rx || tx < previous.tx {
|
||||
return nil, nil, nil
|
||||
}
|
||||
cpu := float64((total-previous.cpuTotal)-(idle-previous.cpuIdle)) * 100 / float64(total-previous.cpuTotal)
|
||||
seconds := now.Sub(previous.at).Seconds()
|
||||
if seconds <= 0 {
|
||||
return &cpu, nil, nil
|
||||
}
|
||||
rxRate, txRate := int64(float64(rx-previous.rx)/seconds), int64(float64(tx-previous.tx)/seconds)
|
||||
return &cpu, &rxRate, &txRate
|
||||
}
|
||||
|
||||
func readCPUCounters(path string) (uint64, uint64, bool) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if !strings.HasPrefix(line, "cpu ") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
return 0, 0, false
|
||||
}
|
||||
var total uint64
|
||||
for _, field := range fields[1:] {
|
||||
value, err := strconv.ParseUint(field, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
total += value
|
||||
}
|
||||
idle, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
if len(fields) > 5 {
|
||||
iowait, _ := strconv.ParseUint(fields[5], 10, 64)
|
||||
idle += iowait
|
||||
}
|
||||
return total, idle, total > 0
|
||||
}
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
func rootDiskUsage() (int64, int64, bool) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs("/", &stat); err != nil || stat.Blocks == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
total := int64(stat.Blocks * uint64(stat.Bsize))
|
||||
free := int64(stat.Bavail * uint64(stat.Bsize))
|
||||
return total - free, total, true
|
||||
}
|
||||
|
||||
func collectProcesses(procRoot string) []wire.ProcessMetric {
|
||||
entries, err := os.ReadDir(procRoot)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
processes := make([]wire.ProcessMetric, 0, maxServerMetricProcesses)
|
||||
for _, entry := range entries {
|
||||
pid, err := strconv.Atoi(entry.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
status, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "status"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
name, rss := "", int64(0)
|
||||
for _, line := range strings.Split(string(status), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch fields[0] {
|
||||
case "Name:":
|
||||
name = fields[1]
|
||||
case "VmRSS:":
|
||||
rss = parseI64(fields[1]) * 1024
|
||||
}
|
||||
}
|
||||
if name != "" {
|
||||
processes = append(processes, wire.ProcessMetric{PID: pid, Name: name, MemoryRSS: rss})
|
||||
}
|
||||
}
|
||||
sort.Slice(processes, func(i, j int) bool { return processes[i].MemoryRSS > processes[j].MemoryRSS })
|
||||
if len(processes) > maxServerMetricProcesses {
|
||||
processes = processes[:maxServerMetricProcesses]
|
||||
}
|
||||
return processes
|
||||
}
|
||||
|
||||
func countProcesses(procRoot string) int {
|
||||
entries, err := os.ReadDir(procRoot)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
if _, err := strconv.Atoi(entry.Name()); err == nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func readLinuxHostMetrics(root string) (total, available int64, load1, load5, load15 float64, uptime int64, ok bool) {
|
||||
mem, err := os.ReadFile(filepath.Join(root, "meminfo"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, line := range strings.Split(string(mem), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch fields[0] {
|
||||
case "MemTotal:":
|
||||
total = parseI64(fields[1]) * 1024
|
||||
case "MemAvailable:":
|
||||
available = parseI64(fields[1]) * 1024
|
||||
}
|
||||
}
|
||||
load, err := os.ReadFile(filepath.Join(root, "loadavg"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fields := strings.Fields(string(load))
|
||||
if len(fields) < 3 {
|
||||
return
|
||||
}
|
||||
load1, _ = strconv.ParseFloat(fields[0], 64)
|
||||
load5, _ = strconv.ParseFloat(fields[1], 64)
|
||||
load15, _ = strconv.ParseFloat(fields[2], 64)
|
||||
up, err := os.ReadFile(filepath.Join(root, "uptime"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fields = strings.Fields(string(up))
|
||||
if len(fields) == 0 {
|
||||
return
|
||||
}
|
||||
seconds, err := strconv.ParseFloat(fields[0], 64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
uptime = int64(seconds)
|
||||
return total, available, load1, load5, load15, uptime, total > 0
|
||||
}
|
||||
|
||||
func parseI64(value string) int64 { out, _ := strconv.ParseInt(value, 10, 64); return out }
|
||||
55
internal/distworker/server_metrics_test.go
Обычный файл
55
internal/distworker/server_metrics_test.go
Обычный файл
@@ -0,0 +1,55 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadLinuxHostMetrics(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "meminfo"), []byte("MemTotal: 100 kB\nMemAvailable: 25 kB\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "loadavg"), []byte("1.5 2.5 3.5 1/1 1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "uptime"), []byte("42.9 0\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
total, available, one, five, fifteen, uptime, ok := readLinuxHostMetrics(root)
|
||||
if !ok || total != 102400 || available != 25600 || one != 1.5 || five != 2.5 || fifteen != 3.5 || uptime != 42 {
|
||||
t.Fatalf("unexpected host metric parse: %d %d %g %g %g %d %t", total, available, one, five, fifteen, uptime, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectProcessesLimitsAndSortsByRSS(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for i := 1; i <= maxServerMetricProcesses+2; i++ {
|
||||
dir := filepath.Join(root, strconv.Itoa(i))
|
||||
if err := os.Mkdir(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status := fmt.Sprintf("Name:\tproc%d\nVmRSS:\t%d kB\n", i, i)
|
||||
if err := os.WriteFile(filepath.Join(dir, "status"), []byte(status), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
processes := collectProcesses(root)
|
||||
if len(processes) != maxServerMetricProcesses || processes[0].PID != maxServerMetricProcesses+2 {
|
||||
t.Fatalf("unexpected bounded process snapshot: %#v", processes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadCPUCounters(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "stat")
|
||||
if err := os.WriteFile(path, []byte("cpu 10 2 8 70 10 0 0 0\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
total, idle, ok := readCPUCounters(path)
|
||||
if !ok || total != 100 || idle != 80 {
|
||||
t.Fatalf("unexpected cpu counters: %d %d %t", total, idle, ok)
|
||||
}
|
||||
}
|
||||
27
internal/distworker/types.go
Обычный файл
27
internal/distworker/types.go
Обычный файл
@@ -0,0 +1,27 @@
|
||||
// Package distworker provides the distributed worker implementation.
|
||||
// This file aliases wire format types for convenience.
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
// notifyResultEnvelope pairs a notification task with its report so the
|
||||
// writer goroutine can log context alongside the result.
|
||||
type notifyResultEnvelope struct {
|
||||
task wire.NotificationTask
|
||||
report wire.NotificationResultReport
|
||||
}
|
||||
|
||||
// RegisterRequest and related types are wire type aliases for convenience.
|
||||
type (
|
||||
RegisterRequest = wire.RegisterRequest
|
||||
RegisterResponse = wire.RegisterResponse
|
||||
HeartbeatRequest = wire.HeartbeatRequest
|
||||
CheckJob = wire.CheckJob
|
||||
JobsResponse = wire.JobsResponse
|
||||
CheckResultReport = wire.CheckResultReport
|
||||
ResultsRequest = wire.ResultsRequest
|
||||
NotificationTask = wire.NotificationTask
|
||||
NotificationResultReport = wire.NotificationResultReport
|
||||
)
|
||||
Ссылка в новой задаче
Block a user