fix(worker): harden control-plane lifecycle
Все проверки выполнены успешно
CI / test (push) Successful in 2m32s
Docker / Build and publish worker image (push) Successful in 18m17s

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.
Этот коммит содержится в:
Gleb Tv
2026-07-19 23:11:43 +03:00
родитель 6937674449
Коммит e987f24903
38 изменённых файлов: 2203 добавлений и 674 удалений

Просмотреть файл

@@ -15,7 +15,6 @@ import (
"syscall"
"time"
"github.com/hashicorp/raft"
"github.com/joho/godotenv"
"rocketgit.ru/rsmon/worker/internal/distworker"
@@ -33,19 +32,6 @@ var (
// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
// install can opt out without recompiling.
webappEnabled = true
// clusterDebugApplyTestConfig, when true, submits the hardcoded
// CriticalCheckConfig from workercluster.DefaultDebugCriticalCheck
// to the cluster on startup. Wired via the
// --cluster-debug-apply-test-config CLI flag; the e2e script
// uses this so it can verify FSM replication without the
// signed-config-adoption producer (which lands in a later phase).
//
// DEBUG: this flag is a placeholder. It must be removed (or
// guarded behind a build tag) before any production build.
//
// TODO(phase-N): remove once the real producer is wired.
clusterDebugApplyTestConfig = false
)
func main() {
@@ -57,8 +43,6 @@ func main() {
versionFlag := flag.Bool("version", false, "Print version and exit")
noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
debugApplyConfig := flag.Bool("cluster-debug-apply-test-config", false,
"Submit a hardcoded CriticalCheckConfig to the cluster on startup. DEBUG: remove once the real config.adopt producer is wired.")
flag.Parse()
if *versionFlag {
@@ -66,7 +50,6 @@ func main() {
os.Exit(0)
}
webappEnabled = !*noWeb
clusterDebugApplyTestConfig = *debugApplyConfig
if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
os.Exit(healthCheck())
@@ -295,46 +278,12 @@ func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.C
log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
nodeID, localAddr, bootstrap, len(peers))
if clusterDebugApplyTestConfig && bootstrap {
// The debug apply only fires on bootstrap nodes; a
// joiner cannot commit a log entry until it has been
// promoted to voter. Wait for this node to win an
// election first (a fresh single-voter cluster elects
// itself immediately but the goroutine may run before
// the state has flipped).
go func() {
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if c.Raft() != nil && c.Raft().State() == raft.Leader {
break
}
time.Sleep(100 * time.Millisecond)
}
if c.Raft() == nil || c.Raft().State() != raft.Leader {
log.Printf("worker cluster: applying debug test config id=%d: not leader after 15s",
workercluster.DefaultDebugCriticalCheck().ID)
return
}
check := workercluster.DefaultDebugCriticalCheck()
applied, err := c.ApplyTestConfig(&check)
if err != nil {
log.Printf("worker cluster: applying debug test config id=%d: %v",
check.ID, err)
return
}
log.Printf("worker cluster: applying debug test config id=%d applied_index=%d",
check.ID, applied)
}()
}
return c, &clusterAdapter{c: c}, nil
}
// clusterAdapter wraps *workercluster.Cluster so it implements the
// webapp.ClusterView interface without webapp importing the raft
// code path. The ApplyTestConfig signature is the one webapp expects
// (no config argument; the cluster subsystem owns the hardcoded
// payload so the two sides cannot drift).
// code path.
type clusterAdapter struct {
c *workercluster.Cluster
}
@@ -359,11 +308,6 @@ func (a *clusterAdapter) Stats() webapp.ClusterStats {
}
}
func (a *clusterAdapter) ApplyTestConfig() (uint64, error) {
check := workercluster.DefaultDebugCriticalCheck()
return a.c.ApplyTestConfig(&check)
}
func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }
@@ -528,13 +472,17 @@ func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow {
out := make([]webapp.NotificationRow, len(src))
for i, r := range src {
out[i] = webapp.NotificationRow{
Kind: r.Kind,
Channel: r.Channel,
Subject: r.Subject,
Body: r.Body,
OK: r.OK,
Error: r.Error,
At: r.At,
Kind: r.Kind,
Channel: r.Channel,
Subject: r.Subject,
Body: r.Body,
OK: r.OK,
Error: r.Error,
At: r.At,
JobID: r.JobID,
Method: r.Method,
Status: r.Status,
DurationMs: r.DurationMs,
}
}
return out

Просмотреть файл

@@ -1,9 +1,13 @@
package main
import (
"errors"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
@@ -52,3 +56,23 @@ func TestProbeLiveness(t *testing.T) {
})
}
}
func TestRemovedClusterDebugApplyTestConfigFlagIsRejected(t *testing.T) {
binary := filepath.Join(t.TempDir(), "rsmon-worker")
build := exec.Command("go", "build", "-o", binary, ".")
if output, err := build.CombinedOutput(); err != nil {
t.Fatalf("build worker binary: %v\n%s", err, output)
}
output, err := exec.Command(binary, "--cluster-debug-apply-test-config").CombinedOutput()
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatalf("removed flag error = %v, want parser exit\n%s", err, output)
}
if exitErr.ExitCode() != 2 {
t.Fatalf("removed flag exit code = %d, want 2\n%s", exitErr.ExitCode(), output)
}
if !strings.Contains(string(output), "flag provided but not defined: -cluster-debug-apply-test-config") {
t.Fatalf("removed flag output = %q, want undefined-flag error", output)
}
}

Просмотреть файл

@@ -36,7 +36,7 @@ func installCommand(args []string) int {
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
fs.StringVar(&opts.URL, "url", installer.DefaultURL, "RSMon server URL")
fs.BoolVar(&opts.Docker, "docker", false, "run the prebuilt Docker image instead of the binary")
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "Docker image used with --docker")
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "immutable Docker repository@sha256 digest required with --docker")
fs.BoolVar(&opts.NoStart, "no-start", false, "install and enable without starting")
fs.Usage = func() {
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker install --token TOKEN [--url URL] [--no-start]")
@@ -89,7 +89,7 @@ func deployCommand(args []string) int {
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
fs.StringVar(&opts.URL, "url", installer.DefaultURL, "RSMon server URL")
fs.BoolVar(&opts.Docker, "docker", false, "install the prebuilt Docker image remotely")
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "Docker image used with --docker")
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "immutable Docker repository@sha256 digest required with --docker")
fs.BoolVar(&opts.NoStart, "no-start", false, "install and enable without starting")
fs.Usage = func() {
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker deploy --host HOST --user USER --token TOKEN [SSH options]")