fix(worker): harden control-plane lifecycle
Все проверки выполнены успешно
CI / test (push) Successful in 2m32s
Docker / Build and publish worker image (push) Successful in 18m17s

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.
Этот коммит содержится в:
Gleb Tv
2026-07-19 23:11:43 +03:00
родитель 6937674449
Коммит e987f24903
38 изменённых файлов: 2203 добавлений и 674 удалений

Просмотреть файл

@@ -15,7 +15,6 @@ import (
"syscall"
"time"
"github.com/hashicorp/raft"
"github.com/joho/godotenv"
"rocketgit.ru/rsmon/worker/internal/distworker"
@@ -33,19 +32,6 @@ var (
// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
// install can opt out without recompiling.
webappEnabled = true
// clusterDebugApplyTestConfig, when true, submits the hardcoded
// CriticalCheckConfig from workercluster.DefaultDebugCriticalCheck
// to the cluster on startup. Wired via the
// --cluster-debug-apply-test-config CLI flag; the e2e script
// uses this so it can verify FSM replication without the
// signed-config-adoption producer (which lands in a later phase).
//
// DEBUG: this flag is a placeholder. It must be removed (or
// guarded behind a build tag) before any production build.
//
// TODO(phase-N): remove once the real producer is wired.
clusterDebugApplyTestConfig = false
)
func main() {
@@ -57,8 +43,6 @@ func main() {
versionFlag := flag.Bool("version", false, "Print version and exit")
noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
debugApplyConfig := flag.Bool("cluster-debug-apply-test-config", false,
"Submit a hardcoded CriticalCheckConfig to the cluster on startup. DEBUG: remove once the real config.adopt producer is wired.")
flag.Parse()
if *versionFlag {
@@ -66,7 +50,6 @@ func main() {
os.Exit(0)
}
webappEnabled = !*noWeb
clusterDebugApplyTestConfig = *debugApplyConfig
if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
os.Exit(healthCheck())
@@ -295,46 +278,12 @@ func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.C
log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
nodeID, localAddr, bootstrap, len(peers))
if clusterDebugApplyTestConfig && bootstrap {
// The debug apply only fires on bootstrap nodes; a
// joiner cannot commit a log entry until it has been
// promoted to voter. Wait for this node to win an
// election first (a fresh single-voter cluster elects
// itself immediately but the goroutine may run before
// the state has flipped).
go func() {
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if c.Raft() != nil && c.Raft().State() == raft.Leader {
break
}
time.Sleep(100 * time.Millisecond)
}
if c.Raft() == nil || c.Raft().State() != raft.Leader {
log.Printf("worker cluster: applying debug test config id=%d: not leader after 15s",
workercluster.DefaultDebugCriticalCheck().ID)
return
}
check := workercluster.DefaultDebugCriticalCheck()
applied, err := c.ApplyTestConfig(&check)
if err != nil {
log.Printf("worker cluster: applying debug test config id=%d: %v",
check.ID, err)
return
}
log.Printf("worker cluster: applying debug test config id=%d applied_index=%d",
check.ID, applied)
}()
}
return c, &clusterAdapter{c: c}, nil
}
// clusterAdapter wraps *workercluster.Cluster so it implements the
// webapp.ClusterView interface without webapp importing the raft
// code path. The ApplyTestConfig signature is the one webapp expects
// (no config argument; the cluster subsystem owns the hardcoded
// payload so the two sides cannot drift).
// code path.
type clusterAdapter struct {
c *workercluster.Cluster
}
@@ -359,11 +308,6 @@ func (a *clusterAdapter) Stats() webapp.ClusterStats {
}
}
func (a *clusterAdapter) ApplyTestConfig() (uint64, error) {
check := workercluster.DefaultDebugCriticalCheck()
return a.c.ApplyTestConfig(&check)
}
func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }
@@ -528,13 +472,17 @@ func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow {
out := make([]webapp.NotificationRow, len(src))
for i, r := range src {
out[i] = webapp.NotificationRow{
Kind: r.Kind,
Channel: r.Channel,
Subject: r.Subject,
Body: r.Body,
OK: r.OK,
Error: r.Error,
At: r.At,
Kind: r.Kind,
Channel: r.Channel,
Subject: r.Subject,
Body: r.Body,
OK: r.OK,
Error: r.Error,
At: r.At,
JobID: r.JobID,
Method: r.Method,
Status: r.Status,
DurationMs: r.DurationMs,
}
}
return out