fix(worker): harden control-plane lifecycle

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.

internal/webapp/constants.go

@@ -24,14 +24,13 @@ const (
 // Environment variable names referenced by ConfigFromEnv. Lifted out
 // so the validator and the cmd binary share the same constants.
 const (
-	envWorkerHost        = "WORKER_HOST"
-	envWorkerPort        = "WORKER_PORT"
-	envWorkerURL         = "WORKER_URL"
-	envWorkerLogin       = "WORKER_LOGIN"
-	envWorkerPassword    = "WORKER_PASSWORD"
-	envClusterEnabled    = "WORKER_CLUSTER_ENABLED"
-	envClusterDebugApply = "WORKER_CLUSTER_DEBUG_APPLY"
-	envReleaseURL        = "WORKER_RELEASE_URL"
+	envWorkerHost     = "WORKER_HOST"
+	envWorkerPort     = "WORKER_PORT"
+	envWorkerURL      = "WORKER_URL"
+	envWorkerLogin    = "WORKER_LOGIN"
+	envWorkerPassword = "WORKER_PASSWORD"
+	envClusterEnabled = "WORKER_CLUSTER_ENABLED"
+	envReleaseURL     = "WORKER_RELEASE_URL"
 )
 
 // Route paths used as redirect targets. Lifted out so goconst stops

internal/webapp/handlers_cluster.go

@@ -74,63 +74,6 @@ func (s *Server) handleClusterStatus(w http.ResponseWriter, _ *http.Request) {
 	}
 }
 
-// handleClusterApplyTestConfig applies a hardcoded config.adopt log
-// entry to the cluster. It exists so the e2e script and any operator
-// debugging session can verify FSM replication without having to wire
-// up the real signed-config-adoption producer (which lives in a later
-// phase).
-//
-// DEBUG: this endpoint is a placeholder for the real producer. It must
-// be replaced (or removed) before any production deployment.
-//
-// The handler is gated behind Config.DebugClusterApply (env
-// WORKER_CLUSTER_DEBUG_APPLY=true). When the flag is false the
-// handler returns 404 — the route is still registered so the auth
-// + CSRF paths are exercised in tests, but no real FSM entry is ever
-// appended from a production webapp.
-//
-// TODO(worker-cluster-real-producer): remove the apply-test-config
-// endpoint entirely once the signed-config-adoption producer ships.
-func (s *Server) handleClusterApplyTestConfig(w http.ResponseWriter, r *http.Request) {
-	writeNoStore(w)
-	if !s.cfg.DebugClusterApply {
-		http.NotFound(w, r)
-		return
-	}
-	if s.cluster == nil {
-		http.Error(w, "cluster not configured", http.StatusServiceUnavailable)
-		return
-	}
-	if !s.requireCSRF(sessionFromContextOrFail(w, r), r) {
-		http.Error(w, "csrf token required", http.StatusForbidden)
-		return
-	}
-	applied, err := s.cluster.ApplyTestConfig()
-	if err != nil {
-		s.deps.Logger.Printf("cluster apply test config: %v", err)
-		http.Error(w, err.Error(), http.StatusBadGateway)
-		return
-	}
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	if err := json.NewEncoder(w).Encode(map[string]uint64{"applied_index": applied}); err != nil {
-		s.deps.Logger.Printf("cluster apply encode: %v", err)
-	}
-}
-
-// sessionFromContextOrFail is a tiny adapter so requireCSRF can be
-// called from this handler without leaking the middleware into the
-// cluster package. If no session is attached (should not happen
-// because requireSession already ran) we return a stub session with
-// no CSRF token, which causes requireCSRF to refuse the request.
-func sessionFromContextOrFail(_ http.ResponseWriter, r *http.Request) *Session {
-	sess, _ := sessionFromContext(r.Context())
-	if sess != nil {
-		return sess
-	}
-	return &Session{}
-}
-
 // ErrClusterNotConfigured is returned when a cluster-admin endpoint is
 // hit on a server without a cluster attached.
 var ErrClusterNotConfigured = errors.New("webapp: cluster not configured")

internal/webapp/handlers_cluster_test.go

@@ -1,16 +1,11 @@
 package webapp
 
 import (
-	"context"
 	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
-	"strings"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -21,23 +16,13 @@ import (
 // pinned without standing up a real raft group.
 type stubCluster struct {
 	stats        ClusterStats
-	applyIndex   uint64
-	applyErr     error
-	applyCalled  int
-	applyMu      sync.Mutex
 	clusterIDOut string
 	addrOut      string
 }
 
 func (s *stubCluster) Stats() ClusterStats { return s.stats }
-func (s *stubCluster) ApplyTestConfig() (uint64, error) {
-	s.applyMu.Lock()
-	defer s.applyMu.Unlock()
-	s.applyCalled++
-	return s.applyIndex, s.applyErr
-}
-func (s *stubCluster) ClusterID() string { return s.clusterIDOut }
-func (s *stubCluster) LocalAddr() string { return s.addrOut }
+func (s *stubCluster) ClusterID() string   { return s.clusterIDOut }
+func (s *stubCluster) LocalAddr() string   { return s.addrOut }
 
 // withClusterServer returns a test server whose ClusterView is the
 // supplied stub. The first-run password path is also exercised so
@@ -166,151 +151,23 @@ func TestClusterStatus_RequiresSession(t *testing.T) {
 	assert.Equal(t, "/web/login", resp.Header.Get("Location"))
 }
 
-// TestClusterApplyTestConfig_NotConfigured verifies the 404 path
-// when WORKER_CLUSTER_DEBUG_APPLY is false (the production default)
-// and no cluster is attached. The handler must refuse before it
-// even checks the cluster because the debug flag is off.
-func TestClusterApplyTestConfig_NotConfigured(t *testing.T) {
-	srv := newTestServer(t, &stubRunner{id: "w-1"})
-	require.False(t, srv.cfg.DebugClusterApply, "default config must leave the debug apply flag off")
-	ts := newHTTPTestServer(t, srv)
-	c, _ := loginAsFirstRun(t, ts.URL, srv)
-	clearRequiresChange(t, srv)
+// TestClusterApplyTestConfig_NotExposed verifies production requests cannot
+// append a hardcoded config through the former debug endpoint.
+func TestClusterApplyTestConfig_NotExposed(t *testing.T) {
+	t.Setenv("WORKER_CLUSTER_DEBUG_APPLY", "true")
+	ts, _, c := withClusterServer(t, &stubCluster{})
 
-	resp, err := c.PostForm(ts.URL+"/web/api/cluster/apply-test-config", url.Values{})
-	require.NoError(t, err)
-	defer resp.Body.Close() //nolint:errcheck
-	assert.Equal(t, http.StatusNotFound, resp.StatusCode,
-		"debug apply must be invisible (404) when WORKER_CLUSTER_DEBUG_APPLY is unset")
-}
-
-// TestClusterApplyTestConfig_DebugOffReturns404 verifies that even
-// with a cluster attached the apply endpoint stays 404 unless the
-// debug flag is on. The flag, not cluster presence, gates the
-// endpoint.
-func TestClusterApplyTestConfig_DebugOffReturns404(t *testing.T) {
-	stub := &stubCluster{applyIndex: 42}
-	ts, _, c := withClusterServer(t, stub)
-
-	resp, err := c.Get(ts.URL + "/overview")
-	require.NoError(t, err)
-	bodyBytes, _ := io.ReadAll(resp.Body)
-	resp.Body.Close() //nolint:errcheck
-	csrf := extractCSRFToken(t, string(bodyBytes))
-
-	form := url.Values{}
-	form.Set("csrf_token", csrf)
-	req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
-		strings.NewReader(form.Encode()))
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err = c.Do(req)
+	resp, err := c.Post(ts.URL+"/web/api/cluster/apply-test-config", "", nil)
 	require.NoError(t, err)
 	defer resp.Body.Close() //nolint:errcheck
 	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-	assert.Equal(t, 0, stub.applyCalled,
-		"ApplyTestConfig must never be called when the debug flag is off")
 }
 
-// TestClusterApplyTestConfig_HappyPath verifies that the apply-test-
-// config endpoint returns the applied index when the cluster
-// subsystem accepts the entry. CSRF is checked. The DebugClusterApply
-// flag must be on for the endpoint to be reachable.
-func TestClusterApplyTestConfig_HappyPath(t *testing.T) {
-	stub := &stubCluster{
-		stats: ClusterStats{
-			NodeID: "worker1", State: "Leader", Leader: "worker1",
-			Voters: []string{"worker1"},
-		},
-		applyIndex:   13,
-		clusterIDOut: "worker1",
-		addrOut:      "127.0.0.1:17401",
-	}
-	ts, srv, c := withClusterServer(t, stub)
-	srv.cfg.DebugClusterApply = true
-
-	// Fetch CSRF token from any authenticated page.
-	resp, err := c.Get(ts.URL + "/overview")
-	require.NoError(t, err)
-	bodyBytes, _ := io.ReadAll(resp.Body)
-	resp.Body.Close() //nolint:errcheck
-	csrf := extractCSRFToken(t, string(bodyBytes))
-
-	form := url.Values{}
-	form.Set("csrf_token", csrf)
-	req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
-		strings.NewReader(form.Encode()))
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err = c.Do(req)
-	require.NoError(t, err)
-	defer resp.Body.Close() //nolint:errcheck
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	bodyBytes, _ = io.ReadAll(resp.Body)
-
-	var got map[string]uint64
-	require.NoError(t, json.Unmarshal(bodyBytes, &got))
-	assert.EqualValues(t, 13, got["applied_index"])
-
-	assert.Equal(t, 1, stub.applyCalled)
-}
-
-// TestClusterApplyTestConfig_PropagatesError verifies that errors
-// from the cluster subsystem surface as 502 Bad Gateway. Debug flag
-// must be on.
-func TestClusterApplyTestConfig_PropagatesError(t *testing.T) {
-	stub := &stubCluster{
-		applyErr:     errStubApply,
-		clusterIDOut: "worker1",
-		addrOut:      "127.0.0.1:17401",
-	}
-	ts, srv, c := withClusterServer(t, stub)
-	srv.cfg.DebugClusterApply = true
-
-	resp, err := c.Get(ts.URL + "/overview")
-	require.NoError(t, err)
-	bodyBytes, _ := io.ReadAll(resp.Body)
-	resp.Body.Close() //nolint:errcheck
-	csrf := extractCSRFToken(t, string(bodyBytes))
-
-	form := url.Values{}
-	form.Set("csrf_token", csrf)
-	req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
-		strings.NewReader(form.Encode()))
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err = c.Do(req)
-	require.NoError(t, err)
-	defer resp.Body.Close() //nolint:errcheck
-	assert.Equal(t, http.StatusBadGateway, resp.StatusCode)
-}
-
-// TestClusterApplyTestConfig_RequiresCSRF ensures the apply-test-
-// config POST is refused without a CSRF token. Debug flag must be
-// on for the endpoint to be reachable; without the flag it returns
-// 404 (priority over CSRF check).
-func TestClusterApplyTestConfig_RequiresCSRF(t *testing.T) {
-	stub := &stubCluster{applyIndex: 99}
-	ts, srv, c := withClusterServer(t, stub)
-	srv.cfg.DebugClusterApply = true
-
-	resp, err := c.PostForm(ts.URL+"/web/api/cluster/apply-test-config", url.Values{})
-	require.NoError(t, err)
-	defer resp.Body.Close() //nolint:errcheck
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode,
-		"apply-test-config without CSRF must be 403")
-	assert.Equal(t, 0, stub.applyCalled, "ApplyTestConfig must not be called without CSRF")
-}
-
-// errStubApply is a sentinel error used by the apply-error test.
-var errStubApply = errApply("worker not leader")
-
-type errApply string
-
-func (e errApply) Error() string { return string(e) }
-
 // TestSetClusterDetaches verifies SetCluster(nil) returns the server
 // to the no-cluster-attached state (503 from the endpoints).
 func TestSetClusterDetaches(t *testing.T) {
 	srv := newTestServer(t, &stubRunner{id: "w-1"})
-	stub := &stubCluster{applyIndex: 7}
+	stub := &stubCluster{}
 	srv.SetCluster(stub)
 	require.NotNil(t, srv.Cluster())
 
@@ -326,11 +183,3 @@ func TestSetClusterDetaches(t *testing.T) {
 	defer resp.Body.Close() //nolint:errcheck
 	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
 }
-
-// _ = context.Background and time.Time keep the linter quiet about
-// unused imports if the file shrinks.
-var (
-	_ = context.Background
-	_ = time.Now
-	_ = url.Parse
-)

internal/webapp/handlers_updates_test.go

@@ -224,7 +224,10 @@ func TestChecksPage_RunNowDisabledButton(t *testing.T) {
 // TestNotificationsPage_ResendDisabledButton mirrors the checks
 // page test for the resend button on /notifications.
 func TestNotificationsPage_ResendDisabledButton(t *testing.T) {
-	srv := newTestServer(t, &stubRunner{id: "w-1"})
+	srv := newTestServer(t, &stubRunner{id: "w-1", notifs: []NotificationRow{
+		{Kind: "email", Channel: "smtp", Subject: "selfcheck", Body: "main API down", OK: true, At: time.Now()},
+		{JobID: "delegated-job", Method: "sms", Status: "permanent", DurationMs: 12, At: time.Now()},
+	}})
 	ts := newHTTPTestServer(t, srv)
 	c, _ := loginAsFirstRun(t, ts.URL, srv)
 	clearRequiresChange(t, srv)
@@ -239,6 +242,11 @@ func TestNotificationsPage_ResendDisabledButton(t *testing.T) {
 	assert.Contains(t, page, "Resend")
 	assert.Contains(t, page, "disabled")
 	assert.Contains(t, page, "worker-notifier-mvp")
+	assert.Contains(t, page, "selfcheck")
+	assert.Contains(t, page, "delegated-job")
+	assert.Contains(t, page, "sms")
+	assert.Contains(t, page, "permanent")
+	assert.Contains(t, page, "12 ms")
 }
 
 // TestAppsPage_ReferencesInventoryPlan ensures the copy on

internal/webapp/routes.go

@@ -62,7 +62,6 @@ func (s *Server) routes() {
 	// Both routes require a session (the worker webapp is single-tenant
 	// so every logged-in operator is effectively an admin).
 	s.mux.Handle("GET /web/api/cluster/status", s.requireSession(s.handleClusterStatus))
-	s.mux.Handle("POST /web/api/cluster/apply-test-config", s.requireSession(s.handleClusterApplyTestConfig))
 
 	// Cross-worker peer status. The path is intentionally under
 	// /api/ (not /web/api/) so the basic-auth middleware does not

internal/webapp/server.go

@@ -80,16 +80,6 @@ type Config struct {
 	BasicAuthLogin    string
 	BasicAuthPassword string
 
-	// DebugClusterApply gates the /web/api/cluster/apply-test-config
-	// endpoint. When false (the default) the route is registered but
-	// the handler returns 404 so the endpoint is invisible in
-	// production. Operators who want to poke the cluster FSM during
-	// development set WORKER_CLUSTER_DEBUG_APPLY=true. The endpoint
-	// must NEVER be reachable in production — it appends hardcoded
-	// log entries to the Raft FSM without going through the real
-	// config-adoption producer.
-	DebugClusterApply bool
-
 	// ReleaseURL is the optional URL the worker polls to discover
 	// the latest published version of the worker binary. When empty
 	// the /updates page shows the placeholder "v1 (dev)". The URL
@@ -186,7 +176,6 @@ func ConfigFromEnv(env map[string]string, defaultDataDir string) (Config, error)
 	if v := env["RSMON_WEBAPP_STORE_PATH"]; v != "" {
 		cfg.StorePath = v
 	}
-	cfg.DebugClusterApply = parseBool(env[envClusterDebugApply])
 	cfg.ReleaseURL = strings.TrimSpace(env[envReleaseURL])
 	return cfg, nil
 }
@@ -212,7 +201,7 @@ func ConfigFromEnvOrDefault() Config {
 		envWorkerHost, envWorkerPort, envWorkerURL, envWorkerLogin, envWorkerPassword,
 		"RSMON_WEBAPP_DATA_DIR", "RSMON_WEBAPP_STORE_PATH",
 		"WORKER_CLUSTER_ENABLED",
-		envClusterDebugApply, envReleaseURL,
+		envReleaseURL,
 	} {
 		if v := os.Getenv(k); v != "" {
 			env[k] = v
@@ -254,7 +243,6 @@ type Deps struct {
 // pulling in the raft package or bbolt.
 type ClusterView interface {
 	Stats() ClusterStats
-	ApplyTestConfig() (uint64, error)
 	ClusterID() string
 	LocalAddr() string
 }
@@ -320,9 +308,7 @@ type ResultRow struct {
 	At         time.Time
 }
 
-// NotificationRow is one row from the worker's in-memory notification
-// ring buffer. Phase 1 only emits selfcheck alerts; main-app-issued
-// notifications still live in the main app's DB.
+// NotificationRow is one row from the worker's in-memory notification ring.
 type NotificationRow struct {
 	Kind    string // "email", "telegram_private", "telegram_group"
 	Channel string
@@ -331,6 +317,11 @@ type NotificationRow struct {
 	OK      bool
 	Error   string
 	At      time.Time
+
+	JobID      string
+	Method     string
+	Status     string
+	DurationMs int
 }
 
 // Server is the local HTTP server for the worker webapp. It owns the

internal/webapp/server_test.go

@@ -88,40 +88,6 @@ func TestConfigFromEnvBasicAuthRejectsXOR(t *testing.T) {
 	assert.Error(t, err, "XOR (password only) must be rejected")
 }
 
-// TestConfigFromEnvDebugClusterApply pins the default-off behavior
-// of the cluster-apply debug gate and verifies the env flag flips
-// it on. Production builds must not accidentally expose the
-// endpoint, so the default is false.
-func TestConfigFromEnvDebugClusterApply(t *testing.T) {
-	cfg, err := ConfigFromEnv(map[string]string{}, t.TempDir())
-	require.NoError(t, err)
-	assert.False(t, cfg.DebugClusterApply, "default must leave the debug flag off")
-
-	cfg, err = ConfigFromEnv(map[string]string{
-		"WORKER_CLUSTER_DEBUG_APPLY": "true",
-	}, t.TempDir())
-	require.NoError(t, err)
-	assert.True(t, cfg.DebugClusterApply)
-
-	// Other truthy spellings accepted.
-	for _, v := range []string{"yes", "1", "TRUE", "YeS"} {
-		cfg, err = ConfigFromEnv(map[string]string{
-			"WORKER_CLUSTER_DEBUG_APPLY": v,
-		}, t.TempDir())
-		require.NoError(t, err)
-		assert.True(t, cfg.DebugClusterApply, "must accept truthy value %q", v)
-	}
-
-	// Empty / unknown values stay false.
-	for _, v := range []string{"", "false", "0", "no"} {
-		cfg, err = ConfigFromEnv(map[string]string{
-			"WORKER_CLUSTER_DEBUG_APPLY": v,
-		}, t.TempDir())
-		require.NoError(t, err)
-		assert.False(t, cfg.DebugClusterApply, "must reject non-truthy value %q", v)
-	}
-}
-
 // TestConfigFromEnvReleaseURL pins the env-driven WORKER_RELEASE_URL
 // plumbing. The handler reads cfg.ReleaseURL when the page renders,
 // so the value must survive ConfigFromEnv exactly.

internal/webapp/templates/notifications.html

@@ -1,15 +1,15 @@
 {{define "body"}}<section class="card">
   <h1>Recent notifications</h1>
-  <p class="muted">Last 50 notifications emitted by this worker (Phase 1: selfcheck alerts only).</p>
+  <p class="muted">Last 50 notification attempts emitted by this worker.</p>
   <p><button type="button" disabled title="{{.ResendTooltip}}">Resend selected</button>
      <span class="muted">{{.ResendTooltip}}</span></p>
   <table>
     <thead>
       <tr>
-        <th>Channel</th>
-        <th>Subject</th>
-        <th>Body</th>
+        <th>Type</th>
+        <th>Details</th>
         <th>Status</th>
+        <th>Duration</th>
         <th>Error</th>
         <th>When</th>
       </tr>
@@ -17,15 +17,23 @@
     <tbody>
       {{range .Rows}}
       <tr>
-        <td>{{.Channel}} ({{.Kind}})</td>
-        <td>{{.Subject}}</td>
-        <td><code>{{.Body}}</code></td>
+        {{if .JobID}}
+        <td>delegated</td>
+        <td>job <code>{{.JobID}}</code>, {{.Method}}</td>
+        <td>{{.Status}}</td>
+        <td>{{.DurationMs}} ms</td>
+        <td></td>
+        {{else}}
+        <td>selfcheck</td>
+        <td>{{.Channel}} ({{.Kind}}): {{.Subject}} <code>{{.Body}}</code></td>
         <td>{{if .OK}}<span class="ok">delivered</span>{{else}}<span class="error">failed</span>{{end}}</td>
+        <td>-</td>
         <td>{{.Error}}</td>
+        {{end}}
         <td>{{fmtTime .At}}</td>
       </tr>
       {{else}}
-      <tr><td colspan="6" class="muted">No notifications yet.</td></tr>
+        <tr><td colspan="6" class="muted">No notifications yet.</td></tr>
       {{end}}
     </tbody>
   </table>