fix(worker): harden control-plane lifecycle

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.

internal/workercluster/admin_test.go

@@ -72,11 +72,9 @@ func TestClusterSmokeStats(t *testing.T) {
 	assert.Equal(t, addr, c.LocalAddr())
 }
 
-// TestClusterApplyTestConfig_Smoke verifies the ApplyTestConfig helper
-// commits a config.adopt entry and the FSM reflects the new version.
-// This is the function the webapp admin endpoint and the CLI flag
-// both go through.
-func TestClusterApplyTestConfig_Smoke(t *testing.T) {
+// TestClusterTestConfig_Smoke verifies the test-only helper commits a
+// config.adopt entry and the FSM reflects the new version.
+func TestClusterTestConfig_Smoke(t *testing.T) {
 	addr := pickPort(t)
 	dataDir := t.TempDir()
 	creds := HTTPCreds{Login: "alice", Password: "secret"}
@@ -108,8 +106,8 @@ func TestClusterApplyTestConfig_Smoke(t *testing.T) {
 	require.Equal(t, raft.Leader, c.Raft().State(),
 		"smoke test requires the local node to be leader")
 
-	check := DefaultDebugCriticalCheck()
-	idx, err := c.ApplyTestConfig(&check)
+	check := defaultTestCriticalCheck()
+	idx, err := c.applyTestConfig(&check)
 	require.NoError(t, err)
 	assert.NotZero(t, idx, "applied index must be non-zero")
 
@@ -164,11 +162,11 @@ func TestClusterStats_FSMFieldsOnFreshCluster(t *testing.T) {
 		"newly created FSM defaults Partition.State to 'steady'")
 }
 
-// TestApplyTestConfig_NonLeaderErrors pins the precondition that the
+// TestTestConfig_NonLeaderErrors pins the precondition that the
 // helper refuses to submit an entry on a non-leader (the raft library
 // would reject the apply anyway, but we want the failure to be
 // deterministic and informative).
-func TestApplyTestConfig_NonLeaderErrors(t *testing.T) {
+func TestTestConfig_NonLeaderErrors(t *testing.T) {
 	// Three-node fixture so we have a clear "not the leader" node.
 	if testing.Short() {
 		t.Skip("3-node smoke skipped in -short mode")
@@ -187,9 +185,9 @@ func TestApplyTestConfig_NonLeaderErrors(t *testing.T) {
 	require.NotNil(t, leader)
 	require.NotNil(t, follower)
 
-	check := DefaultDebugCriticalCheck()
-	_, err := follower.ApplyTestConfig(&check)
-	require.Error(t, err, "non-leader must refuse ApplyTestConfig")
+	check := defaultTestCriticalCheck()
+	_, err := follower.applyTestConfig(&check)
+	require.Error(t, err, "non-leader must refuse test config application")
 	assert.Contains(t, strings.ToLower(err.Error()), "not leader")
 }
 

internal/workercluster/cluster.go

@@ -2,7 +2,6 @@ package workercluster
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -325,63 +324,6 @@ func (c *Cluster) Snapshot() error {
 	return r.Snapshot().Error()
 }
 
-// DefaultDebugCriticalCheck returns the hardcoded CriticalCheckConfig
-// the cluster admin debug endpoint and the
-// --cluster-debug-apply-test-config CLI flag apply. A fresh Epoch is
-// stamped on every call so repeated applies produce distinct entries
-// (handy for verifying replication timing).
-func DefaultDebugCriticalCheck() CriticalCheckConfig {
-	return CriticalCheckConfig{
-		ID:        9999,
-		Kind:      "distributed_critical",
-		IntervalS: 30,
-		Target:    "http://example.com",
-		Epoch:     time.Now().UTC().UnixNano(),
-	}
-}
-
-// ApplyTestConfig submits a hardcoded config.adopt log entry with the
-// supplied CriticalCheckConfig. Returns the applied log index. This
-// is a debug convenience used by the e2e script and the
-// --cluster-debug-apply-test-config CLI flag; production code should
-// build entries from the real signed-config-adoption producer
-// (Phase-N work).
-//
-// DEBUG: this exists only so the e2e shell script can verify FSM
-// replication without a real producer wired in.
-//
-// TODO(phase-N): remove once the real producer lands.
-func (c *Cluster) ApplyTestConfig(check *CriticalCheckConfig) (uint64, error) {
-	c.mu.Lock()
-	r := c.raft
-	c.mu.Unlock()
-	if r == nil {
-		return 0, errors.New("workercluster: not started")
-	}
-	if r.State() != raft.Leader {
-		return 0, errors.New("workercluster: not leader; submit on the leader")
-	}
-
-	payload := ConfigAdoptPayload{
-		Version: 1,
-		Actor:   c.opts.NodeID,
-		Checks:  []CriticalCheckConfig{*check},
-	}
-	raw, err := json.Marshal(payload)
-	if err != nil {
-		return 0, fmt.Errorf("workercluster: encode payload: %w", err)
-	}
-	entry, err := EncodeEntry(&Entry{Type: EntryConfigAdopt, Adopted: raw})
-	if err != nil {
-		return 0, fmt.Errorf("workercluster: encode entry: %w", err)
-	}
-	fut := r.Apply(entry, 10*time.Second)
-	if err := fut.Error(); err != nil {
-		return 0, fmt.Errorf("workercluster: apply test config: %w", err)
-	}
-	return fut.Index(), nil
-}
-
 // ClusterID returns the NodeID this cluster was constructed with. It
 // is exposed so HTTP handlers can label status responses with a
 // stable identifier even when the raft library's own State() reports

internal/workercluster/test_config_test.go

@@ -0,0 +1,49 @@
+package workercluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/raft"
+)
+
+// defaultTestCriticalCheck is test-only scaffold input for replication tests.
+func defaultTestCriticalCheck() CriticalCheckConfig {
+	return CriticalCheckConfig{
+		ID:        9999,
+		Kind:      "distributed_critical",
+		IntervalS: 30,
+		Target:    "http://example.com",
+		Epoch:     time.Now().UTC().UnixNano(),
+	}
+}
+
+// applyTestConfig is deliberately compiled only into workercluster tests.
+func (c *Cluster) applyTestConfig(check *CriticalCheckConfig) (uint64, error) {
+	c.mu.Lock()
+	r := c.raft
+	c.mu.Unlock()
+	if r == nil {
+		return 0, errors.New("workercluster: not started")
+	}
+	if r.State() != raft.Leader {
+		return 0, errors.New("workercluster: not leader; submit on the leader")
+	}
+
+	payload := ConfigAdoptPayload{Version: 1, Actor: c.opts.NodeID, Checks: []CriticalCheckConfig{*check}}
+	raw, err := json.Marshal(payload)
+	if err != nil {
+		return 0, fmt.Errorf("workercluster: encode payload: %w", err)
+	}
+	entry, err := EncodeEntry(&Entry{Type: EntryConfigAdopt, Adopted: raw})
+	if err != nil {
+		return 0, fmt.Errorf("workercluster: encode entry: %w", err)
+	}
+	fut := r.Apply(entry, 10*time.Second)
+	if err := fut.Error(); err != nil {
+		return 0, fmt.Errorf("workercluster: apply test config: %w", err)
+	}
+	return fut.Index(), nil
+}