fix(worker): harden control-plane lifecycle

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.

internal/distworker/runner.go

@@ -25,6 +25,8 @@ const (
 	// minQueueCapacity is the lower bound for the bounded job/result
 	// channels so that a small pool still has some backpressure headroom.
 	minQueueCapacity = 16
+
+	malformedTaskEnvelopeError = "malformed_task_envelope"
 )
 
 // jobPool is the minimal interface the Runner needs from a worker pool. It
@@ -43,6 +45,11 @@ type resultEnvelope struct {
 	reports []wire.CheckResultReport
 }
 
+type metricEnvelope struct {
+	generation uint64
+	report     wire.ServerMetricReport
+}
+
 // Runner manages the worker execution loop.
 //
 // Concurrency design:
@@ -61,23 +68,32 @@ type resultEnvelope struct {
 //     websocket writes thread-safe and removes the per-task goroutine
 //     that previously blocked pool execution while holding the mutex.
 type Runner struct {
-	config         *Config
-	client         *Client
-	pool           jobPool
-	concurrency    int64
-	maxConcurrency int
-	jobQueue       chan wire.CheckJob
-	results        chan resultEnvelope
-	notifyQueue    chan wire.NotificationTask
-	notifyResults  chan notifyResultEnvelope
-	metricResults  chan wire.ServerMetricReport
-	stopCh         chan struct{}
-	wg             sync.WaitGroup
-	started        atomic.Bool
-	queueDepth     int64
-	activeCount    int64
-	notifyDepth    int64
-	notifyActive   int64
+	config           *Config
+	client           *Client
+	pool             jobPool
+	concurrency      int64
+	maxConcurrency   int
+	jobQueue         chan wire.CheckJob
+	results          chan resultEnvelope
+	notifyQueue      chan wire.NotificationTask
+	notifyResults    chan notifyResultEnvelope
+	metricResults    chan metricEnvelope
+	stopCh           chan struct{}
+	wg               sync.WaitGroup
+	controlWG        sync.WaitGroup
+	backgroundWG     sync.WaitGroup
+	lifecycleMu      sync.Mutex
+	controlCtx       context.Context
+	controlCancel    context.CancelFunc
+	backgroundCtx    context.Context
+	backgroundCancel context.CancelFunc
+	rotationCtx      context.Context
+	rotationCancel   context.CancelFunc
+	started          atomic.Bool
+	queueDepth       int64
+	activeCount      int64
+	notifyDepth      int64
+	notifyActive     int64
 
 	// credentialsMu guards credentials during init/config refresh.
 	credentialsMu sync.RWMutex
@@ -121,15 +137,15 @@ type Runner struct {
 	masterStatusUp *bool
 	masterStatusAt time.Time
 
-	// selfcheckCancel terminates the periodic selfcheck goroutine started
-	// by Start(). Nil until Start runs.
-	selfcheckCancel context.CancelFunc
-
 	// executor is the function the pool runs for each job. It is a
 	// field so tests can swap it for a deterministic stub without
 	// touching the websocket plumbing.
 	executor func(payload interface{}) interface{}
 
+	// notificationExecutor is a test seam for the notification task
+	// boundary. Production uses ExecuteNotification's normal delivery path.
+	notificationExecutor func(context.Context, models.Task) wire.NotificationResultReport
+
 	// resultsBuf holds the most recent result rows. The webapp
 	// reads from it via RecentResults(n). Cleared by Stop so the
 	// ring does not leak between worker runs.
@@ -152,26 +168,40 @@ type Runner struct {
 	// workerID / regionCode / workerVersion / workerCaps are
 	// captured from the most recent init/config websocket message
 	// so the webapp can render them read-only.
-	workerIDMu    sync.RWMutex
-	workerID      string
-	regionMu      sync.RWMutex
-	regionCode    string
-	versionMu     sync.RWMutex
-	workerVersion string
-	capsMu        sync.RWMutex
-	workerCaps    []string
-	serverID      atomic.Int64
+	workerIDMu           sync.RWMutex
+	workerID             string
+	regionMu             sync.RWMutex
+	regionCode           string
+	versionMu            sync.RWMutex
+	workerVersion        string
+	capsMu               sync.RWMutex
+	workerCaps           []string
+	serverID             atomic.Int64
+	metricGeneration     atomic.Uint64
+	nextMetricGeneration atomic.Uint64
 
-	// clientMu guards swap of the http/websocket client during
-	// token rotation. The websocket loop reads r.client under the
-	// lock; RotateToken swaps a fresh client in under the lock
-	// before closing the old connection.
-	clientMu sync.Mutex
+	// clientMu guards the control-plane client and its active websocket.
+	// Rotation swaps the client and closes only controlConn, leaving the
+	// runner's global stop signal and local subsystems untouched.
+	clientMu       sync.Mutex
+	controlConn    *websocket.Conn
+	controlWriteMu *sync.Mutex
+	reconnectCh    chan struct{}
+	rotationMu     sync.Mutex
 
-	// closeOnce guards Close against double-close on the websocket
-	// from RotateToken. Phase 1 has a single websocket; RotateToken
-	// closes it so the reconnect loop picks up the new token.
-	closeOnce sync.Once
+	// outbox holds messages removed from a per-connection writer queue that
+	// could not be written before its websocket closed.
+	outboxMu   sync.Mutex
+	outbox     []wire.WorkerMessage
+	outboxWake chan struct{}
+
+	// beforeControlWrite is a test seam used to hold a dequeued message while
+	// a connection rotates.
+	beforeControlWrite func()
+
+	// beforeTokenCommit lets tests make shutdown win between a successful HTTP
+	// rotation response and the lifecycle-protected in-memory commit.
+	beforeTokenCommit func()
 }
 
 // NewRunner creates a new worker runner. Config is taken by pointer to
@@ -183,10 +213,21 @@ func NewRunner(cfg *Config) *Runner {
 	if maxConc <= 0 {
 		maxConc = DefaultMaxConcurrency
 	}
+	controlCtx, controlCancel := context.WithCancel(context.Background())
+	backgroundCtx, backgroundCancel := context.WithCancel(context.Background())
+	rotationCtx, rotationCancel := context.WithCancel(context.Background())
 	return &Runner{
 		config:           cfg,
 		maxConcurrency:   maxConc,
 		stopCh:           make(chan struct{}),
+		reconnectCh:      make(chan struct{}, 1),
+		outboxWake:       make(chan struct{}, 1),
+		controlCtx:       controlCtx,
+		controlCancel:    controlCancel,
+		backgroundCtx:    backgroundCtx,
+		backgroundCancel: backgroundCancel,
+		rotationCtx:      rotationCtx,
+		rotationCancel:   rotationCancel,
 		resultsBuf:       newResultBuffer(),
 		notificationsBuf: newNotificationBuffer(),
 		peerCache:        newPeerCache(),
@@ -196,23 +237,32 @@ func NewRunner(cfg *Config) *Runner {
 // Start begins the worker execution
 func (r *Runner) Start() error {
 	log.Println("worker: starting...")
+	r.lifecycleMu.Lock()
 
 	if r.config.URL == "" || r.config.Token == "" {
+		r.lifecycleMu.Unlock()
 		return fmt.Errorf("RSMON_URL and RSMON_TOKEN must be set")
 	}
 
 	if !r.started.CompareAndSwap(false, true) {
+		r.lifecycleMu.Unlock()
 		return fmt.Errorf("worker: runner already started")
 	}
+	if r.stopped() {
+		r.lifecycleMu.Unlock()
+		return fmt.Errorf("worker: runner stopped")
+	}
 
+	r.clientMu.Lock()
 	r.client = NewClient(r.config.URL, r.config.Token)
+	r.clientMu.Unlock()
 
 	queueCap := r.queueCapacity()
 	r.jobQueue = make(chan wire.CheckJob, queueCap)
 	r.results = make(chan resultEnvelope, queueCap)
 	r.notifyQueue = make(chan wire.NotificationTask, queueCap)
 	r.notifyResults = make(chan notifyResultEnvelope, queueCap)
-	r.metricResults = make(chan wire.ServerMetricReport, queueCap)
+	r.metricResults = make(chan metricEnvelope, queueCap)
 
 	if r.executor == nil {
 		r.executor = r.defaultExecuteJob
@@ -237,31 +287,48 @@ func (r *Runner) Start() error {
 		go r.notifyDispatcher()
 	}
 
-	// Start websocket task loop
-	go r.websocketLoop()
-	go r.serverMetricLoop()
+	// Start websocket task loop.
+	r.controlWG.Add(1)
+	go func() {
+		defer r.controlWG.Done()
+		r.websocketLoop()
+	}()
+	r.backgroundWG.Add(1)
+	go func() {
+		defer r.backgroundWG.Done()
+		r.serverMetricLoop(r.backgroundCtx)
+	}()
 
 	// Start periodic selfcheck loop. This probes the main API and, on
 	// sustained unreachability, notifies system contacts directly via
 	// the cached credentials. Lifetimes of selfcheck goroutines are
 	// bound to stopCh (and the explicit cancel, kept for symmetry).
-	selfcheckCtx, selfcheckCancel := context.WithCancel(context.Background())
-	r.selfcheckCancel = selfcheckCancel
-	go r.startSelfcheck(selfcheckCtx)
+	r.backgroundWG.Add(1)
+	go func() {
+		defer r.backgroundWG.Done()
+		r.startSelfcheck(r.backgroundCtx)
+	}()
 
 	// Start the peer poller. It refreshes r.peerCache with each
 	// peer's latest /api/peer/status verdict. The selfcheck
 	// consumes the cache to drive consensus. The poller is bound
 	// to selfcheckCtx so it shuts down together with the
 	// selfcheck loop on Stop.
-	go r.peerPollerLoop(selfcheckCtx)
+	r.backgroundWG.Add(1)
+	go func() {
+		defer r.backgroundWG.Done()
+		r.peerPollerLoop(r.backgroundCtx)
+	}()
+	r.lifecycleMu.Unlock()
 
 	// Wait for stop signal
 	<-r.stopCh
 	log.Println("worker: shutting down...")
-	if r.selfcheckCancel != nil {
-		r.selfcheckCancel()
+	if r.backgroundCancel != nil {
+		r.backgroundCancel()
 	}
+	r.controlWG.Wait()
+	r.backgroundWG.Wait()
 
 	// Dispatchers exit via stopCh. Do not close jobQueue here: the websocket
 	// reader can still be unwinding and may otherwise race with a send.
@@ -271,21 +338,42 @@ func (r *Runner) Start() error {
 	// otherwise an in-flight pool.Process would panic.
 	r.pool.Close()
 
-	// Close results so any future writers exit promptly. (At this point
-	// the websocket connection is also gone, so this is just defensive.)
-	close(r.results)
-
 	return nil
 }
 
 // Stop gracefully stops the worker
 func (r *Runner) Stop() {
+	r.lifecycleMu.Lock()
 	select {
 	case <-r.stopCh:
 		// already closed
 	default:
 		close(r.stopCh)
 	}
+	if r.controlCancel != nil {
+		r.controlCancel()
+	}
+	if r.backgroundCancel != nil {
+		r.backgroundCancel()
+	}
+	if r.rotationCancel != nil {
+		r.rotationCancel()
+	}
+	r.clientMu.Lock()
+	conn := r.controlConn
+	r.clientMu.Unlock()
+	if conn != nil {
+		_ = conn.Close()
+	}
+	r.lifecycleMu.Unlock()
+
+	// Cancel and close before waiting for rotation. A rotation can be waiting
+	// on controlWriteMu while a blocked writer needs that close to return.
+	r.rotationMu.Lock()
+	r.rotationMu.Unlock()
+	r.controlWG.Wait()
+	r.backgroundWG.Wait()
+	r.wg.Wait()
 }
 
 // Enqueue submits a job to the worker pool. It returns false if the runner
@@ -295,6 +383,9 @@ func (r *Runner) Enqueue(job wire.CheckJob) bool { //nolint:lll,gocritic // wire
 	if r.jobQueue == nil {
 		return false
 	}
+	if r.stopped() {
+		return false
+	}
 	select {
 	case r.jobQueue <- job:
 		atomic.AddInt64(&r.queueDepth, 1)
@@ -318,6 +409,9 @@ func (r *Runner) EnqueueNotification(task wire.NotificationTask) bool {
 	if r.notifyQueue == nil {
 		return false
 	}
+	if r.stopped() {
+		return false
+	}
 	select {
 	case r.notifyQueue <- task:
 		atomic.AddInt64(&r.notifyDepth, 1)
@@ -350,23 +444,30 @@ func (r *Runner) notifyDispatcher() {
 // shape, runs the executor, and pushes the result into notifyResults. The
 // writer goroutine picks it up and serializes the websocket write.
 func (r *Runner) executeAndForwardNotification(task wire.NotificationTask) { //nolint:gocritic // wire payload is shared
-	if r.notifyResults == nil {
-		return
-	}
-	deadline := time.Now().Add(models.DefaultNotificationExecutionTimeout)
+	started := time.Now()
+	deadline := started.Add(models.DefaultNotificationExecutionTimeout)
+	var taskDeadline *time.Time
 	if task.Deadline != nil {
-		if taskDeadline, err := time.Parse(time.RFC3339Nano, *task.Deadline); err == nil && taskDeadline.Before(deadline) {
-			deadline = taskDeadline
+		parsedDeadline, err := time.Parse(time.RFC3339Nano, *task.Deadline)
+		if err != nil {
+			r.completeNotification(task, notificationPermanentReport(task, "invalid notification deadline", started))
+			return
+		}
+		taskDeadline = &parsedDeadline
+		if !parsedDeadline.After(started) {
+			r.completeNotification(task, notificationPermanentReport(task, "notification deadline expired", started))
+			return
+		}
+		if parsedDeadline.Before(deadline) {
+			deadline = parsedDeadline
 		}
 	}
 	ctx, cancel := context.WithDeadline(context.Background(), deadline)
 	defer cancel()
 	payload, _ := json.Marshal(task)
 	dbTask := models.Task{JobID: task.JobID, LeaseToken: task.LeaseToken, Payload: payload}
-	if task.Deadline != nil {
-		if deadline, err := time.Parse(time.RFC3339Nano, *task.Deadline); err == nil {
-			dbTask.Deadline = &deadline
-		}
+	if taskDeadline != nil {
+		dbTask.Deadline = taskDeadline
 	}
 	if task.MessageID != 0 {
 		msgID := task.MessageID
@@ -377,15 +478,33 @@ func (r *Runner) executeAndForwardNotification(task wire.NotificationTask) { //n
 		_ = ev
 	}
 	report := r.ExecuteNotification(ctx, dbTask)
-	env := notifyResultEnvelope{task: task, report: report}
+	r.forwardNotificationResult(task, report)
+}
 
-	select {
-	case <-r.stopCh:
+func notificationPermanentReport(task wire.NotificationTask, message string, started time.Time) wire.NotificationResultReport {
+	return wire.NotificationResultReport{
+		JobID:      task.JobID,
+		LeaseToken: task.LeaseToken,
+		MessageID:  task.MessageID,
+		Status:     wire.NotificationResultPermanent,
+		Error:      stringPtr(message),
+		DurationMs: int(time.Since(started) / time.Millisecond),
+	}
+}
+
+// completeNotification records the terminal local outcome before forwarding
+// the result. Rejected tasks use this path because they do not enter the executor.
+func (r *Runner) completeNotification(task wire.NotificationTask, report wire.NotificationResultReport) {
+	r.recordDelegatedNotification(report.JobID, task.Method, report)
+	r.forwardNotificationResult(task, report)
+}
+
+func (r *Runner) forwardNotificationResult(task wire.NotificationTask, report wire.NotificationResultReport) {
+	if r.notifyResults == nil || r.stopped() {
 		return
-	default:
 	}
 	select {
-	case r.notifyResults <- env:
+	case r.notifyResults <- notifyResultEnvelope{task: task, report: report}:
 	case <-r.stopCh:
 	}
 }
@@ -482,6 +601,7 @@ func (r *Runner) websocketLoop() {
 
 		select {
 		case <-time.After(3 * time.Second):
+		case <-r.reconnectCh:
 		case <-r.stopCh:
 			return
 		}
@@ -489,29 +609,75 @@ func (r *Runner) websocketLoop() {
 }
 
 func (r *Runner) runWebsocket() error {
-	conn, err := r.client.WorkerSocket()
+	var writeMu sync.Mutex
+	r.clientMu.Lock()
+	client := r.client
+	r.clientMu.Unlock()
+	if client == nil {
+		return fmt.Errorf("worker: client not initialized")
+	}
+
+	conn, err := client.WorkerSocketContext(r.controlCtx)
 	if err != nil {
 		return err
 	}
-	defer conn.Close() //nolint:errcheck
+
+	// A rotation can complete while the websocket dial is in flight. Do not
+	// install a connection authenticated with the superseded token.
+	r.clientMu.Lock()
+	if r.client != client || r.stopped() {
+		r.clientMu.Unlock()
+		_ = conn.Close()
+		return nil
+	}
+	r.controlConn = conn
+	r.controlWriteMu = &writeMu
+	metricGeneration := r.nextMetricGeneration.Add(1)
+	r.metricGeneration.Store(metricGeneration)
+	r.clientMu.Unlock()
+	defer func() {
+		r.clientMu.Lock()
+		if r.controlConn == conn {
+			r.controlConn = nil
+			r.controlWriteMu = nil
+			r.metricGeneration.CompareAndSwap(metricGeneration, 0)
+		}
+		r.clientMu.Unlock()
+		_ = conn.Close()
+	}()
 	log.Println("worker: websocket connected")
 
-	var writeMu sync.Mutex
 	done := make(chan struct{})
+	var doneOnce sync.Once
+	closeDone := func() { doneOnce.Do(func() { close(done) }) }
+	var connectionWG sync.WaitGroup
+	defer func() {
+		closeDone()
+		_ = conn.Close()
+		connectionWG.Wait()
+	}()
 
 	// Heartbeat goroutine — shares writeMu with the writer.
-	go r.heartbeat(conn, &writeMu, done)
+	connectionWG.Add(1)
+	go func() {
+		defer connectionWG.Done()
+		r.heartbeat(conn, &writeMu, done)
+	}()
 
 	// Single writer goroutine for this connection: serializes result
 	// and heartbeat writes through writeMu so websocket.WriteJSON is
 	// never called concurrently. The dispatcher loop feeds it via the
 	// bounded results channel.
-	go r.writer(conn, &writeMu, done)
+	connectionWG.Add(1)
+	go func() {
+		defer connectionWG.Done()
+		r.writer(conn, &writeMu, done, metricGeneration)
+	}()
 
 	for {
 		var msg wire.WorkerMessage
 		if err := conn.ReadJSON(&msg); err != nil {
-			close(done)
+			closeDone()
 			return err
 		}
 		if (msg.Kind == "init" || msg.Kind == "config") && msg.Init != nil {
@@ -522,32 +688,113 @@ func (r *Runner) runWebsocket() error {
 			continue
 		}
 		if !r.enqueueTaskMessage(msg) {
-			close(done)
+			closeDone()
 			return nil
 		}
 	}
 }
 
-// enqueueTaskMessage prefers the v2 envelope over sibling legacy fields. Some
-// rollout frames contain both check representations; executing the first match
-// only keeps a current runner from running one check twice.
+// enqueueTaskMessage prefers the v2 envelope over sibling legacy fields. An
+// envelope is accepted only when it selects exactly one matching payload with
+// the same non-empty outer and inner job IDs. Invalid envelopes never fall
+// back to a sibling legacy payload, which could otherwise execute a task the
+// control plane did not intend to send.
 func (r *Runner) enqueueTaskMessage(msg wire.WorkerMessage) bool { //nolint:gocritic // wire envelope is the dispatcher boundary
+	if msg.TaskEnvelope != nil {
+		return r.enqueueTaskEnvelope(msg.TaskEnvelope)
+	}
+
 	switch {
-	case msg.TaskEnvelope != nil && msg.TaskEnvelope.Type == wire.TaskTypeNotification && msg.TaskEnvelope.Notify != nil:
-		return r.EnqueueNotification(*msg.TaskEnvelope.Notify)
-	case msg.TaskEnvelope != nil && msg.TaskEnvelope.Type == wire.TaskTypeCheck && msg.TaskEnvelope.Job != nil:
-		return r.Enqueue(*msg.TaskEnvelope.Job)
 	case msg.NotificationTask != nil:
+		if msg.NotificationTask.JobID == "" || msg.NotificationTask.LeaseToken == "" {
+			return true
+		}
 		log.Printf("worker: received websocket notification task %s method=%s", msg.NotificationTask.JobID, msg.NotificationTask.Method)
 		return r.EnqueueNotification(*msg.NotificationTask)
 	case msg.Task != nil:
+		if msg.Task.JobID == "" || msg.Task.LeaseToken == "" {
+			return true
+		}
 		log.Printf("worker: received websocket task %s", msg.Task.JobID)
+		if !checkexec.SupportsKind(msg.Task.Kind) {
+			return r.enqueueUnsupportedCheck(*msg.Task)
+		}
 		return r.Enqueue(*msg.Task)
 	default:
 		return true
 	}
 }
 
+func (r *Runner) enqueueTaskEnvelope(envelope *wire.TaskEnvelope) bool {
+	if (envelope.Job == nil) == (envelope.Notify == nil) {
+		return true
+	}
+	if envelope.Job != nil {
+		if !matchingEnvelopeJobID(envelope.JobID, envelope.Job.JobID) || envelope.Job.LeaseToken == "" {
+			return true
+		}
+		if envelope.Type != wire.TaskTypeCheck {
+			return r.enqueueFailedCheck(*envelope.Job, malformedTaskEnvelopeError)
+		}
+		if !checkexec.SupportsKind(envelope.Job.Kind) {
+			return r.enqueueUnsupportedCheck(*envelope.Job)
+		}
+		return r.Enqueue(*envelope.Job)
+	}
+	if !matchingEnvelopeJobID(envelope.JobID, envelope.Notify.JobID) || envelope.Notify.LeaseToken == "" {
+		return true
+	}
+	if envelope.Type != wire.TaskTypeNotification {
+		return r.enqueueFailedNotification(*envelope.Notify, malformedTaskEnvelopeError)
+	}
+	return r.EnqueueNotification(*envelope.Notify)
+}
+
+func matchingEnvelopeJobID(outer, inner string) bool {
+	return outer != "" && outer == inner
+}
+
+func (r *Runner) enqueueUnsupportedCheck(job wire.CheckJob) bool {
+	return r.enqueueFailedCheck(job, "unsupported_kind: "+job.Kind)
+}
+
+func (r *Runner) enqueueFailedCheck(job wire.CheckJob, errorCode string) bool {
+	if r.results == nil || r.stopped() {
+		return false
+	}
+	report := wire.CheckResultReport{
+		JobID:      job.JobID,
+		CheckID:    job.CheckID,
+		MonitorID:  job.MonitorID,
+		State:      "FAIL",
+		Error:      stringPtr(errorCode),
+		DurationMs: 0,
+	}
+	select {
+	case r.results <- resultEnvelope{job: job, reports: []wire.CheckResultReport{report}}:
+		return true
+	case <-r.stopCh:
+		return false
+	}
+}
+
+func (r *Runner) enqueueFailedNotification(task wire.NotificationTask, errorCode string) bool {
+	if r.stopped() {
+		return false
+	}
+	r.completeNotification(task, notificationPermanentReport(task, errorCode, time.Now()))
+	return true
+}
+
+func (r *Runner) stopped() bool {
+	select {
+	case <-r.stopCh:
+		return true
+	default:
+		return false
+	}
+}
+
 func (r *Runner) heartbeat(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan struct{}) {
 	ticker := time.NewTicker(heartbeatInterval)
 	defer ticker.Stop()
@@ -601,23 +848,39 @@ func (r *Runner) LastHeartbeatAck() time.Time {
 	return r.lastHeartbeatAt
 }
 
-func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan struct{}) {
+func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan struct{}, metricGeneration uint64) {
 	for {
+		if message, ok := r.takeOutbox(); ok {
+			if err := r.writeControlMessage(conn, writeMu, message); err != nil {
+				r.requeueOutbox(message)
+				_ = conn.Close()
+				return
+			}
+			continue
+		}
+
 		select {
 		case env, ok := <-r.results:
 			if !ok {
 				return
 			}
-			writeMu.Lock()
 			for i := range env.reports {
 				report := &env.reports[i]
 				report.LeaseToken = env.job.LeaseToken
-				if err := conn.WriteJSON(wire.WorkerMessage{Kind: "result", Result: report}); err != nil {
+				message := wire.WorkerMessage{Kind: "result", Result: report}
+				if err := r.writeControlMessage(conn, writeMu, message); err != nil {
+					remaining := []wire.WorkerMessage{message}
+					for j := i + 1; j < len(env.reports); j++ {
+						next := env.reports[j]
+						next.LeaseToken = env.job.LeaseToken
+						remaining = append(remaining, wire.WorkerMessage{Kind: "result", Result: &next})
+					}
+					r.requeueOutbox(remaining...)
 					log.Printf(
 						"worker: failed to report result job=%s check=%d kind=%s state=%s: %v",
 						report.JobID, report.CheckID, env.job.Kind, report.State, err,
 					)
-					writeMu.Unlock()
+					_ = conn.Close()
 					return
 				}
 				log.Printf(
@@ -626,32 +889,37 @@ func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan s
 				)
 				r.resultsBuf.add(resultRowFromReport(&env, report, time.Now().UTC()))
 			}
-			writeMu.Unlock()
 		case env, ok := <-r.notifyResults:
 			if !ok {
 				return
 			}
-			writeMu.Lock()
-			if err := conn.WriteJSON(wire.WorkerMessage{Kind: "result", NotificationResult: &env.report}); err != nil {
+			message := wire.WorkerMessage{Kind: "result", NotificationResult: &env.report}
+			if err := r.writeControlMessage(conn, writeMu, message); err != nil {
+				r.requeueOutbox(message)
 				log.Printf(
 					"worker: failed to report notification result job=%s method=%s status=%s: %v",
 					env.report.JobID, env.task.Method, env.report.Status, err,
 				)
-				writeMu.Unlock()
+				_ = conn.Close()
 				return
 			}
 			log.Printf(
 				"worker: completed notification job=%s method=%s status=%s message=%d duration_ms=%d",
 				env.report.JobID, env.task.Method, env.report.Status, env.task.MessageID, env.report.DurationMs,
 			)
-			writeMu.Unlock()
-		case report := <-r.metricResults:
-			writeMu.Lock()
-			if err := conn.WriteJSON(wire.WorkerMessage{Kind: "result", ServerMetric: &report}); err != nil {
-				writeMu.Unlock()
+		case metric := <-r.metricResults:
+			if metric.generation != metricGeneration {
+				continue
+			}
+			message := wire.WorkerMessage{Kind: "result", ServerMetric: &metric.report}
+			if err := r.writeControlMessage(conn, writeMu, message); err != nil {
+				// Metrics are periodic snapshots without a lease or idempotency key.
+				// Dropping a failed snapshot avoids duplicate control-plane inserts;
+				// the next collection tick supplies a fresh replacement.
+				_ = conn.Close()
 				return
 			}
-			writeMu.Unlock()
+		case <-r.outboxWake:
 		case <-done:
 			return
 		case <-r.stopCh:
@@ -660,6 +928,39 @@ func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan s
 	}
 }
 
+func (r *Runner) takeOutbox() (wire.WorkerMessage, bool) {
+	r.outboxMu.Lock()
+	defer r.outboxMu.Unlock()
+	if len(r.outbox) == 0 {
+		return wire.WorkerMessage{}, false
+	}
+	message := r.outbox[0]
+	r.outbox = r.outbox[1:]
+	return message, true
+}
+
+func (r *Runner) requeueOutbox(messages ...wire.WorkerMessage) {
+	if len(messages) == 0 {
+		return
+	}
+	r.outboxMu.Lock()
+	r.outbox = append(messages, r.outbox...)
+	r.outboxMu.Unlock()
+	select {
+	case r.outboxWake <- struct{}{}:
+	default:
+	}
+}
+
+func (r *Runner) writeControlMessage(conn *websocket.Conn, writeMu *sync.Mutex, message wire.WorkerMessage) error {
+	writeMu.Lock()
+	defer writeMu.Unlock()
+	if r.beforeControlWrite != nil {
+		r.beforeControlWrite()
+	}
+	return conn.WriteJSON(message)
+}
+
 func (r *Runner) applyInit(init *wire.WorkerInit) {
 	if init.Concurrency > 0 && init.Concurrency != r.Concurrency() {
 		size := init.Concurrency
@@ -881,9 +1182,6 @@ func (r *Runner) RecentResults(n int) []ResultRow {
 }
 
 // RecentNotifications returns the most recent n notification rows.
-// Phase 1 only fills this buffer from selfcheck alerts via
-// RecordNotification; the main-app-issued notifications still live
-// in the main app's database.
 func (r *Runner) RecentNotifications(n int) []NotificationRow {
 	if r == nil || r.notificationsBuf == nil {
 		return nil
@@ -892,8 +1190,7 @@ func (r *Runner) RecentNotifications(n int) []NotificationRow {
 }
 
 // RecordNotification appends one row to the notification ring buffer.
-// Called from selfcheck.sendSystemAlert so the webapp /notifications
-// page can show what the worker emitted. Safe before Start.
+// Called from selfcheck.sendSystemAlert. Safe before Start.
 func (r *Runner) RecordNotification(n *NotificationRow) {
 	if r == nil || r.notificationsBuf == nil {
 		return
@@ -904,12 +1201,24 @@ func (r *Runner) RecordNotification(n *NotificationRow) {
 	r.notificationsBuf.add(n)
 }
 
+func (r *Runner) recordDelegatedNotification(jobID, method string, report wire.NotificationResultReport) {
+	r.RecordNotification(&NotificationRow{
+		JobID:      jobID,
+		Method:     method,
+		Status:     report.Status,
+		DurationMs: report.DurationMs,
+		At:         time.Now().UTC(),
+	})
+}
+
 // Token returns the current bearer token. The webapp settings page
 // masks this for display.
 func (r *Runner) Token() string {
 	if r == nil || r.config == nil {
 		return ""
 	}
+	r.clientMu.Lock()
+	defer r.clientMu.Unlock()
 	return r.config.Token
 }
 
@@ -983,48 +1292,95 @@ func (r *Runner) HTTPConfig() HTTPConfig {
 //
 // Returns the new token string. On any failure the old token and
 // client are kept untouched.
-func (r *Runner) RotateToken(_ context.Context) (string, error) {
+func (r *Runner) RotateToken(ctx context.Context) (string, error) {
 	if r == nil || r.config == nil {
 		return "", fmt.Errorf("worker: runner not initialized")
 	}
-	if r.client == nil {
+
+	// The endpoint authenticates with the current token, so concurrent
+	// rotations must not mint replacements from the same stale client.
+	r.rotationMu.Lock()
+	defer r.rotationMu.Unlock()
+	if r.stopped() {
+		return "", fmt.Errorf("worker: runner stopped")
+	}
+	requestCtx, cancelRequest := context.WithCancel(ctx)
+	rotationDone := make(chan struct{})
+	defer func() {
+		close(rotationDone)
+		cancelRequest()
+	}()
+	go func() {
+		select {
+		case <-r.rotationCtx.Done():
+			cancelRequest()
+		case <-rotationDone:
+		}
+	}()
+
+	r.clientMu.Lock()
+	client := r.client
+	r.clientMu.Unlock()
+	if client == nil {
 		return "", fmt.Errorf("worker: client not yet started")
 	}
-	newToken, err := r.client.RotateToken()
+	newToken, err := client.RotateToken(requestCtx)
 	if err != nil {
 		return "", err
 	}
+	if r.beforeTokenCommit != nil {
+		r.beforeTokenCommit()
+	}
+
+	// Stop and the post-HTTP commit share lifecycleMu. Once Stop has closed
+	// stopCh, this rotation cannot install or report a replacement token.
+	r.lifecycleMu.Lock()
+	r.clientMu.Lock()
+	if r.stopped() {
+		r.clientMu.Unlock()
+		r.lifecycleMu.Unlock()
+		return "", fmt.Errorf("worker: runner stopped")
+	}
 	if newToken == "" || newToken == r.config.Token {
+		r.clientMu.Unlock()
+		r.lifecycleMu.Unlock()
 		return "", fmt.Errorf("worker: rotate-token returned unchanged or empty token")
 	}
 
-	// Swap config + client under lock so a concurrent heartbeat
-	// cannot race with the rotation.
-	r.clientMu.Lock()
+	// Swap client and detach only the active control-plane connection. stopCh
+	// remains exclusively owned by Stop, so dispatch, metrics, selfcheck, peer
+	// polling, and services started alongside the runner keep running.
 	r.config.Token = newToken
-	oldClient := r.client
 	r.client = NewClient(r.config.URL, newToken)
+	conn := r.controlConn
+	writeMu := r.controlWriteMu
 	r.clientMu.Unlock()
+	r.lifecycleMu.Unlock()
 
 	// Stamp the rotation time so the settings page can show it.
 	r.tokenRotatedMu.Lock()
 	r.tokenRotatedAt = time.Now().UTC()
 	r.tokenRotatedMu.Unlock()
 
-	// Force the websocket loop to reconnect with the new token. The
-	// old connection's next heartbeat will fail with 401; closing
-	// the connection now shortens that window.
-	r.closeOnce.Do(func() {
-		// Close the underlying websocket by triggering the runner's
-		// normal stop path; the websocketLoop goroutine will reconnect
-		// after we re-arm stopCh. This is the cleanest way to drive
-		// the loop without exposing internals.
-		select {
-		case <-r.stopCh:
-		default:
-			close(r.stopCh)
-		}
-	})
-	_ = oldClient // client has no Close; the websocket layer owns it.
+	// Interrupt the current connection before waiting for its writer. A stalled
+	// WriteJSON holds writeMu; closing the socket makes that write fail so the
+	// writer can requeue its dequeued envelope and release the mutex.
+	if conn != nil {
+		_ = conn.Close()
+	}
+	if writeMu != nil {
+		writeMu.Lock()
+		writeMu.Unlock()
+	}
+	r.lifecycleMu.Lock()
+	stopped := r.stopped()
+	r.lifecycleMu.Unlock()
+	if stopped {
+		return "", fmt.Errorf("worker: runner stopped")
+	}
+	select {
+	case r.reconnectCh <- struct{}{}:
+	default:
+	}
 	return newToken, nil
 }