fix(worker): harden control-plane lifecycle

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.

internal/distworker/runner_test.go

@@ -1,6 +1,9 @@
 package distworker
 
 import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -8,6 +11,7 @@ import (
 	"time"
 
 	"github.com/Jeffail/tunny"
+	"github.com/gorilla/websocket"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -174,6 +178,835 @@ func TestEnqueueRespectsBackpressure(t *testing.T) {
 	}
 }
 
+func TestStopRejectsTerminalResultsWithoutClosingResultChannel(t *testing.T) {
+	r := NewRunner(&Config{MaxConcurrency: 1})
+	r.results = make(chan resultEnvelope, 1)
+	r.Stop()
+
+	assert.False(t, r.enqueueFailedCheck(wire.CheckJob{JobID: "job-1", LeaseToken: "lease-1"}, "unsupported_kind: rkn"))
+	assert.Empty(t, r.results)
+
+	select {
+	case r.results <- resultEnvelope{}:
+	default:
+		t.Fatal("results channel should remain open after Stop")
+	}
+}
+
+func TestRotateTokenReconnectsWithoutStoppingRunner(t *testing.T) {
+	const (
+		oldToken = "old-token"
+		newToken = "new-token"
+	)
+
+	connections := make(chan string, 2)
+	results := make(chan wire.WorkerMessage, 1)
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case "/api/internal/workers/rotate-token":
+			if req.Header.Get("Authorization") != "Bearer "+oldToken {
+				http.Error(w, "unexpected rotation token", http.StatusUnauthorized)
+				return
+			}
+			_ = json.NewEncoder(w).Encode(struct {
+				AuthToken string `json:"auth_token"`
+			}{AuthToken: newToken})
+		case "/worker":
+			conn, err := upgrader.Upgrade(w, req, nil)
+			if err != nil {
+				return
+			}
+			defer conn.Close()
+			token := req.URL.Query().Get("token")
+			connections <- token
+			if token == oldToken {
+				_, _, _ = conn.ReadMessage() // Rotation must close this connection.
+				return
+			}
+			if token != newToken {
+				return
+			}
+			if conn.WriteJSON(wire.WorkerMessage{Kind: "task", Task: &wire.CheckJob{
+				JobID: "after-rotation", LeaseToken: "lease", CheckID: 1, Kind: "http",
+			}}) != nil {
+				return
+			}
+			for {
+				var message wire.WorkerMessage
+				if err := conn.ReadJSON(&message); err != nil {
+					return
+				}
+				if message.Kind == "result" && message.Result != nil && message.Result.JobID == "after-rotation" {
+					results <- message
+					return
+				}
+			}
+		default:
+			http.NotFound(w, req)
+		}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: oldToken, MaxConcurrency: 1})
+	r.executor = func(payload interface{}) interface{} {
+		job := payload.(wire.CheckJob)
+		return []wire.CheckResultReport{{JobID: job.JobID, CheckID: job.CheckID, State: "OK"}}
+	}
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() {
+		r.Stop()
+		select {
+		case err := <-startDone:
+			require.NoError(t, err)
+		case <-time.After(time.Second):
+			t.Fatal("runner did not stop")
+		}
+	})
+
+	select {
+	case token := <-connections:
+		require.Equal(t, oldToken, token)
+	case <-time.After(time.Second):
+		t.Fatal("worker did not establish its initial control connection")
+	}
+
+	gotToken, err := r.RotateToken(t.Context())
+	require.NoError(t, err)
+	require.Equal(t, newToken, gotToken)
+	require.Equal(t, newToken, r.Token())
+
+	select {
+	case token := <-connections:
+		require.Equal(t, newToken, token)
+	case <-time.After(time.Second):
+		t.Fatal("worker did not reconnect with the replacement token")
+	}
+	select {
+	case result := <-results:
+		require.NotNil(t, result.Result)
+		assert.Equal(t, "OK", result.Result.State)
+	case <-time.After(time.Second):
+		t.Fatal("runner did not execute work after token rotation")
+	}
+	assert.False(t, r.stopped(), "rotation must not stop runner-owned subsystems")
+}
+
+func TestStopClosesAndJoinsIdleControlConnection(t *testing.T) {
+	closed := make(chan struct{}, 1)
+	connected := make(chan struct{}, 1)
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if req.URL.Path != "/worker" {
+			http.NotFound(w, req)
+			return
+		}
+		conn, err := upgrader.Upgrade(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		connected <- struct{}{}
+		_, _, _ = conn.ReadMessage()
+		closed <- struct{}{}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() { r.Stop() })
+
+	select {
+	case <-time.After(time.Second):
+		t.Fatal("worker did not establish idle control connection")
+	case <-connected:
+	}
+	r.Stop()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not close the idle control connection")
+	}
+	select {
+	case err := <-startDone:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not join the control loop")
+	}
+}
+
+func TestStopCancelsDialInProgress(t *testing.T) {
+	dialStarted := make(chan struct{})
+	allowUpgrade := make(chan struct{})
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if req.URL.Path != "/worker" {
+			http.NotFound(w, req)
+			return
+		}
+		close(dialStarted)
+		<-allowUpgrade
+		_, _ = upgrader.Upgrade(w, req, nil)
+	}))
+	defer func() {
+		close(allowUpgrade)
+		server.Close()
+	}()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	select {
+	case <-dialStarted:
+	case <-time.After(time.Second):
+		t.Fatal("worker did not begin websocket dial")
+	}
+	r.Stop()
+	select {
+	case err := <-startDone:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not join a canceled websocket dial")
+	}
+}
+
+func TestStartAndStopRegisterControlLoopSafely(t *testing.T) {
+	server := httptest.NewServer(http.NotFoundHandler())
+	defer server.Close()
+	for i := 0; i < 25; i++ {
+		r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
+		startDone := make(chan error, 1)
+		stopDone := make(chan struct{})
+		go func() { startDone <- r.Start() }()
+		go func() {
+			r.Stop()
+			close(stopDone)
+		}()
+		select {
+		case <-stopDone:
+		case <-time.After(time.Second):
+			t.Fatal("Stop did not complete")
+		}
+		select {
+		case <-startDone:
+		case <-time.After(time.Second):
+			t.Fatal("Start did not return after concurrent Stop")
+		}
+	}
+}
+
+func TestRotateTokenPreservesDequeuedResult(t *testing.T) {
+	const (
+		oldToken = "old-token"
+		newToken = "new-token"
+	)
+	connected := make(chan string, 2)
+	delivered := make(chan wire.WorkerMessage, 1)
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case "/api/internal/workers/rotate-token":
+			_ = json.NewEncoder(w).Encode(struct {
+				AuthToken string `json:"auth_token"`
+			}{AuthToken: newToken})
+		case "/worker":
+			conn, err := upgrader.Upgrade(w, req, nil)
+			if err != nil {
+				return
+			}
+			defer conn.Close()
+			token := req.URL.Query().Get("token")
+			connected <- token
+			if token == oldToken {
+				var message wire.WorkerMessage
+				if conn.ReadJSON(&message) == nil {
+					delivered <- message
+				}
+				return
+			}
+			if token != newToken {
+				return
+			}
+			var message wire.WorkerMessage
+			if conn.ReadJSON(&message) == nil {
+				delivered <- message
+			}
+		}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: oldToken, MaxConcurrency: 1})
+	enteredWrite := make(chan struct{})
+	releaseWrite := make(chan struct{})
+	var once sync.Once
+	r.beforeControlWrite = func() {
+		once.Do(func() {
+			close(enteredWrite)
+			<-releaseWrite
+		})
+	}
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() {
+		r.Stop()
+		select {
+		case <-startDone:
+		case <-time.After(time.Second):
+			t.Fatal("runner did not stop")
+		}
+	})
+
+	select {
+	case token := <-connected:
+		require.Equal(t, oldToken, token)
+	case <-time.After(time.Second):
+		t.Fatal("worker did not establish its initial control connection")
+	}
+	r.results <- resultEnvelope{job: wire.CheckJob{JobID: "result", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "result", State: "OK"}}}
+	select {
+	case <-enteredWrite:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not dequeue result")
+	}
+	rotated := make(chan error, 1)
+	go func() {
+		_, err := r.RotateToken(t.Context())
+		rotated <- err
+	}()
+	close(releaseWrite)
+	require.NoError(t, <-rotated)
+
+	select {
+	case message := <-delivered:
+		require.NotNil(t, message.Result)
+		assert.Equal(t, "result", message.Result.JobID)
+		assert.Equal(t, "lease", message.Result.LeaseToken)
+	case <-time.After(time.Second):
+		t.Fatal("dequeued result was lost during rotation")
+	}
+}
+
+func TestRotateTokenSerializesWithStop(t *testing.T) {
+	rotationStarted := make(chan struct{})
+	releaseHandler := make(chan struct{})
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case "/api/internal/workers/rotate-token":
+			close(rotationStarted)
+			<-releaseHandler
+		case "/worker":
+			conn, err := upgrader.Upgrade(w, req, nil)
+			if err == nil {
+				defer conn.Close()
+				_, _, _ = conn.ReadMessage()
+			}
+		}
+	}))
+	defer func() {
+		close(releaseHandler)
+		server.Close()
+	}()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() { r.Stop() })
+
+	// Wait for Start to install its client before beginning rotation.
+	deadline := time.After(time.Second)
+	for {
+		r.clientMu.Lock()
+		started := r.client != nil
+		r.clientMu.Unlock()
+		if started {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("runner did not start")
+		default:
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	rotated := make(chan error, 1)
+	go func() {
+		_, err := r.RotateToken(t.Context())
+		rotated <- err
+	}()
+	select {
+	case <-rotationStarted:
+	case <-time.After(time.Second):
+		t.Fatal("rotation request did not start")
+	}
+	stopped := make(chan struct{})
+	go func() {
+		r.Stop()
+		close(stopped)
+	}()
+	require.Error(t, <-rotated, "Stop must cancel an in-flight rotation request")
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not complete after canceling rotation")
+	}
+	select {
+	case err := <-startDone:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("runner did not stop")
+	}
+	_, err := r.RotateToken(t.Context())
+	require.Error(t, err, "rotation cannot succeed after shutdown")
+}
+
+func TestStopUnblocksRotationWaitingForWriter(t *testing.T) {
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	connected := make(chan struct{}, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case "/api/internal/workers/rotate-token":
+			_ = json.NewEncoder(w).Encode(struct {
+				AuthToken string `json:"auth_token"`
+			}{AuthToken: "new-token"})
+		case "/worker":
+			conn, err := upgrader.Upgrade(w, req, nil)
+			if err != nil {
+				return
+			}
+			defer conn.Close()
+			connected <- struct{}{}
+			_, _, _ = conn.ReadMessage()
+		}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
+	writeBlocked := make(chan struct{})
+	var once sync.Once
+	r.beforeControlWrite = func() {
+		once.Do(func() {
+			close(writeBlocked)
+			<-r.controlCtx.Done()
+		})
+	}
+	rotationReady := make(chan struct{})
+	r.beforeTokenCommit = func() { close(rotationReady) }
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() { r.Stop() })
+	select {
+	case <-connected:
+	case <-time.After(time.Second):
+		t.Fatal("worker did not connect")
+	}
+	r.results <- resultEnvelope{job: wire.CheckJob{JobID: "blocked", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "blocked", State: "OK"}}}
+	select {
+	case <-writeBlocked:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not block")
+	}
+	rotated := make(chan error, 1)
+	go func() {
+		_, err := r.RotateToken(t.Context())
+		rotated <- err
+	}()
+	select {
+	case <-rotationReady:
+	case <-time.After(time.Second):
+		t.Fatal("rotation did not reach writer serialization")
+	}
+	stopped := make(chan struct{})
+	go func() {
+		r.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop deadlocked behind rotation waiting for writer")
+	}
+	select {
+	case <-rotated:
+	case <-time.After(time.Second):
+		t.Fatal("rotation did not unblock after Stop closed the connection")
+	}
+	select {
+	case err := <-startDone:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("runner did not stop")
+	}
+}
+
+func TestStopWinsBeforePostHTTPRotationCommit(t *testing.T) {
+	commitReady := make(chan struct{})
+	releaseCommit := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if req.URL.Path != "/api/internal/workers/rotate-token" {
+			http.NotFound(w, req)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(struct {
+			AuthToken string `json:"auth_token"`
+		}{AuthToken: "new-token"})
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
+	r.client = NewClient(server.URL, "old-token")
+	r.beforeTokenCommit = func() {
+		close(commitReady)
+		<-releaseCommit
+	}
+	rotated := make(chan error, 1)
+	go func() {
+		_, err := r.RotateToken(t.Context())
+		rotated <- err
+	}()
+	select {
+	case <-commitReady:
+	case <-time.After(time.Second):
+		t.Fatal("rotation did not reach post-HTTP commit")
+	}
+	stopped := make(chan struct{})
+	go func() {
+		r.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-r.stopCh:
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not win lifecycle ownership")
+	}
+	close(releaseCommit)
+	require.Error(t, <-rotated, "rotation cannot succeed after Stop wins")
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop did not finish")
+	}
+	assert.Equal(t, "old-token", r.Token())
+}
+
+func TestRotateTokenUnchangedResponseReleasesLifecycle(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		_ = json.NewEncoder(w).Encode(struct {
+			AuthToken string `json:"auth_token"`
+		}{AuthToken: "old-token"})
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
+	r.client = NewClient(server.URL, "old-token")
+	_, err := r.RotateToken(t.Context())
+	require.Error(t, err)
+
+	stopped := make(chan struct{})
+	go func() {
+		r.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop deadlocked after unchanged rotation response")
+	}
+	_, err = r.RotateToken(t.Context())
+	require.Error(t, err, "later rotation must observe shutdown")
+}
+
+func TestRotateTokenClosesStalledWriterBeforeWaiting(t *testing.T) {
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	connected := make(chan struct{}, 1)
+	connectionClosed := make(chan struct{})
+	resent := make(chan wire.WorkerMessage, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case "/api/internal/workers/rotate-token":
+			_ = json.NewEncoder(w).Encode(struct {
+				AuthToken string `json:"auth_token"`
+			}{AuthToken: "new-token"})
+		case "/worker":
+			conn, err := upgrader.Upgrade(w, req, nil)
+			if err != nil {
+				return
+			}
+			defer conn.Close()
+			if req.URL.Query().Get("token") == "old-token" {
+				connected <- struct{}{}
+				_, _, _ = conn.ReadMessage()
+				connectionClosed <- struct{}{}
+				return
+			}
+			var message wire.WorkerMessage
+			if conn.ReadJSON(&message) == nil {
+				resent <- message
+			}
+		}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
+	writerBlocked := make(chan struct{})
+	var once sync.Once
+	r.beforeControlWrite = func() {
+		once.Do(func() {
+			close(writerBlocked)
+			<-connectionClosed
+		})
+	}
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() { r.Stop() })
+	select {
+	case <-connected:
+	case <-time.After(time.Second):
+		t.Fatal("worker did not connect")
+	}
+	r.results <- resultEnvelope{job: wire.CheckJob{JobID: "stalled", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "stalled", State: "OK"}}}
+	select {
+	case <-writerBlocked:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not stall")
+	}
+	rotated := make(chan error, 1)
+	go func() {
+		_, err := r.RotateToken(t.Context())
+		rotated <- err
+	}()
+	select {
+	case err := <-rotated:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("rotation waited for stalled writer before closing its connection")
+	}
+	select {
+	case message := <-resent:
+		require.NotNil(t, message.Result)
+		assert.Equal(t, "stalled", message.Result.JobID)
+	case <-time.After(time.Second):
+		t.Fatal("failed check result was not requeued after rotation")
+	}
+	r.Stop()
+	select {
+	case err := <-startDone:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("runner did not stop")
+	}
+}
+
+func TestWriterDropsFailedServerMetricSnapshot(t *testing.T) {
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	upgraded := make(chan struct{})
+	closeServer := make(chan struct{})
+	serverClosed := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		conn, err := upgrader.Upgrade(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		close(upgraded)
+		<-closeServer
+		close(serverClosed)
+	}))
+	defer server.Close()
+
+	conn, err := NewClient(server.URL, "token").WorkerSocket()
+	require.NoError(t, err)
+	defer conn.Close()
+	select {
+	case <-upgraded:
+	case <-time.After(time.Second):
+		t.Fatal("websocket did not connect")
+	}
+
+	r := NewRunner(&Config{})
+	r.metricResults = make(chan metricEnvelope, 1)
+	enteredWrite := make(chan struct{})
+	releaseWrite := make(chan struct{})
+	r.beforeControlWrite = func() {
+		close(enteredWrite)
+		<-releaseWrite
+	}
+	done := make(chan struct{})
+	writerDone := make(chan struct{})
+	go func() {
+		var writeMu sync.Mutex
+		r.writer(conn, &writeMu, done, 1)
+		close(writerDone)
+	}()
+	r.metricResults <- metricEnvelope{generation: 1, report: wire.ServerMetricReport{ServerID: 1}}
+	select {
+	case <-enteredWrite:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not dequeue metric snapshot")
+	}
+	close(closeServer)
+	select {
+	case <-serverClosed:
+	case <-time.After(time.Second):
+		t.Fatal("server did not close websocket")
+	}
+	_ = conn.Close()
+	close(releaseWrite)
+	select {
+	case <-writerDone:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not return after failed metric write")
+	}
+	_, replayable := r.takeOutbox()
+	assert.False(t, replayable, "failed metric snapshot must not enter result outbox")
+}
+
+func TestWriterRequeuesFailedNotificationResult(t *testing.T) {
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	upgraded := make(chan struct{})
+	closeServer := make(chan struct{})
+	serverClosed := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		conn, err := upgrader.Upgrade(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		close(upgraded)
+		<-closeServer
+		close(serverClosed)
+	}))
+	defer server.Close()
+
+	conn, err := NewClient(server.URL, "token").WorkerSocket()
+	require.NoError(t, err)
+	defer conn.Close()
+	select {
+	case <-upgraded:
+	case <-time.After(time.Second):
+		t.Fatal("websocket did not connect")
+	}
+
+	r := NewRunner(&Config{})
+	r.notifyResults = make(chan notifyResultEnvelope, 1)
+	enteredWrite := make(chan struct{})
+	releaseWrite := make(chan struct{})
+	r.beforeControlWrite = func() {
+		close(enteredWrite)
+		<-releaseWrite
+	}
+	done := make(chan struct{})
+	writerDone := make(chan struct{})
+	go func() {
+		var writeMu sync.Mutex
+		r.writer(conn, &writeMu, done, 0)
+		close(writerDone)
+	}()
+	r.notifyResults <- notifyResultEnvelope{report: wire.NotificationResultReport{JobID: "notification", LeaseToken: "lease"}}
+	select {
+	case <-enteredWrite:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not dequeue notification result")
+	}
+	close(closeServer)
+	select {
+	case <-serverClosed:
+	case <-time.After(time.Second):
+		t.Fatal("server did not close websocket")
+	}
+	_ = conn.Close()
+	close(releaseWrite)
+	select {
+	case <-writerDone:
+	case <-time.After(time.Second):
+		t.Fatal("writer did not return after failed notification write")
+	}
+	message, replayable := r.takeOutbox()
+	require.True(t, replayable, "failed notification result must enter result outbox")
+	require.NotNil(t, message.NotificationResult)
+	assert.Equal(t, "notification", message.NotificationResult.JobID)
+}
+
+func TestMetricGenerationRejectsDisconnectedAndSendsFreshMetric(t *testing.T) {
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	firstClosed := make(chan struct{})
+	secondConnected := make(chan struct{})
+	received := make(chan wire.WorkerMessage, 1)
+	var connections atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		conn, err := upgrader.Upgrade(w, req, nil)
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		if connections.Add(1) == 1 {
+			close(firstClosed)
+			return
+		}
+		close(secondConnected)
+		var message wire.WorkerMessage
+		if conn.ReadJSON(&message) == nil {
+			received <- message
+		}
+	}))
+	defer server.Close()
+
+	r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
+	startDone := make(chan error, 1)
+	go func() { startDone <- r.Start() }()
+	t.Cleanup(func() { r.Stop() })
+	select {
+	case <-firstClosed:
+	case <-time.After(time.Second):
+		t.Fatal("worker did not establish initial websocket")
+	}
+	// Wait until the old connection has fully torn down, then try to enqueue a
+	// metric in the old-drain/disconnected interleaving.
+	deadline := time.After(time.Second)
+	for r.metricGeneration.Load() != 0 {
+		select {
+		case <-deadline:
+			t.Fatal("old control generation did not clear")
+		default:
+			time.Sleep(time.Millisecond)
+		}
+	}
+	staleCount := 1
+	r.enqueueMetric(wire.ServerMetricReport{ServerID: 1, ProcessCount: &staleCount})
+	assert.Empty(t, r.metricResults, "disconnected metric must not enter bounded channel")
+	select {
+	case r.reconnectCh <- struct{}{}:
+	default:
+	}
+	select {
+	case <-secondConnected:
+	case <-time.After(time.Second):
+		t.Fatal("worker did not reconnect")
+	}
+	deadline = time.After(time.Second)
+	for r.metricGeneration.Load() == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("new control generation did not install")
+		default:
+			time.Sleep(time.Millisecond)
+		}
+	}
+	// This enqueue occurs immediately after the new connection installation.
+	freshCount := 2
+	r.enqueueMetric(wire.ServerMetricReport{ServerID: 1, ProcessCount: &freshCount})
+	select {
+	case message := <-received:
+		require.NotNil(t, message.ServerMetric)
+		assert.Equal(t, 2, *message.ServerMetric.ProcessCount)
+	case <-time.After(time.Second):
+		t.Fatal("fresh metric was not sent after reconnect")
+	}
+}
+
 func TestApplyInitResizesPool(t *testing.T) {
 	executor := func(payload interface{}) interface{} {
 		return []wire.CheckResultReport{}