fix(worker): harden control-plane lifecycle

- reconnect safely after token rotation and retry leased results
- reject malformed tasks and remove production cluster debug mutation
- validate environment files and require immutable container images

BREAKING CHANGE: Docker install, deploy, and Compose now require an
immutable repository@sha256 image reference.

internal/distworker/notification_test.go

@@ -2,7 +2,10 @@ package distworker
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -228,6 +231,177 @@ func TestExecuteNotification_ReportsJobIDFromPayload(t *testing.T) {
 	assert.Equal(t, "outer-job", report.JobID)
 }
 
+func TestDelegatedNotificationAuditCoexistsWithSelfcheckWithoutSecrets(t *testing.T) {
+	const (
+		recipient        = "ops-private@example.com"
+		renderedBody     = "private rendered body"
+		credentialSecret = "smtp-password-secret"
+		hookURL          = "https://hooks.example.com/private"
+		authorization    = "Bearer private-authorization"
+		providerResponse = "private provider response"
+	)
+	r := runnerWithCreds(&wire.NotificationCredentials{
+		SMTP: []wire.SMTPCredential{{Password: credentialSecret}},
+	})
+	r.notifyResults = make(chan notifyResultEnvelope, 1)
+	r.RecordNotification(&NotificationRow{
+		Kind: "email", Channel: notificationChannelSMTP, Subject: "selfcheck", Body: "main API down", OK: true,
+	})
+	r.executeAndForwardNotification(wire.NotificationTask{
+		JobID: "delegated-job", Method: "sms", Subject: authorization,
+		BodyText: renderedBody, BodyHTML: providerResponse,
+		Contact: wire.NotificationContact{Kind: "sms", Value: recipient, Name: hookURL},
+	})
+
+	rows := r.RecentNotifications(10)
+	require.Len(t, rows, 2)
+	assert.Equal(t, "selfcheck", rows[0].Subject)
+	delegated := rows[1]
+	assert.Equal(t, "delegated-job", delegated.JobID)
+	assert.Equal(t, "sms", delegated.Method)
+	assert.Equal(t, wire.NotificationResultPermanent, delegated.Status)
+	assert.Empty(t, delegated.Subject)
+	assert.Empty(t, delegated.Body)
+	assert.Empty(t, delegated.Error)
+	assert.False(t, delegated.At.IsZero())
+
+	stored, err := json.Marshal(delegated)
+	require.NoError(t, err)
+	for _, secret := range []string{recipient, renderedBody, credentialSecret, hookURL, authorization, providerResponse} {
+		assert.NotContains(t, string(stored), secret)
+	}
+}
+
+func TestDelegatedNotificationAuditRecordsOneTerminalRow(t *testing.T) {
+	const secret = "private-notification-secret"
+	validTask := func(jobID string) wire.NotificationTask {
+		return wire.NotificationTask{
+			JobID: jobID, LeaseToken: "lease", Method: "sms", Subject: secret, BodyText: secret,
+			Contact: wire.NotificationContact{Value: secret, Name: secret},
+		}
+	}
+	tests := []struct {
+		name   string
+		invoke func(*Runner)
+		method string
+	}{
+		{
+			name:   "normal result",
+			invoke: func(r *Runner) { r.executeAndForwardNotification(validTask("normal")) },
+			method: "sms",
+		},
+		{
+			name: "invalid deadline",
+			invoke: func(r *Runner) {
+				task := validTask("invalid-deadline")
+				deadline := "not-a-timestamp"
+				task.Deadline = &deadline
+				r.executeAndForwardNotification(task)
+			},
+			method: "sms",
+		},
+		{
+			name: "expired deadline",
+			invoke: func(r *Runner) {
+				task := validTask("expired-deadline")
+				deadline := time.Now().Add(-time.Second).Format(time.RFC3339Nano)
+				task.Deadline = &deadline
+				r.executeAndForwardNotification(task)
+			},
+			method: "sms",
+		},
+		{
+			name: "malformed payload",
+			invoke: func(r *Runner) {
+				r.ExecuteNotification(context.Background(), models.Task{JobID: "malformed-payload", Payload: []byte(`{"subject":"` + secret)})
+			},
+		},
+		{
+			name: "executor panic",
+			invoke: func(r *Runner) {
+				r.notificationExecutor = func(context.Context, models.Task) wire.NotificationResultReport {
+					panic(secret)
+				}
+				r.ExecuteNotification(context.Background(), models.Task{JobID: "panic", Payload: []byte(`{}`)})
+			},
+		},
+		{
+			name: "malformed envelope",
+			invoke: func(r *Runner) {
+				r.enqueueTaskMessage(wire.WorkerMessage{TaskEnvelope: &wire.TaskEnvelope{
+					Type: "invalid", JobID: "bad-envelope", Notify: ptrNotificationTask(validTask("bad-envelope")),
+				}})
+			},
+			method: "sms",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			r := runnerWithCreds(&wire.NotificationCredentials{})
+			r.notifyResults = make(chan notifyResultEnvelope, 1)
+			tc.invoke(r)
+
+			rows := r.RecentNotifications(10)
+			require.Len(t, rows, 1)
+			assert.Equal(t, wire.NotificationResultPermanent, rows[0].Status)
+			assert.Equal(t, tc.method, rows[0].Method)
+			stored, err := json.Marshal(rows[0])
+			require.NoError(t, err)
+			assert.NotContains(t, string(stored), secret)
+		})
+	}
+}
+
+func TestDelegatedNotificationAuditConcurrentRowsAreSecretFree(t *testing.T) {
+	const attempts = 32
+	const secret = "concurrent-private-secret"
+	r := runnerWithCreds(&wire.NotificationCredentials{})
+	var wg sync.WaitGroup
+	for i := 0; i < attempts; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			r.ExecuteNotification(context.Background(), models.Task{
+				JobID:   fmt.Sprintf("job-%d", i),
+				Payload: []byte(`{"job_id":"job-` + fmt.Sprint(i) + `","method":"sms","body_text":"` + secret + `","contact":{"value":"` + secret + `"}}`),
+			})
+		}(i)
+	}
+	wg.Wait()
+
+	rows := r.RecentNotifications(attempts)
+	require.Len(t, rows, attempts)
+	for _, row := range rows {
+		stored, err := json.Marshal(row)
+		require.NoError(t, err)
+		assert.NotContains(t, string(stored), secret)
+		assert.Equal(t, wire.NotificationResultPermanent, row.Status)
+	}
+}
+
+func TestInvalidNotificationDeadlineIsNotExecuted(t *testing.T) {
+	r := runnerWithCreds(&wire.NotificationCredentials{})
+	r.notifyResults = make(chan notifyResultEnvelope, 1)
+	called := false
+	r.notificationExecutor = func(context.Context, models.Task) wire.NotificationResultReport {
+		called = true
+		return wire.NotificationResultReport{Status: wire.NotificationResultDelivered}
+	}
+	deadline := "not-a-timestamp"
+	r.executeAndForwardNotification(wire.NotificationTask{
+		JobID: "invalid-deadline", LeaseToken: "lease", Method: "sms", Deadline: &deadline,
+	})
+
+	assert.False(t, called)
+	env := <-r.notifyResults
+	assert.Equal(t, wire.NotificationResultPermanent, env.report.Status)
+	require.NotNil(t, env.report.Error)
+	assert.Equal(t, "invalid notification deadline", *env.report.Error)
+	require.Len(t, r.RecentNotifications(10), 1)
+}
+
+func ptrNotificationTask(task wire.NotificationTask) *wire.NotificationTask { return &task }
+
 // guard against time import being unused if the above compile-time helpers
 // are dropped in a future refactor.
 var _ = time.Second