feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.

app/models/dead_worker_reaper.go

@@ -0,0 +1,120 @@
+package models
+
+import (
+	"context"
+	"log"
+	"time"
+)
+
+// DeadWorkerHeartbeatTimeout is the threshold for ReapDeadWorkers — a worker
+// whose last_seen is older than this is considered dead and any leased tasks
+// it owns are reassigned to the pool. Five minutes mirrors the StaleWorkers()
+// check in check_jobs.go so the two reapers cannot disagree about who is
+// dead. See docs/todo.md Phase 4 §5.
+const DeadWorkerHeartbeatTimeout = 5 * time.Minute
+
+// ReapDeadWorkers marks any non-dead worker whose last_seen is older than
+// DeadWorkerHeartbeatTimeout as "dead", then reassigns its leased tasks
+// back to the queued pool so other workers (or freshly registered ones)
+// can pick them up. It mirrors the structure of ReapExpiredTasks — two
+// short UPDATE statements, cheap enough to run from the web process
+// every 30s.
+//
+// The returned tuple is (reaped, reassigned, err): reaped counts the
+// workers that flipped to dead during this call; reassigned counts the
+// leased tasks that were given back to the pool. A zero count on either
+// is normal — the reaper is idempotent and the call is silent when
+// nothing is due.
+func ReapDeadWorkers() (reaped int, reassigned int, err error) {
+	now := time.Now()
+	cutoff := now.Add(-DeadWorkerHeartbeatTimeout)
+
+	// First flip the workers to dead so the second UPDATE can match the
+	// freshly-stamped ids without having to re-derive them in Go.
+	r := DB().Exec(`
+		UPDATE worker_nodes
+		SET status = ?, updated_at = ?
+		WHERE status <> ? AND last_seen IS NOT NULL AND last_seen < ?`,
+		"dead", now, "dead", cutoff,
+	)
+	if r.Error != nil {
+		return 0, 0, r.Error
+	}
+	reaped = int(r.RowsAffected)
+
+	// Nothing flipped → no tasks to return. Cheaper than running an
+	// UPDATE that touches 0 rows on every tick when the fleet is healthy.
+	if reaped == 0 {
+		return 0, 0, nil
+	}
+
+	// Second: clear any leased tasks owned by the now-dead workers.
+	// The selector stashes the worker's WorkerID string in lease_owner;
+	// matching against the (now-stale) worker row's WorkerID is the
+	// same identifier the selector uses, so we don't need an extra
+	// join. Tasks in other states (queued, succeeded, dead, …) are
+	// unaffected — only leased work the dead worker still owned has
+	// to go back to the queue.
+	r2 := DB().Exec(`
+		UPDATE tasks
+		SET state = ?, lease_owner = '', lease_expires_at = NULL, updated_at = ?
+		WHERE state = ? AND lease_owner IN (
+			SELECT worker_id FROM worker_nodes WHERE status = ?
+		)`,
+		TaskStateQueued, now, TaskStateLeased, "dead",
+	)
+	if r2.Error != nil {
+		return reaped, 0, r2.Error
+	}
+	reassigned = int(r2.RowsAffected)
+	return reaped, reassigned, nil
+}
+
+// StartDeadWorkerReaper launches a goroutine that runs ReapDeadWorkers
+// on the given interval until ctx is canceled. Mirrors StartTaskReaper
+// in this package: same ticker pattern, same logging style, same
+// recover() safety net so a malformed row cannot crash the web process.
+//
+// The default interval is 30s; values <= 0 fall back to the default so
+// the helper is safe to call from any call site without a guard. Wire
+// from main.init() once per process — the reaper uses short row-level
+// locks and is cheap under load (two indexed UPDATEs of <= a few
+// hundred rows in steady state).
+//
+// Passing a nil context falls back to context.Background() so callers
+// can write `models.StartDeadWorkerReaper(nil, ...)` in one-liners
+// (main, tests) without having to import "context" first.
+func StartDeadWorkerReaper(ctx context.Context, interval time.Duration) {
+	if interval <= 0 {
+		interval = 30 * time.Second
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				func() {
+					defer func() {
+						if r := recover(); r != nil {
+							log.Printf("dead_worker_reaper: panic recovered: %v", r)
+						}
+					}()
+					reaped, reassigned, err := ReapDeadWorkers()
+					if err != nil {
+						log.Printf("dead_worker_reaper: error: %v", err)
+						return
+					}
+					if reaped > 0 || reassigned > 0 {
+						log.Printf("dead_worker_reaper: reaped=%d reassigned=%d", reaped, reassigned)
+					}
+				}()
+			}
+		}
+	}()
+}