feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.

internal/distworker/consensus.go

@@ -0,0 +1,237 @@
+package distworker
+
+import (
+	"sync"
+	"time"
+)
+
+// consensusDecision is the outcome of one consensus round. The
+// selfcheck module drives its alert state machine off Down/Up; NoQuorum
+// is the "we don't know yet" verdict returned when too few voters
+// reported for either side to be a majority.
+type consensusDecision int
+
+const (
+	// consensusNoQuorum means we have fewer than the minimum voters
+	// required to reach a majority. The selfcheck holds the
+	// previous state instead of flipping.
+	consensusNoQuorum consensusDecision = iota
+	// consensusUp means a majority of (self + known peers) reported
+	// the master API as up.
+	consensusUp
+	// consensusDown means a majority reported the master API as
+	// down. The selfcheck starts (or continues) the down timer.
+	consensusDown
+)
+
+// String makes the verdict easy to log without bespoke formatting.
+func (d consensusDecision) String() string {
+	switch d {
+	case consensusUp:
+		return "up"
+	case consensusDown:
+		return "down"
+	default:
+		return "no-quorum"
+	}
+}
+
+// tallyConsensus counts how many of self+peers are up vs down. The
+// returned sizes are useful for logging/debugging and let the
+// selfcheck report "2 of 3 voters say down" in its log line.
+//
+// selfUp == nil means "self vote is unknown" (e.g. the first tick
+// has not completed). The cluster size is the number of known votes
+// (self, if known, plus each peer observation).
+func tallyConsensus(selfUp *bool, peers []peerObservation) (up, down, total int) {
+	if selfUp != nil {
+		total++
+		if *selfUp {
+			up++
+		} else {
+			down++
+		}
+	}
+	for i := range peers {
+		total++
+		if peers[i].Up {
+			up++
+		} else {
+			down++
+		}
+	}
+	return up, down, total
+}
+
+// decideConsensus applies the simple-majority rule: whichever side
+// (up or down) has at least floor(total/2)+1 votes wins. When total
+// is 0 (no votes at all) or neither side reaches that threshold the
+// result is consensusNoQuorum.
+//
+// minVotes is the minimum number of fresh votes (self + peers) that
+// must be present before any verdict is reported. It is the gate
+// that turns simple-majority into a peer-backed cluster quorum: a
+// multi-worker deployment (WorkerInit.Peers non-empty) passes
+// minVotes=2 so a lone self vote — no fresh peer observations yet —
+// is consensusNoQuorum, both for up and down. A single-worker /
+// no-peer deployment passes minVotes=1 to preserve the prior
+// single-node behavior. Values below 1 are clamped to 1.
+//
+// The "self" vote is required to reach quorum: a worker that has not
+// produced its own first probe cannot make a down consensus call
+// (its own vote would be missing). Pass nil for selfUp to model the
+// pre-first-probe window.
+func decideConsensus(selfUp *bool, peers []peerObservation, minVotes int) consensusDecision {
+	if selfUp == nil {
+		return consensusNoQuorum
+	}
+	if minVotes < 1 {
+		minVotes = 1
+	}
+	up, down, total := tallyConsensus(selfUp, peers)
+	if total < minVotes {
+		return consensusNoQuorum
+	}
+	// Standard majority for a non-empty set: floor(total/2)+1.
+	// For 1 voter (self only, no peers yet) this collapses to 1
+	// which still requires unanimous agreement with self.
+	majority := total/2 + 1
+	if up >= majority {
+		return consensusUp
+	}
+	if down >= majority {
+		return consensusDown
+	}
+	return consensusNoQuorum
+}
+
+// consensusState tracks the cluster-level up/down verdict over time
+// so the selfcheck module can fire alerts only after the verdict has
+// held for selfcheckConsensusWait. This mirrors the "incident
+// state machine" idea from
+// docs/distributed/worker-to-worker-raft.md §9.1 in miniature: we
+// only have two states (down / clear) and one wait threshold, but
+// the structure is the same so the next slice can swap the rule for
+// the full FSM without changing the alert site.
+type consensusState struct {
+	mu sync.Mutex
+	// downSince records the wall-clock time the cluster verdict
+	// first flipped to consensusDown. Cleared when the verdict
+	// flips back to consensusUp. nil means "not currently down".
+	downSince *time.Time
+	// alertActive mirrors the prior selfcheckState.sent* flags. It
+	// is true between the down-alert firing and the recovery alert
+	// firing so a duplicate probe does not re-send the same
+	// "master is down" notification.
+	alertActive bool
+	// lastVerdict keeps the most recent decision for the next tick
+	// to compare against without recomputing from scratch.
+	lastVerdict consensusDecision
+	// notificationLeader is the last deterministic worker elected to
+	// send system-contact notifications for this local view of the
+	// cluster. The first non-empty leader only initializes the field;
+	// later changes are alert-worthy.
+	notificationLeader string
+}
+
+func (s *consensusState) notificationLeaderChanged(leader string) (old string, changed bool) {
+	if s == nil || leader == "" {
+		return "", false
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.notificationLeader == "" {
+		s.notificationLeader = leader
+		return "", false
+	}
+	if s.notificationLeader == leader {
+		return "", false
+	}
+	old = s.notificationLeader
+	s.notificationLeader = leader
+	return old, true
+}
+
+// isDownConsensusHeld reports whether the supplied verdict means
+// "down for at least selfcheckConsensusWait" given the current
+// state. A fresh down verdict sets downSince; a subsequent down
+// verdict keeps the original timestamp so the wait is measured from
+// the first observation, not the most recent.
+func (s *consensusState) isDownConsensusHeld(verdict consensusDecision, now time.Time) bool {
+	if s == nil {
+		return false
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	switch verdict {
+	case consensusDown:
+		if s.downSince == nil {
+			t := now
+			s.downSince = &t
+		}
+		s.lastVerdict = verdict
+		return now.Sub(*s.downSince) >= selfcheckConsensusWait
+	case consensusUp:
+		s.downSince = nil
+		s.alertActive = false
+		s.lastVerdict = verdict
+		return false
+	default:
+		// NoQuorum: hold the existing state. Do not reset
+		// downSince (a transient blip should not extend the
+		// timer, but it should not erase progress either).
+		return false
+	}
+}
+
+// markDownAlertFired records that the down alert has been emitted so
+// the next tick does not fire it again. Idempotent.
+func (s *consensusState) markDownAlertFired() {
+	if s == nil {
+		return
+	}
+	s.mu.Lock()
+	s.alertActive = true
+	s.mu.Unlock()
+}
+
+// shouldFireRecovery reports whether the cluster has been up long
+// enough to fire a recovery alert. Recovery uses the same wait
+// window as the down alert so a flapping verdict does not spam
+// recovery notifications.
+func (s *consensusState) shouldFireRecovery(verdict consensusDecision, now time.Time) bool {
+	if s == nil {
+		return false
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if verdict != consensusUp {
+		return false
+	}
+	if !s.alertActive {
+		return false
+	}
+	if s.lastVerdict != consensusUp {
+		// Just flipped from down to up; stamp the recovery timer.
+		s.lastVerdict = verdict
+		t := now
+		s.downSince = &t
+		return false
+	}
+	if s.downSince == nil {
+		return false
+	}
+	return now.Sub(*s.downSince) >= selfcheckConsensusWait
+}
+
+// markRecoveryFired clears the alert-active flag so a future
+// down verdict can fire the down alert again.
+func (s *consensusState) markRecoveryFired() {
+	if s == nil {
+		return
+	}
+	s.mu.Lock()
+	s.alertActive = false
+	s.downSince = nil
+	s.mu.Unlock()
+}