worker/internal/distworker/consensus.go

package distworker

import (
	"sync"
	"time"
)

// consensusDecision is the outcome of one consensus round. The
// selfcheck module drives its alert state machine off Down/Up; NoQuorum
// is the "we don't know yet" verdict returned when too few voters
// reported for either side to be a majority.
type consensusDecision int

const (
	// consensusNoQuorum means we have fewer than the minimum voters
	// required to reach a majority. The selfcheck holds the
	// previous state instead of flipping.
	consensusNoQuorum consensusDecision = iota
	// consensusUp means a majority of (self + known peers) reported
	// the master API as up.
	consensusUp
	// consensusDown means a majority reported the master API as
	// down. The selfcheck starts (or continues) the down timer.
	consensusDown
)

// String makes the verdict easy to log without bespoke formatting.
func (d consensusDecision) String() string {
	switch d {
	case consensusUp:
		return "up"
	case consensusDown:
		return "down"
	default:
		return "no-quorum"
	}
}

// tallyConsensus counts how many of self+peers are up vs down. The
// returned sizes are useful for logging/debugging and let the
// selfcheck report "2 of 3 voters say down" in its log line.
//
// selfUp == nil means "self vote is unknown" (e.g. the first tick
// has not completed). The cluster size is the number of known votes
// (self, if known, plus each peer observation).
func tallyConsensus(selfUp *bool, peers []peerObservation) (up, down, total int) {
	if selfUp != nil {
		total++
		if *selfUp {
			up++
		} else {
			down++
		}
	}
	for i := range peers {
		total++
		if peers[i].Up {
			up++
		} else {
			down++
		}
	}
	return up, down, total
}

// decideConsensus applies the simple-majority rule: whichever side
// (up or down) has at least floor(total/2)+1 votes wins. When total
// is 0 (no votes at all) or neither side reaches that threshold the
// result is consensusNoQuorum.
//
// minVotes is the minimum number of fresh votes (self + peers) that
// must be present before any verdict is reported. It is the gate
// that turns simple-majority into a peer-backed cluster quorum: a
// multi-worker deployment (WorkerInit.Peers non-empty) passes
// minVotes=2 so a lone self vote — no fresh peer observations yet —
// is consensusNoQuorum, both for up and down. A single-worker /
// no-peer deployment passes minVotes=1 to preserve the prior
// single-node behavior. Values below 1 are clamped to 1.
//
// The "self" vote is required to reach quorum: a worker that has not
// produced its own first probe cannot make a down consensus call
// (its own vote would be missing). Pass nil for selfUp to model the
// pre-first-probe window.
func decideConsensus(selfUp *bool, peers []peerObservation, minVotes int) consensusDecision {
	if selfUp == nil {
		return consensusNoQuorum
	}
	if minVotes < 1 {
		minVotes = 1
	}
	up, down, total := tallyConsensus(selfUp, peers)
	if total < minVotes {
		return consensusNoQuorum
	}
	// Standard majority for a non-empty set: floor(total/2)+1.
	// For 1 voter (self only, no peers yet) this collapses to 1
	// which still requires unanimous agreement with self.
	majority := total/2 + 1
	if up >= majority {
		return consensusUp
	}
	if down >= majority {
		return consensusDown
	}
	return consensusNoQuorum
}

// consensusState tracks the cluster-level up/down verdict over time
// so the selfcheck module can fire alerts only after the verdict has
// held for selfcheckConsensusWait. This mirrors the "incident
// state machine" idea from
// docs/distributed/worker-to-worker-raft.md §9.1 in miniature: we
// only have two states (down / clear) and one wait threshold, but
// the structure is the same so the next slice can swap the rule for
// the full FSM without changing the alert site.
type consensusState struct {
	mu sync.Mutex
	// downSince records the wall-clock time the cluster verdict
	// first flipped to consensusDown. Cleared when the verdict
	// flips back to consensusUp. nil means "not currently down".
	downSince *time.Time
	// alertActive mirrors the prior selfcheckState.sent* flags. It
	// is true between the down-alert firing and the recovery alert
	// firing so a duplicate probe does not re-send the same
	// "master is down" notification.
	alertActive bool
	// lastVerdict keeps the most recent decision for the next tick
	// to compare against without recomputing from scratch.
	lastVerdict consensusDecision
	// notificationLeader is the last deterministic worker elected to
	// send system-contact notifications for this local view of the
	// cluster. The first non-empty leader only initializes the field;
	// later changes are alert-worthy.
	notificationLeader string
}

func (s *consensusState) notificationLeaderChanged(leader string) (old string, changed bool) {
	if s == nil || leader == "" {
		return "", false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.notificationLeader == "" {
		s.notificationLeader = leader
		return "", false
	}
	if s.notificationLeader == leader {
		return "", false
	}
	old = s.notificationLeader
	s.notificationLeader = leader
	return old, true
}

// isDownConsensusHeld reports whether the supplied verdict means
// "down for at least selfcheckConsensusWait" given the current
// state. A fresh down verdict sets downSince; a subsequent down
// verdict keeps the original timestamp so the wait is measured from
// the first observation, not the most recent.
func (s *consensusState) isDownConsensusHeld(verdict consensusDecision, now time.Time) bool {
	if s == nil {
		return false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	switch verdict {
	case consensusDown:
		if s.downSince == nil {
			t := now
			s.downSince = &t
		}
		s.lastVerdict = verdict
		return now.Sub(*s.downSince) >= selfcheckConsensusWait
	case consensusUp:
		s.downSince = nil
		s.alertActive = false
		s.lastVerdict = verdict
		return false
	default:
		// NoQuorum: hold the existing state. Do not reset
		// downSince (a transient blip should not extend the
		// timer, but it should not erase progress either).
		return false
	}
}

// markDownAlertFired records that the down alert has been emitted so
// the next tick does not fire it again. Idempotent.
func (s *consensusState) markDownAlertFired() {
	if s == nil {
		return
	}
	s.mu.Lock()
	s.alertActive = true
	s.mu.Unlock()
}

// shouldFireRecovery reports whether the cluster has been up long
// enough to fire a recovery alert. Recovery uses the same wait
// window as the down alert so a flapping verdict does not spam
// recovery notifications.
func (s *consensusState) shouldFireRecovery(verdict consensusDecision, now time.Time) bool {
	if s == nil {
		return false
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if verdict != consensusUp {
		return false
	}
	if !s.alertActive {
		return false
	}
	if s.lastVerdict != consensusUp {
		// Just flipped from down to up; stamp the recovery timer.
		s.lastVerdict = verdict
		t := now
		s.downSince = &t
		return false
	}
	if s.downSince == nil {
		return false
	}
	return now.Sub(*s.downSince) >= selfcheckConsensusWait
}

// markRecoveryFired clears the alert-active flag so a future
// down verdict can fire the down alert again.
func (s *consensusState) markRecoveryFired() {
	if s == nil {
		return
	}
	s.mu.Lock()
	s.alertActive = false
	s.downSince = nil
	s.mu.Unlock()
}