package distworker import ( "context" "encoding/json" "net/http" "net/http/httptest" "sync" "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "rocketgit.ru/rsmon/worker/internal/wire" ) func TestDecideConsensus(t *testing.T) { up := true down := false peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}} peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}} cases := []struct { name string self *bool peers []peerObservation minVotes int want consensusDecision }{ { name: "self only up (single-node, minVotes=1)", self: &up, peers: nil, minVotes: 1, want: consensusUp, }, { name: "self only down (single-node, minVotes=1)", self: &down, peers: nil, minVotes: 1, want: consensusDown, }, { name: "self down + 2 peers down = majority down", self: &down, peers: []peerObservation{ {WorkerID: "p1", Up: false, ObservedAt: time.Now()}, {WorkerID: "p2", Up: false, ObservedAt: time.Now()}, }, minVotes: 2, want: consensusDown, }, { name: "self up + 2 peers up = majority up", self: &up, peers: []peerObservation{ {WorkerID: "p1", Up: true, ObservedAt: time.Now()}, {WorkerID: "p2", Up: true, ObservedAt: time.Now()}, }, minVotes: 2, want: consensusUp, }, { name: "self up + 1 up + 1 down = majority up (2 of 3)", self: &up, peers: []peerObservation{ {WorkerID: "p1", Up: true, ObservedAt: time.Now()}, {WorkerID: "p2", Up: false, ObservedAt: time.Now()}, }, minVotes: 2, want: consensusUp, }, { name: "self up + 1 down (1 up, 1 down) = tie (no quorum)", self: &up, peers: []peerObservation{ {WorkerID: "p1", Up: false, ObservedAt: time.Now()}, }, minVotes: 2, want: consensusNoQuorum, }, { name: "self down + 1 up + 1 down = majority down (2 of 3)", self: &down, peers: []peerObservation{ {WorkerID: "p1", Up: true, ObservedAt: time.Now()}, {WorkerID: "p2", Up: false, ObservedAt: time.Now()}, }, minVotes: 2, want: consensusDown, }, { name: "self vote missing = no quorum", self: nil, peers: peersDown, minVotes: 2, want: consensusNoQuorum, }, { name: "self vote missing + 1 up peer = still no quorum", self: nil, peers: peersUp, minVotes: 2, want: consensusNoQuorum, }, { name: "minVotes=0 clamped to 1 (single-node self up)", self: &up, peers: nil, minVotes: 0, want: consensusUp, }, { name: "minVotes<0 clamped to 1 (single-node self down)", self: &down, peers: nil, minVotes: -3, want: consensusDown, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { assert.Equal(t, tc.want, decideConsensus(tc.self, tc.peers, tc.minVotes)) }) } } func TestConsensusState_FiresAfterWait(t *testing.T) { state := &consensusState{} now := time.Now() // Stamp the initial down verdict at the baseline so subsequent // ticks measure against a known start time. held := state.isDownConsensusHeld(consensusDown, now) assert.False(t, held, "must not fire on the first down tick") assert.NotNil(t, state.downSince, "downSince should be stamped on first down verdict") // Not yet fired even after 4m59s. held = state.isDownConsensusHeld(consensusDown, now.Add(4*time.Minute+59*time.Second)) assert.False(t, held, "must not fire before the wait elapses") // Fire after the wait elapses. held = state.isDownConsensusHeld(consensusDown, now.Add(5*time.Minute+time.Second)) assert.True(t, held, "must fire once the wait has elapsed") state.markDownAlertFired() assert.True(t, state.alertActive) // Subsequent down verdicts do not re-stamp downSince. stampBefore := *state.downSince _ = state.isDownConsensusHeld(consensusDown, now.Add(10*time.Minute)) assert.Equal(t, stampBefore, *state.downSince, "downSince must be set on the first observation, not re-stamped") // Recovery clears the down timer and the alert flag. _ = state.isDownConsensusHeld(consensusUp, now.Add(11*time.Minute)) assert.Nil(t, state.downSince) assert.False(t, state.alertActive) } func TestConsensusState_RecoveryRequiresHeld(t *testing.T) { state := &consensusState{} now := time.Now() // Pretend we already fired a down alert. state.alertActive = true t0 := now state.downSince = &t0 // First up verdict stamps a fresh recovery timer; the alert // must NOT fire yet. assert.False(t, state.shouldFireRecovery(consensusUp, now.Add(time.Second))) assert.Equal(t, consensusUp, state.lastVerdict) // Stale up verdict (verdict != up) does not advance the // recovery timer. assert.False(t, state.shouldFireRecovery(consensusDown, now.Add(time.Minute))) // Held long enough, recovery fires. assert.True(t, state.shouldFireRecovery(consensusUp, now.Add(selfcheckConsensusWait+time.Second))) state.markRecoveryFired() assert.False(t, state.alertActive) assert.Nil(t, state.downSince) } func TestConsensusState_NoQuorumHoldsState(t *testing.T) { state := &consensusState{} now := time.Now() // First establish a down verdict + timer. _ = state.isDownConsensusHeld(consensusDown, now) stampBefore := *state.downSince state.markDownAlertFired() // NoQuorum must not reset the timer; the alert stays fired // so the next down verdict does not double-fire. _ = state.isDownConsensusHeld(consensusNoQuorum, now.Add(2*time.Minute)) assert.Equal(t, stampBefore, *state.downSince) assert.True(t, state.alertActive) } func TestTallyConsensus(t *testing.T) { up, down := true, false peers := []peerObservation{ {Up: true}, {Up: false}, {Up: true}, } gotUp, gotDown, gotTotal := tallyConsensus(&up, peers) assert.Equal(t, 3, gotUp) assert.Equal(t, 1, gotDown) assert.Equal(t, 4, gotTotal) gotUp, gotDown, gotTotal = tallyConsensus(&down, nil) assert.Equal(t, 0, gotUp) assert.Equal(t, 1, gotDown) assert.Equal(t, 1, gotTotal) gotUp, gotDown, gotTotal = tallyConsensus(nil, peers) assert.Equal(t, 2, gotUp) assert.Equal(t, 1, gotDown) assert.Equal(t, 3, gotTotal) } func TestPeerCacheSnapshotAndPut(t *testing.T) { c := newPeerCache() c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()}) c.put(peerObservation{WorkerID: "b", Up: false, ObservedAt: time.Now()}) snap := c.snapshot() assert.Len(t, snap, 2) } func TestPeerCacheResetForDropsRemovedPeers(t *testing.T) { c := newPeerCache() c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()}) c.put(peerObservation{WorkerID: "b", Up: true, ObservedAt: time.Now()}) c.resetFor([]wire.PeerInfo{{WorkerID: "a"}, {WorkerID: "c"}}) snap := c.snapshot() ids := map[string]peerObservation{} for _, s := range snap { ids[s.WorkerID] = s } _, hasA := ids["a"] _, hasB := ids["b"] _, hasC := ids["c"] assert.True(t, hasA, "peer 'a' retained") assert.False(t, hasB, "removed peer 'b' dropped") assert.True(t, hasC, "new peer 'c' added") } func TestApplyInitStoresPeersAndResetsCache(t *testing.T) { executor := func(interface{}) interface{} { return []wire.CheckResultReport{} } r := newTestRunner(t, 4, 1, executor) require.NotNil(t, r.peerCache, "runner must own a peer cache after NewRunner") r.applyInit(&wire.WorkerInit{ WorkerID: "w-1", Concurrency: 2, Peers: []wire.PeerInfo{ {WorkerID: "w-2", URL: "http://127.0.0.1:27402"}, {WorkerID: "w-3", URL: "http://127.0.0.1:27403"}, }, }) got := r.Peers() require.Len(t, got, 2) assert.Equal(t, "w-2", got[0].WorkerID) assert.Equal(t, "w-3", got[1].WorkerID) // Cache should be primed with empty observations for the new // peers; the per-peer entries are visible via snapshot(). snap := r.peerCache.snapshot() assert.Len(t, snap, 2) for _, s := range snap { assert.True(t, s.ObservedAt.IsZero(), "freshly-reset peer %s should have a zero observed_at", s.WorkerID) } } func TestFetchPeerStatus_DecodesUpResponse(t *testing.T) { up := true now := time.Now().UTC().Truncate(time.Second) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(PeerStatus{ WorkerID: "remote", Up: up, ObservedAt: now, }) })) defer srv.Close() obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{ WorkerID: "remote", URL: srv.URL, }) assert.True(t, obs.Up) assert.NoError(t, obs.Err) assert.Equal(t, "remote", obs.WorkerID) assert.Equal(t, now, obs.ObservedAt) } func TestFetchPeerStatus_NetworkErrorIsRecorded(t *testing.T) { obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{ WorkerID: "unreachable", URL: "http://127.0.0.1:1", }) assert.Error(t, obs.Err) assert.False(t, obs.Up) } func TestFetchPeerStatus_BadURLRejected(t *testing.T) { obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{ WorkerID: "weird", URL: "ftp://example.com", }) assert.Error(t, obs.Err) } func TestDecideConsensus_MultiWorkerQuorumGate(t *testing.T) { // When the control plane has configured peers (a multi-worker // install), a lone self vote must NOT be treated as a // cluster-wide verdict in either direction. A second, // independent vote is required to reach any verdict. up := true down := false peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}} peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}} t.Run("self down + configured peers but no fresh peer = no quorum", func(t *testing.T) { assert.Equal(t, consensusNoQuorum, decideConsensus(&down, nil, 2), "a lone down vote must not page when peers are configured but not reporting") assert.Equal(t, consensusNoQuorum, decideConsensus(&up, nil, 2), "a lone up vote must not clear an active alert when peers are configured but not reporting") }) t.Run("self down + one fresh down peer = down consensus", func(t *testing.T) { assert.Equal(t, consensusDown, decideConsensus(&down, peersDown, 2)) }) t.Run("self up + one fresh up peer = up consensus", func(t *testing.T) { assert.Equal(t, consensusUp, decideConsensus(&up, peersUp, 2)) }) t.Run("self down + one fresh up peer = no quorum (1/1 split)", func(t *testing.T) { // With two voters and one of each, neither side has a // majority, so the verdict stays NoQuorum. This is the // intended conservative behavior — we don't want a // single dissenting peer to override self. assert.Equal(t, consensusNoQuorum, decideConsensus(&down, peersUp, 2)) }) } func TestDecideConsensus_SingleNodePreserved(t *testing.T) { // When no peers are configured (single-worker / no-peer // deployment) the caller passes minVotes=1, which preserves // the prior single-node behavior: a self-only vote is enough // to drive a verdict in either direction. up := true down := false assert.Equal(t, consensusUp, decideConsensus(&up, nil, 1), "single-node self up must still reach up consensus") assert.Equal(t, consensusDown, decideConsensus(&down, nil, 1), "single-node self down must still reach down consensus") } func TestPeerObservationsForConsensus_DropsStale(t *testing.T) { executor := func(interface{}) interface{} { return []wire.CheckResultReport{} } r := newTestRunner(t, 4, 1, executor) now := time.Now() fresh := peerObservation{WorkerID: "p1", Up: true, ObservedAt: now.Add(-30 * time.Second)} stale := peerObservation{WorkerID: "p2", Up: true, ObservedAt: now.Add(-5 * time.Minute)} errored := peerObservation{WorkerID: "p3", Up: true, Err: assert.AnError} unknown := peerObservation{WorkerID: "p4"} r.peerCache.put(fresh) r.peerCache.put(stale) r.peerCache.put(errored) r.peerCache.put(unknown) got := r.peerObservationsForConsensus(now) require.Len(t, got, 1) assert.Equal(t, "p1", got[0].WorkerID) } func TestPollPeersOnceIsConcurrencySafe(t *testing.T) { var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { atomic.AddInt32(&hits, 1) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(PeerStatus{ WorkerID: "remote", Up: true, ObservedAt: time.Now().UTC(), }) })) defer srv.Close() executor := func(interface{}) interface{} { return []wire.CheckResultReport{} } r := newTestRunner(t, 4, 1, executor) r.applyInit(&wire.WorkerInit{ WorkerID: "w-1", URL: "http://127.0.0.1:1", // local self probe will fail, no effect on peer poll Peers: []wire.PeerInfo{ {WorkerID: "p1", URL: srv.URL}, }, }) // Run multiple concurrent poll cycles. The cache uses a single // RWMutex; this verifies there is no data race under that // access pattern. var wg sync.WaitGroup for i := 0; i < 8; i++ { wg.Add(1) go func() { defer wg.Done() r.pollPeersOnce(context.Background()) }() } wg.Wait() assert.GreaterOrEqual(t, atomic.LoadInt32(&hits), int32(1)) }