worker/internal/distworker/peer_test.go

package distworker

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"rocketgit.ru/rsmon/worker/internal/wire"
)

func TestDecideConsensus(t *testing.T) {
	up := true
	down := false
	peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
	peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}

	cases := []struct {
		name     string
		self     *bool
		peers    []peerObservation
		minVotes int
		want     consensusDecision
	}{
		{
			name:     "self only up (single-node, minVotes=1)",
			self:     &up,
			peers:    nil,
			minVotes: 1,
			want:     consensusUp,
		},
		{
			name:     "self only down (single-node, minVotes=1)",
			self:     &down,
			peers:    nil,
			minVotes: 1,
			want:     consensusDown,
		},
		{
			name: "self down + 2 peers down = majority down",
			self: &down,
			peers: []peerObservation{
				{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
			},
			minVotes: 2,
			want:     consensusDown,
		},
		{
			name: "self up + 2 peers up = majority up",
			self: &up,
			peers: []peerObservation{
				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
				{WorkerID: "p2", Up: true, ObservedAt: time.Now()},
			},
			minVotes: 2,
			want:     consensusUp,
		},
		{
			name: "self up + 1 up + 1 down = majority up (2 of 3)",
			self: &up,
			peers: []peerObservation{
				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
			},
			minVotes: 2,
			want:     consensusUp,
		},
		{
			name: "self up + 1 down (1 up, 1 down) = tie (no quorum)",
			self: &up,
			peers: []peerObservation{
				{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
			},
			minVotes: 2,
			want:     consensusNoQuorum,
		},
		{
			name: "self down + 1 up + 1 down = majority down (2 of 3)",
			self: &down,
			peers: []peerObservation{
				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
			},
			minVotes: 2,
			want:     consensusDown,
		},
		{
			name:     "self vote missing = no quorum",
			self:     nil,
			peers:    peersDown,
			minVotes: 2,
			want:     consensusNoQuorum,
		},
		{
			name:     "self vote missing + 1 up peer = still no quorum",
			self:     nil,
			peers:    peersUp,
			minVotes: 2,
			want:     consensusNoQuorum,
		},
		{
			name:     "minVotes=0 clamped to 1 (single-node self up)",
			self:     &up,
			peers:    nil,
			minVotes: 0,
			want:     consensusUp,
		},
		{
			name:     "minVotes<0 clamped to 1 (single-node self down)",
			self:     &down,
			peers:    nil,
			minVotes: -3,
			want:     consensusDown,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			assert.Equal(t, tc.want, decideConsensus(tc.self, tc.peers, tc.minVotes))
		})
	}
}

func TestConsensusState_FiresAfterWait(t *testing.T) {
	state := &consensusState{}
	now := time.Now()

	// Stamp the initial down verdict at the baseline so subsequent
	// ticks measure against a known start time.
	held := state.isDownConsensusHeld(consensusDown, now)
	assert.False(t, held, "must not fire on the first down tick")
	assert.NotNil(t, state.downSince, "downSince should be stamped on first down verdict")

	// Not yet fired even after 4m59s.
	held = state.isDownConsensusHeld(consensusDown, now.Add(4*time.Minute+59*time.Second))
	assert.False(t, held, "must not fire before the wait elapses")

	// Fire after the wait elapses.
	held = state.isDownConsensusHeld(consensusDown, now.Add(5*time.Minute+time.Second))
	assert.True(t, held, "must fire once the wait has elapsed")
	state.markDownAlertFired()
	assert.True(t, state.alertActive)

	// Subsequent down verdicts do not re-stamp downSince.
	stampBefore := *state.downSince
	_ = state.isDownConsensusHeld(consensusDown, now.Add(10*time.Minute))
	assert.Equal(t, stampBefore, *state.downSince,
		"downSince must be set on the first observation, not re-stamped")

	// Recovery clears the down timer and the alert flag.
	_ = state.isDownConsensusHeld(consensusUp, now.Add(11*time.Minute))
	assert.Nil(t, state.downSince)
	assert.False(t, state.alertActive)
}

func TestConsensusState_RecoveryRequiresHeld(t *testing.T) {
	state := &consensusState{}
	now := time.Now()

	// Pretend we already fired a down alert.
	state.alertActive = true
	t0 := now
	state.downSince = &t0

	// First up verdict stamps a fresh recovery timer; the alert
	// must NOT fire yet.
	assert.False(t, state.shouldFireRecovery(consensusUp, now.Add(time.Second)))
	assert.Equal(t, consensusUp, state.lastVerdict)

	// Stale up verdict (verdict != up) does not advance the
	// recovery timer.
	assert.False(t, state.shouldFireRecovery(consensusDown, now.Add(time.Minute)))

	// Held long enough, recovery fires.
	assert.True(t, state.shouldFireRecovery(consensusUp, now.Add(selfcheckConsensusWait+time.Second)))
	state.markRecoveryFired()
	assert.False(t, state.alertActive)
	assert.Nil(t, state.downSince)
}

func TestConsensusState_NoQuorumHoldsState(t *testing.T) {
	state := &consensusState{}
	now := time.Now()

	// First establish a down verdict + timer.
	_ = state.isDownConsensusHeld(consensusDown, now)
	stampBefore := *state.downSince
	state.markDownAlertFired()

	// NoQuorum must not reset the timer; the alert stays fired
	// so the next down verdict does not double-fire.
	_ = state.isDownConsensusHeld(consensusNoQuorum, now.Add(2*time.Minute))
	assert.Equal(t, stampBefore, *state.downSince)
	assert.True(t, state.alertActive)
}

func TestTallyConsensus(t *testing.T) {
	up, down := true, false
	peers := []peerObservation{
		{Up: true}, {Up: false}, {Up: true},
	}
	gotUp, gotDown, gotTotal := tallyConsensus(&up, peers)
	assert.Equal(t, 3, gotUp)
	assert.Equal(t, 1, gotDown)
	assert.Equal(t, 4, gotTotal)

	gotUp, gotDown, gotTotal = tallyConsensus(&down, nil)
	assert.Equal(t, 0, gotUp)
	assert.Equal(t, 1, gotDown)
	assert.Equal(t, 1, gotTotal)

	gotUp, gotDown, gotTotal = tallyConsensus(nil, peers)
	assert.Equal(t, 2, gotUp)
	assert.Equal(t, 1, gotDown)
	assert.Equal(t, 3, gotTotal)
}

func TestPeerCacheSnapshotAndPut(t *testing.T) {
	c := newPeerCache()
	c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
	c.put(peerObservation{WorkerID: "b", Up: false, ObservedAt: time.Now()})

	snap := c.snapshot()
	assert.Len(t, snap, 2)
}

func TestPeerCacheResetForDropsRemovedPeers(t *testing.T) {
	c := newPeerCache()
	c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
	c.put(peerObservation{WorkerID: "b", Up: true, ObservedAt: time.Now()})

	c.resetFor([]wire.PeerInfo{{WorkerID: "a"}, {WorkerID: "c"}})
	snap := c.snapshot()
	ids := map[string]peerObservation{}
	for _, s := range snap {
		ids[s.WorkerID] = s
	}
	_, hasA := ids["a"]
	_, hasB := ids["b"]
	_, hasC := ids["c"]
	assert.True(t, hasA, "peer 'a' retained")
	assert.False(t, hasB, "removed peer 'b' dropped")
	assert.True(t, hasC, "new peer 'c' added")
}

func TestApplyInitStoresPeersAndResetsCache(t *testing.T) {
	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
	r := newTestRunner(t, 4, 1, executor)
	require.NotNil(t, r.peerCache, "runner must own a peer cache after NewRunner")

	r.applyInit(&wire.WorkerInit{
		WorkerID:    "w-1",
		Concurrency: 2,
		Peers: []wire.PeerInfo{
			{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
			{WorkerID: "w-3", URL: "http://127.0.0.1:27403"},
		},
	})
	got := r.Peers()
	require.Len(t, got, 2)
	assert.Equal(t, "w-2", got[0].WorkerID)
	assert.Equal(t, "w-3", got[1].WorkerID)

	// Cache should be primed with empty observations for the new
	// peers; the per-peer entries are visible via snapshot().
	snap := r.peerCache.snapshot()
	assert.Len(t, snap, 2)
	for _, s := range snap {
		assert.True(t, s.ObservedAt.IsZero(),
			"freshly-reset peer %s should have a zero observed_at", s.WorkerID)
	}
}

func TestFetchPeerStatus_DecodesUpResponse(t *testing.T) {
	up := true
	now := time.Now().UTC().Truncate(time.Second)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(PeerStatus{
			WorkerID:   "remote",
			Up:         up,
			ObservedAt: now,
		})
	}))
	defer srv.Close()

	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
		WorkerID: "remote",
		URL:      srv.URL,
	})
	assert.True(t, obs.Up)
	assert.NoError(t, obs.Err)
	assert.Equal(t, "remote", obs.WorkerID)
	assert.Equal(t, now, obs.ObservedAt)
}

func TestFetchPeerStatus_NetworkErrorIsRecorded(t *testing.T) {
	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
		WorkerID: "unreachable",
		URL:      "http://127.0.0.1:1",
	})
	assert.Error(t, obs.Err)
	assert.False(t, obs.Up)
}

func TestFetchPeerStatus_BadURLRejected(t *testing.T) {
	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
		WorkerID: "weird",
		URL:      "ftp://example.com",
	})
	assert.Error(t, obs.Err)
}

func TestDecideConsensus_MultiWorkerQuorumGate(t *testing.T) {
	// When the control plane has configured peers (a multi-worker
	// install), a lone self vote must NOT be treated as a
	// cluster-wide verdict in either direction. A second,
	// independent vote is required to reach any verdict.
	up := true
	down := false
	peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}
	peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}

	t.Run("self down + configured peers but no fresh peer = no quorum", func(t *testing.T) {
		assert.Equal(t, consensusNoQuorum, decideConsensus(&down, nil, 2),
			"a lone down vote must not page when peers are configured but not reporting")
		assert.Equal(t, consensusNoQuorum, decideConsensus(&up, nil, 2),
			"a lone up vote must not clear an active alert when peers are configured but not reporting")
	})

	t.Run("self down + one fresh down peer = down consensus", func(t *testing.T) {
		assert.Equal(t, consensusDown, decideConsensus(&down, peersDown, 2))
	})

	t.Run("self up + one fresh up peer = up consensus", func(t *testing.T) {
		assert.Equal(t, consensusUp, decideConsensus(&up, peersUp, 2))
	})

	t.Run("self down + one fresh up peer = no quorum (1/1 split)", func(t *testing.T) {
		// With two voters and one of each, neither side has a
		// majority, so the verdict stays NoQuorum. This is the
		// intended conservative behavior — we don't want a
		// single dissenting peer to override self.
		assert.Equal(t, consensusNoQuorum, decideConsensus(&down, peersUp, 2))
	})
}

func TestDecideConsensus_SingleNodePreserved(t *testing.T) {
	// When no peers are configured (single-worker / no-peer
	// deployment) the caller passes minVotes=1, which preserves
	// the prior single-node behavior: a self-only vote is enough
	// to drive a verdict in either direction.
	up := true
	down := false

	assert.Equal(t, consensusUp, decideConsensus(&up, nil, 1),
		"single-node self up must still reach up consensus")
	assert.Equal(t, consensusDown, decideConsensus(&down, nil, 1),
		"single-node self down must still reach down consensus")
}

func TestPeerObservationsForConsensus_DropsStale(t *testing.T) {
	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
	r := newTestRunner(t, 4, 1, executor)

	now := time.Now()
	fresh := peerObservation{WorkerID: "p1", Up: true, ObservedAt: now.Add(-30 * time.Second)}
	stale := peerObservation{WorkerID: "p2", Up: true, ObservedAt: now.Add(-5 * time.Minute)}
	errored := peerObservation{WorkerID: "p3", Up: true, Err: assert.AnError}
	unknown := peerObservation{WorkerID: "p4"}
	r.peerCache.put(fresh)
	r.peerCache.put(stale)
	r.peerCache.put(errored)
	r.peerCache.put(unknown)

	got := r.peerObservationsForConsensus(now)
	require.Len(t, got, 1)
	assert.Equal(t, "p1", got[0].WorkerID)
}

func TestPollPeersOnceIsConcurrencySafe(t *testing.T) {
	var hits int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		atomic.AddInt32(&hits, 1)
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(PeerStatus{
			WorkerID:   "remote",
			Up:         true,
			ObservedAt: time.Now().UTC(),
		})
	}))
	defer srv.Close()

	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
	r := newTestRunner(t, 4, 1, executor)
	r.applyInit(&wire.WorkerInit{
		WorkerID: "w-1",
		URL:      "http://127.0.0.1:1", // local self probe will fail, no effect on peer poll
		Peers: []wire.PeerInfo{
			{WorkerID: "p1", URL: srv.URL},
		},
	})

	// Run multiple concurrent poll cycles. The cache uses a single
	// RWMutex; this verifies there is no data race under that
	// access pattern.
	var wg sync.WaitGroup
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			r.pollPeersOnce(context.Background())
		}()
	}
	wg.Wait()
	assert.GreaterOrEqual(t, atomic.LoadInt32(&hits), int32(1))
}