feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.

internal/distworker/peer_test.go

@@ -0,0 +1,424 @@
+package distworker
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"rsgit.ru/rsmon/rsmon/internal/wire"
+)
+
+func TestDecideConsensus(t *testing.T) {
+	up := true
+	down := false
+	peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
+	peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}
+
+	cases := []struct {
+		name     string
+		self     *bool
+		peers    []peerObservation
+		minVotes int
+		want     consensusDecision
+	}{
+		{
+			name:     "self only up (single-node, minVotes=1)",
+			self:     &up,
+			peers:    nil,
+			minVotes: 1,
+			want:     consensusUp,
+		},
+		{
+			name:     "self only down (single-node, minVotes=1)",
+			self:     &down,
+			peers:    nil,
+			minVotes: 1,
+			want:     consensusDown,
+		},
+		{
+			name: "self down + 2 peers down = majority down",
+			self: &down,
+			peers: []peerObservation{
+				{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
+				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
+			},
+			minVotes: 2,
+			want:     consensusDown,
+		},
+		{
+			name: "self up + 2 peers up = majority up",
+			self: &up,
+			peers: []peerObservation{
+				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
+				{WorkerID: "p2", Up: true, ObservedAt: time.Now()},
+			},
+			minVotes: 2,
+			want:     consensusUp,
+		},
+		{
+			name: "self up + 1 up + 1 down = majority up (2 of 3)",
+			self: &up,
+			peers: []peerObservation{
+				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
+				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
+			},
+			minVotes: 2,
+			want:     consensusUp,
+		},
+		{
+			name: "self up + 1 down (1 up, 1 down) = tie (no quorum)",
+			self: &up,
+			peers: []peerObservation{
+				{WorkerID: "p1", Up: false, ObservedAt: time.Now()},
+			},
+			minVotes: 2,
+			want:     consensusNoQuorum,
+		},
+		{
+			name: "self down + 1 up + 1 down = majority down (2 of 3)",
+			self: &down,
+			peers: []peerObservation{
+				{WorkerID: "p1", Up: true, ObservedAt: time.Now()},
+				{WorkerID: "p2", Up: false, ObservedAt: time.Now()},
+			},
+			minVotes: 2,
+			want:     consensusDown,
+		},
+		{
+			name:     "self vote missing = no quorum",
+			self:     nil,
+			peers:    peersDown,
+			minVotes: 2,
+			want:     consensusNoQuorum,
+		},
+		{
+			name:     "self vote missing + 1 up peer = still no quorum",
+			self:     nil,
+			peers:    peersUp,
+			minVotes: 2,
+			want:     consensusNoQuorum,
+		},
+		{
+			name:     "minVotes=0 clamped to 1 (single-node self up)",
+			self:     &up,
+			peers:    nil,
+			minVotes: 0,
+			want:     consensusUp,
+		},
+		{
+			name:     "minVotes<0 clamped to 1 (single-node self down)",
+			self:     &down,
+			peers:    nil,
+			minVotes: -3,
+			want:     consensusDown,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, decideConsensus(tc.self, tc.peers, tc.minVotes))
+		})
+	}
+}
+
+func TestConsensusState_FiresAfterWait(t *testing.T) {
+	state := &consensusState{}
+	now := time.Now()
+
+	// Stamp the initial down verdict at the baseline so subsequent
+	// ticks measure against a known start time.
+	held := state.isDownConsensusHeld(consensusDown, now)
+	assert.False(t, held, "must not fire on the first down tick")
+	assert.NotNil(t, state.downSince, "downSince should be stamped on first down verdict")
+
+	// Not yet fired even after 4m59s.
+	held = state.isDownConsensusHeld(consensusDown, now.Add(4*time.Minute+59*time.Second))
+	assert.False(t, held, "must not fire before the wait elapses")
+
+	// Fire after the wait elapses.
+	held = state.isDownConsensusHeld(consensusDown, now.Add(5*time.Minute+time.Second))
+	assert.True(t, held, "must fire once the wait has elapsed")
+	state.markDownAlertFired()
+	assert.True(t, state.alertActive)
+
+	// Subsequent down verdicts do not re-stamp downSince.
+	stampBefore := *state.downSince
+	_ = state.isDownConsensusHeld(consensusDown, now.Add(10*time.Minute))
+	assert.Equal(t, stampBefore, *state.downSince,
+		"downSince must be set on the first observation, not re-stamped")
+
+	// Recovery clears the down timer and the alert flag.
+	_ = state.isDownConsensusHeld(consensusUp, now.Add(11*time.Minute))
+	assert.Nil(t, state.downSince)
+	assert.False(t, state.alertActive)
+}
+
+func TestConsensusState_RecoveryRequiresHeld(t *testing.T) {
+	state := &consensusState{}
+	now := time.Now()
+
+	// Pretend we already fired a down alert.
+	state.alertActive = true
+	t0 := now
+	state.downSince = &t0
+
+	// First up verdict stamps a fresh recovery timer; the alert
+	// must NOT fire yet.
+	assert.False(t, state.shouldFireRecovery(consensusUp, now.Add(time.Second)))
+	assert.Equal(t, consensusUp, state.lastVerdict)
+
+	// Stale up verdict (verdict != up) does not advance the
+	// recovery timer.
+	assert.False(t, state.shouldFireRecovery(consensusDown, now.Add(time.Minute)))
+
+	// Held long enough, recovery fires.
+	assert.True(t, state.shouldFireRecovery(consensusUp, now.Add(selfcheckConsensusWait+time.Second)))
+	state.markRecoveryFired()
+	assert.False(t, state.alertActive)
+	assert.Nil(t, state.downSince)
+}
+
+func TestConsensusState_NoQuorumHoldsState(t *testing.T) {
+	state := &consensusState{}
+	now := time.Now()
+
+	// First establish a down verdict + timer.
+	_ = state.isDownConsensusHeld(consensusDown, now)
+	stampBefore := *state.downSince
+	state.markDownAlertFired()
+
+	// NoQuorum must not reset the timer; the alert stays fired
+	// so the next down verdict does not double-fire.
+	_ = state.isDownConsensusHeld(consensusNoQuorum, now.Add(2*time.Minute))
+	assert.Equal(t, stampBefore, *state.downSince)
+	assert.True(t, state.alertActive)
+}
+
+func TestTallyConsensus(t *testing.T) {
+	up, down := true, false
+	peers := []peerObservation{
+		{Up: true}, {Up: false}, {Up: true},
+	}
+	gotUp, gotDown, gotTotal := tallyConsensus(&up, peers)
+	assert.Equal(t, 3, gotUp)
+	assert.Equal(t, 1, gotDown)
+	assert.Equal(t, 4, gotTotal)
+
+	gotUp, gotDown, gotTotal = tallyConsensus(&down, nil)
+	assert.Equal(t, 0, gotUp)
+	assert.Equal(t, 1, gotDown)
+	assert.Equal(t, 1, gotTotal)
+
+	gotUp, gotDown, gotTotal = tallyConsensus(nil, peers)
+	assert.Equal(t, 2, gotUp)
+	assert.Equal(t, 1, gotDown)
+	assert.Equal(t, 3, gotTotal)
+}
+
+func TestPeerCacheSnapshotAndPut(t *testing.T) {
+	c := newPeerCache()
+	c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
+	c.put(peerObservation{WorkerID: "b", Up: false, ObservedAt: time.Now()})
+
+	snap := c.snapshot()
+	assert.Len(t, snap, 2)
+}
+
+func TestPeerCacheResetForDropsRemovedPeers(t *testing.T) {
+	c := newPeerCache()
+	c.put(peerObservation{WorkerID: "a", Up: true, ObservedAt: time.Now()})
+	c.put(peerObservation{WorkerID: "b", Up: true, ObservedAt: time.Now()})
+
+	c.resetFor([]wire.PeerInfo{{WorkerID: "a"}, {WorkerID: "c"}})
+	snap := c.snapshot()
+	ids := map[string]peerObservation{}
+	for _, s := range snap {
+		ids[s.WorkerID] = s
+	}
+	_, hasA := ids["a"]
+	_, hasB := ids["b"]
+	_, hasC := ids["c"]
+	assert.True(t, hasA, "peer 'a' retained")
+	assert.False(t, hasB, "removed peer 'b' dropped")
+	assert.True(t, hasC, "new peer 'c' added")
+}
+
+func TestApplyInitStoresPeersAndResetsCache(t *testing.T) {
+	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
+	r := newTestRunner(t, 4, 1, executor)
+	require.NotNil(t, r.peerCache, "runner must own a peer cache after NewRunner")
+
+	r.applyInit(&wire.WorkerInit{
+		WorkerID:    "w-1",
+		Concurrency: 2,
+		Peers: []wire.PeerInfo{
+			{WorkerID: "w-2", URL: "http://127.0.0.1:27402"},
+			{WorkerID: "w-3", URL: "http://127.0.0.1:27403"},
+		},
+	})
+	got := r.Peers()
+	require.Len(t, got, 2)
+	assert.Equal(t, "w-2", got[0].WorkerID)
+	assert.Equal(t, "w-3", got[1].WorkerID)
+
+	// Cache should be primed with empty observations for the new
+	// peers; the per-peer entries are visible via snapshot().
+	snap := r.peerCache.snapshot()
+	assert.Len(t, snap, 2)
+	for _, s := range snap {
+		assert.True(t, s.ObservedAt.IsZero(),
+			"freshly-reset peer %s should have a zero observed_at", s.WorkerID)
+	}
+}
+
+func TestFetchPeerStatus_DecodesUpResponse(t *testing.T) {
+	up := true
+	now := time.Now().UTC().Truncate(time.Second)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(PeerStatus{
+			WorkerID:   "remote",
+			Up:         up,
+			ObservedAt: now,
+		})
+	}))
+	defer srv.Close()
+
+	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
+		WorkerID: "remote",
+		URL:      srv.URL,
+	})
+	assert.True(t, obs.Up)
+	assert.NoError(t, obs.Err)
+	assert.Equal(t, "remote", obs.WorkerID)
+	assert.Equal(t, now, obs.ObservedAt)
+}
+
+func TestFetchPeerStatus_NetworkErrorIsRecorded(t *testing.T) {
+	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
+		WorkerID: "unreachable",
+		URL:      "http://127.0.0.1:1",
+	})
+	assert.Error(t, obs.Err)
+	assert.False(t, obs.Up)
+}
+
+func TestFetchPeerStatus_BadURLRejected(t *testing.T) {
+	obs := fetchPeerStatus(context.Background(), &wire.PeerInfo{
+		WorkerID: "weird",
+		URL:      "ftp://example.com",
+	})
+	assert.Error(t, obs.Err)
+}
+
+func TestDecideConsensus_MultiWorkerQuorumGate(t *testing.T) {
+	// When the control plane has configured peers (a multi-worker
+	// install), a lone self vote must NOT be treated as a
+	// cluster-wide verdict in either direction. A second,
+	// independent vote is required to reach any verdict.
+	up := true
+	down := false
+	peersDown := []peerObservation{{WorkerID: "p1", Up: false, ObservedAt: time.Now()}}
+	peersUp := []peerObservation{{WorkerID: "p1", Up: true, ObservedAt: time.Now()}}
+
+	t.Run("self down + configured peers but no fresh peer = no quorum", func(t *testing.T) {
+		assert.Equal(t, consensusNoQuorum, decideConsensus(&down, nil, 2),
+			"a lone down vote must not page when peers are configured but not reporting")
+		assert.Equal(t, consensusNoQuorum, decideConsensus(&up, nil, 2),
+			"a lone up vote must not clear an active alert when peers are configured but not reporting")
+	})
+
+	t.Run("self down + one fresh down peer = down consensus", func(t *testing.T) {
+		assert.Equal(t, consensusDown, decideConsensus(&down, peersDown, 2))
+	})
+
+	t.Run("self up + one fresh up peer = up consensus", func(t *testing.T) {
+		assert.Equal(t, consensusUp, decideConsensus(&up, peersUp, 2))
+	})
+
+	t.Run("self down + one fresh up peer = no quorum (1/1 split)", func(t *testing.T) {
+		// With two voters and one of each, neither side has a
+		// majority, so the verdict stays NoQuorum. This is the
+		// intended conservative behavior — we don't want a
+		// single dissenting peer to override self.
+		assert.Equal(t, consensusNoQuorum, decideConsensus(&down, peersUp, 2))
+	})
+}
+
+func TestDecideConsensus_SingleNodePreserved(t *testing.T) {
+	// When no peers are configured (single-worker / no-peer
+	// deployment) the caller passes minVotes=1, which preserves
+	// the prior single-node behavior: a self-only vote is enough
+	// to drive a verdict in either direction.
+	up := true
+	down := false
+
+	assert.Equal(t, consensusUp, decideConsensus(&up, nil, 1),
+		"single-node self up must still reach up consensus")
+	assert.Equal(t, consensusDown, decideConsensus(&down, nil, 1),
+		"single-node self down must still reach down consensus")
+}
+
+func TestPeerObservationsForConsensus_DropsStale(t *testing.T) {
+	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
+	r := newTestRunner(t, 4, 1, executor)
+
+	now := time.Now()
+	fresh := peerObservation{WorkerID: "p1", Up: true, ObservedAt: now.Add(-30 * time.Second)}
+	stale := peerObservation{WorkerID: "p2", Up: true, ObservedAt: now.Add(-5 * time.Minute)}
+	errored := peerObservation{WorkerID: "p3", Up: true, Err: assert.AnError}
+	unknown := peerObservation{WorkerID: "p4"}
+	r.peerCache.put(fresh)
+	r.peerCache.put(stale)
+	r.peerCache.put(errored)
+	r.peerCache.put(unknown)
+
+	got := r.peerObservationsForConsensus(now)
+	require.Len(t, got, 1)
+	assert.Equal(t, "p1", got[0].WorkerID)
+}
+
+func TestPollPeersOnceIsConcurrencySafe(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(PeerStatus{
+			WorkerID:   "remote",
+			Up:         true,
+			ObservedAt: time.Now().UTC(),
+		})
+	}))
+	defer srv.Close()
+
+	executor := func(interface{}) interface{} { return []wire.CheckResultReport{} }
+	r := newTestRunner(t, 4, 1, executor)
+	r.applyInit(&wire.WorkerInit{
+		WorkerID: "w-1",
+		URL:      "http://127.0.0.1:1", // local self probe will fail, no effect on peer poll
+		Peers: []wire.PeerInfo{
+			{WorkerID: "p1", URL: srv.URL},
+		},
+	})
+
+	// Run multiple concurrent poll cycles. The cache uses a single
+	// RWMutex; this verifies there is no data race under that
+	// access pattern.
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			r.pollPeersOnce(context.Background())
+		}()
+	}
+	wg.Wait()
+	assert.GreaterOrEqual(t, atomic.LoadInt32(&hits), int32(1))
+}