worker/internal/workercluster/fsm_test.go

package workercluster

import (
	"testing"

	"github.com/hashicorp/raft"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestFSMApply_ConfigAdopt verifies the FSM accepts a config.adopt entry
// and replaces the Config + ConfigVersion.
func TestFSMApply_ConfigAdopt(t *testing.T) {
	fsm := NewFSM()

	checks := []CriticalCheckConfig{
		{ID: 1, MonitorID: 11, Kind: "http", Target: "https://a", IntervalS: 30},
		{ID: 2, MonitorID: 12, Kind: "ssl", Target: "b", IntervalS: 60},
	}
	raw, err := jsonMarshal(ConfigAdoptPayload{Version: 7, Actor: "ctrl", Checks: checks})
	require.NoError(t, err)

	resp := fsm.Apply(&raft.Log{Index: 1, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryConfigAdopt, nil, nil, nil, raw, nil, nil, nil)})
	require.Nil(t, resp)

	stats := fsm.Stats()
	assert.EqualValues(t, 7, stats.ConfigVersion)
	assert.Equal(t, 2, stats.ConfigCount)
}

// TestFSMApply_ObserverSetUpdate verifies the observer-set update path.
func TestFSMApply_ObserverSetUpdate(t *testing.T) {
	fsm := NewFSM()

	set := ObserverSet{Version: 3, ConfigVersion: 7, Voters: []string{"w1", "w2", "w3"}, Observers: []string{"w4"}}
	raw, err := jsonMarshal(ObserverSetPayload{Set: set})
	require.NoError(t, err)

	resp := fsm.Apply(&raft.Log{Index: 2, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryObserverSetUpdate, raw, nil, nil, nil, nil, nil, nil)})
	require.Nil(t, resp)

	stats := fsm.Stats()
	assert.EqualValues(t, 3, stats.ObserverSet.Version)
	assert.Equal(t, []string{"w1", "w2", "w3"}, stats.ObserverSet.Voters)
	assert.EqualValues(t, 2, stats.ObserverSet.AdoptedAtIx)
}

// TestFSMApply_MembershipProposeAdd verifies propose_add populates the
// membership cache with the worker's joined-at index.
func TestFSMApply_MembershipProposeAdd(t *testing.T) {
	fsm := NewFSM()

	m := Member{WorkerID: "w-1", WorkerURL: "http://w1", RaftAddress: "127.0.0.1:8001", Role: "voter"}
	raw, err := jsonMarshal(MemberPayload{Member: m})
	require.NoError(t, err)

	resp := fsm.Apply(&raft.Log{Index: 3, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)})
	require.Nil(t, resp)

	stats := fsm.Stats()
	require.Contains(t, stats.Membership, "w-1")
	got := stats.Membership["w-1"]
	assert.Equal(t, "voter", got.Role)
	assert.EqualValues(t, 3, got.JoinedAtIx)
}

// TestFSMApply_MembershipDemote flips a voter to observer.
func TestFSMApply_MembershipDemote(t *testing.T) {
	fsm := NewFSM()

	addRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, addRaw, nil, nil, nil, nil, nil)}))

	demRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "observer"}, Reason: "drain"})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipDemote, nil, demRaw, nil, nil, nil, nil, nil)}))

	got := fsm.Membership["w-1"]
	assert.Equal(t, "observer", got.Role)
	assert.EqualValues(t, 2, got.LastSeenIx)
}

// TestFSMApply_MembershipRemove drops a member from the cache.
func TestFSMApply_MembershipRemove(t *testing.T) {
	fsm := NewFSM()

	addRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, addRaw, nil, nil, nil, nil, nil)}))

	rmRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1"}, Reason: "gone"})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipRemove, nil, rmRaw, nil, nil, nil, nil, nil)}))

	_, ok := fsm.Membership["w-1"]
	assert.False(t, ok, "member should be removed")
}

// TestFSMApply_OutboxEnqueue verifies the outbox grows and the seq is
// stamped automatically.
func TestFSMApply_OutboxEnqueue(t *testing.T) {
	fsm := NewFSM()

	raw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, TenantID: 9, Channel: "telegram", DedupKey: "notif:1:telegram:c"}})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, raw)}))

	require.Len(t, fsm.Outbox, 1)
	assert.EqualValues(t, 1, fsm.Outbox[0].Seq)
	assert.Equal(t, "pending", fsm.Outbox[0].State)
}

// TestFSMApply_OutboxDelivered verifies the delivered update path.
func TestFSMApply_OutboxDelivered(t *testing.T) {
	fsm := NewFSM()

	raw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, Channel: "telegram"}})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, raw)}))

	upd, err := jsonMarshal(OutboxUpdatePayload{Seq: 1, State: "sent", Attempts: 1})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxDelivered, nil, nil, nil, nil, nil, nil, upd)}))

	assert.Equal(t, "sent", fsm.Outbox[0].State)
	assert.Equal(t, 1, fsm.Outbox[0].Attempts)
}

// TestFSMSnapshot_Restore exercises the snapshot+restore cycle. We
// populate the FSM, snapshot it, restore into a fresh FSM, and verify
// the state matches exactly.
func TestFSMSnapshot_Restore(t *testing.T) {
	src := NewFSM()

	checks := []CriticalCheckConfig{
		{ID: 1, MonitorID: 11, Kind: "http", Target: "https://a", IntervalS: 30},
	}
	rawC, err := jsonMarshal(ConfigAdoptPayload{Version: 9, Checks: checks})
	require.NoError(t, err)
	require.Nil(t, src.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryConfigAdopt, nil, nil, nil, rawC, nil, nil, nil)}))

	memberRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter", WorkerURL: "http://w1"}})
	require.NoError(t, err)
	require.Nil(t, src.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, memberRaw, nil, nil, nil, nil, nil)}))

	setRaw, err := jsonMarshal(ObserverSetPayload{Set: ObserverSet{Version: 4, Voters: []string{"w-1"}}})
	require.NoError(t, err)
	require.Nil(t, src.Apply(&raft.Log{Index: 3, Type: raft.LogCommand, Data: mustEntry(t, EntryObserverSetUpdate, setRaw, nil, nil, nil, nil, nil, nil)}))

	outRaw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, Channel: "telegram"}})
	require.NoError(t, err)
	require.Nil(t, src.Apply(&raft.Log{Index: 4, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, outRaw)}))

	snap, err := src.Snapshot()
	require.NoError(t, err)

	var buf bytesBuffer
	require.NoError(t, snap.Persist(&buf))

	dst := NewFSM()
	require.NoError(t, dst.Restore(&buf))

	s := dst.Stats()
	assert.EqualValues(t, 9, s.ConfigVersion)
	assert.Equal(t, 1, s.ConfigCount)
	assert.Equal(t, 1, s.Members)
	assert.Equal(t, 1, s.OutboxLen)
	assert.EqualValues(t, 4, s.ObserverSet.Version)
	assert.Equal(t, "w-1", s.Membership["w-1"].WorkerID)
}

// TestFSMSnapshot_SnapshotIsIsolatedFromLive confirms that a snapshot
// taken from one FSM does not see updates made after the snapshot is
// captured.
func TestFSMSnapshot_SnapshotIsIsolatedFromLive(t *testing.T) {
	fsm := NewFSM()

	snap, err := fsm.Snapshot()
	require.NoError(t, err)

	raw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}})
	require.NoError(t, err)
	require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)}))

	var buf bytesBuffer
	require.NoError(t, snap.Persist(&buf))

	dst := NewFSM()
	require.NoError(t, dst.Restore(&buf))
	assert.Equal(t, 0, dst.Stats().Members, "snapshot taken before Apply must not contain the new member")
}

// TestFSMApply_UnknownEntryTypeReturnsError verifies Apply returns an
// error for unknown entry kinds.
func TestFSMApply_UnknownEntryTypeReturnsError(t *testing.T) {
	fsm := NewFSM()
	resp := fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, "weird.thing", nil, nil, nil, nil, nil, nil, nil)})
	err, _ := resp.(error)
	require.Error(t, err)
}

// TestFSMSnapshot_StatsIsConsistent confirms Stats() returns a coherent
// view after a batch of Applies. The raft library calls Apply
// serially, so we do not need to lock inside Apply; this test
// exercises the same pattern at the application level.
func TestFSMSnapshot_StatsIsConsistent(t *testing.T) {
	fsm := NewFSM()

	raw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}})
	require.NoError(t, err)

	for i := uint64(1); i <= 50; i++ {
		_ = fsm.Apply(&raft.Log{Index: i, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)})
	}
	s := fsm.Stats()
	require.Equal(t, 1, s.Members, "single member")
	require.Equal(t, "w-1", s.Membership["w-1"].WorkerID)
}