package workercluster import ( "testing" "github.com/hashicorp/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestFSMApply_ConfigAdopt verifies the FSM accepts a config.adopt entry // and replaces the Config + ConfigVersion. func TestFSMApply_ConfigAdopt(t *testing.T) { fsm := NewFSM() checks := []CriticalCheckConfig{ {ID: 1, MonitorID: 11, Kind: "http", Target: "https://a", IntervalS: 30}, {ID: 2, MonitorID: 12, Kind: "ssl", Target: "b", IntervalS: 60}, } raw, err := jsonMarshal(ConfigAdoptPayload{Version: 7, Actor: "ctrl", Checks: checks}) require.NoError(t, err) resp := fsm.Apply(&raft.Log{Index: 1, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryConfigAdopt, nil, nil, nil, raw, nil, nil, nil)}) require.Nil(t, resp) stats := fsm.Stats() assert.EqualValues(t, 7, stats.ConfigVersion) assert.Equal(t, 2, stats.ConfigCount) } // TestFSMApply_ObserverSetUpdate verifies the observer-set update path. func TestFSMApply_ObserverSetUpdate(t *testing.T) { fsm := NewFSM() set := ObserverSet{Version: 3, ConfigVersion: 7, Voters: []string{"w1", "w2", "w3"}, Observers: []string{"w4"}} raw, err := jsonMarshal(ObserverSetPayload{Set: set}) require.NoError(t, err) resp := fsm.Apply(&raft.Log{Index: 2, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryObserverSetUpdate, raw, nil, nil, nil, nil, nil, nil)}) require.Nil(t, resp) stats := fsm.Stats() assert.EqualValues(t, 3, stats.ObserverSet.Version) assert.Equal(t, []string{"w1", "w2", "w3"}, stats.ObserverSet.Voters) assert.EqualValues(t, 2, stats.ObserverSet.AdoptedAtIx) } // TestFSMApply_MembershipProposeAdd verifies propose_add populates the // membership cache with the worker's joined-at index. func TestFSMApply_MembershipProposeAdd(t *testing.T) { fsm := NewFSM() m := Member{WorkerID: "w-1", WorkerURL: "http://w1", RaftAddress: "127.0.0.1:8001", Role: "voter"} raw, err := jsonMarshal(MemberPayload{Member: m}) require.NoError(t, err) resp := fsm.Apply(&raft.Log{Index: 3, Term: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)}) require.Nil(t, resp) stats := fsm.Stats() require.Contains(t, stats.Membership, "w-1") got := stats.Membership["w-1"] assert.Equal(t, "voter", got.Role) assert.EqualValues(t, 3, got.JoinedAtIx) } // TestFSMApply_MembershipDemote flips a voter to observer. func TestFSMApply_MembershipDemote(t *testing.T) { fsm := NewFSM() addRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, addRaw, nil, nil, nil, nil, nil)})) demRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "observer"}, Reason: "drain"}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipDemote, nil, demRaw, nil, nil, nil, nil, nil)})) got := fsm.Membership["w-1"] assert.Equal(t, "observer", got.Role) assert.EqualValues(t, 2, got.LastSeenIx) } // TestFSMApply_MembershipRemove drops a member from the cache. func TestFSMApply_MembershipRemove(t *testing.T) { fsm := NewFSM() addRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, addRaw, nil, nil, nil, nil, nil)})) rmRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1"}, Reason: "gone"}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipRemove, nil, rmRaw, nil, nil, nil, nil, nil)})) _, ok := fsm.Membership["w-1"] assert.False(t, ok, "member should be removed") } // TestFSMApply_OutboxEnqueue verifies the outbox grows and the seq is // stamped automatically. func TestFSMApply_OutboxEnqueue(t *testing.T) { fsm := NewFSM() raw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, TenantID: 9, Channel: "telegram", DedupKey: "notif:1:telegram:c"}}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, raw)})) require.Len(t, fsm.Outbox, 1) assert.EqualValues(t, 1, fsm.Outbox[0].Seq) assert.Equal(t, "pending", fsm.Outbox[0].State) } // TestFSMApply_OutboxDelivered verifies the delivered update path. func TestFSMApply_OutboxDelivered(t *testing.T) { fsm := NewFSM() raw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, Channel: "telegram"}}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, raw)})) upd, err := jsonMarshal(OutboxUpdatePayload{Seq: 1, State: "sent", Attempts: 1}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxDelivered, nil, nil, nil, nil, nil, nil, upd)})) assert.Equal(t, "sent", fsm.Outbox[0].State) assert.Equal(t, 1, fsm.Outbox[0].Attempts) } // TestFSMSnapshot_Restore exercises the snapshot+restore cycle. We // populate the FSM, snapshot it, restore into a fresh FSM, and verify // the state matches exactly. func TestFSMSnapshot_Restore(t *testing.T) { src := NewFSM() checks := []CriticalCheckConfig{ {ID: 1, MonitorID: 11, Kind: "http", Target: "https://a", IntervalS: 30}, } rawC, err := jsonMarshal(ConfigAdoptPayload{Version: 9, Checks: checks}) require.NoError(t, err) require.Nil(t, src.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryConfigAdopt, nil, nil, nil, rawC, nil, nil, nil)})) memberRaw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter", WorkerURL: "http://w1"}}) require.NoError(t, err) require.Nil(t, src.Apply(&raft.Log{Index: 2, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, memberRaw, nil, nil, nil, nil, nil)})) setRaw, err := jsonMarshal(ObserverSetPayload{Set: ObserverSet{Version: 4, Voters: []string{"w-1"}}}) require.NoError(t, err) require.Nil(t, src.Apply(&raft.Log{Index: 3, Type: raft.LogCommand, Data: mustEntry(t, EntryObserverSetUpdate, setRaw, nil, nil, nil, nil, nil, nil)})) outRaw, err := jsonMarshal(OutboxPayload{Entry: OutboxMeta{IncidentID: 1, Channel: "telegram"}}) require.NoError(t, err) require.Nil(t, src.Apply(&raft.Log{Index: 4, Type: raft.LogCommand, Data: mustEntry(t, EntryOutboxEnqueue, nil, nil, nil, nil, nil, nil, outRaw)})) snap, err := src.Snapshot() require.NoError(t, err) var buf bytesBuffer require.NoError(t, snap.Persist(&buf)) dst := NewFSM() require.NoError(t, dst.Restore(&buf)) s := dst.Stats() assert.EqualValues(t, 9, s.ConfigVersion) assert.Equal(t, 1, s.ConfigCount) assert.Equal(t, 1, s.Members) assert.Equal(t, 1, s.OutboxLen) assert.EqualValues(t, 4, s.ObserverSet.Version) assert.Equal(t, "w-1", s.Membership["w-1"].WorkerID) } // TestFSMSnapshot_SnapshotIsIsolatedFromLive confirms that a snapshot // taken from one FSM does not see updates made after the snapshot is // captured. func TestFSMSnapshot_SnapshotIsIsolatedFromLive(t *testing.T) { fsm := NewFSM() snap, err := fsm.Snapshot() require.NoError(t, err) raw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}}) require.NoError(t, err) require.Nil(t, fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)})) var buf bytesBuffer require.NoError(t, snap.Persist(&buf)) dst := NewFSM() require.NoError(t, dst.Restore(&buf)) assert.Equal(t, 0, dst.Stats().Members, "snapshot taken before Apply must not contain the new member") } // TestFSMApply_UnknownEntryTypeReturnsError verifies Apply returns an // error for unknown entry kinds. func TestFSMApply_UnknownEntryTypeReturnsError(t *testing.T) { fsm := NewFSM() resp := fsm.Apply(&raft.Log{Index: 1, Type: raft.LogCommand, Data: mustEntry(t, "weird.thing", nil, nil, nil, nil, nil, nil, nil)}) err, _ := resp.(error) require.Error(t, err) } // TestFSMSnapshot_StatsIsConsistent confirms Stats() returns a coherent // view after a batch of Applies. The raft library calls Apply // serially, so we do not need to lock inside Apply; this test // exercises the same pattern at the application level. func TestFSMSnapshot_StatsIsConsistent(t *testing.T) { fsm := NewFSM() raw, err := jsonMarshal(MemberPayload{Member: Member{WorkerID: "w-1", Role: "voter"}}) require.NoError(t, err) for i := uint64(1); i <= 50; i++ { _ = fsm.Apply(&raft.Log{Index: i, Type: raft.LogCommand, Data: mustEntry(t, EntryMembershipProposeAdd, nil, raw, nil, nil, nil, nil, nil)}) } s := fsm.Stats() require.Equal(t, 1, s.Members, "single member") require.Equal(t, "w-1", s.Membership["w-1"].WorkerID) }