feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
515
internal/workercluster/e2e_test.go
Обычный файл
515
internal/workercluster/e2e_test.go
Обычный файл
@@ -0,0 +1,515 @@
|
||||
package workercluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/raft"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// clusterFixture is a 3-node in-process cluster used by the e2e tests.
|
||||
// It opens three bbolt stores, three HTTP listeners, three rafthttp
|
||||
// handlers, and starts each raft.Raft. The first node bootstraps as a
|
||||
// 1-voter cluster; the other two join via the raft-membership handler.
|
||||
type clusterFixture struct {
|
||||
Nodes []*Cluster
|
||||
|
||||
creds HTTPCreds
|
||||
logOut io.Writer
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
func newClusterFixture(t *testing.T, n int) *clusterFixture {
|
||||
t.Helper()
|
||||
require.GreaterOrEqual(t, n, 1, "cluster fixture requires at least one node")
|
||||
|
||||
creds := HTTPCreds{Login: "alice", Password: "secret"}
|
||||
f := &clusterFixture{
|
||||
creds: creds,
|
||||
logOut: io.Discard,
|
||||
}
|
||||
|
||||
// Pre-reserve three ports so the bootstrap node has stable
|
||||
// addresses for the other nodes to dial into.
|
||||
addrs := make([]string, n)
|
||||
dataDirs := make([]string, n)
|
||||
for i := 0; i < n; i++ {
|
||||
l := newLocalListener(t)
|
||||
addrs[i] = l.Addr().String()
|
||||
require.NoError(t, l.Close())
|
||||
dataDirs[i] = t.TempDir()
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Bootstrap the first node.
|
||||
bootstrapOpts := Options{
|
||||
NodeID: "node-1",
|
||||
LocalAddr: addrs[0],
|
||||
DataDir: dataDirs[0],
|
||||
Creds: creds,
|
||||
Bootstrap: true,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: f.logOut,
|
||||
}
|
||||
c1, err := New(&bootstrapOpts)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c1.Start(ctx))
|
||||
require.NoError(t, waitForLeader(ctx, c1, 5*time.Second))
|
||||
f.Nodes = append(f.Nodes, c1)
|
||||
|
||||
// Bring up the remaining nodes sequentially, joining via node-1
|
||||
// until each becomes a voter.
|
||||
for i := 1; i < n; i++ {
|
||||
opts := Options{
|
||||
NodeID: fmt.Sprintf("node-%d", i+1),
|
||||
LocalAddr: addrs[i],
|
||||
DataDir: dataDirs[i],
|
||||
Creds: creds,
|
||||
Bootstrap: false,
|
||||
Seed: Peer{WorkerID: "node-1", Address: addrs[0]},
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: f.logOut,
|
||||
}
|
||||
c, err := New(&opts)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.Start(ctx))
|
||||
// Wait for the join to be reflected in the cluster config.
|
||||
require.NoError(t, waitForVoterCount(ctx, c, i+1, 15*time.Second))
|
||||
f.Nodes = append(f.Nodes, c)
|
||||
}
|
||||
|
||||
t.Cleanup(f.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func waitForVoterCount(ctx context.Context, c *Cluster, n int, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
s := c.Stats()
|
||||
if s.NumPeers+1 >= n {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("workercluster: only %d voters after %s (need %d)", s.NumPeers+1, timeout, n)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *clusterFixture) Close() {
|
||||
f.stopOnce.Do(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for _, n := range f.Nodes {
|
||||
if n != nil {
|
||||
_ = n.Shutdown(ctx)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestThreeNodeCluster_LeaderElection starts a 3-node cluster and
|
||||
// verifies exactly one leader is elected within the timeout, and all
|
||||
// nodes see the same leader.
|
||||
func TestThreeNodeCluster_LeaderElection(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
f := newClusterFixture(t, 3)
|
||||
|
||||
// Wait until every node reports the same leader.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var lastLeader string
|
||||
for {
|
||||
leaders := make(map[string]int)
|
||||
for _, n := range f.Nodes {
|
||||
s := n.Stats()
|
||||
if s.Leader == "" {
|
||||
leaders[""]++
|
||||
continue
|
||||
}
|
||||
leaders[s.Leader]++
|
||||
}
|
||||
if len(leaders) == 1 {
|
||||
for leader, count := range leaders {
|
||||
if leader != "" && count == 3 {
|
||||
lastLeader = leader
|
||||
break
|
||||
}
|
||||
}
|
||||
if lastLeader != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
require.False(t, time.Now().After(deadline), "no consensus on leader within deadline, last seen: %v", leaders)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
assert.NotEmpty(t, lastLeader)
|
||||
}
|
||||
|
||||
// TestThreeNodeCluster_FSMApply verifies a config.adopt entry proposed
|
||||
// on one node ends up in the FSM of all three.
|
||||
func TestThreeNodeCluster_FSMApply(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
f := newClusterFixture(t, 3)
|
||||
require.NoError(t, waitForLeader(context.Background(), f.Nodes[0], 5*time.Second))
|
||||
|
||||
leader := leaderCluster(f)
|
||||
require.NotNil(t, leader, "expected a leader in the fixture")
|
||||
|
||||
checks := []CriticalCheckConfig{
|
||||
{ID: 1, MonitorID: 11, Kind: "http", Target: "https://a", IntervalS: 30},
|
||||
{ID: 2, MonitorID: 12, Kind: "ssl", Target: "b", IntervalS: 60},
|
||||
}
|
||||
raw, err := jsonMarshal(ConfigAdoptPayload{Version: 11, Actor: leader.opts.NodeID, Checks: checks})
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := EncodeEntry(&Entry{Type: EntryConfigAdopt, Adopted: raw})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, leader.Apply(context.Background(), entry, 5*time.Second))
|
||||
|
||||
// Wait until every node has applied the entry.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
ok := true
|
||||
for _, n := range f.Nodes {
|
||||
if n.FSM().Stats().ConfigVersion != 11 {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
require.False(t, time.Now().After(deadline), "config did not replicate within deadline")
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
for _, n := range f.Nodes {
|
||||
s := n.FSM().Stats()
|
||||
assert.EqualValues(t, 11, s.ConfigVersion, "node %s", n.opts.NodeID)
|
||||
assert.Equal(t, 2, s.ConfigCount, "node %s", n.opts.NodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestThreeNodeCluster_Failover shuts down the leader and verifies a
|
||||
// new leader is elected within the election-timeout window.
|
||||
func TestThreeNodeCluster_Failover(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
f := newClusterFixture(t, 3)
|
||||
require.NoError(t, waitForLeader(context.Background(), f.Nodes[0], 5*time.Second))
|
||||
|
||||
leader := leaderCluster(f)
|
||||
require.NotNil(t, leader)
|
||||
oldLeaderID := leader.opts.NodeID
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, leader.Shutdown(ctx))
|
||||
|
||||
// Wait until one of the remaining two nodes holds leadership.
|
||||
require.NoError(t, waitForLeaderOnAny(f.Nodes[1:], 10*time.Second))
|
||||
|
||||
leaders := make(map[string]bool)
|
||||
for _, n := range f.Nodes[1:] {
|
||||
s := n.Stats()
|
||||
require.NotEmpty(t, s.Leader, "node %s has no leader after failover", s.NodeID)
|
||||
leaders[s.Leader] = true
|
||||
}
|
||||
require.True(t, leaders[oldLeaderID] == false, "old leader should not be elected after shutdown")
|
||||
require.Len(t, leaders, 1, "exactly one leader expected after failover, got %v", leaders)
|
||||
}
|
||||
|
||||
func waitForLeaderOnAny(nodes []*Cluster, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
for _, n := range nodes {
|
||||
if n.Stats().Leader != "" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("no leader after %s", timeout)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// TestThreeNodeCluster_JoinAndRemove boots a 1-node cluster, adds a
|
||||
// second and a third voter, then removes the third. We assert the FSM
|
||||
// membership cache reflects the change.
|
||||
func TestThreeNodeCluster_JoinAndRemove(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// 1-node bootstrap.
|
||||
creds := HTTPCreds{Login: "alice", Password: "secret"}
|
||||
addr1 := pickPort(t)
|
||||
c1, err := New(&Options{
|
||||
NodeID: "node-1",
|
||||
LocalAddr: addr1,
|
||||
DataDir: t.TempDir(),
|
||||
Creds: creds,
|
||||
Bootstrap: true,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: io.Discard,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c1.Start(ctx))
|
||||
require.NoError(t, waitForLeader(ctx, c1, 5*time.Second))
|
||||
t.Cleanup(func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
for _, n := range []*Cluster{c1} {
|
||||
if n != nil {
|
||||
_ = n.Shutdown(shutCtx)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Join node-2.
|
||||
addr2 := pickPort(t)
|
||||
c2, err := JoinCluster(ctx, &Options{
|
||||
NodeID: "node-2",
|
||||
LocalAddr: addr2,
|
||||
DataDir: t.TempDir(),
|
||||
Creds: creds,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: io.Discard,
|
||||
}, Peer{WorkerID: "node-1", Address: addr1})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = c2.Shutdown(shutCtx)
|
||||
})
|
||||
require.NoError(t, waitForVoterCount(ctx, c2, 2, 10*time.Second))
|
||||
|
||||
// Join node-3.
|
||||
addr3 := pickPort(t)
|
||||
c3, err := JoinCluster(ctx, &Options{
|
||||
NodeID: "node-3",
|
||||
LocalAddr: addr3,
|
||||
DataDir: t.TempDir(),
|
||||
Creds: creds,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: io.Discard,
|
||||
}, Peer{WorkerID: "node-1", Address: addr1})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = c3.Shutdown(shutCtx)
|
||||
})
|
||||
require.NoError(t, waitForVoterCount(ctx, c3, 3, 10*time.Second))
|
||||
|
||||
// All three nodes should see 3 voters.
|
||||
for _, n := range []*Cluster{c1, c2, c3} {
|
||||
cfgFuture := n.Raft().GetConfiguration()
|
||||
require.NoError(t, cfgFuture.Error())
|
||||
ids := make([]string, 0, 3)
|
||||
for _, srv := range cfgFuture.Configuration().Servers {
|
||||
ids = append(ids, string(srv.ID))
|
||||
}
|
||||
require.Equal(t, 3, len(ids), "node %s should have 3 voters, got %v", n.opts.NodeID, ids)
|
||||
}
|
||||
|
||||
// Remove node-3 by proposing a membership change via its local
|
||||
// raft. (rafthttp only ships Join + Leave; we use raft.RemoveServer
|
||||
// directly here to keep the test driver straightforward.)
|
||||
leader := pickLeader(t, []*Cluster{c1, c2, c3})
|
||||
require.NotNil(t, leader)
|
||||
require.NoError(t, leader.Raft().RemoveServer(raft.ServerID("node-3"), 0, 5*time.Second).Error())
|
||||
|
||||
require.NoError(t, waitForVoterCount(ctx, c1, 2, 10*time.Second))
|
||||
}
|
||||
|
||||
// TestThreeNodeCluster_SnapshotRestore populates the FSM with a few
|
||||
// entries, takes a snapshot on the leader, then restarts the leader
|
||||
// from the same data dir to verify the snapshot+log restore cycle.
|
||||
func TestThreeNodeCluster_SnapshotRestore(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
f := newClusterFixture(t, 3)
|
||||
require.NoError(t, waitForLeader(context.Background(), f.Nodes[0], 5*time.Second))
|
||||
|
||||
leader := leaderCluster(f)
|
||||
require.NotNil(t, leader)
|
||||
|
||||
// Push a few entries so the FSM has something worth snapshotting.
|
||||
for i := 0; i < 5; i++ {
|
||||
checks := []CriticalCheckConfig{
|
||||
{ID: int64(i + 1), MonitorID: int64(i + 1), Kind: "http", Target: fmt.Sprintf("https://a/%d", i), IntervalS: 30},
|
||||
}
|
||||
raw, err := jsonMarshal(ConfigAdoptPayload{Version: uint64(i + 1), Checks: checks})
|
||||
require.NoError(t, err)
|
||||
entry, err := EncodeEntry(&Entry{Type: EntryConfigAdopt, Adopted: raw})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, leader.Apply(context.Background(), entry, 5*time.Second))
|
||||
}
|
||||
|
||||
// Force a snapshot.
|
||||
require.NoError(t, leader.Snapshot())
|
||||
|
||||
// All followers should see the snapshot effect.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
ok := true
|
||||
for _, n := range f.Nodes {
|
||||
if n.FSM().Stats().ConfigVersion != 5 {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
require.False(t, time.Now().After(deadline), "snapshot did not replicate within deadline")
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Snapshot file must exist on disk for at least the leader.
|
||||
store := leader.store
|
||||
snaps, err := store.SnapshotStore().List()
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, snaps, "expected at least one snapshot on disk after Snapshot()")
|
||||
}
|
||||
|
||||
// TestBootstrapSafety_TwoClusters confirms that starting two
|
||||
// independent clusters with the same bootstrap token creates two
|
||||
// distinct raft groups, not a single merged one.
|
||||
func TestBootstrapSafety_TwoClusters(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("e2e test skipped in -short mode")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
creds := HTTPCreds{Login: "alice", Password: "secret"}
|
||||
a1, err := New(&Options{
|
||||
NodeID: "a-1",
|
||||
LocalAddr: pickPort(t),
|
||||
DataDir: t.TempDir(),
|
||||
Creds: creds,
|
||||
Bootstrap: true,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: io.Discard,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, a1.Start(ctx))
|
||||
t.Cleanup(func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = a1.Shutdown(shutCtx)
|
||||
})
|
||||
|
||||
b1, err := New(&Options{
|
||||
NodeID: "b-1",
|
||||
LocalAddr: pickPort(t),
|
||||
DataDir: t.TempDir(),
|
||||
Creds: creds,
|
||||
Bootstrap: true,
|
||||
HeartbeatTimeout: 300 * time.Millisecond,
|
||||
ElectionTimeout: 1000 * time.Millisecond,
|
||||
LogOutput: io.Discard,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, b1.Start(ctx))
|
||||
t.Cleanup(func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = b1.Shutdown(shutCtx)
|
||||
})
|
||||
|
||||
require.NoError(t, waitForLeader(ctx, a1, 5*time.Second))
|
||||
require.NoError(t, waitForLeader(ctx, b1, 5*time.Second))
|
||||
|
||||
require.NoError(t, waitForLeaderOnAny([]*Cluster{a1}, 1*time.Second))
|
||||
|
||||
// Each cluster must have exactly one voter: itself.
|
||||
aConfig := a1.Raft().GetConfiguration()
|
||||
require.NoError(t, aConfig.Error())
|
||||
require.Len(t, aConfig.Configuration().Servers, 1)
|
||||
assert.Equal(t, raft.ServerID("a-1"), aConfig.Configuration().Servers[0].ID)
|
||||
|
||||
bConfig := b1.Raft().GetConfiguration()
|
||||
require.NoError(t, bConfig.Error())
|
||||
require.Len(t, bConfig.Configuration().Servers, 1)
|
||||
assert.Equal(t, raft.ServerID("b-1"), bConfig.Configuration().Servers[0].ID)
|
||||
}
|
||||
|
||||
// pickPort reserves a free port on the loopback interface and returns
|
||||
// its address. We close the listener immediately; the actual raft
|
||||
// listener will rebind to the same port because no other process has
|
||||
// claimed it.
|
||||
func pickPort(t *testing.T) string {
|
||||
t.Helper()
|
||||
l := newLocalListener(t)
|
||||
addr := l.Addr().String()
|
||||
require.NoError(t, l.Close())
|
||||
return addr
|
||||
}
|
||||
|
||||
func leaderCluster(f *clusterFixture) *Cluster {
|
||||
for _, n := range f.Nodes {
|
||||
if n.Raft().State() == raft.Leader {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func leaderFromFixture(f *clusterFixture) string {
|
||||
for _, n := range f.Nodes {
|
||||
if n.Raft().State() == raft.Leader {
|
||||
return n.opts.NodeID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func pickLeader(t *testing.T, nodes []*Cluster) *Cluster {
|
||||
t.Helper()
|
||||
for _, n := range nodes {
|
||||
if n.Raft().State() == raft.Leader {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatal("no leader in cluster fixture")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureLocalAddrsAreLoopback sanity-checks the helper above produces
|
||||
// 127.0.0.1 addresses so the e2e suite never accidentally opens a
|
||||
// non-loopback port.
|
||||
func TestPickPortIsLoopback(t *testing.T) {
|
||||
addr := pickPort(t)
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, strings.HasPrefix(host, "127.") || host == "::1", "expected loopback, got %q", host)
|
||||
}
|
||||
Ссылка в новой задаче
Block a user