func TestFailedElection(t *testing.T) {
log.SetOutput(&bytes.Buffer{})
defer log.SetOutput(os.Stdout)
oldMin, oldMax := raft.ResetElectionTimeoutMs(25, 50)
defer raft.ResetElectionTimeoutMs(oldMin, oldMax)
noop := func([]byte) ([]byte, error) { return []byte{}, nil }
server := raft.NewServer(1, &bytes.Buffer{}, noop)
server.SetPeers(raft.MakePeers(disapprovingPeer(2), nonresponsivePeer(3)))
server.Start()
defer func() { server.Stop(); t.Logf("server stopped") }()
time.Sleep(2 * raft.ElectionTimeout())
if server.State() == raft.Leader {
t.Fatalf("erroneously became Leader")
}
t.Logf("remained %s", server.State())
}
func testOrder(t *testing.T, nServers int) {
values := rand.Perm(8 + rand.Intn(16))
// command and response
type send struct {
Send int `json:"send"`
}
type recv struct {
Recv int `json:"recv"`
}
do := func(sb *synchronizedBuffer) func(buf []byte) ([]byte, error) {
return func(buf []byte) ([]byte, error) {
sb.Write(buf) // write incoming message
var s send // decode incoming message
json.Unmarshal(buf, &s) // ...
return json.Marshal(recv{Recv: s.Send}) // write outgoing message
}
}
// set up the cluster
servers := []*raft.Server{} // server components
storage := []*bytes.Buffer{} // persistent log storage
buffers := []*synchronizedBuffer{} // the "state machine" for each server
for i := 0; i < nServers; i++ {
buffers = append(buffers, &synchronizedBuffer{})
storage = append(storage, &bytes.Buffer{})
servers = append(servers, raft.NewServer(uint64(i+1), storage[i], do(buffers[i])))
}
peers := raft.Peers{}
for _, server := range servers {
peers[server.Id()] = raft.NewLocalPeer(server)
}
for _, server := range servers {
server.SetPeers(peers)
}
// define cmds
cmds := []send{}
for _, v := range values {
cmds = append(cmds, send{v})
}
// the expected "state-machine" output of applying each command
expectedBuffer := &synchronizedBuffer{}
for _, cmd := range cmds {
buf, _ := json.Marshal(cmd)
expectedBuffer.Write(buf)
}
// boot up the cluster
for _, server := range servers {
server.Start()
defer func(server0 *raft.Server) {
log.Printf("issuing stop command to server %d", server0.Id())
server0.Stop()
}(server)
}
// send commands
for i, cmd := range cmds {
id := uint64(rand.Intn(nServers)) + 1
peer := peers[id]
buf, _ := json.Marshal(cmd)
response := make(chan []byte, 1)
retry:
for {
log.Printf("command=%d/%d peer=%d: sending %s", i+1, len(cmds), id, buf)
switch err := peer.Command(buf, response); err {
case nil:
log.Printf("command=%d/%d peer=%d: OK", i+1, len(cmds), id)
break retry
case raft.ErrUnknownLeader, raft.ErrDeposed:
log.Printf("command=%d/%d peer=%d: failed (%s) -- will retry", i+1, len(cmds), id, err)
time.Sleep(raft.ElectionTimeout())
continue
case raft.ErrTimeout:
log.Printf("command=%d/%d peer=%d: timed out -- assume it went through", i+1, len(cmds), id)
break retry
default:
t.Fatalf("command=%d/%d peer=%d: failed (%s) -- fatal", i+1, len(cmds), id, err)
}
}
r, ok := <-response
if !ok {
log.Printf("command=%d/%d peer=%d: truncated, will retry", i+1, len(cmds), id)
response = make(chan []byte, 1) // channel was closed, must re-make
goto retry
}
log.Printf("command=%d/%d peer=%d: OK, got response %s", i+1, len(cmds), id, string(r))
}
// done sending
log.Printf("testOrder done sending %d command(s) to network", len(cmds))
// check the buffers (state machines)
for i, sb := range buffers {
for {
expected, got := expectedBuffer.String(), sb.String()
if len(got) < len(expected) {
t.Logf("server %d: not yet fully replicated, will check again", i+1)
time.Sleep(raft.BroadcastInterval())
continue // retry
}
if expected != got {
t.Errorf("server %d: fully replicated, expected\n\t%s, got\n\t%s", i+1, expected, got)
break
}
t.Logf("server %d: %s OK", i+1, got)
break
}
}
}