// #RFS-L2a: If Command received from client: append entry to local log
func TestCM_Leader_AppendCommand(t *testing.T) {
mcm, _ := testSetupMCM_Leader_Figure7LeaderLine(t)
// pre check
iole, err := mcm.pcm.LogRO.GetIndexOfLastEntry()
if err != nil {
t.Fatal()
}
if iole != 10 {
t.Fatal()
}
ioleAC, err := mcm.pcm.AppendCommand(testhelpers.DummyCommand(1101))
if err != nil || ioleAC != 11 {
t.Fatal()
}
iole, err = mcm.pcm.LogRO.GetIndexOfLastEntry()
if err != nil {
t.Fatal()
}
if iole != 11 {
t.Fatal()
}
le := testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 11)
if !reflect.DeepEqual(le, LogEntry{8, Command("c1101")}) {
t.Fatal(le)
}
}
func TestConsensusModule_AppendCommand_Leader(t *testing.T) {
cm, mrs := setupConsensusModuleR2(t, testdata.TestUtil_MakeFigure7LeaderLineTerms())
defer cm.Stop()
testConsensusModule_RpcReplyCallback_AndBecomeLeader(t, cm, mrs)
// pre check
iole, err := cm.passiveConsensusModule.LogRO.GetIndexOfLastEntry()
if err != nil {
t.Fatal()
}
if iole != 10 {
t.Fatal()
}
ioleAC, err := cm.AppendCommand(testhelpers.DummyCommand(1101))
if cm.IsStopped() {
t.Error()
}
if err != nil {
t.Fatal()
}
if ioleAC != 11 {
t.Fatal()
}
iole, err = cm.passiveConsensusModule.LogRO.GetIndexOfLastEntry()
if err != nil {
t.Fatal()
}
if iole != 11 {
t.Fatal()
}
le := testhelpers.TestHelper_GetLogEntryAtIndex(cm.passiveConsensusModule.LogRO, 11)
if !reflect.DeepEqual(le, LogEntry{8, Command("c1101")}) {
t.Fatal(le)
}
}
func TestCluster_SOLO_Command_And_CommitIndexAdvance(t *testing.T) {
cm, diml, dsm := testSetup_SOLO_Leader(t)
defer cm.Stop()
// Apply a command on the leader
ioleAC, result := cm.AppendCommand(testhelpers.DummyCommand(101))
// FIXME: sleep just enough!
time.Sleep(testdata.SleepToLetGoroutineRun)
if result != nil {
t.Fatal()
}
if ioleAC != 1 {
t.Fatal()
}
if iole, err := diml.GetIndexOfLastEntry(); err != nil || iole != 1 {
t.Fatal()
}
expectedLe := LogEntry{1, Command("c101")}
// Command is in the leader's log
le := testhelpers.TestHelper_GetLogEntryAtIndex(diml, 1)
if !reflect.DeepEqual(le, expectedLe) {
t.Fatal(le)
}
// but not yet committed
if dsm.GetCommitIndex() != 0 {
t.Fatal()
}
// A tick allows command to be committed
time.Sleep(testdata.TickerDuration)
if dsm.GetCommitIndex() != 1 {
t.Fatal()
}
}
func TestCluster_CommandIsReplicatedVsMissingNode(t *testing.T) {
imrsh, cm1, diml1, dsm1, cm2, diml2, dsm2, cm3, _, _ := testSetupClusterWithLeader(t)
defer cm1.Stop()
defer cm2.Stop()
// Simulate a follower crash
imrsh.cms["s3"] = nil
cm3.Stop()
cm3 = nil
// Apply a command on the leader
ioleAC, result := cm1.AppendCommand(testhelpers.DummyCommand(101))
if result != nil {
t.Fatal()
}
if ioleAC != 1 {
t.Fatal()
}
if iole, err := diml1.GetIndexOfLastEntry(); err != nil || iole != 1 {
t.Fatal()
}
expectedLe := LogEntry{1, Command("c101")}
// Command is in the leader's log
le := testhelpers.TestHelper_GetLogEntryAtIndex(diml1, 1)
if !reflect.DeepEqual(le, expectedLe) {
t.Fatal(le)
}
// but not yet in connected follower's
iole, err := diml2.GetIndexOfLastEntry()
if err != nil || iole != 0 {
t.Fatal()
}
// A tick allows command to be replicated to connected followers
time.Sleep(testdata.TickerDuration)
iole, err = diml2.GetIndexOfLastEntry()
if err != nil || iole != 1 {
t.Fatal(iole)
}
le = testhelpers.TestHelper_GetLogEntryAtIndex(diml2, 1)
if !reflect.DeepEqual(le, expectedLe) {
t.Fatal(le)
}
// and committed on the leader
if dsm1.GetCommitIndex() != 1 {
t.Fatal()
}
// but not yet on the connected followers
if dsm2.GetCommitIndex() != 0 {
t.Fatal()
}
// Another tick propagates the commit to the connected followers
time.Sleep(testdata.TickerDuration)
if dsm2.GetCommitIndex() != 1 {
t.Fatal()
}
// Crashed follower restarts
cm3b, diml3b, dsm3b := setupConsensusModuleR3(
t,
"s3",
testdata.ElectionTimeoutLow,
nil,
imrsh.getRpcService("s3"),
)
defer cm3b.Stop()
imrsh.cms["s3"] = cm3b
if dsm3b.GetCommitIndex() != 0 {
t.Fatal()
}
// A tick propagates the command and the commit to the recovered follower
time.Sleep(testdata.TickerDuration)
// FIXME: err if cm3b.GetLeader() != "s1"
le = testhelpers.TestHelper_GetLogEntryAtIndex(diml3b, 1)
if !reflect.DeepEqual(le, expectedLe) {
t.Fatal(le)
}
if dsm3b.GetCommitIndex() != 1 {
t.Fatal()
}
}
// Variant of TestRpcAEAppendNewEntries to test alternate path for step 5.
// Note: this test case based on Figure 7, case (b) in the Raft paper
func TestCM_RpcAE_AppendNewEntriesB(t *testing.T) {
f := func(
setup func(t *testing.T, terms []TermNo) (mcm *managedConsensusModule, mrs *testhelpers.MockRpcSender),
senderTermIsNewer bool,
expectedVotedFor ServerId,
expectedErr string,
) {
mcm, _ := setup(
t,
[]TermNo{1, 1, 1, 4},
)
err := mcm.pcm.setCommitIndex(3)
if err != nil {
t.Fatal(err)
}
serverTerm := mcm.pcm.RaftPersistentState.GetCurrentTerm()
electionTimeoutTime1 := mcm.pcm.ElectionTimeoutTracker.GetElectionTimeoutTime()
senderTerm := serverTerm
if senderTermIsNewer {
senderTerm += 1
}
if !testhelpers.DummyCommandEquals(testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 4).Command, 4) {
t.Error()
}
sentLogEntries := []LogEntry{
{4, Command("c501")},
{5, Command("c601")},
}
appendEntries := &RpcAppendEntries{senderTerm, 4, 4, sentLogEntries, 7}
reply, err := mcm.Rpc_RpcAppendEntries("s4", appendEntries)
if expectedErr != "" {
if err != nil && err.Error() == expectedErr {
return
}
t.Fatal(err)
} else {
if err != nil {
t.Fatal(err)
}
}
expectedRpc := RpcAppendEntriesReply{senderTerm, true}
if *reply != expectedRpc {
t.Fatal(reply)
}
iole, err := mcm.pcm.LogRO.GetIndexOfLastEntry()
if err != nil {
t.Fatal()
}
if iole != 6 {
t.Fatal(iole)
}
addedLogEntry := testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 6)
if addedLogEntry.TermNo != 5 {
t.Error()
}
if !testhelpers.DummyCommandEquals(addedLogEntry.Command, 601) {
t.Error()
}
if mcm.pcm.GetCommitIndex() != 6 {
t.Error()
}
if mcm.pcm.RaftPersistentState.GetVotedFor() != expectedVotedFor {
t.Fatal()
}
// #RFS-F2: (paraphrasing) AppendEntries RPC from current leader should
// prevent election timeout
if mcm.pcm.ElectionTimeoutTracker.GetElectionTimeoutTime() == electionTimeoutTime1 {
t.Fatal()
}
}
// Follower
f(testSetupMCM_Follower_WithTerms, true, "", "")
f(testSetupMCM_Follower_WithTerms, false, "", "")
// Candidate
f(testSetupMCM_Candidate_WithTerms, true, "", "")
f(testSetupMCM_Candidate_WithTerms, false, "s1", "")
// Leader
f(testSetupMCM_Leader_WithTerms, true, "", "")
f(
testSetupMCM_Leader_WithTerms, false, "s1",
"FATAL: two leaders with same term - got AppendEntries from: s4 with term: 8",
)
}