// #RFS-L2a: If Command received from client: append entry to local log func TestCM_Leader_AppendCommand(t *testing.T) { mcm, _ := testSetupMCM_Leader_Figure7LeaderLine(t) // pre check iole, err := mcm.pcm.LogRO.GetIndexOfLastEntry() if err != nil { t.Fatal() } if iole != 10 { t.Fatal() } ioleAC, err := mcm.pcm.AppendCommand(testhelpers.DummyCommand(1101)) if err != nil || ioleAC != 11 { t.Fatal() } iole, err = mcm.pcm.LogRO.GetIndexOfLastEntry() if err != nil { t.Fatal() } if iole != 11 { t.Fatal() } le := testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 11) if !reflect.DeepEqual(le, LogEntry{8, Command("c1101")}) { t.Fatal(le) } }
func TestConsensusModule_AppendCommand_Leader(t *testing.T) { cm, mrs := setupConsensusModuleR2(t, testdata.TestUtil_MakeFigure7LeaderLineTerms()) defer cm.Stop() testConsensusModule_RpcReplyCallback_AndBecomeLeader(t, cm, mrs) // pre check iole, err := cm.passiveConsensusModule.LogRO.GetIndexOfLastEntry() if err != nil { t.Fatal() } if iole != 10 { t.Fatal() } ioleAC, err := cm.AppendCommand(testhelpers.DummyCommand(1101)) if cm.IsStopped() { t.Error() } if err != nil { t.Fatal() } if ioleAC != 11 { t.Fatal() } iole, err = cm.passiveConsensusModule.LogRO.GetIndexOfLastEntry() if err != nil { t.Fatal() } if iole != 11 { t.Fatal() } le := testhelpers.TestHelper_GetLogEntryAtIndex(cm.passiveConsensusModule.LogRO, 11) if !reflect.DeepEqual(le, LogEntry{8, Command("c1101")}) { t.Fatal(le) } }
func TestCluster_SOLO_Command_And_CommitIndexAdvance(t *testing.T) { cm, diml, dsm := testSetup_SOLO_Leader(t) defer cm.Stop() // Apply a command on the leader ioleAC, result := cm.AppendCommand(testhelpers.DummyCommand(101)) // FIXME: sleep just enough! time.Sleep(testdata.SleepToLetGoroutineRun) if result != nil { t.Fatal() } if ioleAC != 1 { t.Fatal() } if iole, err := diml.GetIndexOfLastEntry(); err != nil || iole != 1 { t.Fatal() } expectedLe := LogEntry{1, Command("c101")} // Command is in the leader's log le := testhelpers.TestHelper_GetLogEntryAtIndex(diml, 1) if !reflect.DeepEqual(le, expectedLe) { t.Fatal(le) } // but not yet committed if dsm.GetCommitIndex() != 0 { t.Fatal() } // A tick allows command to be committed time.Sleep(testdata.TickerDuration) if dsm.GetCommitIndex() != 1 { t.Fatal() } }
func TestCluster_CommandIsReplicatedVsMissingNode(t *testing.T) { imrsh, cm1, diml1, dsm1, cm2, diml2, dsm2, cm3, _, _ := testSetupClusterWithLeader(t) defer cm1.Stop() defer cm2.Stop() // Simulate a follower crash imrsh.cms["s3"] = nil cm3.Stop() cm3 = nil // Apply a command on the leader ioleAC, result := cm1.AppendCommand(testhelpers.DummyCommand(101)) if result != nil { t.Fatal() } if ioleAC != 1 { t.Fatal() } if iole, err := diml1.GetIndexOfLastEntry(); err != nil || iole != 1 { t.Fatal() } expectedLe := LogEntry{1, Command("c101")} // Command is in the leader's log le := testhelpers.TestHelper_GetLogEntryAtIndex(diml1, 1) if !reflect.DeepEqual(le, expectedLe) { t.Fatal(le) } // but not yet in connected follower's iole, err := diml2.GetIndexOfLastEntry() if err != nil || iole != 0 { t.Fatal() } // A tick allows command to be replicated to connected followers time.Sleep(testdata.TickerDuration) iole, err = diml2.GetIndexOfLastEntry() if err != nil || iole != 1 { t.Fatal(iole) } le = testhelpers.TestHelper_GetLogEntryAtIndex(diml2, 1) if !reflect.DeepEqual(le, expectedLe) { t.Fatal(le) } // and committed on the leader if dsm1.GetCommitIndex() != 1 { t.Fatal() } // but not yet on the connected followers if dsm2.GetCommitIndex() != 0 { t.Fatal() } // Another tick propagates the commit to the connected followers time.Sleep(testdata.TickerDuration) if dsm2.GetCommitIndex() != 1 { t.Fatal() } // Crashed follower restarts cm3b, diml3b, dsm3b := setupConsensusModuleR3( t, "s3", testdata.ElectionTimeoutLow, nil, imrsh.getRpcService("s3"), ) defer cm3b.Stop() imrsh.cms["s3"] = cm3b if dsm3b.GetCommitIndex() != 0 { t.Fatal() } // A tick propagates the command and the commit to the recovered follower time.Sleep(testdata.TickerDuration) // FIXME: err if cm3b.GetLeader() != "s1" le = testhelpers.TestHelper_GetLogEntryAtIndex(diml3b, 1) if !reflect.DeepEqual(le, expectedLe) { t.Fatal(le) } if dsm3b.GetCommitIndex() != 1 { t.Fatal() } }
// Variant of TestRpcAEAppendNewEntries to test alternate path for step 5. // Note: this test case based on Figure 7, case (b) in the Raft paper func TestCM_RpcAE_AppendNewEntriesB(t *testing.T) { f := func( setup func(t *testing.T, terms []TermNo) (mcm *managedConsensusModule, mrs *testhelpers.MockRpcSender), senderTermIsNewer bool, expectedVotedFor ServerId, expectedErr string, ) { mcm, _ := setup( t, []TermNo{1, 1, 1, 4}, ) err := mcm.pcm.setCommitIndex(3) if err != nil { t.Fatal(err) } serverTerm := mcm.pcm.RaftPersistentState.GetCurrentTerm() electionTimeoutTime1 := mcm.pcm.ElectionTimeoutTracker.GetElectionTimeoutTime() senderTerm := serverTerm if senderTermIsNewer { senderTerm += 1 } if !testhelpers.DummyCommandEquals(testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 4).Command, 4) { t.Error() } sentLogEntries := []LogEntry{ {4, Command("c501")}, {5, Command("c601")}, } appendEntries := &RpcAppendEntries{senderTerm, 4, 4, sentLogEntries, 7} reply, err := mcm.Rpc_RpcAppendEntries("s4", appendEntries) if expectedErr != "" { if err != nil && err.Error() == expectedErr { return } t.Fatal(err) } else { if err != nil { t.Fatal(err) } } expectedRpc := RpcAppendEntriesReply{senderTerm, true} if *reply != expectedRpc { t.Fatal(reply) } iole, err := mcm.pcm.LogRO.GetIndexOfLastEntry() if err != nil { t.Fatal() } if iole != 6 { t.Fatal(iole) } addedLogEntry := testhelpers.TestHelper_GetLogEntryAtIndex(mcm.pcm.LogRO, 6) if addedLogEntry.TermNo != 5 { t.Error() } if !testhelpers.DummyCommandEquals(addedLogEntry.Command, 601) { t.Error() } if mcm.pcm.GetCommitIndex() != 6 { t.Error() } if mcm.pcm.RaftPersistentState.GetVotedFor() != expectedVotedFor { t.Fatal() } // #RFS-F2: (paraphrasing) AppendEntries RPC from current leader should // prevent election timeout if mcm.pcm.ElectionTimeoutTracker.GetElectionTimeoutTime() == electionTimeoutTime1 { t.Fatal() } } // Follower f(testSetupMCM_Follower_WithTerms, true, "", "") f(testSetupMCM_Follower_WithTerms, false, "", "") // Candidate f(testSetupMCM_Candidate_WithTerms, true, "", "") f(testSetupMCM_Candidate_WithTerms, false, "s1", "") // Leader f(testSetupMCM_Leader_WithTerms, true, "", "") f( testSetupMCM_Leader_WithTerms, false, "s1", "FATAL: two leaders with same term - got AppendEntries from: s4 with term: 8", ) }