// TestElect tests normal behavior that leader should
// be elected under normal condition and everyone
// should agree upon current leader
func TestElect(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:9999", PeerSocket: "127.0.0.1:9009", TimeoutInMillis: 1500, HbTimeoutInMillis: 50, LogDirectoryPath: "logs", StableStoreDirectoryPath: "./stable"}
// delete stored state to avoid unnecessary effect on following test cases
deleteState(raftConf.StableStoreDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
fmt.Println("Started Proxy")
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]Raft, serverCount+1)
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 5000+i, RaftToClusterConf(raftConf))
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
s, err := NewWithConfig(clusterServer, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s
}
// there should be a leader after sufficiently long duration
count := 0
time.Sleep(10 * time.Second)
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].isLeader() {
fmt.Println("Server " + strconv.Itoa(i) + " was chosen as leader.")
count++
}
}
if count != 1 {
t.Errorf("No leader was chosen")
}
}
func CreateRaftCluster() ([]raft.Raft, error) {
raftConf, err := raftImpl.ReadConfig("./config.json")
if err != nil {
return nil, err
}
initState(raftConf.StableStoreDirectoryPath, raftConf.LogDirectoryPath, raftConf.RaftLogDirectoryPath)
cluster.NewProxyWithConfig(raftImpl.RaftToClusterConf(raftConf))
registerTypes()
raftServers := make([]raft.Raft, serverCount)
for i := 0; i < serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i+1, "127.0.0.1", 7500+i, raftImpl.RaftToClusterConf(raftConf))
if err != nil {
return nil, err
}
logStore, err := llog.Create(raftConf.RaftLogDirectoryPath + "/" + strconv.Itoa(i+1))
if err != nil {
return nil, err
}
s, err := raftImpl.NewWithConfig(clusterServer, logStore, raftConf)
if err != nil {
return nil, err
}
raftServers[i] = s
}
// wait for some time to ensure that the servers
// have elected a leader
time.Sleep(5 * time.Second)
return raftServers, err
}
// TestState checks whether server correctly
// reads persistent state from the disk
func TestState(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:9999", PeerSocket: "127.0.0.1:9009", TimeoutInMillis: 1500, HbTimeoutInMillis: 50, LogDirectoryPath: "logs", StableStoreDirectoryPath: "./stable"}
// delete stored state to avoid unnecessary effect on following test cases
deleteState(raftConf.StableStoreDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]Raft, serverCount+1)
// create persistent state for one of the servers
// state is created in such a way that one of the
// servers will be way ahead of the others in
// logical time and hence it should eventually
// be elected as reader
pStateBytes, err := PersistentStateToBytes(&PersistentState{Term: 500, VotedFor: 1})
if err != nil {
//TODO: Add the state that was encoded. This might help in case of an error
t.Errorf("Cannot encode PersistentState")
}
err = ioutil.WriteFile(raftConf.StableStoreDirectoryPath+"/1", pStateBytes, UserReadWriteMode)
if err != nil {
t.Errorf("Could not create state to storage on file " + raftConf.StableStoreDirectoryPath)
}
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 5400+i, RaftToClusterConf(raftConf))
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
s, err := NewWithConfig(clusterServer, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s
}
// there should be a leader after sufficiently long duration
count := 0
time.Sleep(10 * time.Second)
leader := 0
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].isLeader() {
fmt.Println("Server " + strconv.Itoa(i) + " was chosen as leader.")
leader = i
count++
}
}
if count > 1 || leader != 1 {
t.Errorf("Persistent state read was not utilized")
}
}
// TestLeaderSeparation tests that new leader gets elected when
// elected leader crashes. Leader crash is simulated using
// PseudoCluster service. It then re-enables the links between
// old leader and rest of the cluster and checks whether the
// old leader reverts back to follower.
func TestLeaderSeparation(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:9999", PeerSocket: "127.0.0.1:9009", TimeoutInMillis: 1500, HbTimeoutInMillis: 50, LogDirectoryPath: "logs", StableStoreDirectoryPath: "./stable"}
// delete stored state to avoid unnecessary effect on following test cases
deleteState(raftConf.StableStoreDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
fmt.Println("Started Proxy")
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]*raftServer, serverCount+1)
pseudoClusters := make([]*test.PseudoCluster, serverCount+1)
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 5100+i, RaftToClusterConf(raftConf))
pseudoCluster := test.NewPseudoCluster(clusterServer)
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
s, err := NewWithConfig(pseudoCluster, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s.(*raftServer)
pseudoClusters[i] = pseudoCluster
}
// wait for leader to be elected
time.Sleep(4 * time.Second)
count := 0
oldLeader := 0
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].isLeader() {
fmt.Println("Server " + strconv.Itoa(i) + " was chosen as leader.")
oldLeader = i
count++
}
}
if count != 1 {
t.Errorf("No leader was chosen in 1 minute")
}
// isolate Leader
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].Pid() != oldLeader {
pseudoClusters[oldLeader].AddToInboxFilter(raftServers[i].Pid())
pseudoClusters[oldLeader].AddToOutboxFilter(raftServers[i].Pid())
}
}
// prevent broadcasts from leader too
pseudoClusters[oldLeader].AddToOutboxFilter(cluster.BROADCAST)
// wait for other servers to discover that leader
// has crashed and to elect a new leader
time.Sleep(5 * time.Second)
count = 0
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].isLeader() && i != oldLeader {
fmt.Println("Server " + strconv.Itoa(i) + " was chosen as new leader.")
count++
}
}
// new leader must be chosen
if count != 1 {
t.Errorf("No leader was chosen")
}
// re-enable link between old leader and other servers
for i := 1; i <= serverCount; i += 1 {
s := raftServers[i]
if i != oldLeader {
pseudoClusters[oldLeader].RemoveFromInboxFilter(s.Pid())
pseudoClusters[oldLeader].RemoveFromOutboxFilter(s.Pid())
}
}
// wait for oldLeader to discover new leader
time.Sleep(2 * time.Second)
// state of the old leader must be FOLLOWER
// since a new leader has been elected
// with a greater term
if raftServers[oldLeader].State() != FOLLOWER {
t.Errorf("Old leader is still in leader state.")
}
}
// TestReplicateScale tests that 1000 messages
// are replicated in reasonable amount of time
// Reasonable is considering 5 servers doing
// disk IO for each message
func TestReplicateScale(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:8145", PeerSocket: "127.0.0.1:9100", TimeoutInMillis: 1500, HbTimeoutInMillis: 100, LogDirectoryPath: "logs", StableStoreDirectoryPath: "./stable", RaftLogDirectoryPath: "../LocalLog"}
// delete stored state to avoid unnecessary effect on following test cases
initState(raftConf.StableStoreDirectoryPath, raftConf.LogDirectoryPath, raftConf.RaftLogDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
fmt.Println("Started Proxy")
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]raft.Raft, serverCount+1)
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 7000+i, RaftToClusterConf(raftConf))
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
logStore, err := llog.Create(raftConf.RaftLogDirectoryPath + "/" + strconv.Itoa(i))
if err != nil {
t.Errorf("Error in creating log. " + err.Error())
return
}
s, err := NewWithConfig(clusterServer, logStore, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s
}
// there should be a leader after sufficiently long duration
time.Sleep(5 * time.Second)
leaderId := raftServers[2].Leader()
if leaderId == NONE {
t.Errorf("No leader was chosen")
return
}
fmt.Println("Leader pid: " + strconv.Itoa(leaderId))
// replicate an entry
// data := "Woah, it works !!"
leader := raftServers[leaderId]
var follower raft.Raft = nil
for i, server := range raftServers {
if i > 0 && server.Pid() != leaderId {
follower = server
fmt.Println("Follower is " + strconv.Itoa(follower.Pid()))
break
}
}
count := 100
// launch a goroutine to consume messages for other followers and leader
for i, server := range raftServers {
if i > 0 && server.Pid() != follower.Pid() {
go readMessages(server)
}
}
done := make(chan bool, 1)
go replicateVerification(follower, count, done)
for count != 0 {
leader.Outbox() <- strconv.Itoa(count)
count -= 1
}
select {
case <-done:
case <-time.After(5 * time.Minute):
t.Errorf("Message replication took more than 5 minutes !")
}
}
// TestReplicate tests that a single
// message is correctly replicated
func _TestReplicate(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:9999", PeerSocket: "127.0.0.1:9009", TimeoutInMillis: 1500, HbTimeoutInMillis: 50, LogDirectoryPath: "logs", StableStoreDirectoryPath: "./stable", RaftLogDirectoryPath: "../LocalLog"}
// delete stored state to avoid unnecessary effect on following test cases
initState(raftConf.StableStoreDirectoryPath, raftConf.LogDirectoryPath, raftConf.RaftLogDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
fmt.Println("Started Proxy")
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]raft.Raft, serverCount+1)
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 5000+i, RaftToClusterConf(raftConf))
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
logStore, err := llog.Create(raftConf.RaftLogDirectoryPath + "/" + strconv.Itoa(i))
if err != nil {
t.Errorf("Error in creating log. " + err.Error())
return
}
s, err := NewWithConfig(clusterServer, logStore, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s
}
// there should be a leader after sufficiently long duration
time.Sleep(5 * time.Second)
leaderId := raftServers[2].Leader()
if leaderId == NONE {
t.Errorf("No leader was chosen")
return
}
fmt.Println("Leader pid: " + strconv.Itoa(leaderId))
// replicate an entry
data := "Woah, it works !!"
leader := raftServers[leaderId]
leader.Outbox() <- data
var follower raft.Raft = nil
for i, server := range raftServers {
if i > 0 && server.Pid() != leaderId {
follower = server
break
}
}
select {
case msg := <-follower.Inbox():
fmt.Println("Received message: ")
fmt.Println(msg)
case <-time.After(15 * time.Second):
fmt.Println("Message replication took more than 15 seconds !")
}
}
// TestPartition tests that in case of network partition
// the partition with majority servers elect leader
func TestPartition(t *testing.T) {
raftConf := &RaftConfig{MemberRegSocket: "127.0.0.1:8124", PeerSocket: "127.0.0.1:9987", TimeoutInMillis: 1500, HbTimeoutInMillis: 150, LogDirectoryPath: "logs1", StableStoreDirectoryPath: "./stable1", RaftLogDirectoryPath: "../LocalLog1"}
// delete stored state to avoid unnecessary effect on following test cases
initState(raftConf.StableStoreDirectoryPath, raftConf.LogDirectoryPath, raftConf.RaftLogDirectoryPath)
// launch cluster proxy servers
cluster.NewProxyWithConfig(RaftToClusterConf(raftConf))
fmt.Println("Started Proxy")
time.Sleep(100 * time.Millisecond)
serverCount := 5
raftServers := make([]raft.Raft, serverCount+1)
pseudoClusters := make([]*test.PseudoCluster, serverCount+1)
for i := 1; i <= serverCount; i += 1 {
// create cluster.Server
clusterServer, err := cluster.NewWithConfig(i, "127.0.0.1", 8500+i, RaftToClusterConf(raftConf))
pseudoCluster := test.NewPseudoCluster(clusterServer)
if err != nil {
t.Errorf("Error in creating cluster server. " + err.Error())
return
}
logStore, err := llog.Create(raftConf.RaftLogDirectoryPath + "/" + strconv.Itoa(i))
if err != nil {
t.Errorf("Error in creating log. " + err.Error())
return
}
s, err := NewWithConfig(pseudoCluster, logStore, raftConf)
if err != nil {
t.Errorf("Error in creating Raft servers. " + err.Error())
return
}
raftServers[i] = s
pseudoClusters[i] = pseudoCluster
}
// wait for leader to be elected
time.Sleep(20 * time.Second)
count := 0
oldLeader := 0
for i := 1; i <= serverCount; i += 1 {
if raftServers[i].Leader() == raftServers[i].Pid() {
oldLeader = i
count++
}
}
if count != 1 {
t.Errorf("No leader was chosen in 1 minute")
return
}
// isolate Leader and any one follower
follower := 0
for i := 1; i <= serverCount; i += 1 {
if i != oldLeader {
follower = i
break
}
}
fmt.Println("Server " + strconv.Itoa(follower) + " was chosen as follower in minority partition")
for i := 1; i <= serverCount; i += 1 {
pseudoClusters[oldLeader].AddToInboxFilter(raftServers[i].Pid())
pseudoClusters[oldLeader].AddToOutboxFilter(raftServers[i].Pid())
pseudoClusters[follower].AddToInboxFilter(raftServers[i].Pid())
pseudoClusters[follower].AddToOutboxFilter(raftServers[i].Pid())
}
pseudoClusters[oldLeader].AddToOutboxFilter(cluster.BROADCAST)
pseudoClusters[follower].AddToOutboxFilter(cluster.BROADCAST)
// wait for other servers to discover that leader
// has crashed and to elect a new leader
time.Sleep(20 * time.Second)
count = 0
for i := 1; i <= serverCount; i += 1 {
if i != oldLeader && i != follower && raftServers[i].Leader() == raftServers[i].Pid() {
fmt.Println("Server " + strconv.Itoa(i) + " was chosen as new leader in majority partition.")
count++
}
}
// new leader must be chosen
if count != 1 {
t.Errorf("No leader was chosen in majority partition")
}
}