// ElectLeader performs a leader election for the next round.
func (this *Election) ElectLeader(timeout time.Duration) (
string, int64, error) {
lock := this.ctlr.ReadLockAll()
target := this.msn.UID()
if !this.InCommittee() {
target = this.committee[rand.Intn(len(this.committee))]
}
lock.Unlock()
request := thispb.ElectRequest{}
request.ElectionRound = proto.Int64(this.CurrentRound() + 1)
message := thispb.ElectionMessage{}
message.ElectRequest = &request
reqHeader := this.msn.NewRequest(this.namespace, this.uid, "Election.Elect",
timeout)
defer this.msn.CloseMessage(reqHeader)
if err := msg.SendProto(this.msn, target, reqHeader, &message); err != nil {
this.Errorf("could not send elect request to %s: %v", target, err)
return "", -1, err
}
message.Reset()
_, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Errorf("could not receive elect response from %s: %v", target,
errRecv)
return "", -1, errRecv
}
if message.ElectResponse == nil {
this.Errorf("elect response from %s is empty", target)
return "", -1, errs.ErrCorrupt
}
response := message.GetElectResponse()
change := thispb.ElectionChange{}
change.ElectionRound = response.ElectionRound
change.ElectionWinner = response.ElectionWinner
if err := this.UpdateElection(&change); err != nil {
this.Errorf("could not update to new election round: %v", err)
return "", -1, err
}
newRound := response.GetElectionRound()
newLeader := response.GetElectionWinner()
return newLeader, newRound, nil
}
// Propose proposes given value for a consensus.
//
// value: The value to propose for consensus.
//
// timeout: Maximum time duration for the propose operation.
//
// Returns the chosen value on success.
func (this *Paxos) Propose(value []byte, timeout time.Duration) (
[]byte, error) {
// If local instance is not a proposer, find a random proposer.
proposer := this.msn.UID()
if !this.IsProposer() {
proposer = this.proposerList[rand.Intn(len(this.proposerList))]
this.Infof("using %s as the proposer", proposer)
}
// Send the propose request.
request := thispb.ProposeRequest{}
request.ProposedValue = value
message := thispb.PaxosMessage{}
message.ProposeRequest = &request
reqHeader := this.msn.NewRequest(this.namespace, this.uid,
"ClassicPaxos.Propose", timeout)
errSend := msg.SendProto(this.msn, proposer, reqHeader, &message)
if errSend != nil {
this.Errorf("could not send propose request to %s: %v", proposer, errSend)
return nil, errSend
}
// Wait for the response.
_, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Errorf("could not receive propose response from %s: %v", proposer,
errRecv)
return nil, errRecv
}
if message.ProposeResponse == nil {
this.Errorf("propose response from %s is empty", proposer)
return nil, errs.ErrCorrupt
}
response := message.GetProposeResponse()
return response.GetChosenValue(), nil
}
// NotifyAllLearners sends the current vote to all learners.
func (this *Paxos) NotifyAllLearners() (status error) {
if !this.IsConfigured() || !this.IsAcceptor() {
return nil
}
defer func() {
if status != nil && !errs.IsClosed(status) {
now := time.Now()
next := now.Add(this.opts.LearnRetryInterval)
_ = this.alarm.ScheduleAt(this.uid, next, this.NotifyAllLearners)
}
}()
rlock := this.ctlr.ReadLock("acceptor", "config")
// Stop notifications when all learners know the consensus value.
if len(this.doneLearnerSet) == len(this.learnerList) {
rlock.Unlock()
return nil
}
// Make a copy of what we need: learners and the vote map.
numLearners := len(this.learnerList)
learnerList := append([]string{}, this.learnerList...)
votedValueMap := make(map[int64][]byte)
for ballot, value := range this.votedValueMap {
if ackMap := this.learnerAckMap[ballot]; len(ackMap) < numLearners {
votedValueMap[ballot] = value
}
}
rlock.Unlock()
request := thispb.LearnRequest{}
for ballot, value := range votedValueMap {
request.VotedBallotList = append(request.VotedBallotList, ballot)
request.VotedValueList = append(request.VotedValueList, value)
}
message := thispb.PaxosMessage{}
message.LearnRequest = &request
// Send notification to all learners.
reqHeader := this.msn.NewRequest(this.namespace, this.uid,
"ClassicPaxos.Learn", this.opts.LearnTimeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, learnerList, reqHeader,
&message)
if errSend != nil {
this.Errorf("could not send learn request to all learners: %v", errSend)
return errSend
}
// Wait for responses from all learners.
for ii := 0; ii < count; ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive learner responses: %v", errRecv)
break
}
learner := resHeader.GetMessengerId()
if message.LearnResponse == nil {
continue
}
response := message.GetLearnResponse()
// Save the learner acknowledgment to the wal.
change := thispb.AcceptorChange{}
change.AckedLearner = proto.String(learner)
if response.GetKnowsChosenValue() {
change.AckedChosenValue = proto.Bool(true)
} else {
for ballot := range votedValueMap {
change.AckedBallotList = append(change.AckedBallotList, ballot)
}
}
if err := this.UpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
}
return nil
}
// doPhase2 performs classic paxos phase2 steps.
func (this *Paxos) doPhase2(ctxHeader *msgpb.Header, ballot int64,
value []byte, acceptorList []string) error {
phase2request := thispb.Phase2Request{}
phase2request.BallotNumber = proto.Int64(ballot)
phase2request.ProposedValue = value
message := thispb.PaxosMessage{}
message.Phase2Request = &phase2request
header := msg.NewNestedRequest(this.msn, ctxHeader, this.namespace, this.uid,
"ClassicPaxos.Phase2")
defer this.msn.CloseMessage(header)
count, errSend := msg.SendAllProto(this.msn, acceptorList, header, &message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send phase2 request to majority acceptors: %v",
errSend)
return errSend
}
this.Infof("send phase2 request %s to acceptors: %v", header, acceptorList)
responseMap := make(map[string]*thispb.Phase2Response)
for ii := 0; ii < count && len(responseMap) < this.MajoritySize(); ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, header, &message)
if errRecv != nil {
break
}
acceptor := resHeader.GetMessengerId()
if _, ok := responseMap[acceptor]; ok {
this.Warningf("duplicate phase2 response from %s (ignored)", acceptor)
continue
}
if message.Phase2Response == nil {
this.Warningf("phase2 response data is empty from %s (ignored)",
acceptor)
continue
}
response := message.GetPhase2Response()
promisedBallot := response.GetPromisedBallot()
if promisedBallot < ballot {
this.Errorf("as per phase2 response, acceptor %s seems to have rolled "+
"back on his phase1 promise to ballot %d", acceptor, ballot)
continue
}
if promisedBallot > ballot {
this.Warningf("acceptor %s has moved on to higher ballot %d", acceptor,
promisedBallot)
break
}
responseMap[acceptor] = response
}
if len(responseMap) < this.MajoritySize() {
this.Warningf("could not get majority phase2 votes %v for value [%s] "+
"ballot %d", responseMap, value, ballot)
return errs.ErrRetry
}
this.Infof("value [%s] is chosen by phase2 responses %v", value, responseMap)
return nil
}
// doPhase1 performs classic paxos phase1 steps.
func (this *Paxos) doPhase1(ctxHeader *msgpb.Header, ballot int64,
phase1AcceptorList []string) ([]byte, []string, error) {
phase1request := thispb.Phase1Request{}
phase1request.BallotNumber = proto.Int64(ballot)
message := thispb.PaxosMessage{}
message.Phase1Request = &phase1request
reqHeader := msg.NewNestedRequest(this.msn, ctxHeader, this.namespace,
this.uid, "ClassicPaxos.Phase1")
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, phase1AcceptorList, reqHeader,
&message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send phase1 request to majority nodes: %v", errSend)
return nil, nil, errSend
}
this.Infof("sent phase1 request %s to acceptors %v", reqHeader,
phase1AcceptorList)
var acceptorList []string
maxVotedBallot := int64(-1)
var maxVotedValue []byte
responseMap := make(map[string]*thispb.Phase1Response)
for ii := 0; ii < count && len(responseMap) < this.MajoritySize(); ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive more phase1 responses for %s: %v",
reqHeader, errRecv)
break
}
acceptor := resHeader.GetMessengerId()
if _, ok := responseMap[acceptor]; ok {
this.Warningf("duplicate phase1 response from %s (ignored)", acceptor)
continue
}
if message.Phase1Response == nil {
this.Warningf("phase1 response data is empty from %s (ignored)",
acceptor)
continue
}
response := message.GetPhase1Response()
if response.PromisedBallot == nil {
this.Warningf("phase1 response from %s has no promise ballot", acceptor)
continue
}
promisedBallot := response.GetPromisedBallot()
if promisedBallot > ballot {
this.Warningf("acceptor %s has moved on to ballot %d", acceptor,
promisedBallot)
break
}
if promisedBallot < ballot {
this.Errorf("acceptor %s did not promise this ballot %d", acceptor,
ballot)
continue
}
// We received a promise from this acceptor.
acceptorList = append(acceptorList, acceptor)
responseMap[acceptor] = response
// If there was a voted value already, we need to pick the max voted value.
if response.VotedBallot != nil {
votedBallot := response.GetVotedBallot()
if votedBallot > maxVotedBallot {
maxVotedBallot = votedBallot
maxVotedValue = response.GetVotedValue()
}
}
}
if len(responseMap) < this.MajoritySize() {
this.Warningf("could not get majority phase1 votes %v for ballot %d",
responseMap, ballot)
return nil, nil, errs.ErrRetry
}
if maxVotedValue == nil {
this.Infof("no prior value was chosen as per phase1 responses %v",
responseMap)
} else {
this.Infof("value [%s] could have been chosen as per phase1 responses %v",
maxVotedValue, responseMap)
}
return maxVotedValue, acceptorList, nil
}
// RefreshLeader queries committee members for the latest election information.
func (this *Election) RefreshLeader(timeout time.Duration) (
string, int64, error) {
lock := this.ctlr.ReadLock("config")
committee := append([]string{}, this.committee...)
lock.Unlock()
request := thispb.StatusRequest{}
message := thispb.ElectionMessage{}
message.StatusRequest = &request
reqHeader := this.msn.NewRequest(this.namespace, this.uid, "Election.Status",
timeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, committee, reqHeader, &message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send status request to committee: %v", errSend)
return "", -1, errSend
}
// Wait for majority number of status responses and pick the winner for the
// largest election round. It may also happen that no election round exists
// because this instance is just created.
maxElectionRound := int64(-1)
maxElectionWinner := ""
remoteSet := make(map[string]struct{})
for ii := 0; ii < count && len(remoteSet) < this.MajoritySize(); ii++ {
message := thispb.ElectionMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive any more status responses for %s: %v",
reqHeader, errRecv)
break
}
memberID := resHeader.GetMessengerId()
if _, ok := remoteSet[memberID]; ok {
this.Warningf("duplicate status response from %s (ignored)", memberID)
continue
}
// TODO Check the response header for errors.
if message.StatusResponse == nil {
this.Warningf("status response data is empty from %s (ignored)",
memberID)
continue
}
response := message.GetStatusResponse()
if response.ElectionRound != nil {
round := response.GetElectionRound()
if round > maxElectionRound {
maxElectionRound = round
maxElectionWinner = response.GetElectionWinner()
}
}
remoteSet[memberID] = struct{}{}
}
if len(remoteSet) < this.MajoritySize() {
this.Errorf("could not get majority responses %v in finding leader",
remoteSet)
return "", -1, errs.ErrRetry
}
if maxElectionRound > this.CurrentRound() {
change := thispb.ElectionChange{}
change.ElectionRound = proto.Int64(maxElectionRound)
change.ElectionWinner = proto.String(maxElectionWinner)
if err := this.UpdateElection(&change); err != nil {
this.Errorf("could not update election status: %v", err)
return maxElectionWinner, maxElectionRound, err
}
}
this.Infof("found %s as the last known leader in round %d",
maxElectionWinner, maxElectionRound)
return maxElectionWinner, maxElectionRound, nil
}