// NotifyAllLearners sends the current vote to all learners.
func (this *Paxos) NotifyAllLearners() (status error) {
if !this.IsConfigured() || !this.IsAcceptor() {
return nil
}
defer func() {
if status != nil && !errs.IsClosed(status) {
now := time.Now()
next := now.Add(this.opts.LearnRetryInterval)
_ = this.alarm.ScheduleAt(this.uid, next, this.NotifyAllLearners)
}
}()
rlock := this.ctlr.ReadLock("acceptor", "config")
// Stop notifications when all learners know the consensus value.
if len(this.doneLearnerSet) == len(this.learnerList) {
rlock.Unlock()
return nil
}
// Make a copy of what we need: learners and the vote map.
numLearners := len(this.learnerList)
learnerList := append([]string{}, this.learnerList...)
votedValueMap := make(map[int64][]byte)
for ballot, value := range this.votedValueMap {
if ackMap := this.learnerAckMap[ballot]; len(ackMap) < numLearners {
votedValueMap[ballot] = value
}
}
rlock.Unlock()
request := thispb.LearnRequest{}
for ballot, value := range votedValueMap {
request.VotedBallotList = append(request.VotedBallotList, ballot)
request.VotedValueList = append(request.VotedValueList, value)
}
message := thispb.PaxosMessage{}
message.LearnRequest = &request
// Send notification to all learners.
reqHeader := this.msn.NewRequest(this.namespace, this.uid,
"ClassicPaxos.Learn", this.opts.LearnTimeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, learnerList, reqHeader,
&message)
if errSend != nil {
this.Errorf("could not send learn request to all learners: %v", errSend)
return errSend
}
// Wait for responses from all learners.
for ii := 0; ii < count; ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive learner responses: %v", errRecv)
break
}
learner := resHeader.GetMessengerId()
if message.LearnResponse == nil {
continue
}
response := message.GetLearnResponse()
// Save the learner acknowledgment to the wal.
change := thispb.AcceptorChange{}
change.AckedLearner = proto.String(learner)
if response.GetKnowsChosenValue() {
change.AckedChosenValue = proto.Bool(true)
} else {
for ballot := range votedValueMap {
change.AckedBallotList = append(change.AckedBallotList, ballot)
}
}
if err := this.UpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
}
return nil
}
// doPhase1 performs classic paxos phase1 steps.
func (this *Paxos) doPhase1(ctxHeader *msgpb.Header, ballot int64,
phase1AcceptorList []string) ([]byte, []string, error) {
phase1request := thispb.Phase1Request{}
phase1request.BallotNumber = proto.Int64(ballot)
message := thispb.PaxosMessage{}
message.Phase1Request = &phase1request
reqHeader := msg.NewNestedRequest(this.msn, ctxHeader, this.namespace,
this.uid, "ClassicPaxos.Phase1")
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, phase1AcceptorList, reqHeader,
&message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send phase1 request to majority nodes: %v", errSend)
return nil, nil, errSend
}
this.Infof("sent phase1 request %s to acceptors %v", reqHeader,
phase1AcceptorList)
var acceptorList []string
maxVotedBallot := int64(-1)
var maxVotedValue []byte
responseMap := make(map[string]*thispb.Phase1Response)
for ii := 0; ii < count && len(responseMap) < this.MajoritySize(); ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive more phase1 responses for %s: %v",
reqHeader, errRecv)
break
}
acceptor := resHeader.GetMessengerId()
if _, ok := responseMap[acceptor]; ok {
this.Warningf("duplicate phase1 response from %s (ignored)", acceptor)
continue
}
if message.Phase1Response == nil {
this.Warningf("phase1 response data is empty from %s (ignored)",
acceptor)
continue
}
response := message.GetPhase1Response()
if response.PromisedBallot == nil {
this.Warningf("phase1 response from %s has no promise ballot", acceptor)
continue
}
promisedBallot := response.GetPromisedBallot()
if promisedBallot > ballot {
this.Warningf("acceptor %s has moved on to ballot %d", acceptor,
promisedBallot)
break
}
if promisedBallot < ballot {
this.Errorf("acceptor %s did not promise this ballot %d", acceptor,
ballot)
continue
}
// We received a promise from this acceptor.
acceptorList = append(acceptorList, acceptor)
responseMap[acceptor] = response
// If there was a voted value already, we need to pick the max voted value.
if response.VotedBallot != nil {
votedBallot := response.GetVotedBallot()
if votedBallot > maxVotedBallot {
maxVotedBallot = votedBallot
maxVotedValue = response.GetVotedValue()
}
}
}
if len(responseMap) < this.MajoritySize() {
this.Warningf("could not get majority phase1 votes %v for ballot %d",
responseMap, ballot)
return nil, nil, errs.ErrRetry
}
if maxVotedValue == nil {
this.Infof("no prior value was chosen as per phase1 responses %v",
responseMap)
} else {
this.Infof("value [%s] could have been chosen as per phase1 responses %v",
maxVotedValue, responseMap)
}
return maxVotedValue, acceptorList, nil
}
// doPhase2 performs classic paxos phase2 steps.
func (this *Paxos) doPhase2(ctxHeader *msgpb.Header, ballot int64,
value []byte, acceptorList []string) error {
phase2request := thispb.Phase2Request{}
phase2request.BallotNumber = proto.Int64(ballot)
phase2request.ProposedValue = value
message := thispb.PaxosMessage{}
message.Phase2Request = &phase2request
header := msg.NewNestedRequest(this.msn, ctxHeader, this.namespace, this.uid,
"ClassicPaxos.Phase2")
defer this.msn.CloseMessage(header)
count, errSend := msg.SendAllProto(this.msn, acceptorList, header, &message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send phase2 request to majority acceptors: %v",
errSend)
return errSend
}
this.Infof("send phase2 request %s to acceptors: %v", header, acceptorList)
responseMap := make(map[string]*thispb.Phase2Response)
for ii := 0; ii < count && len(responseMap) < this.MajoritySize(); ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, header, &message)
if errRecv != nil {
break
}
acceptor := resHeader.GetMessengerId()
if _, ok := responseMap[acceptor]; ok {
this.Warningf("duplicate phase2 response from %s (ignored)", acceptor)
continue
}
if message.Phase2Response == nil {
this.Warningf("phase2 response data is empty from %s (ignored)",
acceptor)
continue
}
response := message.GetPhase2Response()
promisedBallot := response.GetPromisedBallot()
if promisedBallot < ballot {
this.Errorf("as per phase2 response, acceptor %s seems to have rolled "+
"back on his phase1 promise to ballot %d", acceptor, ballot)
continue
}
if promisedBallot > ballot {
this.Warningf("acceptor %s has moved on to higher ballot %d", acceptor,
promisedBallot)
break
}
responseMap[acceptor] = response
}
if len(responseMap) < this.MajoritySize() {
this.Warningf("could not get majority phase2 votes %v for value [%s] "+
"ballot %d", responseMap, value, ballot)
return errs.ErrRetry
}
this.Infof("value [%s] is chosen by phase2 responses %v", value, responseMap)
return nil
}
// RefreshLeader queries committee members for the latest election information.
func (this *Election) RefreshLeader(timeout time.Duration) (
string, int64, error) {
lock := this.ctlr.ReadLock("config")
committee := append([]string{}, this.committee...)
lock.Unlock()
request := thispb.StatusRequest{}
message := thispb.ElectionMessage{}
message.StatusRequest = &request
reqHeader := this.msn.NewRequest(this.namespace, this.uid, "Election.Status",
timeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, committee, reqHeader, &message)
if errSend != nil && count < this.MajoritySize() {
this.Errorf("could not send status request to committee: %v", errSend)
return "", -1, errSend
}
// Wait for majority number of status responses and pick the winner for the
// largest election round. It may also happen that no election round exists
// because this instance is just created.
maxElectionRound := int64(-1)
maxElectionWinner := ""
remoteSet := make(map[string]struct{})
for ii := 0; ii < count && len(remoteSet) < this.MajoritySize(); ii++ {
message := thispb.ElectionMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive any more status responses for %s: %v",
reqHeader, errRecv)
break
}
memberID := resHeader.GetMessengerId()
if _, ok := remoteSet[memberID]; ok {
this.Warningf("duplicate status response from %s (ignored)", memberID)
continue
}
// TODO Check the response header for errors.
if message.StatusResponse == nil {
this.Warningf("status response data is empty from %s (ignored)",
memberID)
continue
}
response := message.GetStatusResponse()
if response.ElectionRound != nil {
round := response.GetElectionRound()
if round > maxElectionRound {
maxElectionRound = round
maxElectionWinner = response.GetElectionWinner()
}
}
remoteSet[memberID] = struct{}{}
}
if len(remoteSet) < this.MajoritySize() {
this.Errorf("could not get majority responses %v in finding leader",
remoteSet)
return "", -1, errs.ErrRetry
}
if maxElectionRound > this.CurrentRound() {
change := thispb.ElectionChange{}
change.ElectionRound = proto.Int64(maxElectionRound)
change.ElectionWinner = proto.String(maxElectionWinner)
if err := this.UpdateElection(&change); err != nil {
this.Errorf("could not update election status: %v", err)
return maxElectionWinner, maxElectionRound, err
}
}
this.Infof("found %s as the last known leader in round %d",
maxElectionWinner, maxElectionRound)
return maxElectionWinner, maxElectionRound, nil
}
