// Phase1RPC handles ClassicPaxos.Phase1 rpc.
func (this *Paxos) Phase1RPC(header *msgpb.Header,
request *thispb.Phase1Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header),
"acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase1Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase1Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase1 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase1 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot == this.promisedBallot {
this.Warningf("duplicate phase1 request from client %s with an already "+
"promised ballot number %d", clientID, ballot)
return respond()
}
// Save the promise into the wal.
change := thispb.AcceptorChange{}
change.PromisedBallot = proto.Int64(ballot)
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has now promised higher ballot %d from %s", ballot,
clientID)
return respond()
}
// NotifyAllLearners sends the current vote to all learners.
func (this *Paxos) NotifyAllLearners() (status error) {
if !this.IsConfigured() || !this.IsAcceptor() {
return nil
}
defer func() {
if status != nil && !errs.IsClosed(status) {
now := time.Now()
next := now.Add(this.opts.LearnRetryInterval)
_ = this.alarm.ScheduleAt(this.uid, next, this.NotifyAllLearners)
}
}()
rlock := this.ctlr.ReadLock("acceptor", "config")
// Stop notifications when all learners know the consensus value.
if len(this.doneLearnerSet) == len(this.learnerList) {
rlock.Unlock()
return nil
}
// Make a copy of what we need: learners and the vote map.
numLearners := len(this.learnerList)
learnerList := append([]string{}, this.learnerList...)
votedValueMap := make(map[int64][]byte)
for ballot, value := range this.votedValueMap {
if ackMap := this.learnerAckMap[ballot]; len(ackMap) < numLearners {
votedValueMap[ballot] = value
}
}
rlock.Unlock()
request := thispb.LearnRequest{}
for ballot, value := range votedValueMap {
request.VotedBallotList = append(request.VotedBallotList, ballot)
request.VotedValueList = append(request.VotedValueList, value)
}
message := thispb.PaxosMessage{}
message.LearnRequest = &request
// Send notification to all learners.
reqHeader := this.msn.NewRequest(this.namespace, this.uid,
"ClassicPaxos.Learn", this.opts.LearnTimeout)
defer this.msn.CloseMessage(reqHeader)
count, errSend := msg.SendAllProto(this.msn, learnerList, reqHeader,
&message)
if errSend != nil {
this.Errorf("could not send learn request to all learners: %v", errSend)
return errSend
}
// Wait for responses from all learners.
for ii := 0; ii < count; ii++ {
message := thispb.PaxosMessage{}
resHeader, errRecv := msg.ReceiveProto(this.msn, reqHeader, &message)
if errRecv != nil {
this.Warningf("could not receive learner responses: %v", errRecv)
break
}
learner := resHeader.GetMessengerId()
if message.LearnResponse == nil {
continue
}
response := message.GetLearnResponse()
// Save the learner acknowledgment to the wal.
change := thispb.AcceptorChange{}
change.AckedLearner = proto.String(learner)
if response.GetKnowsChosenValue() {
change.AckedChosenValue = proto.Bool(true)
} else {
for ballot := range votedValueMap {
change.AckedBallotList = append(change.AckedBallotList, ballot)
}
}
if err := this.UpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
}
return nil
}
// Phase2RPC handles ClassicPaxos.Phase2 rpc.
func (this *Paxos) Phase2RPC(header *msgpb.Header,
request *thispb.Phase2Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header), "acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase2Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase2Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase2 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase2 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot > this.promisedBallot {
this.Errorf("phase2 request from client %s without acquiring a prior "+
"promise", clientID)
return respond()
}
value := request.GetProposedValue()
// Save the phase2 vote into the wal.
change := thispb.AcceptorChange{}
change.VotedBallot = proto.Int64(ballot)
change.VotedValue = value
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has voted for %d in ballot %s", ballot, value)
if err := respond(); err != nil {
return err
}
// Schedule a notification to all learners.
_ = this.alarm.ScheduleAt(this.uid, time.Now(), this.NotifyAllLearners)
return nil
}
func (this *Paxos) doUpdateAcceptor(change *thispb.AcceptorChange) error {
if !this.wal.IsRecovering() {
walRecord := thispb.WALRecord{}
walRecord.AcceptorChange = change
_, errSync := wal.SyncChangeProto(this.wal, this.uid, &walRecord)
if errSync != nil {
this.Errorf("could not write acceptor change record: %v", errSync)
return errSync
}
}
if change.PromisedBallot != nil {
this.promisedBallot = change.GetPromisedBallot()
}
if change.VotedBallot != nil {
ballot := change.GetVotedBallot()
value := change.GetVotedValue()
this.votedBallot = ballot
this.votedValue = value
this.votedValueMap[ballot] = value
}
if change.AckedLearner != nil {
learner := change.GetAckedLearner()
if change.GetAckedChosenValue() {
this.doneLearnerSet[learner] = struct{}{}
} else {
for _, ballot := range change.GetAckedBallotList() {
learnerSet, found := this.learnerAckMap[ballot]
if !found {
learnerSet = make(map[string]struct{})
this.learnerAckMap[ballot] = learnerSet
}
learnerSet[learner] = struct{}{}
}
}
}
return nil
}
