// CloseMessage releases a message header.
//
// header: Message header.
//
// Returns nil on success.
func (this *Messenger) CloseMessage(header *msgpb.Header) error {
if header.GetMessengerId() != this.uid {
this.Errorf("message header %s is not created by this messenger", header)
return errs.ErrInvalid
}
if header.Request == nil {
return nil
}
//
// Remove the request and its response channel from live requests map.
//
lock, errLock := this.ctlr.Lock("this.requestMap")
if errLock != nil {
return errLock
}
defer lock.Unlock()
requestID := header.GetMessageId()
responseCh, found := this.requestMap[requestID]
if !found {
return nil
}
close(responseCh)
delete(this.requestMap, requestID)
return nil
}
// Dispatch handles all incoming rpc requests for this object.
func (this *Election) Dispatch(header *msgpb.Header, data []byte) error {
request := header.GetRequest()
objectID := request.GetObjectId()
// Dispatch classic paxos rpcs to the paxos instances.
if objectID != this.uid {
if this.IsPaxosUID(objectID) {
return this.DispatchPaxos(header, data)
}
this.Errorf("rpc request %s doesn't belong to this instance", header)
return errs.ErrInvalid
}
message := thispb.ElectionMessage{}
if err := proto.Unmarshal(data, &message); err != nil {
this.Errorf("could not parse incoming message %s", header)
return err
}
switch {
case message.ElectRequest != nil:
return this.ElectRPC(header, message.GetElectRequest())
case message.StatusRequest != nil:
return this.StatusRPC(header, message.GetStatusRequest())
default:
this.Errorf("unknown/invalid rpc reqest %s", header)
return errs.ErrInvalid
}
}
// DispatchResponse enqueues an incoming response into the response channel for
// the corresponding request.
//
// header: Message header for the response.
//
// data: User data in the response message.
//
// Returns nil on success.
func (this *Messenger) DispatchResponse(header *msgpb.Header,
data []byte) error {
lock, errLock := this.ctlr.Lock("this.requestMap")
if errLock != nil {
return errLock
}
defer lock.Unlock()
response := header.GetResponse()
requestID := response.GetRequestId()
responseCh, found := this.requestMap[requestID]
if !found {
this.Errorf("could not find request %d to dispatch response %s",
requestID, header)
return errs.ErrNotExist
}
// Close the lock early to release the resources.
lock.Unlock()
entry := &Entry{header: header, data: data}
select {
case responseCh <- entry:
return nil
default:
this.Errorf("could not queue response %s to request %d", header,
requestID)
return errs.ErrOverflow
}
}
// StatusRPC implements the Election.Status rpc.
func (this *Election) StatusRPC(reqHeader *msgpb.Header,
request *thispb.StatusRequest) error {
if !this.InCommittee() {
this.Errorf("this election object is not a committee member")
return errs.ErrNotExist
}
lock := this.ctlr.ReadLock("election")
round := this.CurrentRound()
winner := this.currentWinner
lock.Unlock()
// Committee members always have up to date election information.
response := thispb.StatusResponse{}
response.ElectionRound = proto.Int64(round)
response.ElectionWinner = proto.String(winner)
message := thispb.ElectionMessage{}
message.StatusResponse = &response
clientID := reqHeader.GetMessengerId()
if err := msg.SendResponseProto(this.msn, reqHeader, &message); err != nil {
this.Errorf("could not send status response to %s: %v", clientID, err)
return err
}
return nil
}
// Phase1RPC handles ClassicPaxos.Phase1 rpc.
func (this *Paxos) Phase1RPC(header *msgpb.Header,
request *thispb.Phase1Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header),
"acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase1Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase1Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase1 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase1 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot == this.promisedBallot {
this.Warningf("duplicate phase1 request from client %s with an already "+
"promised ballot number %d", clientID, ballot)
return respond()
}
// Save the promise into the wal.
change := thispb.AcceptorChange{}
change.PromisedBallot = proto.Int64(ballot)
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has now promised higher ballot %d from %s", ballot,
clientID)
return respond()
}
// SendResponse sends a response message to the source of a request.
//
// msn: The Messenger.
//
// reqHeader: Message header for a request.
//
// data: User data to include in the response.
//
// Returns nil on success.
func SendResponse(msn Messenger, reqHeader *msgpb.Header, data []byte) error {
if reqHeader.Request == nil {
msn.Errorf("message header %s is not a request", reqHeader)
return errs.ErrInvalid
}
resHeader := msn.NewResponse(reqHeader)
return msn.Send(reqHeader.GetMessengerId(), resHeader, data)
}
func (this *RPCServer) Dispatch(header *msgpb.Header, data []byte) error {
response := this.NewResponse(header)
sourceID := header.GetMessengerId()
if err := this.Send(sourceID, response, []byte("response-data")); err != nil {
this.Errorf("could not send response for %s to messenger %s: %v", header,
sourceID, err)
return err
}
return nil
}
// DispatchPaxos handles incoming rpc requests for the paxos instances.
func (this *Election) DispatchPaxos(header *msgpb.Header, data []byte) error {
request := header.GetRequest()
objectID := request.GetObjectId()
instance, errGet := this.GetPaxosInstance(objectID)
if errGet != nil {
this.Errorf("could not get paxos instance with object id %s", objectID)
return errGet
}
return instance.Dispatch(header, data)
}
// Dispatch implements the msg.Handler interface for Paxos objects.
//
// header: Message header for an incoming request.
//
// data: User data in the message.
//
// Returns the result of perform the incoming request.
func (this *Paxos) Dispatch(header *msgpb.Header, data []byte) error {
request := header.GetRequest()
if request == nil {
this.Error("rpc message %s is not a request", header)
return errs.ErrInvalid
}
if request.GetObjectId() != this.uid {
this.Errorf("rpc request [%s] doesn't belong to this paxos instance %s",
header, this.uid)
return errs.ErrInvalid
}
message := &thispb.PaxosMessage{}
if err := proto.Unmarshal(data, message); err != nil {
this.Errorf("could not parse message [%s]: %v", header, err)
return err
}
switch {
case message.ProposeRequest != nil:
if !this.IsProposer() {
return errs.ErrInvalid
}
return this.ProposeRPC(header, message.ProposeRequest)
case message.Phase1Request != nil:
if !this.IsAcceptor() {
return errs.ErrInvalid
}
return this.Phase1RPC(header, message.Phase1Request)
case message.Phase2Request != nil:
if !this.IsAcceptor() {
return errs.ErrInvalid
}
return this.Phase2RPC(header, message.Phase2Request)
case message.LearnRequest != nil:
if !this.IsLearner() {
return errs.ErrInvalid
}
return this.LearnRPC(header, message.LearnRequest)
default:
this.Errorf("rpc request [%s] has no request parameters", header)
return errs.ErrInvalid
}
}
// ElectRPC implements the Election.Elect rpc.
func (this *Election) ElectRPC(reqHeader *msgpb.Header,
request *thispb.ElectRequest) (status error) {
if !this.InCommittee() {
return errs.ErrInvalid
}
round := request.GetElectionRound()
current := this.CurrentRound()
if round != current+1 {
this.Errorf("could not begin election round %d because current round "+
"is only at %d", round, current)
return errs.ErrStale
}
uid := this.PaxosUIDFromElectionRound(round)
paxos, errGet := this.GetPaxosInstance(uid)
if errGet != nil {
this.Errorf("could not find paxos instance for uid %s: %v", uid, errGet)
return errGet
}
this.Infof("starting new election for round %d using paxos instance %s",
round, uid)
clientID := reqHeader.GetMessengerId()
proposedValue := []byte(clientID)
chosenValue, errPropose := paxos.Propose(proposedValue,
msg.RequestTimeout(reqHeader))
if errPropose != nil {
this.Errorf("could not propose %s as the leader for election round %d: %v",
proposedValue, round, errPropose)
return errPropose
}
winner := string(chosenValue)
// Send the election result.
response := thispb.ElectResponse{}
response.ElectionRound = proto.Int64(round)
response.ElectionWinner = proto.String(winner)
message := thispb.ElectionMessage{}
message.ElectResponse = &response
if err := msg.SendResponseProto(this.msn, reqHeader, &message); err != nil {
this.Errorf("could not send election response to %s: %v", clientID, err)
return err
}
return nil
}
// Receive waits for a response to a live request.
//
// request: Message header of a live request.
//
// Returns response message header and response data on success.
func (this *Messenger) Receive(request *msgpb.Header) (*msgpb.Header,
[]byte, error) {
timeout := msg.RequestTimeout(request)
lock, errLock := this.ctlr.TimedLock(timeout, "this.requestMap")
if errLock != nil {
return nil, nil, errLock
}
defer lock.Unlock()
requestID := request.GetMessageId()
responseCh, found := this.requestMap[requestID]
if !found {
this.Errorf("no live request [%s] exist", request)
return nil, nil, errs.ErrNotExist
}
// Close the lock early to unblock others.
lock.Unlock()
// Perform a non-blocking receive if timeout is zero.
if timeout == 0 {
select {
case entry := <-responseCh:
return entry.header, entry.data, nil
default:
return nil, nil, errs.ErrRetry
}
}
// Perform a blocking receive with a timeout.
select {
case <-time.After(timeout):
this.Warningf("timedout waiting for response to %s", request)
return nil, nil, errs.ErrTimeout
case entry := <-responseCh:
if entry == nil {
return nil, nil, errs.ErrRetry
}
return entry.header, entry.data, nil
}
}
// LearnRPC handles ClassicPaxos.Learn rpc.
func (this *Paxos) LearnRPC(header *msgpb.Header,
request *thispb.LearnRequest) (status error) {
if !this.IsLearner() {
this.Errorf("this paxos instance is not a learner; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header), "learner")
if errLock != nil {
return errLock
}
defer lock.Unlock()
acceptor := header.GetMessengerId()
if this.chosenValue == nil {
change := thispb.LearnerChange{}
change.VotedBallotList = request.VotedBallotList
change.VotedValueList = request.VotedValueList
change.VotedAcceptor = proto.String(acceptor)
if err := this.doUpdateLearner(&change); err != nil {
this.Errorf("could not update learner state: %v", err)
return err
}
}
response := thispb.LearnResponse{}
if this.chosenValue != nil {
response.KnowsChosenValue = proto.Bool(true)
}
message := thispb.PaxosMessage{}
message.LearnResponse = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not respond to learn request from %s: %v", acceptor,
errSend)
return errSend
}
return nil
}
// RequestTimeout returns the pending time left to complete the operations
// represented by the request header. Timeout is measured from the creation
// timestamp if the request header is created locally; otherwise, timeout is
// measured from the time this request is received from the socket (optionally,
// removing any transfer latency if available.)
func RequestTimeout(header *msgpb.Header) time.Duration {
if header.Request == nil {
// This is not a request.
return 0
}
start := header.GetCreateTimestampNsecs()
if header.ReceiverTimestampNsecs != nil {
start = header.GetReceiverTimestampNsecs()
}
request := header.GetRequest()
timeout := time.Duration(request.GetTimeoutNsecs())
elapsed := time.Now().UnixNano() - start
return timeout - time.Duration(elapsed)
}
// NewResponse creates a message header for a response message.
//
// request: Message header for the received request.
func (this *Messenger) NewResponse(request *msgpb.Header) *msgpb.Header {
header := this.NewPost()
header.Response = &msgpb.Header_Response{}
header.Response.RequestId = proto.Int64(request.GetMessageId())
return header
}
// DispatchRequest invokes the target (exported) function identified in the
// request message.
//
// header: Message header for the request.
//
// data: User data in the request.
//
// Returns dispatch operation status and the handler status.
func (this *Messenger) DispatchRequest(header *msgpb.Header,
data []byte) (msnStatus, appStatus error) {
defer func() {
if msnStatus != nil || appStatus != nil {
failure := this.NewResponse(header)
if msnStatus != nil {
failure.Response.MessengerStatus = errs.MakeProtoFromError(msnStatus)
} else {
failure.Response.HandlerStatus = errs.MakeProtoFromError(appStatus)
}
sourceID := header.GetMessengerId()
if err := this.Send(sourceID, failure, nil); err != nil {
this.Errorf("could not reply failure %s for %s from %s (ignored)",
failure, header, sourceID)
}
}
}()
lock, errLock := this.ctlr.Lock("this.exportMap")
if errLock != nil {
return errLock, nil
}
defer lock.Unlock()
request := header.GetRequest()
classID := request.GetClassId()
methodName := request.GetMethodName()
if len(classID) == 0 || len(methodName) == 0 {
this.Errorf("header %s is invalid because class id and/or method name "+
"are empty", header)
msnStatus = errs.NewErrInvalid("class id and/or method name cannot be " +
"empty")
return msnStatus, nil
}
methodMap, found := this.exportMap[classID]
if !found {
this.Errorf("no class with id %s was registered", classID)
msnStatus = errs.NewErrNotExist("no class with id %s was registered",
classID)
return msnStatus, nil
}
handler, found := methodMap[methodName]
if !found {
this.Errorf("class %s has no method named %s", classID, methodName)
msnStatus = errs.NewErrNotExist("class %s has no method named %s", classID,
methodName)
return msnStatus, nil
}
// Close the lock early to release the resources.
lock.Unlock()
appStatus = handler.Dispatch(header, data)
if appStatus != nil {
this.Warningf("request %s failed with status %v", header, appStatus)
}
return nil, appStatus
}
// Phase2RPC handles ClassicPaxos.Phase2 rpc.
func (this *Paxos) Phase2RPC(header *msgpb.Header,
request *thispb.Phase2Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header), "acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase2Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase2Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase2 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase2 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot > this.promisedBallot {
this.Errorf("phase2 request from client %s without acquiring a prior "+
"promise", clientID)
return respond()
}
value := request.GetProposedValue()
// Save the phase2 vote into the wal.
change := thispb.AcceptorChange{}
change.VotedBallot = proto.Int64(ballot)
change.VotedValue = value
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has voted for %d in ballot %s", ballot, value)
if err := respond(); err != nil {
return err
}
// Schedule a notification to all learners.
_ = this.alarm.ScheduleAt(this.uid, time.Now(), this.NotifyAllLearners)
return nil
}