func (this *Messenger) doStop() (status error) {
for address, listener := range this.listenerMap {
if err := listener.Close(); err != nil {
this.Errorf("could not close listener %s: %v", address, err)
status = errs.MergeErrors(status, err)
}
}
this.listenerMap = make(map[string]net.Listener)
for _, entryCh := range this.requestMap {
close(entryCh)
}
this.requestMap = make(map[int64]chan *Entry)
for _, peer := range this.peerMap {
for tport := range peer.transportMap {
if err := tport.connection.Close(); err != nil {
this.Errorf("could not close transport %s to peer %s: %v", tport.name,
peer.peerID, err)
status = errs.MergeErrors(status, err)
}
tport.connection = nil
}
this.Infof("closed %d sockets to peer %s", len(peer.transportMap),
peer.peerID)
peer.transportMap = make(map[*Transport]struct{})
}
return status
}
// Close releases all wal resources and destroys the object.
func (this *WriteAheadLog) Close() (status error) {
this.mutex.Lock()
defer this.mutex.Unlock()
if this.changeFileMap == nil {
this.Errorf("wal is already closed")
return errs.ErrClosed
}
for fileID, file := range this.changeFileMap {
if err := file.Close(); err != nil {
this.Errorf("could not close change records file %d: %v", fileID, err)
status = errs.MergeErrors(status, err)
}
}
this.changeFileMap = nil
if this.checkpointFile != nil {
if err := this.checkpointFile.Close(); err != nil {
this.Errorf("could not close checkpoints file %d: %v",
this.beginCheckpointOffset, err)
status = errs.MergeErrors(status, err)
}
this.checkpointFile = nil
}
return status
}
// Validate checks user configuration options for invalid settings.
func (this *Options) Validate() (status error) {
if this.MaxElectionHistory < 0 {
err := errs.NewErrInvalid("election round history size cannot be -ve")
status = errs.MergeErrors(status, err)
}
if err := this.PaxosOptions.Validate(); err != nil {
status = errs.MergeErrors(status, err)
}
return status
}
// ClosePeer closes all open transports to a remote messenger instance and
// releases its resources.
//
// peerID: Globally unique id for the remote peer instance.
//
// Returns nil on success.
func (this *Messenger) ClosePeer(peerID string) (status error) {
lock, errLock := this.ctlr.Lock("this.peerMap", peerID)
if errLock != nil {
return errLock
}
defer lock.Unlock()
peer, found := this.peerMap[peerID]
if !found {
return nil
}
delete(this.peerMap, peerID)
lock.Unlock("this.peerMap")
close(peer.outCh)
for tport := range peer.transportMap {
if tport.connection != nil {
if err := tport.connection.Close(); err != nil {
this.Errorf("could not close transport connection for %s: %v",
tport.name, err)
status = errs.MergeErrors(status, err)
}
}
tport.connection = nil
}
peer.transportMap = make(map[*Transport]struct{})
return status
}
// Initialize initializes an election object.
func (this *Election) Initialize(opts *Options, namespace, uid string,
msn msg.Messenger, wal wal.WriteAheadLog) (status error) {
re := regexp.MustCompile(uid)
if err := wal.ConfigureRecoverer(re, this); err != nil {
this.Errorf("could not configure wal recoverer: %v", err)
return err
}
defer func() {
if status != nil {
if err := wal.ConfigureRecoverer(re, nil); err != nil {
this.Errorf("could not unconfigure wal recoverer: %v", err)
status = errs.MergeErrors(status, err)
}
}
}()
this.uid = uid
this.namespace = namespace
this.opts = *opts
this.msn = msn
this.wal = wal
this.majoritySize = -1
this.currentRound = -1
this.classicPaxosMap = make(map[int64]*classic.Paxos)
this.electionHistoryMap = make(map[int64]string)
this.Logger = this.NewLogger("election-%s-%s", this.msn.UID(), this.uid)
this.ctlr.Initialize(this)
return nil
}
// Close releases all resources and destroys the object.
func (this *Paxos) Close() (status error) {
if err := this.ctlr.Close(); err != nil {
return err
}
if err := this.alarm.Close(); err != nil {
this.Errorf("could not close alarm handler: %v", err)
status = errs.MergeErrors(status, err)
}
re := regexp.MustCompile(this.uid)
if err := this.wal.ConfigureRecoverer(re, nil); err != nil {
this.Errorf("could not unconfigure wal recoverer: %v", err)
status = errs.MergeErrors(status, err)
}
return status
}
// abortCheckpoint cancels currently active checkpoint.
//
// Returns nil on success. Since partial abort may not always be undone, abort
// should be considered successful even after a failure.
func (this *WriteAheadLog) abortCheckpoint() (status error) {
if this.checkpointFile == nil {
return nil
}
if err := this.checkpointFile.Close(); err != nil {
this.Errorf("could not close checkpoint file: %v", err)
status = errs.MergeErrors(status, err)
}
this.checkpointFile = nil
filePath := fmt.Sprintf("%s/%s.checkpoint.temp", this.dir, this.name)
if err := os.Remove(filePath); err != nil {
this.Errorf("could not remove temporary checkpoint file %s: %v", filePath,
err)
status = errs.MergeErrors(status, err)
}
return status
}
// Validate verifies user options for correctness.
func (this *Options) Validate() (status error) {
if this.MaxReadSize < 8 {
err := errs.NewErrInvalid("minimum read size must be at least 8 bytes")
status = errs.MergeErrors(status, err)
}
if this.MaxWriteSize < 8 {
err := errs.NewErrInvalid("minimum write size must be at least 8 bytes")
status = errs.MergeErrors(status, err)
}
if this.MaxFileSize < 8 {
err := errs.NewErrInvalid("minimum file size must be at least 8 bytes")
status = errs.MergeErrors(status, err)
}
if this.MaxReadDirNames < 1 {
err := errs.NewErrInvalid("readier must read at least one name per call")
status = errs.MergeErrors(status, err)
}
return status
}
// Close releases all resources and destroys the object.
func (this *Election) Close() (status error) {
if err := this.ctlr.Close(); err != nil {
return err
}
re := regexp.MustCompile(this.uid)
if err := this.wal.ConfigureRecoverer(re, nil); err != nil {
this.Errorf("could not unconfigure wal recoverer: %v", err)
status = errs.MergeErrors(status, err)
}
for round, paxos := range this.classicPaxosMap {
if err := paxos.Close(); err != nil {
this.Errorf("could not close paxos instance for round %d: %v", round,
err)
status = errs.MergeErrors(status, err)
}
}
return status
}
// Validate checks if user configuration items are all valid.
func (this *Options) Validate() (status error) {
if this.ProposeRetryInterval < time.Millisecond {
err := errs.NewErrInvalid("propose retry should wait for at least a " +
"millisecond")
status = errs.MergeErrors(status, err)
}
if this.NumExtraPhase1Acceptors < 0 {
err := errs.NewErrInvalid("number of extra phase1 acceptors cannot be -ve")
status = errs.MergeErrors(status, err)
}
if this.LearnTimeout < time.Millisecond {
err := errs.NewErrInvalid("learn timeout must be at least a millisecond")
status = errs.MergeErrors(status, err)
}
if this.LearnRetryInterval < time.Millisecond {
err := errs.NewErrInvalid("learn notfication retry interval is too small")
status = errs.MergeErrors(status, err)
}
return status
}
func (this *Messenger) stopListener(address string) (status error) {
listener, found := this.listenerMap[address]
if !found {
this.Errorf("could not find listener for address %s", address)
return errs.ErrNotExist
}
delete(this.listenerMap, address)
if err := listener.Close(); err != nil {
this.Errorf("could not close the listener %s: %v", address, err)
status = errs.MergeErrors(status, err)
}
return status
}
// Validate checks for invalid user configuration settings.
func (this *Options) Validate() (status error) {
if this.ResponseQueueSize < 1 {
err := errs.NewErrInvalid("response queue size should be at least one")
status = errs.MergeErrors(status, err)
}
if this.SendQueueSize < 1 {
err := errs.NewErrInvalid("outbox queue size should at least one")
status = errs.MergeErrors(status, err)
}
if this.NegotiationTimeout < time.Millisecond {
err := errs.NewErrInvalid("connection negotiation timeout is too small")
status = errs.MergeErrors(status, err)
}
if this.SendRetryTimeout < 10*time.Millisecond {
err := errs.NewErrInvalid("send retry timeout is too small")
status = errs.MergeErrors(status, err)
}
if this.MaxDispatchRequests < 1 {
err := errs.NewErrInvalid("at least one request should be dispatched")
status = errs.MergeErrors(status, err)
}
return status
}
// Close releases all messenger resources and destroys it.
func (this *Messenger) Close() (status error) {
if err := this.ctlr.Close(); err != nil {
return err
}
if this.started {
if err := this.doStop(); err != nil {
this.Errorf("could not stop messenger service (ignored): %v", err)
status = errs.MergeErrors(status, err)
}
}
return status
}
// SendAll sends a message to multiple nodes.
//
// msn: The Messenger.
//
// targetList: List of targets to send the message.
//
// header: The message header.
//
// data: User data for the message.
//
// Returns the number of targets where message is sent successfully.
func SendAll(msn Messenger, targetList []string, header *msgpb.Header,
data []byte) (int, error) {
count := 0
var errSend error
for _, target := range targetList {
if err := msn.Send(target, header, data); err != nil {
msn.Warningf("could not send %s to %s: %v", header, target, err)
errSend = errs.MergeErrors(errSend, err)
continue
}
count++
}
return count, errSend
}
func (this *Election) doGetPaxosInstance(uid string) (
instance *classic.Paxos, status error) {
round := this.ElectionRoundFromPaxosUID(uid)
if round < 0 {
this.Errorf("could not parse election round from paxos uid %s", uid)
return nil, errs.ErrCorrupt
}
if instance, ok := this.classicPaxosMap[round]; ok {
return instance, nil
}
paxos := &classic.Paxos{Logger: this}
errInit := paxos.Initialize(&this.opts.PaxosOptions, this.namespace, uid,
this.msn, this.wal)
if errInit != nil {
this.Errorf("could not initialize paxos instance %s: %v", uid, errInit)
return nil, errInit
}
errConfig := paxos.Configure(this.committee, this.committee, this.committee)
if errConfig != nil {
this.Errorf("could not configure paxos instance %s: %v", uid, errConfig)
return nil, errConfig
}
defer func() {
if status != nil {
if err := paxos.Close(); err != nil {
this.Errorf("could not close paxos instance %s: %v", uid, err)
status = errs.MergeErrors(status, err)
}
}
}()
if err := paxos.SetLearnerWatch(this); err != nil {
this.Errorf("could not configure watch on the paxos instance %s: %v", uid,
err)
return nil, err
}
this.classicPaxosMap[round] = paxos
return paxos, nil
}
func (this *Messenger) doStart() (status error) {
// Open a listener on all listen addresses.
for _, address := range this.state.ListenerAddressList {
if err := this.startListener(address); err != nil {
this.Errorf("could not start listener at %s: %v", address, err)
return err
}
defer func() {
if status != nil {
if err := this.stopListener(address); err != nil {
this.Errorf("could not stop listener at %s: %v", address, err)
status = errs.MergeErrors(status, err)
}
}
}()
}
return status
}
// Initialize initializes a classic paxos instance.
func (this *Paxos) Initialize(opts *Options, namespace, uid string,
msn msg.Messenger, wal wal.WriteAheadLog) (status error) {
if err := opts.Validate(); err != nil {
this.Errorf("invalid user options: %v", err)
return err
}
re := regexp.MustCompile(uid)
if err := wal.ConfigureRecoverer(re, this); err != nil {
this.Errorf("could not configure wal recoverer: %v", err)
return err
}
defer func() {
if status != nil {
if err := wal.ConfigureRecoverer(re, nil); err != nil {
this.Errorf("could not unconfigure wal recoverer: %v", err)
status = errs.MergeErrors(status, err)
}
}
}()
this.wal = wal
this.opts = *opts
this.msn = msn
this.uid = uid
this.namespace = namespace
this.proposerIndex = -1
this.promisedBallot = -1
this.votedBallot = -1
this.majoritySize = -1
this.proposalBallot = -1
this.doneLearnerSet = make(map[string]struct{})
this.votedValueMap = make(map[int64][]byte)
this.ballotValueMap = make(map[int64][]byte)
this.ballotAcceptorsMap = make(map[int64]map[string]struct{})
this.learnerAckMap = make(map[int64]map[string]struct{})
this.Logger = this.NewLogger("classic-paxos:%s-%s", this.msn.UID(), uid)
this.ctlr.Initialize(this)
this.alarm.Initialize()
return nil
}
// CloseTransport removes a transport to a remote messenger instance.
//
// peer: The remote messenger instance.
//
// tport: Transport to remove.
//
// Returns nil on success.
func (this *Messenger) CloseTransport(peer *Peer, tport *Transport) (
status error) {
lock, errLock := this.ctlr.Lock(peer.peerID)
if errLock != nil {
return errLock
}
delete(peer.transportMap, tport)
lock.Unlock()
if tport.connection == nil {
return nil
}
if err := tport.connection.Close(); err != nil {
this.Errorf("could not close transport connection: %v", err)
status = errs.MergeErrors(status, err)
}
tport.connection = nil
return status
}