//
// Synchronize with the leader.
//
func syncWithPeer(pipe *common.PeerPipe,
handler ActionHandler,
factory MsgFactory,
killch <-chan bool) (success bool, isKilled bool) {
log.Current.Debugf("WatcherServer.syncWithPeer(): Watcher start synchronization with peer (TCP %s)", pipe.GetAddr())
proxy := NewFollowerSyncProxy(pipe, handler, factory, false)
donech := proxy.GetDoneChannel()
go proxy.Start()
defer proxy.Terminate()
// This will block until NewWatcherSyncProxy has sychronized with the peer (a bool is pushed to donech)
select {
case success = <-donech:
if success {
log.Current.Debugf("WatcherServer.syncWithPeer(): Watcher done synchronization with peer (TCP %s)", pipe.GetAddr())
}
return success, false
case <-killch:
// simply return. The pipe will eventually be closed and
// cause WatcherSyncProxy to err out.
log.Current.Debugf("WatcherServer.syncWithPeer(): Recieve kill singal. Synchronization with peer (TCP %s) terminated.",
pipe.GetAddr())
return false, true
}
}
func send(packet common.Packet, pipe *common.PeerPipe) error {
log.Current.Tracef("SyncProxy.send(): sending packet %s to peer (TCP %s)", packet.Name(), pipe.GetAddr())
if !pipe.Send(packet) {
return common.NewError(common.SERVER_ERROR, fmt.Sprintf("SyncProxy.listen(): Fail to send packet %s to peer (TCP %s)",
packet.Name(), pipe.GetAddr()))
}
return nil
}
//
// Synchronize with the leader.
//
func syncWithLeader(naddr string,
pipe *common.PeerPipe,
handler ActionHandler,
factory MsgFactory,
killch <-chan bool) bool {
log.Current.Debugf("FollowerServer.syncWithLeader(): Follower %s start synchronization with leader (TCP %s)",
naddr, pipe.GetAddr())
proxy := NewFollowerSyncProxy(pipe, handler, factory, true)
donech := proxy.GetDoneChannel()
go proxy.Start()
defer proxy.Terminate()
// This will block until NewFollowerSyncProxy has sychronized with the leader (a bool is pushed to donech)
select {
case success := <-donech:
if success {
log.Current.Debugf("FollowerServer.syncWithLeader(): Follower %s done synchronization with leader (TCP %s)",
naddr, pipe.GetAddr())
}
return success
case <-killch:
// simply return. The pipe will eventually be closed and
// cause FollowerSyncProxy to err out.
log.Current.Debugf("FollowerServer.syncWithLeader(): Recieve kill singal. Synchronization with leader (TCP %s) terminated.",
pipe.GetAddr())
}
return false
}
//
// Start a LeaderSyncProxy to synchornize the leader
// and follower state.
//
func (l *LeaderServer) startProxy(peer *common.PeerPipe) {
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in LeaderServer.startProxy() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
} else {
log.Current.Debugf("LeaderServer.startProxy() : Terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
// deregister the proxy with the leader Server upon exit
l.deregisterOutstandingProxy(peer.GetAddr())
}()
// create a proxy that will sycnhronize with the peer.
log.Current.Debugf("LeaderServer.startProxy(): Start synchronization with follower. Peer TCP connection (%s)", peer.GetAddr())
proxy := NewLeaderSyncProxy(l.leader, l.consentState, peer, l.handler, l.factory)
donech := proxy.GetDoneChannel()
// Create an observer for the leader. The leader will put on-going proposal msg and commit msg
// onto the observer queue. This ensure that we can won't miss those mutations as the leader is
// sync'ign withe follower. The messages in observer queue will eventually route to follower.
o := NewObserver()
l.leader.AddObserver(peer.GetAddr(), o)
defer l.leader.RemoveObserver(peer.GetAddr())
// start the proxy
go proxy.Start(o)
defer proxy.Terminate()
// Get the killch for this go-routine
killch := l.getProxyKillChan(peer.GetAddr())
if killch == nil {
log.Current.Debugf("LeaderServer.startProxy(): Cannot find killch for proxy (TCP connection = %s).", peer.GetAddr())
log.Current.Debugf("LeaderServer.startProxy(): Cannot start follower sync.")
return
}
// this go-routine will be blocked until handshake is completed between the
// leader and the follower. By then, the leader will also get majority
// confirmation that it is a leader.
select {
case success := <-donech:
if success {
// tell the leader to add this follower for processing request. If there is a follower running already,
// AddFollower() will terminate the existing follower instance, and then create a new one.
fid := proxy.GetFid()
if proxy.CanFollowerVote() {
l.leader.AddFollower(fid, peer, o)
log.Current.Debugf("LeaderServer.startProxy(): Synchronization with follower %s done (TCP conn = %s). Add follower.",
fid, peer.GetAddr())
// At this point, the follower has voted this server as the leader.
// Notify the request processor to start processing new request for this host
l.notifyReady()
} else {
l.leader.AddWatcher(fid, peer, o)
log.Current.Debugf("LeaderServer.startProxy(): Sync with watcher done. Add Watcher %s (TCP conn = %s)",
fid, peer.GetAddr())
}
} else {
log.Current.Errorf("LeaderServer:startProxy(): Leader Fail to synchronization with follower (TCP conn = %s)", peer.GetAddr())
}
case <-killch:
log.Current.Infof("LeaderServer:startProxy(): Sync proxy is killed while synchronizing with follower (TCP conn == %s)",
peer.GetAddr())
}
}