func runOnce(peer string,
requestMgr RequestMgr,
handler ActionHandler,
factory MsgFactory,
killch <-chan bool,
readych chan<- bool,
alivech chan<- bool,
pingch <-chan bool,
once *sync.Once) (isKilled bool) {
// Catch panic at the main entry point for WatcherServer
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in WatcherServer.runOnce() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
} else {
log.Current.Debugf("WatcherServer.runOnce() terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
if requestMgr != nil {
requestMgr.CleanupOnError()
}
}()
// create connection with a peer
conn, err := createConnection(peer)
if err != nil {
log.Current.Errorf("WatcherServer.runOnce() error : %s", err)
return false
}
pipe := common.NewPeerPipe(conn)
log.Current.Debugf("WatcherServer.runOnce() : Watcher successfully created TCP connection to peer %s", peer)
// close the connection to the peer. If connection is closed,
// sync proxy and watcher will also terminate by err-ing out.
// If sync proxy and watcher terminates the pipe upon termination,
// it is ok to close it again here.
defer common.SafeRun("WatcherServer.runOnce()",
func() {
pipe.Close()
})
// start syncrhorniziing with the metadata server
success, isKilled := syncWithPeer(pipe, handler, factory, killch)
// run watcher after synchronization
if success {
if !runWatcher(pipe, requestMgr, handler, factory, killch, readych, alivech, pingch, once) {
log.Current.Errorf("WatcherServer.runOnce() : Watcher terminated unexpectedly.")
return false
}
} else if !isKilled {
log.Current.Errorf("WatcherServer.runOnce() : Watcher fail to synchronized with peer %s", peer)
return false
}
return true
}
//
// Create a new FollowerServer. This is a blocking call until
// the FollowerServer terminates. Make sure the kilch is a buffered
// channel such that if the goroutine running RunFollowerServer goes
// away, the sender won't get blocked.
//
func RunFollowerServer(naddr string,
leader string,
ss RequestMgr,
handler ActionHandler,
factory MsgFactory,
killch <-chan bool) (err error) {
// Catch panic at the main entry point for FollowerServer
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in RunFollowerServer() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
err = r.(error)
} else {
log.Current.Debugf("%s", "RunFollowerServer terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
}()
// create connection to leader
conn, err := createConnection(leader)
if err != nil {
return err
}
pipe := common.NewPeerPipe(conn)
log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower %s successfully "+
"created TCP connection to leader %s, local address %s", naddr, leader, conn.LocalAddr())
// close the connection to the leader. If connection is closed,
// sync proxy and follower will also terminate by err-ing out.
// If sync proxy and follower terminates the pipe upon termination,
// it is ok to close it again here.
defer common.SafeRun("FollowerServer.runFollowerServer()",
func() {
pipe.Close()
})
// start syncrhorniziing with the leader
success := syncWithLeader(naddr, pipe, handler, factory, killch)
// run server after synchronization
if success {
runFollower(pipe, ss, handler, factory, killch)
log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower Server %s terminate", naddr)
err = nil
} else {
err = common.NewError(common.SERVER_ERROR, fmt.Sprintf("Follower %s fail to synchronized with leader %s",
naddr, leader))
}
return err
}
//
// Listen to new connection request from the follower/peer.
// Start a new LeaderSyncProxy to synchronize the state
// between the leader and the peer.
//
func (l *LeaderServer) listenFollower(listenerState *ListenerState) {
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in LeaderServer.listenFollower() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
} else {
log.Current.Debugf("LeaderServer.listenFollower() terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
common.SafeRun("LeaderServer.listenFollower()",
func() {
l.terminateAllOutstandingProxies()
})
common.SafeRun("LeaderServer.listenFollower()",
func() {
listenerState.donech <- true
})
}()
connCh := l.listener.ConnChannel()
if connCh == nil {
// It should not happen unless the listener is closed
return
}
// if there is a single server, then we don't need to wait for follower
// for the server to be ready to process request.
if l.handler.GetEnsembleSize() == 1 {
if err := l.incrementEpoch(); err != nil {
log.Current.Errorf("LeaderServer.listenFollower(): Error when boostraping leader with ensembleSize=1. Error = %s", err)
return
}
l.notifyReady()
}
for {
select {
case conn, ok := <-connCh:
{
if !ok {
// channel close. Simply return.
return
}
// There is a new peer connection request from the follower. Start a proxy to synchronize with the follower.
// The leader does not proactively connect to follower:
// 1) The ensemble is stable, but a follower may just reboot and needs to connect to the leader
// 2) Even if the leader receives votes from the leader, the leader cannot tell for sure that the follower does
// not change its vote. Only if the follower connects, the leader can confirm the follower's alliance.
//
log.Current.Debugf("LeaderServer.listenFollower(): Receive connection request from follower %s", conn.RemoteAddr())
if l.registerOutstandingProxy(conn.RemoteAddr().String()) {
pipe := common.NewPeerPipe(conn)
go l.startProxy(pipe)
} else {
log.Current.Infof("LeaderServer.listenFollower(): Sync Proxy already running for %s. Ignore new request.", conn.RemoteAddr())
conn.Close()
}
}
case <-listenerState.killch:
log.Current.Debugf("LeaderServer.listenFollower(): Receive kill signal. Terminate.")
return
}
}
}