Example #1
0
func runOnce(peer string,
	requestMgr RequestMgr,
	handler ActionHandler,
	factory MsgFactory,
	killch <-chan bool,
	readych chan<- bool,
	alivech chan<- bool,
	pingch <-chan bool,
	once *sync.Once) (isKilled bool) {

	// Catch panic at the main entry point for WatcherServer
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("panic in WatcherServer.runOnce() : %s\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		} else {
			log.Current.Debugf("WatcherServer.runOnce() terminates.")
			log.Current.Tracef(log.Current.StackTrace())
		}

		if requestMgr != nil {
			requestMgr.CleanupOnError()
		}
	}()

	// create connection with a peer
	conn, err := createConnection(peer)
	if err != nil {
		log.Current.Errorf("WatcherServer.runOnce() error : %s", err)
		return false
	}
	pipe := common.NewPeerPipe(conn)
	log.Current.Debugf("WatcherServer.runOnce() : Watcher successfully created TCP connection to peer %s", peer)

	// close the connection to the peer. If connection is closed,
	// sync proxy and watcher will also terminate by err-ing out.
	// If sync proxy and watcher terminates the pipe upon termination,
	// it is ok to close it again here.
	defer common.SafeRun("WatcherServer.runOnce()",
		func() {
			pipe.Close()
		})

	// start syncrhorniziing with the metadata server
	success, isKilled := syncWithPeer(pipe, handler, factory, killch)

	// run watcher after synchronization
	if success {
		if !runWatcher(pipe, requestMgr, handler, factory, killch, readych, alivech, pingch, once) {
			log.Current.Errorf("WatcherServer.runOnce() : Watcher terminated unexpectedly.")
			return false
		}

	} else if !isKilled {
		log.Current.Errorf("WatcherServer.runOnce() : Watcher fail to synchronized with peer %s", peer)
		return false
	}

	return true
}
Example #2
0
//
// Create a new FollowerServer. This is a blocking call until
// the FollowerServer terminates. Make sure the kilch is a buffered
// channel such that if the goroutine running RunFollowerServer goes
// away, the sender won't get blocked.
//
func RunFollowerServer(naddr string,
	leader string,
	ss RequestMgr,
	handler ActionHandler,
	factory MsgFactory,
	killch <-chan bool) (err error) {

	// Catch panic at the main entry point for FollowerServer
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("panic in RunFollowerServer() : %s\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
			err = r.(error)
		} else {
			log.Current.Debugf("%s", "RunFollowerServer terminates.")
			log.Current.Tracef(log.Current.StackTrace())
		}
	}()

	// create connection to leader
	conn, err := createConnection(leader)
	if err != nil {
		return err
	}

	pipe := common.NewPeerPipe(conn)
	log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower %s successfully "+
		"created TCP connection to leader %s, local address %s", naddr, leader, conn.LocalAddr())

	// close the connection to the leader. If connection is closed,
	// sync proxy and follower will also terminate by err-ing out.
	// If sync proxy and follower terminates the pipe upon termination,
	// it is ok to close it again here.
	defer common.SafeRun("FollowerServer.runFollowerServer()",
		func() {
			pipe.Close()
		})

	// start syncrhorniziing with the leader
	success := syncWithLeader(naddr, pipe, handler, factory, killch)

	// run server after synchronization
	if success {
		runFollower(pipe, ss, handler, factory, killch)
		log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower Server %s terminate", naddr)
		err = nil
	} else {
		err = common.NewError(common.SERVER_ERROR, fmt.Sprintf("Follower %s fail to synchronized with leader %s",
			naddr, leader))
	}

	return err
}
Example #3
0
//
// Listen to new connection request from the follower/peer.
// Start a new LeaderSyncProxy to synchronize the state
// between the leader and the peer.
//
func (l *LeaderServer) listenFollower(listenerState *ListenerState) {

	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("panic in LeaderServer.listenFollower() : %s\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		} else {
			log.Current.Debugf("LeaderServer.listenFollower() terminates.")
			log.Current.Tracef(log.Current.StackTrace())
		}

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				l.terminateAllOutstandingProxies()
			})

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				listenerState.donech <- true
			})
	}()

	connCh := l.listener.ConnChannel()
	if connCh == nil {
		// It should not happen unless the listener is closed
		return
	}

	// if there is a single server, then we don't need to wait for follower
	// for the server to be ready to process request.
	if l.handler.GetEnsembleSize() == 1 {
		if err := l.incrementEpoch(); err != nil {
			log.Current.Errorf("LeaderServer.listenFollower(): Error when boostraping leader with ensembleSize=1. Error = %s", err)
			return
		}

		l.notifyReady()
	}

	for {
		select {
		case conn, ok := <-connCh:
			{
				if !ok {
					// channel close.  Simply return.
					return
				}

				// There is a new peer connection request from the follower.  Start a proxy to synchronize with the follower.
				// The leader does not proactively connect to follower:
				// 1) The ensemble is stable, but a follower may just reboot and needs to connect to the leader
				// 2) Even if the leader receives votes from the leader, the leader cannot tell for sure that the follower does
				//    not change its vote.  Only if the follower connects, the leader can confirm the follower's alliance.
				//
				log.Current.Debugf("LeaderServer.listenFollower(): Receive connection request from follower %s", conn.RemoteAddr())
				if l.registerOutstandingProxy(conn.RemoteAddr().String()) {
					pipe := common.NewPeerPipe(conn)
					go l.startProxy(pipe)
				} else {
					log.Current.Infof("LeaderServer.listenFollower(): Sync Proxy already running for %s. Ignore new request.", conn.RemoteAddr())
					conn.Close()
				}
			}
		case <-listenerState.killch:
			log.Current.Debugf("LeaderServer.listenFollower(): Receive kill signal. Terminate.")
			return
		}
	}
}