//
// run election
//
func (s *Server) runElection() (leader string, err error) {
host := GetHostUDPAddr()
peers := GetPeerUDPAddr()
// Create an election site to start leader election.
log.Current.Debugf("Server.runElection(): Local Server %s start election", host)
log.Current.Debugf("Server.runElection(): Peer in election")
for _, peer := range peers {
log.Current.Debugf(" peer : %s", peer)
}
s.site, err = protocol.CreateElectionSite(host, peers, s.factory, s.handler, false)
if err != nil {
return "", err
}
resultCh := s.site.StartElection()
if resultCh == nil {
return "", common.NewError(common.SERVER_ERROR, "Election Site is in progress or is closed.")
}
leader, ok := <-resultCh // blocked until leader is elected
if !ok {
return "", common.NewError(common.SERVER_ERROR, "Election Fails")
}
return leader, nil
}
//
// Handle a new incoming request
//
//func (s *RequestReceiver) NewRequest(message []byte, reply *[]byte) error {
func (s *RequestReceiver) NewRequest(req *Request, reply **Reply) error {
if s.server.IsDone() {
return common.NewError(common.SERVER_ERROR, "Server is terminated. Cannot process new request.")
}
log.Current.Debugf("RequestReceiver.NewRequest(): Receive request from client")
log.Current.Debugf("RequestReceiver.NewRequest(): opCode %s key %s value %s", req.OpCode, req.Key, req.Value)
opCode := common.GetOpCode(req.OpCode)
if opCode == common.OPCODE_GET {
result, err := s.server.GetValue(req.Key)
if err != nil {
return err
}
log.Current.Debugf("RequestReceiver.NewRequest(): Receive response from server, len(value) = %d", len(result))
*reply = &Reply{Result: result}
return nil
} else if opCode == common.OPCODE_ADD ||
opCode == common.OPCODE_SET ||
opCode == common.OPCODE_DELETE {
if req.Value == nil {
req.Value = ([]byte)("")
}
id, err := common.NewUUID()
if err != nil {
return err
}
request := s.server.factory.CreateRequest(id,
uint32(common.GetOpCode(req.OpCode)),
req.Key,
req.Value)
handle := newRequestHandle(request)
handle.CondVar.L.Lock()
defer handle.CondVar.L.Unlock()
// push the request to a channel
log.Current.Debugf("Handing new request to server. Key %s", req.Key)
s.server.state.incomings <- handle
// This goroutine will wait until the request has been processed.
handle.CondVar.Wait()
log.Current.Debugf("Receive Response for request. Key %s", req.Key)
*reply = &Reply{Result: nil}
return handle.Err
} else {
return common.NewError(common.CLIENT_ERROR, fmt.Sprintf("Invalid Op code %s", req.OpCode))
}
}
//
// Cleanup internal state upon exit
//
func (s *EmbeddedServer) cleanupState() {
s.state.mutex.Lock()
defer s.state.mutex.Unlock()
common.SafeRun("EmbeddedServer.cleanupState()",
func() {
if s.listener != nil {
s.listener.Close()
}
})
common.SafeRun("EmbeddedServer.cleanupState()",
func() {
if s.repo != nil {
s.repo.Close()
}
})
for len(s.state.incomings) > 0 {
request := <-s.state.incomings
request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination")
common.SafeRun("EmbeddedServer.cleanupState()",
func() {
request.CondVar.L.Lock()
defer request.CondVar.L.Unlock()
request.CondVar.Signal()
})
}
for _, request := range s.state.pendings {
request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination")
common.SafeRun("EmbeddedServer.cleanupState()",
func() {
request.CondVar.L.Lock()
defer request.CondVar.L.Unlock()
request.CondVar.Signal()
})
}
for _, request := range s.state.proposals {
request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination")
common.SafeRun("EmbeddedServer.cleanupState()",
func() {
request.CondVar.L.Lock()
defer request.CondVar.L.Unlock()
request.CondVar.Signal()
})
}
}
func listen(name string, pipe *common.PeerPipe) (common.Packet, error) {
reqch := pipe.ReceiveChannel()
req, ok := <-reqch
if !ok {
return nil, common.NewError(common.SERVER_ERROR, "SyncProxy.listen(): channel closed. Terminate")
}
if req.Name() != name {
return nil, common.NewError(common.PROTOCOL_ERROR,
"SyncProxy.listen(): Expect message "+name+", Receive message "+req.Name())
}
return req, nil
}
func (l *LeaderSyncProxy) updateAcceptedEpochAfterQuorum() error {
log.Current.Debugf("LeaderSyncProxy.updateAcceptedEpochAfterQuroum()")
// Get my follower's vote for the accepted epoch
packet, err := listen("FollowerInfo", l.follower)
if err != nil {
return err
}
// Get epoch from follower message
info := packet.(FollowerInfoMsg)
epoch := info.GetAcceptedEpoch()
fid := info.GetFid()
voting := info.GetVoting()
// initialize the follower state
l.followerState = &followerState{lastLoggedTxid: 0, currentEpoch: 0, fid: fid, voting: voting}
// update my vote and wait for epoch to reach quorum
newEpoch, ok := l.state.voteAcceptedEpoch(l.GetFid(), epoch, l.followerState.voting)
if !ok {
return common.NewError(common.ELECTION_ERROR,
"LeaderSyncProxy.updateAcceptedEpochAfterQuorum(): Fail to reach quorum on accepted epoch (FollowerInfo)")
}
// update the accepted epoch based on the quorum result. This function
// will perform update only if the new epoch is larger than existing value.
err = l.handler.NotifyNewAcceptedEpoch(newEpoch)
if err != nil {
return err
}
return nil
}
func (l *LeaderSyncProxy) declareNewLeaderAfterQuorum() error {
log.Current.Debugf("LeaderSyncProxy.declareNewLeaderAfterQuorum()")
// return the new epoch to the follower
epoch, err := l.handler.GetCurrentEpoch()
if err != nil {
return err
}
packet := l.factory.CreateNewLeader(epoch)
err = send(packet, l.follower)
if err != nil {
return err
}
// Get the new leader ack
ack, err := listen("NewLeaderAck", l.follower)
if err != nil {
return err
}
// TODO : Verify the ack
ack = ack // TODO : just to get around compile error
// update my vote and wait for quorum of ack from followers
ok := l.state.voteNewLeaderAck(l.GetFid(), l.followerState.voting)
if !ok {
return common.NewError(common.ELECTION_ERROR,
"LeaderSyncProxy.declareNewLeaderAfterQuorum(): Fail to reach quorum on NewLeaderAck")
}
return nil
}
//
// This is the API for client that is co-located withe gometa server
// in the same process.
//
func NewClientRequest(req *Request, reply **Reply) error {
if gHandler == nil {
return common.NewError(common.SERVER_ERROR, "Server is not ready to receive new request.")
}
return gHandler.NewRequest(req, reply)
}
//
// commit proposal
//
func (l *Leader) commit(txid common.Txnid) error {
// We are skipping proposal. The protocol expects that each follower must
// send Accept Msg in the order of Proposal being received. Since the
// message is sent out a reliable TCP connection, it is not possible to reach
// quorum out of order. Particularly, if a new follower leaves and rejoins,
// the leader is responsible for resending all the pending proposals to the
// followers. So if we see the txid is out-of-order there, then it is
// a fatal condition due to protocol error.
//
if !common.IsNextInSequence(txid, l.lastCommitted) {
log.Current.Debugf("Proposal must committed in sequential order for the same leader term. "+
"Found out-of-order commit. Leader last committed txid %d, commit msg %d", l.lastCommitted, txid)
return common.NewError(common.PROTOCOL_ERROR,
fmt.Sprintf("Proposal must committed in sequential order for the same leader term. "+
"Found out-of-order commit. Leader last committed txid %d, commit msg %d", l.lastCommitted, txid))
}
_, ok := l.proposals[txid]
if !ok {
return common.NewError(common.SERVER_ERROR,
fmt.Sprintf("Cannot find a proposal for the txid %d. Fail to commit the proposal.", txid))
}
// marking the proposal as committed. Always do this first before sending to followers.
err := l.handler.Commit(txid)
if err != nil {
return err
}
// remove the votes
delete(l.quorums, txid)
delete(l.proposals, txid)
// Update lastCommitted
l.lastCommitted = txid
// Send the commit to followers
l.sendCommit(txid)
// TODO: do we need to update election site? I don't think so, but need to double check.
return nil
}
//
// Create a new iterator. This is used by LeaderSyncProxy to replicate log
// entries to a folower/watcher. txid1 is last logged commited txid from follower/watcher.
// txid2 is the txid of the first packet in the observer queue for this follower/watcher.
// If the observer queue is empty, then txid2 is 0.
//
// For any LogEntry returned form the iterator, the following condition must be satisifed:
// 1) The txid of any LogEntry returned from the iterator must be greater than or equal to txid1.
// 2) The last LogEntry must have txid >= txid2. In other words, the iterator can gaurantee
// that it will cover any mutation by txid2, but cannot guarantee that it stops at txid2.
//
// Since TransientCommitLog does not keep track of history, it only supports the following case:
// 1) the beginning txid (txid1) is 0 (sending the full repository)
//
func (r *TransientCommitLog) NewIterator(txid1, txid2 common.Txnid) (CommitLogIterator, error) {
if txid1 != 0 {
return nil, common.NewError(common.REPO_ERROR, "TransientLCommitLog.NewIterator: cannot support beginning txid > 0")
}
// iter can be nil if nothing has been logged
retry := true
retryLabel:
txnid, iter, err := r.repo.AcquireSnapshot(MAIN)
if err != nil {
return nil, err
}
// The snapshot txnid must be at least match the ending txid (txid2). If not, it will retry.
// This is to ensure that it takes care race condition in which the first msg in an observer's
// queue is a commit, and the caller asks for the state of the repository which is at least as
// recent as that commit msg (txid2). If retry fails, return an error.
if txnid < txid2 {
if retry {
time.Sleep(common.XACT_COMMIT_WAIT_TIME)
retry = false
goto retryLabel
}
return nil, common.NewError(common.REPO_ERROR,
fmt.Sprintf("TransientLCommitLog.NewIterator: cannot support ending txid > %d", txnid))
}
result := &TransientLogIterator{
iter: iter,
txnid: txnid,
repo: r.repo,
curTxnid: common.Txnid(txid1 + 1),
curKey: "",
curContent: nil,
curError: nil}
if result.iter != nil {
result.curKey, result.curContent, result.curError = result.iter.Next()
}
return result, nil
}
func send(packet common.Packet, pipe *common.PeerPipe) error {
log.Current.Tracef("SyncProxy.send(): sending packet %s to peer (TCP %s)", packet.Name(), pipe.GetAddr())
if !pipe.Send(packet) {
return common.NewError(common.SERVER_ERROR, fmt.Sprintf("SyncProxy.listen(): Fail to send packet %s to peer (TCP %s)",
packet.Name(), pipe.GetAddr()))
}
return nil
}
//
// Create a new FollowerServer. This is a blocking call until
// the FollowerServer terminates. Make sure the kilch is a buffered
// channel such that if the goroutine running RunFollowerServer goes
// away, the sender won't get blocked.
//
func RunFollowerServer(naddr string,
leader string,
ss RequestMgr,
handler ActionHandler,
factory MsgFactory,
killch <-chan bool) (err error) {
// Catch panic at the main entry point for FollowerServer
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in RunFollowerServer() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
err = r.(error)
} else {
log.Current.Debugf("%s", "RunFollowerServer terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
}()
// create connection to leader
conn, err := createConnection(leader)
if err != nil {
return err
}
pipe := common.NewPeerPipe(conn)
log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower %s successfully "+
"created TCP connection to leader %s, local address %s", naddr, leader, conn.LocalAddr())
// close the connection to the leader. If connection is closed,
// sync proxy and follower will also terminate by err-ing out.
// If sync proxy and follower terminates the pipe upon termination,
// it is ok to close it again here.
defer common.SafeRun("FollowerServer.runFollowerServer()",
func() {
pipe.Close()
})
// start syncrhorniziing with the leader
success := syncWithLeader(naddr, pipe, handler, factory, killch)
// run server after synchronization
if success {
runFollower(pipe, ss, handler, factory, killch)
log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower Server %s terminate", naddr)
err = nil
} else {
err = common.NewError(common.SERVER_ERROR, fmt.Sprintf("Follower %s fail to synchronized with leader %s",
naddr, leader))
}
return err
}
//
// Retrieve entry from commit log
//
func (r *TransientCommitLog) Get(txid common.Txnid) (common.OpCode, string, []byte, error) {
msg, ok := r.logs[txid]
if !ok || msg == nil {
err := common.NewError(common.REPO_ERROR, fmt.Sprintf("LogEntry for txid %d does not exist in commit log", txid))
return common.OPCODE_INVALID, "", nil, err
}
// msg is a protobuf object. Directly pointing slices/strings into a struct
// may cause problems for cgo calls
key := string([]byte(msg.GetKey()))
content := []byte(string(msg.GetContent()))
return common.GetOpCodeFromInt(msg.GetOpCode()), key, content, nil
}
func (a *ServerAction) persistChange(op common.OpCode, key string, content []byte) error {
if op == common.OPCODE_ADD {
return a.repo.Set(repo.MAIN, key, content)
}
if op == common.OPCODE_SET {
return a.repo.Set(repo.MAIN, key, content)
}
if op == common.OPCODE_DELETE {
return a.repo.Delete(repo.MAIN, key)
}
return common.NewError(common.PROTOCOL_ERROR, fmt.Sprintf("ServerAction.persistChange() : Unknown op code %d", op))
}
func (e *Env) initWithArgs() error {
if len(os.Args) < 3 {
return common.NewError(common.ARG_ERROR, "Missing command line argument")
}
err := e.resolveHostAddr()
if err != nil {
return err
}
if len(os.Args) >= 6 {
e.resolvePeerAddr()
if err != nil {
return err
}
}
return nil
}
//
// run server (as leader or follower)
//
func (s *Server) runServer(leader string) (err error) {
host := GetHostUDPAddr()
// If this host is the leader, then start the leader server.
// Otherwise, start the followerServer.
if leader == host {
log.Current.Debugf("Server.runServer() : Local Server %s is elected as leader. Leading ...", leader)
s.state.setStatus(protocol.LEADING)
err = protocol.RunLeaderServer(GetHostTCPAddr(), s.listener, s.state, s.handler, s.factory, s.skillch)
} else {
log.Current.Debugf("Server.runServer() : Remote Server %s is elected as leader. Following ...", leader)
s.state.setStatus(protocol.FOLLOWING)
leaderAddr := findMatchingPeerTCPAddr(leader)
if len(leaderAddr) == 0 {
return common.NewError(common.SERVER_ERROR, "Cannot find matching TCP addr for leader "+leader)
}
err = protocol.RunFollowerServer(GetHostTCPAddr(), leaderAddr, s.state, s.handler, s.factory, s.skillch)
}
return err
}
func (l *LeaderSyncProxy) updateCurrentEpochAfterQuorum() error {
log.Current.Debugf("LeaderSyncProxy.updateCurrentEpochAfterQuorum()")
// Get my follower's vote for the epoch ack
packet, err := listen("EpochAck", l.follower)
if err != nil {
return err
}
// Get epoch from follower message
// TODO : Validate follower epoch
info := packet.(EpochAckMsg)
l.followerState.currentEpoch = info.GetCurrentEpoch()
l.followerState.lastLoggedTxid = common.Txnid(info.GetLastLoggedTxid())
// update my vote and wait for quorum of ack from followers
ok := l.state.voteEpochAck(l.GetFid(), l.followerState.voting)
if !ok {
return common.NewError(common.ELECTION_ERROR,
"LeaderSyncProxy.updateCurrentEpochAfterQuorum(): Fail to reach quorum on current epoch (EpochAck)")
}
// update the current epock after quorum of followers have ack'ed
epoch, err := l.handler.GetAcceptedEpoch()
if err != nil {
return err
}
// update the current epoch based on the quorum result. This function
// will perform update only if the new epoch is larger than existing value.
err = l.handler.NotifyNewCurrentEpoch(epoch)
if err != nil {
return err
}
return nil
}
//
// Goroutine for processing each request one-by-one
//
func (s *LeaderServer) processRequest(killch <-chan bool,
listenerState *ListenerState,
reqHandler CustomRequestHandler) (err error) {
defer func() {
if r := recover(); r != nil {
log.Current.Errorf("panic in LeaderServer.processRequest() : %s\n", r)
log.Current.Errorf("%s", log.Current.StackTrace())
err = r.(error)
} else {
log.Current.Debugf("LeaderServer.processRequest() : Terminates.")
log.Current.Tracef(log.Current.StackTrace())
}
common.SafeRun("LeaderServer.processRequest()",
func() {
listenerState.killch <- true
})
}()
// start processing loop after I am being confirmed as a leader (there
// is a quorum of followers that have sync'ed with me)
if !s.waitTillReady() {
return common.NewError(common.ELECTION_ERROR,
"LeaderServer.processRequest(): Leader times out waiting for quorum of followers. Terminate")
}
// At this point, the leader has gotten a majority of followers to follow, so it
// can proceed. It is possible that it may loose quorum of followers. But in that
// case, the leader will not be able to process any request.
log.Current.Debugf("LeaderServer.processRequest(): Leader Server is ready to proces request")
// Leader is ready at this time. This implies that there is a quorum of follower has
// followed this leader. Get the change channel to keep track of number of followers.
// If the leader no longer has quorum, it needs to let go of its leadership.
leaderchangech := s.leader.GetEnsembleChangeChannel()
ensembleSize := s.handler.GetEnsembleSize()
// notify the request processor to start processing new request
incomings := s.state.requestMgr.GetRequestChannel()
var outgoings <-chan common.Packet = nil
if reqHandler != nil {
outgoings = reqHandler.GetResponseChannel()
} else {
outgoings = make(<-chan common.Packet)
}
for {
select {
case handle, ok := <-incomings:
if ok {
// de-queue the request
s.state.requestMgr.AddPendingRequest(handle)
// forward request to the leader
s.leader.QueueRequest(s.leader.GetFollowerId(), handle.Request)
} else {
// server shutdown.
log.Current.Debugf("LeaderServer.processRequest(): channel for receiving client request is closed. Terminate.")
return nil
}
case msg, ok := <-outgoings:
if ok {
// forward msg to the leader
s.leader.QueueResponse(msg)
} else {
log.Current.Infof("LeaderServer.processRequest(): channel for receiving custom response is closed. Ignore.")
}
case <-killch:
// server shutdown
log.Current.Debugf("LeaderServer.processRequest(): receive kill signal. Stop Client request processing.")
return nil
case <-listenerState.donech:
// listener is down. Terminate this request processing loop as well.
log.Current.Infof("LeaderServer.processRequest(): follower listener terminates. Stop client request processing.")
return nil
case <-leaderchangech:
// Listen to any change to the leader's active ensemble, and to ensure that the leader maintain majority.
// The active ensemble is the set of running followers connected to the leader.
numFollowers := s.leader.GetActiveEnsembleSize()
if numFollowers <= int(ensembleSize/2) {
// leader looses majority of follower.
log.Current.Infof("LeaderServer.processRequest(): leader looses majority of follower. Stop client request processing.")
return nil
}
}
}
return nil
}
func (l *FollowerSyncProxy) receiveAndUpdateAcceptedEpoch() error {
log.Current.Debugf("FollowerSyncProxy.receiveAndUpdateAcceptedEpoch()")
// Get the accepted epoch from the leader. This epoch
// is already being voted on by multiple followers (the highest
// epoch among the quorum of followers).
packet, err := listen("LeaderInfo", l.leader)
if err != nil {
return err
}
// Get epoch from leader message
info := packet.(LeaderInfoMsg)
epoch := info.GetAcceptedEpoch()
if err != nil {
return err
}
acceptedEpoch, err := l.handler.GetAcceptedEpoch()
if err != nil {
return err
}
if epoch > acceptedEpoch {
// Update the accepted epoch based on the quorum result. This function
// will perform update only if the new epoch is larger than existing value.
// Once the accepted epoch is updated, it will not be reset even if the
// sychornization with the leader fails. Therefore, the follower will always
// remember the largest accepted epoch known to it, such that it can be used
// in the next round of voting. Note that the leader derives this new accepted
// epoch only after it has polled from a quorum of followers. So even if sync fails,
// it is unlikey that in the next sync, the leader will give a new accepted epoch smaller
// than what is being stored now.
err = l.handler.NotifyNewAcceptedEpoch(epoch)
if err != nil {
return err
}
} else if epoch == acceptedEpoch {
// In ZK, if the local epoch (acceptedEpoch) == leader's epoch (epoch), it will replly an EpochAck with epoch = -1.
// This is to tell the leader that it should not count this EpockAck when computing quorum of EpochAck.
// This is to ensure that this follower does not "double ack" to the leader (e.g. when this follower rejoins a
// stable ensemble). In our implementation for ConsentState, it should not be affected by double ack from the same host.
} else {
return common.NewError(common.PROTOCOL_ERROR, "Accepted Epoch from leader is smaller or equal to my epoch.")
}
// Notify the leader that I have accepted the epoch. Send
// the last logged txid and current epoch to the leader.
txid, err := l.handler.GetLastLoggedTxid()
if err != nil {
return err
}
currentEpoch, err := l.handler.GetCurrentEpoch()
if err != nil {
return err
}
l.state.lastLoggedTxid = common.Txnid(txid)
l.state.currentEpoch = currentEpoch
packet = l.factory.CreateEpochAck(uint64(txid), currentEpoch)
return send(packet, l.leader)
}