func serve(conn *net.TCPConn, clientId int64, cmdChan chan *Response, rn RaftNode, fsStruct *fs.FS, gversion *int, jsonFile string) {
//fmt.Println("Inside Serve")
reader := bufio.NewReader(conn)
for {
msg, msgerr, fatalerr := fs.GetMsg(reader)
if fatalerr != nil {
reply(conn, &fs.Msg{Kind: 'M'})
conn.Close()
break
}
if msgerr != nil {
if (!reply(conn, &fs.Msg{Kind: 'M'})) {
conn.Close()
break
}
continue
}
var respMsg *fs.Msg
// replicate command to all other servers
// reads need not be replicated
if msg.Kind != 'r' {
// set client id to identify client channel to whom commit channel should forward
msg.ClientId = clientId
// replicate command before processing
msgBytes, err := encode(*msg)
// server is not able to encode
if err != nil {
if (!reply(conn, &fs.Msg{Kind: 'I'})) {
conn.Close()
break
}
continue
}
// replicate this command to all other raft nodes
rn.Append(msgBytes)
// wait on channel untill it has been replicated on majority of raft nodes
respStrVar := <-cmdChan
errVal := respStrVar.err
respMsg = respStrVar.resp
if errVal != nil {
// set message kind to ERR_REDIRECT
cont := []byte(getConnStringById(rn.LeaderId(), jsonFile))
reply(conn, &fs.Msg{Kind: 'R', Contents: cont})
conn.Close()
break
}
} else {
respMsg = fs.ProcessMsg(msg, fsStruct, gversion)
}
// response := fs.ProcessMsg(msg)
if !reply(conn, respMsg) {
conn.Close()
break
}
}
}
func serverMain(id int64, peers []NetConfig, jsonFile string) {
// Initialize raft node and spawn independent go routine
rn := initRaftNode(id, peers, jsonFile)
go rn.processEvents()
// In-memory directory that stores all files
var fsStruct = &fs.FS{Dir: make(map[string]*fs.FileInfo, 1000)}
// Global version maintained across all files
var gversion = 0
// open listen port for client connections
connString := getConnStringById(id, jsonFile)
tcpaddr, err := net.ResolveTCPAddr("tcp", connString)
check(err)
tcp_acceptor, err := net.ListenTCP("tcp", tcpaddr)
check(err)
// register MsgStruct for encoding and decoding of Msg structure to binary data
gob.Register(MsgStruct{})
// used as an indentified for different client connections
var clientId int64 = rn.Id() / 100
// manage index that is processed last
var lastIndexPrcsd int64 = -1
// map to maintain client id to channel mapping, so that after reading command
// from commit channel it can be put to appropriate client channel
clientIdToChanMap := make(map[int64]chan *Response)
var mapLock sync.RWMutex
// go routine to listen on commit channel from a raft node
go func() {
for {
cmtInfo := <-rn.CommitChannel()
// we get index as -1 when append has been requested on follower
// in that case - we do not replicate command but we do get commitInfo on commit channel
if cmtInfo.index != -1 {
// check for missing indexes
if cmtInfo.index != lastIndexPrcsd+1 {
// handleMissingCmnds()
for i := lastIndexPrcsd + 1; i < cmtInfo.index; i++ {
//fmt.Printf("Processing missing indexes: %v\n", i)
err, msngMsg := rn.Get(int(i))
if err != nil {
// invalid index has been requested
fmt.Println("Error: invalid index requested")
}
msngMsgDecoded, err := decode(msngMsg)
if err != nil {
// server facing problem with message decoding
fmt.Println("Error: decoding message after replication")
} else {
response := fs.ProcessMsg(&msngMsgDecoded, fsStruct, &gversion)
tempRes := &Response{response, nil}
mapLock.Lock()
if _, ok := clientIdToChanMap[msngMsgDecoded.ClientId]; ok {
clientIdToChanMap[msngMsgDecoded.ClientId] <- tempRes
}
mapLock.Unlock()
}
}
}
}
msg, err := decode(cmtInfo.data)
if err != nil {
// server facing problem with message decoding
fmt.Println("Error: decoding message after replication")
} else {
response := fs.ProcessMsg(&msg, fsStruct, &gversion)
tempRes := &Response{response, cmtInfo.err}
mapLock.Lock()
if _, ok := clientIdToChanMap[msg.ClientId]; ok {
clientIdToChanMap[msg.ClientId] <- tempRes
}
mapLock.Unlock()
}
if cmtInfo.index != -1 {
lastIndexPrcsd = cmtInfo.index
}
}
}()
for {
// fmt.Println("waiting for connections")
tcp_conn, err := tcp_acceptor.AcceptTCP()
check(err)
//fmt.Println("Connection establised")
// make sure that client id's do not coincide across servers
clientId = (clientId + int64(len(peers))) % math.MaxInt64
mapLock.Lock()
clientIdToChanMap[clientId] = make(chan *Response)
mapLock.Unlock()
go serve(tcp_conn, clientId, clientIdToChanMap[clientId], rn, fsStruct, &gversion, jsonFile)
}
}
