func EventLoop(config *string, client int, q chan int) {
configStruct := ParseConfig(config, q)
state := MakeComputePeerState(client, len(configStruct.Clients))
redisConfig := redis.DefaultConfig()
redisConfig.Network = "tcp"
redisConfig.Address = configStruct.Databases[client].Address
redisConfig.Database = configStruct.Databases[client].Database
state.RedisClient = redis.NewClient(redisConfig)
fmt.Printf("Using redis at %s with db %d\n", configStruct.Databases[client].Address, configStruct.Databases[client].Database)
// Create the 0MQ context
ctx, err := zmq.NewContext()
if err != nil {
//fmt.Println("Error creating 0mq context: ", err)
q <- 1
}
fmt.Println("ZMQ context created")
// Establish the PUB-SUB connection that will be used to direct all the computation clusters
state.SubSock, err = ctx.Socket(zmq.Sub)
state.SubSock.SetHWM(ZMQ_HWM)
if err != nil {
//fmt.Println("Error creating PUB socket: ", err)
q <- 1
}
fmt.Println("SUB socket connection created")
err = state.SubSock.Connect(configStruct.PubAddress)
if err != nil {
//fmt.Println("Error binding PUB socket: ", err)
q <- 1
}
fmt.Println("SUB socket connected")
// Establish coordination socket
state.CoordSock, err = ctx.Socket(zmq.Dealer)
state.CoordSock.SetHWM(ZMQ_HWM)
if err != nil {
//fmt.Println("Error creating Dealer socket: ", err)
q <- 1
}
fmt.Println("Coord socket created")
err = state.CoordSock.Connect(configStruct.ControlAddress)
if err != nil {
//fmt.Println("Error connecting ", err)
q <- 1
}
fmt.Println("Coord socket connected")
state.PeerInSock, err = ctx.Socket(zmq.Router)
state.PeerInSock.SetHWM(ZMQ_HWM)
if err != nil {
//fmt.Println("Error creating peer router socket: ", err)
q <- 1
}
fmt.Println("PeerIn socket created")
err = state.PeerInSock.Bind(configStruct.Clients[client]) // Set up something to listen to peers
if err != nil {
//fmt.Println("Error binding peer router socket")
q <- 1
}
fmt.Println("PeerIn socket bound")
for index, value := range configStruct.Clients {
if index != client {
state.PeerOutSocks[index], err = ctx.Socket(zmq.Dealer)
state.PeerOutSocks[index].SetHWM(ZMQ_HWM)
if err != nil {
//fmt.Println("Error creating dealer socket: ", err)
q <- 1
return
}
err = state.PeerOutSocks[index].Connect(value)
if err != nil {
//fmt.Println("Error connection ", err)
q <- 1
return
}
state.PeerOutChannels[index] = state.PeerOutSocks[index].ChannelsBuffer(BUFFER_SIZE)
defer state.PeerOutChannels[index].Close()
}
}
fmt.Println("PeerOut sockets created and connected")
state.PeerInChannel = state.PeerInSock.Channels()
fmt.Println("Calling from ReceiveFromPeers")
go state.NaggleOutChannel()
go state.NaggleCoordChannel()
go state.ReceiveFromPeers()
state.Sync(q)
fmt.Println("Sync finished")
state.IntermediateSync(q)
fmt.Println("IntermediateSync finished")
fmt.Printf("Subscribing to CMD\n")
state.SubSock.Subscribe([]byte("CMD"))
fmt.Printf("Done subscribing to CMD\n")
//fmt.Println("Receiving")
// We cannot create channels before finalizing the set of subscriptions, since sockets are
// not thread safe. Hence first sync, then get channels
state.SubChannel = state.SubSock.ChannelsBuffer(BUFFER_SIZE)
state.CoordChannel = state.CoordSock.ChannelsBuffer(BUFFER_SIZE)
fmt.Println("Created CoordChannel and SubChannel")
defer func() {
state.CoordChannel.Close()
state.SubChannel.Close()
state.SubSock.Close()
state.CoordSock.Close()
for idx := range state.PeerOutSocks {
state.PeerOutSocks[idx].Close()
}
ctx.Close()
//fmt.Println("Closed socket")
}()
keepaliveCh := make(chan bool, 1)
//go func(ch chan bool) {
// for {
// time.Sleep(1 * time.Second)
// ch <- true
// }
//}(keepaliveCh)
_ = keepaliveCh
fmt.Println("Start waiting for messages")
once := false
for true {
if !once {
fmt.Println("Starting to wait for sub and coord channel")
once = true
}
select {
case msg := <-state.SubChannel.In():
//fmt.Println("Message on SubChannel")
state.ActionMsg(msg)
case msg := <-state.CoordChannel.In():
//fmt.Println("Message on CoordChannel")
state.ActionMsg(msg)
case err = <-state.SubChannel.Errors():
//fmt.Println("Error in SubChannel", err)
q <- 1
return
case err = <-state.CoordChannel.Errors():
//fmt.Println("Error in CoordChannel", err)
q <- 1
return
case <-keepaliveCh:
fmt.Printf("Sub and coord channel receive alive\n")
}
}
q <- 0
}
/* The main event loop */
func EventLoop(config *string, state *InputPeerState, q chan int, ready chan bool) {
// Create the 0MQ context
ctx, err := zmq.NewContext()
state.Config = ParseConfig(config, q)
state.InitPeerState(state.Config.Clients)
if err != nil {
fmt.Println("Error creating 0mq context: ", err)
q <- 1
return
}
// Establish the PUB-SUB connection that will be used to direct all the computation clusters
state.PubSock, err = ctx.Socket(zmq.Pub)
state.PubSock.SetHWM(ZMQ_HWM)
if err != nil {
fmt.Println("Error creating PUB socket: ", err)
q <- 1
return
}
err = state.PubSock.Bind(state.Config.PubAddress)
if err != nil {
fmt.Println("Error binding PUB socket: ", err)
q <- 1
return
}
// Establish coordination socket
state.CoordSock, err = ctx.Socket(zmq.Router)
state.CoordSock.SetHWM(ZMQ_HWM)
if err != nil {
fmt.Println("Error creating REP socket: ", err)
q <- 1
return
}
// Strict error checking
//state.CoordSock.SetRouterMandatory()
err = state.CoordSock.Bind(state.Config.ControlAddress)
if err != nil {
fmt.Println("Error binding coordination socket ", err)
q <- 1
return
}
state.CoordChannel = state.CoordSock.ChannelsBuffer(BUFFER_SIZE)
state.PubChannel = state.PubSock.ChannelsBuffer(BUFFER_SIZE)
state.Sync(q)
go state.NaggleCoordChannel()
go state.NagglePubChannel()
go state.MessageLoop()
ready <- true
// Handle errors from here on out
for {
select {
case err = <-state.CoordChannel.Errors():
fmt.Println("Coordination error", err)
case err = <-state.PubChannel.Errors():
fmt.Println("Publishing error", err)
// Do nothing
}
}
defer func() {
state.CoordChannel.Close()
state.PubChannel.Close()
state.PubSock.Close()
state.CoordSock.Close()
ctx.Close()
fmt.Println("Closed socket")
}()
}