func outgoingConn(node uint16) {
conn, err := connect.Dial(connect.CONSENSUS_PROTOCOL, node)
if err != nil {
// Can't reach the other node.
return
}
receivedConnCh <- receivedConn{node: node, conn: conn}
log.Print("core/consensus: made outgoing connection to ", node)
handleConn(node, conn)
}
// Try to make an outgoing connection to the given node ID.
func tryOutgoing(node uint16) {
conn, err := connect.Dial(connect.FOLLOW_PROTOCOL, node)
if err != nil {
// No luck connecting.
return
}
log.Print("shared/follow: made new outgoing connection to ", node)
// Send the new outgoing connection to the processing goroutine.
followConn := new(followConn)
followConn.node = node
followConn.conn = conn
followConn.outgoing = true
receivedConnCh <- followConn
}
// Sends the given change forward message to the given node.
// Must not be called with our own node ID.
// Does not handle adding timeouts, or adding this node ID to the ignore list.
// Must be called from the processing goroutine.
func sendForward(node uint16, forward *chproto.ChangeForward) {
// If we don't have a connection to this node,
// establish an outgoing connection.
if len(connections[node]) == 0 {
conn, err := connect.Dial(connect.CHANGE_REQUEST_PROTOCOL, node)
if err != nil {
// Message is dropped.
return
}
log.Print("shared/chrequest: made new outgoing conn to node ",
node)
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
connections[node][0].SendProto(2, forward)
}
// Top-level function of the processing goroutine.
func process() {
connections = make(map[uint16][]*connect.BaseConn)
// On startup, make an outgoing connection attempt to all other
// core nodes, before continuing.
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(
connect.CONSENSUS_PROTOCOL, node)
if err != nil {
// Can't reach the other node.
continue
}
log.Print("core/consensus: made outgoing connection to ", node)
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// We should try to make an outgoing connection to any node
// that we do not have at least one connection to.
// We need to make these asynchronously, because connections
// are slow.
case <-reconnectTicker:
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
go outgoingConn(node)
}
// New change request, for us to propose as leader.
case req := <-newChangeCh:
store.StartTransaction()
if !amLeader && receivedPromises != nil {
waitingRequests = append(waitingRequests, req)
} else if !amLeader {
waitingRequests = append(waitingRequests, req)
// Start attempting to be leader.
m := make(map[uint16]*coproto.Promise)
receivedPromises = m
proposal, _ := store.Proposal()
proposal++
store.SetProposal(proposal, config.Id())
firstUn := store.InstructionFirstUnapplied()
// Send prepare messages to all other nodes.
var prepare coproto.Prepare
prepare.Proposal = new(uint64)
prepare.FirstUnapplied = new(uint64)
*prepare.Proposal = proposal
*prepare.FirstUnapplied = firstUn
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) != 0 {
c := connections[node][0]
c.SendProto(2, &prepare)
}
}
// Behave as if we got a promise message
// from ourselves.
var promise coproto.Promise
promise.Proposal = prepare.Proposal
promise.PrevProposal = promise.Proposal
promise.Leader = new(uint32)
*promise.Leader = uint32(config.Id())
promise.PrevLeader = promise.Leader
addPromise(config.Id(), &promise)
} else {
newSlot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(newSlot, req)
proposal, leader := store.Proposal()
inst := makeInst(newSlot, proposal, leader,
req)
sendProposal(inst)
}
store.EndTransaction()
// Leadership attempt timed out.
case timedOutProposal := <-leaderTimeoutCh:
store.StartTransaction()
proposal, leader := store.Proposal()
// If the proposal has changed since that
// leadership attempt, ignore it.
if leader != config.Id() {
return
}
if proposal != timedOutProposal {
return
}
// If we successfully became leader, ignore it.
if amLeader {
return
}
// Otherwise, stop our attempt to become leader.
stopBeingLeader()
store.EndTransaction()
// Proposal timed out.
case timeout := <-proposalTimeoutCh:
store.StartTransaction()
// If this timeout was not canceled, a proposal failed.
// We stop being leader.
if proposalTimeouts[timeout.slot] == timeout {
stopBeingLeader()
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
connections[node] = append(connections[node], conn)
// Received message.
case recvMsg := <-receivedMsgCh:
node := recvMsg.node
conn := recvMsg.conn
msg := recvMsg.msg
switch *msg.MsgType {
case 2:
processPrepare(node, conn, msg.Content)
case 3:
processPromise(node, conn, msg.Content)
case 4:
processAccept(node, conn, msg.Content)
case 5:
processAccepted(node, conn, msg.Content)
case 6:
processNack(node, conn, msg.Content)
default:
// Unknown message.
conn.Close()
}
// Terminate received connection.
case terminatedConn := <-terminatedConnCh:
node := terminatedConn.node
conn := terminatedConn.conn
for i, other := range connections[node] {
if other != conn {
continue
}
conns := connections[node]
conns = append(conns[:i], conns[i+1:]...)
connections[node] = conns
break
}
}
}
}
func process() {
// Try to make an outgoing connection to all other client nodes,
// if we're not in a degraded state.
store.StartTransaction()
if !store.Degraded() {
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
}
store.EndTransaction()
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// If not degraded, try to make an outgoing connection to any
// client node that we do not have at least one connection to.
case <-reconnectTicker:
store.StartTransaction()
// Do not attempt to make connections while degraded.
if store.Degraded() {
store.EndTransaction()
break
}
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
conn, err := connect.Dial(
connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] =
append(connections[node], conn)
go handleConn(node, conn)
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// If we are degraded, reject the connection.
if store.Degraded() {
conn.Close()
store.EndTransaction()
break
}
// Add to our connections.
connections[node] = append(connections[node], conn)
store.EndTransaction()
// Terminated connection.
case receivedConn := <-terminatedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// Remove this connection from our connection list.
index := -1
for i, _ := range connections[node] {
if conn == connections[node][i] {
index = i
break
}
}
if index != -1 {
connections[node] =
append(connections[node][:index],
connections[node][index+1:]...)
}
store.EndTransaction()
}
}
}