func handleDegraded() {
// If we become degraded, drop all clients.
if store.Degraded() {
connectionsLock.Lock()
for conn, _ := range connections {
conn.conn.Close()
}
connections = make(map[*userConn]bool)
sessionsLock.Lock()
sessions = make(map[uint64]*userConn)
sessionsLock.Unlock()
waitingLock.Lock()
waiting = make(map[uint64]*userConn)
waitingLock.Unlock()
connectionsLock.Unlock()
} else {
sessionsLock.Lock()
defer sessionsLock.Unlock()
// If we've become undegraded...
// Check for attached sessions without connections.
checkOrphanAttaches()
// Check for nameless users.
checkNameless()
}
}
func Forward(node uint16, userMsg *UserMessage) {
store.StartTransaction()
defer store.EndTransaction()
// While degraded, we drop all messages.
if store.Degraded() {
return
}
// If we have no connection to that node, we drop the message.
if len(connections[node]) == 0 {
return
}
// Otherwise, send to the given node.
var forward rproto.Forward
forward.Sender = new(uint64)
forward.Recipient = new(uint64)
forward.Tag = new(string)
forward.Content = new(string)
forward.Ttl = new(uint32)
*forward.Sender = userMsg.Sender
*forward.Recipient = userMsg.Recipient
*forward.Tag = userMsg.Tag
*forward.Content = userMsg.Content
*forward.Ttl = uint32(userMsg.Ttl)
connections[node][0].SendProto(2, &forward)
}
func handleDegraded() {
// If the node becomes degraded, drop all relay connections.
if store.Degraded() {
for _, nodeConns := range connections {
for _, conn := range nodeConns {
conn.Close()
}
}
connections = make(map[uint16][]*connect.BaseConn)
}
}
func Startup() {
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
// If undegraded, check for attached users with no session,
// or nameless users in the store.
if !store.Degraded() {
checkOrphanAttaches()
checkNameless()
}
// Start accepting client protocol connections.
go connect.Listen(connect.CLIENT_PROTOCOL, incomingConn)
}
func handleDegraded() {
// If we become degraded, close all follow connections
// aside those receiving bursts.
if store.Degraded() {
connectionsLock.Lock()
for _, conn := range connections {
conn.lock.Lock()
if !conn.receivingBurst {
conn.Close()
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
}
func incomingConn(node uint16, conn *connect.BaseConn) {
store.StartTransaction()
if store.Degraded() {
conn.Close()
store.EndTransaction()
return
}
userConn := new(userConn)
userConn.conn = conn
userConn.deliver = make(chan *relay.UserMessage, 100)
// Add to connections set.
connectionsLock.Lock()
connections[userConn] = true
connectionsLock.Unlock()
store.EndTransaction()
go handleConn(userConn)
}
func process() {
for {
select {
// Try to make any needed outgoing connections.
case <-tryOutgoingCh:
store.StartTransaction()
connectionsLock.Lock()
coreNodes := config.CoreNodes()
if !store.Degraded() {
if config.IsCore() {
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if connections[node] == nil {
go tryOutgoing(node)
}
}
} else {
// Settle for less than 2 core nodes,
// if there *aren't* 2 core nodes.
targetConns := 2
if targetConns > len(coreNodes) {
targetConns = len(coreNodes)
}
newOutgoing := targetConns -
len(connections)
used := -1
for newOutgoing > 0 {
r := processRand.Intn(
len(coreNodes))
// Don't do the same node
// twice.
if r == used {
continue
}
node := coreNodes[r]
if connections[node] == nil {
used = r
go tryOutgoing(node)
newOutgoing--
}
}
}
} else {
if len(connections) == 0 {
r := processRand.Intn(len(coreNodes))
node := coreNodes[r]
if connections[node] == nil {
go tryOutgoing(node)
}
}
}
connectionsLock.Unlock()
store.EndTransaction()
// After a random time, check again.
randDur := time.Duration(processRand.Intn(19)+1) * time.Second
tryOutgoingCh = time.NewTimer(randDur).C
// New received connection.
case conn := <-receivedConnCh:
conn.offerTimers = make(map[uint64]*time.Timer)
store.StartTransaction()
connectionsLock.Lock()
// If we are degraded, reject the connection,
// unless we have no other connection,
// and it is outbound.
if store.Degraded() &&
!(len(connections) == 0 && conn.outgoing) {
conn.lock.Lock()
conn.Close()
conn.lock.Unlock()
connectionsLock.Unlock()
store.EndTransaction()
break
}
// If we have an existing connection to this node,
// close it.
if connections[conn.node] != nil {
other := connections[conn.node]
other.lock.Lock()
other.Close()
if other.firstUnappliedTimer != nil {
other.firstUnappliedTimer.Stop()
}
other.lock.Unlock()
}
// Add to our connections.
connections[conn.node] = conn
// Send initial position and leader messages.
posMsg := new(fproto.Position)
posMsg.FirstUnapplied = new(uint64)
posMsg.Degraded = new(bool)
*posMsg.FirstUnapplied =
store.InstructionFirstUnapplied()
*posMsg.Degraded = store.Degraded()
conn.conn.SendProto(2, posMsg)
proposal, leader := store.Proposal()
leaderMsg := new(fproto.Leader)
leaderMsg.Proposal = new(uint64)
leaderMsg.Leader = new(uint32)
*leaderMsg.Proposal = proposal
*leaderMsg.Leader = uint32(leader)
conn.conn.SendProto(11, leaderMsg)
connectionsLock.Unlock()
store.EndTransaction()
// Start timer for sending first unapplied
// instruction updates.
if config.IsCore() && conn.node <= 0x2000 {
conn.lock.Lock()
conn.firstUnappliedTimer = time.AfterFunc(
firstUnappliedTimerDuration,
func() { conn.firstUnappliedTimeout() })
conn.lock.Unlock()
}
// Start handling received messages from the connection.
go handleConn(conn)
// Terminated connection.
case conn := <-terminatedConnCh:
connectionsLock.Lock()
if connections[conn.node] == conn {
delete(connections, conn.node)
conn.lock.Lock()
conn.closed = true
if conn.firstUnappliedTimer != nil {
conn.firstUnappliedTimer.Stop()
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
}
}
func handlePosition(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.Position
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if config.IsCore() && f.node <= 0x2000 {
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
if store.Degraded() && *msg.Degraded {
f.Close()
return
}
start := store.InstructionStart()
if *msg.FirstUnapplied < start {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
burstMsg.Content = []byte{}
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
*msg.Degraded {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.Degraded {
f.Close()
return
}
// Send all chosen instructions above the first unapplied point.
instructions := store.InstructionSlots()
relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
sendInstructionData(f, slot, slotValues[0].ChangeRequest())
}
}
func process() {
// Try to make an outgoing connection to all other client nodes,
// if we're not in a degraded state.
store.StartTransaction()
if !store.Degraded() {
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
}
store.EndTransaction()
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// If not degraded, try to make an outgoing connection to any
// client node that we do not have at least one connection to.
case <-reconnectTicker:
store.StartTransaction()
// Do not attempt to make connections while degraded.
if store.Degraded() {
store.EndTransaction()
break
}
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
conn, err := connect.Dial(
connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] =
append(connections[node], conn)
go handleConn(node, conn)
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// If we are degraded, reject the connection.
if store.Degraded() {
conn.Close()
store.EndTransaction()
break
}
// Add to our connections.
connections[node] = append(connections[node], conn)
store.EndTransaction()
// Terminated connection.
case receivedConn := <-terminatedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// Remove this connection from our connection list.
index := -1
for i, _ := range connections[node] {
if conn == connections[node][i] {
index = i
break
}
}
if index != -1 {
connections[node] =
append(connections[node][:index],
connections[node][index+1:]...)
}
store.EndTransaction()
}
}
}