// Can only be called from the handling goroutine for conn, // inside a transaction. func followUser(conn *userConn, followId uint64) { conn.following = append(conn.following, followId) user := store.GetEntity(followId) // Send the user's username. var dataMsg cliproto_down.UserData dataMsg.UserId = new(uint64) dataMsg.Key = new(string) dataMsg.Value = new(string) dataMsg.FirstUnapplied = new(uint64) *dataMsg.UserId = followId *dataMsg.Key = "name username" *dataMsg.Value = user.Value(*dataMsg.Key) *dataMsg.FirstUnapplied = store.InstructionFirstUnapplied() conn.conn.SendProto(6, &dataMsg) // Send information on all the user's sessions. user.Attached(func(key string) { *dataMsg.Key = key *dataMsg.Value = "true" conn.conn.SendProto(6, &dataMsg) }) // Send a done message to indicate that we are finished. var doneMsg cliproto_down.UserDataDone doneMsg.UserId = dataMsg.UserId doneMsg.FirstUnapplied = dataMsg.FirstUnapplied conn.conn.SendProto(7, &doneMsg) }
// Must be called inside transaction. func sendFollowUserIdFail(conn *userConn, userId uint64, reason string) { var msg cliproto_down.FollowUserIdFailed msg.UserId = new(uint64) msg.Reason = new(string) msg.FirstUnapplied = new(uint64) *msg.UserId = userId *msg.Reason = reason *msg.FirstUnapplied = store.InstructionFirstUnapplied() conn.conn.SendProto(5, &msg) }
// Must be called inside transaction. func sendUserChange(conn *userConn, entityId uint64, key, value string) { var msg cliproto_down.UserDataChange msg.UserId = new(uint64) msg.Key = new(string) msg.Value = new(string) msg.FirstUnapplied = new(uint64) *msg.UserId = entityId *msg.Key = key *msg.Value = value *msg.FirstUnapplied = store.InstructionFirstUnapplied() conn.conn.SendProto(8, &msg) }
func (f *followConn) firstUnappliedTimeout() { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() if f.closed { return } msg := new(fproto.FirstUnapplied) msg.FirstUnapplied = new(uint64) *msg.FirstUnapplied = store.InstructionFirstUnapplied() f.conn.SendProto(10, msg) f.firstUnappliedTimer = time.AfterFunc(firstUnappliedTimerDuration, func() { f.firstUnappliedTimeout() }) }
func process() { for { select { // Try to make any needed outgoing connections. case <-tryOutgoingCh: store.StartTransaction() connectionsLock.Lock() coreNodes := config.CoreNodes() if !store.Degraded() { if config.IsCore() { for _, node := range coreNodes { if node == config.Id() { continue } if connections[node] == nil { go tryOutgoing(node) } } } else { // Settle for less than 2 core nodes, // if there *aren't* 2 core nodes. targetConns := 2 if targetConns > len(coreNodes) { targetConns = len(coreNodes) } newOutgoing := targetConns - len(connections) used := -1 for newOutgoing > 0 { r := processRand.Intn( len(coreNodes)) // Don't do the same node // twice. if r == used { continue } node := coreNodes[r] if connections[node] == nil { used = r go tryOutgoing(node) newOutgoing-- } } } } else { if len(connections) == 0 { r := processRand.Intn(len(coreNodes)) node := coreNodes[r] if connections[node] == nil { go tryOutgoing(node) } } } connectionsLock.Unlock() store.EndTransaction() // After a random time, check again. randDur := time.Duration(processRand.Intn(19)+1) * time.Second tryOutgoingCh = time.NewTimer(randDur).C // New received connection. case conn := <-receivedConnCh: conn.offerTimers = make(map[uint64]*time.Timer) store.StartTransaction() connectionsLock.Lock() // If we are degraded, reject the connection, // unless we have no other connection, // and it is outbound. if store.Degraded() && !(len(connections) == 0 && conn.outgoing) { conn.lock.Lock() conn.Close() conn.lock.Unlock() connectionsLock.Unlock() store.EndTransaction() break } // If we have an existing connection to this node, // close it. if connections[conn.node] != nil { other := connections[conn.node] other.lock.Lock() other.Close() if other.firstUnappliedTimer != nil { other.firstUnappliedTimer.Stop() } other.lock.Unlock() } // Add to our connections. connections[conn.node] = conn // Send initial position and leader messages. posMsg := new(fproto.Position) posMsg.FirstUnapplied = new(uint64) posMsg.Degraded = new(bool) *posMsg.FirstUnapplied = store.InstructionFirstUnapplied() *posMsg.Degraded = store.Degraded() conn.conn.SendProto(2, posMsg) proposal, leader := store.Proposal() leaderMsg := new(fproto.Leader) leaderMsg.Proposal = new(uint64) leaderMsg.Leader = new(uint32) *leaderMsg.Proposal = proposal *leaderMsg.Leader = uint32(leader) conn.conn.SendProto(11, leaderMsg) connectionsLock.Unlock() store.EndTransaction() // Start timer for sending first unapplied // instruction updates. if config.IsCore() && conn.node <= 0x2000 { conn.lock.Lock() conn.firstUnappliedTimer = time.AfterFunc( firstUnappliedTimerDuration, func() { conn.firstUnappliedTimeout() }) conn.lock.Unlock() } // Start handling received messages from the connection. go handleConn(conn) // Terminated connection. case conn := <-terminatedConnCh: connectionsLock.Lock() if connections[conn.node] == conn { delete(connections, conn.node) conn.lock.Lock() conn.closed = true if conn.firstUnappliedTimer != nil { conn.firstUnappliedTimer.Stop() } conn.lock.Unlock() } connectionsLock.Unlock() } } }
// Must be called from the processing goroutine, inside a transaction. func addPromise(node uint16, msg *coproto.Promise) { receivedPromises[node] = msg // If we have promises from a majority of core nodes, // become leader. if len(receivedPromises) > len(config.CoreNodes())/2 { log.Print("core/consensus: became leader") stopLeaderTimeout() amLeader = true proposal, leader := store.Proposal() // Find a slot number above all those in promise messages, // and above our first unapplied. firstUnapplied := store.InstructionFirstUnapplied() limit := firstUnapplied for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot >= limit { limit = *accepted.Slot + 1 } } } // Start our next slot after the limit. nextProposalSlot = limit // For all slots between this and our first unapplied, // submit a previously accepted instruction unless we // know an instruction was already chosen. // Fills these slots with proposals. // This is O(n^2) in the number of instructions between our // first unapplied and limit. // TODO: Improve worst-case complexity. start := store.InstructionStart() slots := store.InstructionSlots() for i := firstUnapplied; i < limit; i++ { // If we already have a chosen instruction, skip. rel := int(i - start) if len(slots[rel]) == 1 && slots[rel][0].IsChosen() { continue } // Find the previously accepted instruction // accepted with the highest proposal number. var bestInst *coproto.Instruction var bp uint64 // Best proposal var bl uint16 // Best leader for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot != i { continue } if bestInst == nil { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) continue } // TODO: This indent is just absurd. p := *accepted.Proposal l := uint16(*accepted.Leader) if store.CompareProposals(p, l, bp, bl) { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) } } } // If we didn't find an instruction, make an empty one. if bestInst == nil { empty := new(coproto.ChangeRequest) empty.RequestEntity = new(uint64) empty.RequestNode = new(uint32) *empty.RequestEntity = uint64(config.Id()) *empty.RequestNode = uint32(config.Id()) bestInst := new(coproto.Instruction) bestInst.Slot = new(uint64) *bestInst.Slot = i bestInst.Request = empty } // Add proposal timeout. req := makeExtChangeRequest(bestInst.Request) addProposalTimeout(i, req) // Send proposal. bestInst.Proposal = new(uint64) bestInst.Leader = new(uint32) *bestInst.Proposal = proposal *bestInst.Leader = uint32(leader) sendProposal(bestInst) } // Discard received promise messages. receivedPromises = nil // Make an instruction proposal for each waiting change. for _, req := range waitingRequests { slot := nextProposalSlot nextProposalSlot++ addProposalTimeout(slot, req) inst := makeInst(slot, proposal, leader, req) sendProposal(inst) } // Clear waiting changes. waitingRequests = nil } }
// Top-level function of the processing goroutine. func process() { connections = make(map[uint16][]*connect.BaseConn) // On startup, make an outgoing connection attempt to all other // core nodes, before continuing. for _, node := range config.CoreNodes() { if node == config.Id() { continue } conn, err := connect.Dial( connect.CONSENSUS_PROTOCOL, node) if err != nil { // Can't reach the other node. continue } log.Print("core/consensus: made outgoing connection to ", node) connections[node] = append(connections[node], conn) go handleConn(node, conn) } // Retry connections once per config.CHANGE_TIMEOUT_PERIOD. // Largely arbitrary. reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD) for { select { // Connection retry tick. // We should try to make an outgoing connection to any node // that we do not have at least one connection to. // We need to make these asynchronously, because connections // are slow. case <-reconnectTicker: for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) > 0 { continue } go outgoingConn(node) } // New change request, for us to propose as leader. case req := <-newChangeCh: store.StartTransaction() if !amLeader && receivedPromises != nil { waitingRequests = append(waitingRequests, req) } else if !amLeader { waitingRequests = append(waitingRequests, req) // Start attempting to be leader. m := make(map[uint16]*coproto.Promise) receivedPromises = m proposal, _ := store.Proposal() proposal++ store.SetProposal(proposal, config.Id()) firstUn := store.InstructionFirstUnapplied() // Send prepare messages to all other nodes. var prepare coproto.Prepare prepare.Proposal = new(uint64) prepare.FirstUnapplied = new(uint64) *prepare.Proposal = proposal *prepare.FirstUnapplied = firstUn for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) != 0 { c := connections[node][0] c.SendProto(2, &prepare) } } // Behave as if we got a promise message // from ourselves. var promise coproto.Promise promise.Proposal = prepare.Proposal promise.PrevProposal = promise.Proposal promise.Leader = new(uint32) *promise.Leader = uint32(config.Id()) promise.PrevLeader = promise.Leader addPromise(config.Id(), &promise) } else { newSlot := nextProposalSlot nextProposalSlot++ addProposalTimeout(newSlot, req) proposal, leader := store.Proposal() inst := makeInst(newSlot, proposal, leader, req) sendProposal(inst) } store.EndTransaction() // Leadership attempt timed out. case timedOutProposal := <-leaderTimeoutCh: store.StartTransaction() proposal, leader := store.Proposal() // If the proposal has changed since that // leadership attempt, ignore it. if leader != config.Id() { return } if proposal != timedOutProposal { return } // If we successfully became leader, ignore it. if amLeader { return } // Otherwise, stop our attempt to become leader. stopBeingLeader() store.EndTransaction() // Proposal timed out. case timeout := <-proposalTimeoutCh: store.StartTransaction() // If this timeout was not canceled, a proposal failed. // We stop being leader. if proposalTimeouts[timeout.slot] == timeout { stopBeingLeader() } store.EndTransaction() // New received connection. case receivedConn := <-receivedConnCh: node := receivedConn.node conn := receivedConn.conn connections[node] = append(connections[node], conn) // Received message. case recvMsg := <-receivedMsgCh: node := recvMsg.node conn := recvMsg.conn msg := recvMsg.msg switch *msg.MsgType { case 2: processPrepare(node, conn, msg.Content) case 3: processPromise(node, conn, msg.Content) case 4: processAccept(node, conn, msg.Content) case 5: processAccepted(node, conn, msg.Content) case 6: processNack(node, conn, msg.Content) default: // Unknown message. conn.Close() } // Terminate received connection. case terminatedConn := <-terminatedConnCh: node := terminatedConn.node conn := terminatedConn.conn for i, other := range connections[node] { if other != conn { continue } conns := connections[node] conns = append(conns[:i], conns[i+1:]...) connections[node] = conns break } } } }
func handlePosition(f *followConn, content []byte) { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() if f.closed { return } var msg fproto.Position if err := proto.Unmarshal(content, &msg); err != nil { f.Close() return } if config.IsCore() && f.node <= 0x2000 { store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied) } if store.Degraded() && *msg.Degraded { f.Close() return } start := store.InstructionStart() if *msg.FirstUnapplied < start { f.sendingBurst = true burstMsg := new(baseproto.Message) burstMsg.MsgType = new(uint32) burstMsg.Content = []byte{} *burstMsg.MsgType = 3 f.conn.Send(burstMsg) go sendBurst(f) return } else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() && *msg.Degraded { f.sendingBurst = true burstMsg := new(baseproto.Message) burstMsg.MsgType = new(uint32) *burstMsg.MsgType = 3 f.conn.Send(burstMsg) go sendBurst(f) return } else if *msg.Degraded { f.Close() return } // Send all chosen instructions above the first unapplied point. instructions := store.InstructionSlots() relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart()) for ; relativeSlot < len(instructions); relativeSlot++ { slot := store.InstructionStart() + uint64(relativeSlot) slotValues := instructions[relativeSlot] if len(slotValues) != 1 || !slotValues[0].IsChosen() { continue } // Convert the change request to our internal format. sendInstructionData(f, slot, slotValues[0].ChangeRequest()) } }
// Sends a burst to the given connection. // The Bursting message should already have been sent, // and f.sendingBurst set to true, before calling this asynchronously. func sendBurst(f *followConn) { log.Print("shared/follow: sending burst to ", f.node) globalProp := new(fproto.GlobalProperty) globalProp.Key = new(string) globalProp.Value = new(string) entityProp := new(fproto.EntityProperty) entityProp.Entity = new(uint64) entityProp.Key = new(string) entityProp.Value = new(string) aborted := false store.Burst(func(key, value string) bool { *globalProp.Key = key *globalProp.Value = value if !f.conn.SendProto(4, globalProp) { aborted = true } return !aborted }, func(entity uint64, key, value string) bool { *entityProp.Entity = entity *entityProp.Key = key *entityProp.Value = value if !f.conn.SendProto(5, entityProp) { aborted = true } return !aborted }) if aborted { return } store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() f.sendingBurst = false burstDone := new(fproto.BurstDone) burstDone.FirstUnapplied = new(uint64) *burstDone.FirstUnapplied = store.InstructionFirstUnapplied() f.conn.SendProto(6, burstDone) // Send an instruction chosen message for all chosen instructions above // our first unapplied point. instructions := store.InstructionSlots() firstUnapplied := store.InstructionFirstUnapplied() relativeSlot := int(firstUnapplied - store.InstructionStart()) for ; relativeSlot < len(instructions); relativeSlot++ { slot := store.InstructionStart() + uint64(relativeSlot) slotValues := instructions[relativeSlot] if len(slotValues) != 1 || !slotValues[0].IsChosen() { continue } // Convert the change request to our internal format. chosen := new(fproto.InstructionChosen) chosen.Slot = new(uint64) *chosen.Slot = slot f.conn.SendProto(7, chosen) } }