func handleApplied(slot uint64, idempotentChanges []store.Change) { relativeSlot := int(slot - store.InstructionStart()) slots := store.InstructionSlots() origReq := slots[relativeSlot][0].ChangeRequest() chrequest := new(store.ChangeRequest) chrequest.RequestEntity = origReq.RequestEntity chrequest.RequestNode = origReq.RequestNode chrequest.RequestId = origReq.RequestId chrequest.Changeset = idempotentChanges connectionsLock.Lock() for _, conn := range connections { conn.lock.Lock() if conn.sendingBurst { sendInstructionData(conn, slot, chrequest) } conn.lock.Unlock() } connectionsLock.Unlock() }
func handleInstructionRequest(f *followConn, content []byte) { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() if f.closed { return } var msg fproto.InstructionRequest if err := proto.Unmarshal(content, &msg); err != nil { f.Close() return } relativeSlot := int(*msg.Slot - store.InstructionStart()) if relativeSlot < 0 { f.Close() return } instructions := store.InstructionSlots() if relativeSlot >= len(instructions) { f.Close() return } slot := instructions[relativeSlot] if len(slot) != 1 || !slot[0].IsChosen() { f.Close() return } // Convert the change request to our internal format. sendInstructionData(f, *msg.Slot, slot[0].ChangeRequest()) }
// Must be called from the processing goroutine, inside a transaction. // Returns false if it discarded the accepted message because of it being // too far in our future. Any other reason for discarding the message, // as well as the accept succeeding, will return true. func addAccepted(node uint16, accepted *coproto.Accepted) bool { slots := store.InstructionSlots() msgProposal, msgLeader := *accepted.Proposal, uint16(*accepted.Leader) newReq := accepted.Instruction.Request slot := *accepted.Instruction.Slot ourStart := store.InstructionStart() relativeSlot := int(slot - ourStart) // If our instruction slot slice is not this long, // we know we do not have any value in this slot yet, // and so need to add a new one. if len(slots) <= relativeSlot { chreq := makeExtChangeRequest(newReq) newInst := store.AddInstructionValue(slot, chreq) if newInst == nil { return false } newInst.Accept(node, msgProposal, msgLeader) return true } // If we have already chosen an instruction in this slot, // we can ignore subsequent Accepted messages for it. if len(slots[relativeSlot]) == 1 && slots[relativeSlot][0].IsChosen() { return true } // Returns if it successfully finds an existing matching value. for _, value := range slots[relativeSlot] { chreq := value.ChangeRequest() if chreq.RequestNode != uint16(*newReq.RequestNode) { continue } if chreq.RequestId != *newReq.RequestId { continue } value.Accept(node, msgProposal, msgLeader) return true } chreq := makeExtChangeRequest(newReq) newInst := store.AddInstructionValue(slot, chreq) if newInst == nil { return false } newInst.Accept(node, msgProposal, msgLeader) return true }
func handleChosenRequests(slot uint64) { relativeSlot := int(slot - store.InstructionStart()) instruction := store.InstructionSlots()[relativeSlot][0] req := instruction.ChangeRequest() requestTimeoutLock.Lock() _, exists := getRequestTimeout(req.RequestNode, req.RequestId) if exists { log.Print("shared/chrequest: successful request ", req.RequestId) cancelRequestTimeout(req.RequestNode, req.RequestId) } requestTimeoutLock.Unlock() }
func handleInstructionChosen(f *followConn, content []byte) { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() var msg fproto.InstructionChosen if err := proto.Unmarshal(content, &msg); err != nil { f.Close() return } if f.closed { return } relativeSlot := int(*msg.Slot - store.InstructionStart()) if relativeSlot < 0 { return } instructions := store.InstructionSlots() if relativeSlot < len(instructions) { slot := instructions[relativeSlot] if len(slot) == 1 && slot[0].IsChosen() { return } } if !config.IsCore() { // TODO: Should only do this on one follow connection. var intReq fproto.InstructionRequest intReq.Slot = msg.Slot f.conn.SendProto(8, &intReq) } timeout := config.ROUND_TRIP_TIMEOUT_PERIOD f.offerTimers[*msg.Slot] = time.AfterFunc( timeout, func() { f.offerTimeout(*msg.Slot, timeout) }) }
func handleApplied(slot uint64, idemChanges []store.Change) { relativeSlot := int(slot - store.InstructionStart()) slots := store.InstructionSlots() requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode requestId := slots[relativeSlot][0].ChangeRequest().RequestId sessionsLock.Lock() defer sessionsLock.Unlock() waitingLock.Lock() defer waitingLock.Unlock() // If there is a connection which has this as its waiting request, // have it complete authentication. if requestNode == config.Id() { waitingConn := waiting[requestId] if waitingConn != nil { handleAuthComplete(waitingConn, idemChanges) } } ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10) for i := range idemChanges { key := idemChanges[i].Key value := idemChanges[i].Value if strings.HasPrefix(key, "attach ") { // If we were just detached from a session, // drop any connections associated with it, // and if it was in our attach timeouts, remove it. if key == ourAttachStr && value == "" { sessionId := idemChanges[i].TargetEntity if conn := sessions[sessionId]; conn != nil { log.Print("client/listener: " + "detached from existing " + "session") conn.conn.Close() } if orphanAttachTimeouts[sessionId] != nil { orphanAttachTimeouts[sessionId].Stop() delete(orphanAttachTimeouts, sessionId) } } // If we were just attached to a session, // but have no connection associated with that session, // add an attach timeout. // and if it was in our attach timeouts, remove it. if key == ourAttachStr && value != "" { sessionId := idemChanges[i].TargetEntity if sessions[sessionId] == nil { startOrphanTimeout(sessionId) } } // Handle a session attaching/detaching from a user. entityId := idemChanges[i].TargetEntity entity := store.GetEntity(entityId) if entity != nil && entity.Value("kind") == "user" { // Tell following connections when a session // attaches or detaches from a followed user. // TODO: Expensive loop, // should maintain index to avoid this. for conn, _ := range connections { following := false for _, fw := range conn.following { if fw == entityId { following = true break } } if !following { continue } sendUserChange(conn, entityId, key, value) } } } } // If we've created a nameless user, start a timer to delete it. for i := range idemChanges { if idemChanges[i].Key != "kind" || idemChanges[i].Value != "user" { continue } newUserId := idemChanges[i].TargetEntity newUser := store.GetEntity(newUserId) if newUser != nil && newUser.Value("name username") == "" { startNamelessTimeout(newUserId) } } }
func handleChosen(slot uint64) { relativeSlot := int(slot - store.InstructionStart()) slots := store.InstructionSlots() cancelProposalTimeout(slot, slots[relativeSlot][0].ChangeRequest()) }
// Must be called from the processing goroutine, inside a transaction. func addPromise(node uint16, msg *coproto.Promise) { receivedPromises[node] = msg // If we have promises from a majority of core nodes, // become leader. if len(receivedPromises) > len(config.CoreNodes())/2 { log.Print("core/consensus: became leader") stopLeaderTimeout() amLeader = true proposal, leader := store.Proposal() // Find a slot number above all those in promise messages, // and above our first unapplied. firstUnapplied := store.InstructionFirstUnapplied() limit := firstUnapplied for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot >= limit { limit = *accepted.Slot + 1 } } } // Start our next slot after the limit. nextProposalSlot = limit // For all slots between this and our first unapplied, // submit a previously accepted instruction unless we // know an instruction was already chosen. // Fills these slots with proposals. // This is O(n^2) in the number of instructions between our // first unapplied and limit. // TODO: Improve worst-case complexity. start := store.InstructionStart() slots := store.InstructionSlots() for i := firstUnapplied; i < limit; i++ { // If we already have a chosen instruction, skip. rel := int(i - start) if len(slots[rel]) == 1 && slots[rel][0].IsChosen() { continue } // Find the previously accepted instruction // accepted with the highest proposal number. var bestInst *coproto.Instruction var bp uint64 // Best proposal var bl uint16 // Best leader for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot != i { continue } if bestInst == nil { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) continue } // TODO: This indent is just absurd. p := *accepted.Proposal l := uint16(*accepted.Leader) if store.CompareProposals(p, l, bp, bl) { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) } } } // If we didn't find an instruction, make an empty one. if bestInst == nil { empty := new(coproto.ChangeRequest) empty.RequestEntity = new(uint64) empty.RequestNode = new(uint32) *empty.RequestEntity = uint64(config.Id()) *empty.RequestNode = uint32(config.Id()) bestInst := new(coproto.Instruction) bestInst.Slot = new(uint64) *bestInst.Slot = i bestInst.Request = empty } // Add proposal timeout. req := makeExtChangeRequest(bestInst.Request) addProposalTimeout(i, req) // Send proposal. bestInst.Proposal = new(uint64) bestInst.Leader = new(uint32) *bestInst.Proposal = proposal *bestInst.Leader = uint32(leader) sendProposal(bestInst) } // Discard received promise messages. receivedPromises = nil // Make an instruction proposal for each waiting change. for _, req := range waitingRequests { slot := nextProposalSlot nextProposalSlot++ addProposalTimeout(slot, req) inst := makeInst(slot, proposal, leader, req) sendProposal(inst) } // Clear waiting changes. waitingRequests = nil } }
// Must be called from the processing goroutine. func processPrepare(node uint16, conn *connect.BaseConn, content []byte) { var msg coproto.Prepare if err := proto.Unmarshal(content, &msg); err != nil { conn.Close() return } store.StartTransaction() defer store.EndTransaction() newProposal, newLeader := *msg.Proposal, node proposal, leader := store.Proposal() if store.CompareProposals(newProposal, newLeader, proposal, leader) { log.Print("core/consensus: sending promise to ", newLeader) // Create a promise message to send back. var promise coproto.Promise promise.Proposal = new(uint64) promise.Leader = new(uint32) promise.PrevProposal = new(uint64) promise.PrevLeader = new(uint32) *promise.Proposal = newProposal *promise.Leader = uint32(newLeader) *promise.PrevProposal = proposal *promise.PrevLeader = uint32(leader) // Add all the instructions we've previously accepted or chosen. slots := store.InstructionSlots() theirFirstUnapplied := *msg.FirstUnapplied ourStart := store.InstructionStart() relativeSlot := int(theirFirstUnapplied - ourStart) if relativeSlot < 0 { relativeSlot = 0 } var accepted []*coproto.Instruction for ; relativeSlot < len(slots); relativeSlot++ { slot := slots[relativeSlot] slotNum := ourStart + uint64(relativeSlot) for i, _ := range slot { if slot[i].IsChosen() { appendInst(&accepted, slotNum, slot[i]) break } weAccepted := false for _, node := range slot[i].Accepted() { if node == config.Id() { weAccepted = true break } } if weAccepted { appendInst(&accepted, slotNum, slot[i]) break } } } // Send promise message. conn.SendProto(3, &promise) // Accept the other node as our new leader. store.SetProposal(newProposal, newLeader) } else { var nack coproto.Nack nack.Proposal = new(uint64) nack.Leader = new(uint32) *nack.Proposal = proposal *nack.Leader = uint32(leader) conn.SendProto(6, &nack) } }
func handlePosition(f *followConn, content []byte) { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() if f.closed { return } var msg fproto.Position if err := proto.Unmarshal(content, &msg); err != nil { f.Close() return } if config.IsCore() && f.node <= 0x2000 { store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied) } if store.Degraded() && *msg.Degraded { f.Close() return } start := store.InstructionStart() if *msg.FirstUnapplied < start { f.sendingBurst = true burstMsg := new(baseproto.Message) burstMsg.MsgType = new(uint32) burstMsg.Content = []byte{} *burstMsg.MsgType = 3 f.conn.Send(burstMsg) go sendBurst(f) return } else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() && *msg.Degraded { f.sendingBurst = true burstMsg := new(baseproto.Message) burstMsg.MsgType = new(uint32) *burstMsg.MsgType = 3 f.conn.Send(burstMsg) go sendBurst(f) return } else if *msg.Degraded { f.Close() return } // Send all chosen instructions above the first unapplied point. instructions := store.InstructionSlots() relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart()) for ; relativeSlot < len(instructions); relativeSlot++ { slot := store.InstructionStart() + uint64(relativeSlot) slotValues := instructions[relativeSlot] if len(slotValues) != 1 || !slotValues[0].IsChosen() { continue } // Convert the change request to our internal format. sendInstructionData(f, slot, slotValues[0].ChangeRequest()) } }
// Sends a burst to the given connection. // The Bursting message should already have been sent, // and f.sendingBurst set to true, before calling this asynchronously. func sendBurst(f *followConn) { log.Print("shared/follow: sending burst to ", f.node) globalProp := new(fproto.GlobalProperty) globalProp.Key = new(string) globalProp.Value = new(string) entityProp := new(fproto.EntityProperty) entityProp.Entity = new(uint64) entityProp.Key = new(string) entityProp.Value = new(string) aborted := false store.Burst(func(key, value string) bool { *globalProp.Key = key *globalProp.Value = value if !f.conn.SendProto(4, globalProp) { aborted = true } return !aborted }, func(entity uint64, key, value string) bool { *entityProp.Entity = entity *entityProp.Key = key *entityProp.Value = value if !f.conn.SendProto(5, entityProp) { aborted = true } return !aborted }) if aborted { return } store.StartTransaction() defer store.EndTransaction() f.lock.Lock() defer f.lock.Unlock() f.sendingBurst = false burstDone := new(fproto.BurstDone) burstDone.FirstUnapplied = new(uint64) *burstDone.FirstUnapplied = store.InstructionFirstUnapplied() f.conn.SendProto(6, burstDone) // Send an instruction chosen message for all chosen instructions above // our first unapplied point. instructions := store.InstructionSlots() firstUnapplied := store.InstructionFirstUnapplied() relativeSlot := int(firstUnapplied - store.InstructionStart()) for ; relativeSlot < len(instructions); relativeSlot++ { slot := store.InstructionStart() + uint64(relativeSlot) slotValues := instructions[relativeSlot] if len(slotValues) != 1 || !slotValues[0].IsChosen() { continue } // Convert the change request to our internal format. chosen := new(fproto.InstructionChosen) chosen.Slot = new(uint64) *chosen.Slot = slot f.conn.SendProto(7, chosen) } }
func handleInstructionData(f *followConn, content []byte) { store.StartTransaction() defer store.EndTransaction() f.lock.Lock() if f.closed { f.lock.Unlock() return } var msg fproto.InstructionData if err := proto.Unmarshal(content, &msg); err != nil { f.Close() f.lock.Unlock() return } if f.receivingBurst { f.waiting = append(f.waiting, &msg) f.lock.Unlock() return } // We need to unlock before we start mutating the store, // due to callbacks from the store package to elsewhere. f.lock.Unlock() // If we have a chosen instruction in this slot already, // or it is prior to our instruction start number, // discard this message. relativeSlot := int(*msg.Slot - store.InstructionStart()) if relativeSlot < 0 { return } instructions := store.InstructionSlots() if relativeSlot < len(instructions) && len(instructions[relativeSlot]) == 1 && instructions[relativeSlot][0].IsChosen() { return } // Construct a store.ChangeRequest from our // internal ChangeRequest. chrequest := new(store.ChangeRequest) chrequest.RequestEntity = *msg.Request.RequestEntity chrequest.RequestNode = uint16(*msg.Request.RequestNode) chrequest.RequestId = *msg.Request.RequestId chrequest.Changeset = make([]store.Change, len(msg.Request.Changeset)) for i, ch := range msg.Request.Changeset { chrequest.Changeset[i].TargetEntity = *ch.TargetEntity chrequest.Changeset[i].Key = *ch.Key chrequest.Changeset[i].Value = *ch.Value } // Add instruction value and immediately choose it. store.AddInstructionValue(*msg.Slot, chrequest).Choose() }