func handleApplied(slot uint64, idempotentChanges []store.Change) {
relativeSlot := int(slot - store.InstructionStart())
slots := store.InstructionSlots()
origReq := slots[relativeSlot][0].ChangeRequest()
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = origReq.RequestEntity
chrequest.RequestNode = origReq.RequestNode
chrequest.RequestId = origReq.RequestId
chrequest.Changeset = idempotentChanges
connectionsLock.Lock()
for _, conn := range connections {
conn.lock.Lock()
if conn.sendingBurst {
sendInstructionData(conn, slot, chrequest)
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
func handleInstructionRequest(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.InstructionRequest
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
f.Close()
return
}
instructions := store.InstructionSlots()
if relativeSlot >= len(instructions) {
f.Close()
return
}
slot := instructions[relativeSlot]
if len(slot) != 1 || !slot[0].IsChosen() {
f.Close()
return
}
// Convert the change request to our internal format.
sendInstructionData(f, *msg.Slot, slot[0].ChangeRequest())
}
// Must be called from the processing goroutine, inside a transaction.
// Returns false if it discarded the accepted message because of it being
// too far in our future. Any other reason for discarding the message,
// as well as the accept succeeding, will return true.
func addAccepted(node uint16, accepted *coproto.Accepted) bool {
slots := store.InstructionSlots()
msgProposal, msgLeader := *accepted.Proposal, uint16(*accepted.Leader)
newReq := accepted.Instruction.Request
slot := *accepted.Instruction.Slot
ourStart := store.InstructionStart()
relativeSlot := int(slot - ourStart)
// If our instruction slot slice is not this long,
// we know we do not have any value in this slot yet,
// and so need to add a new one.
if len(slots) <= relativeSlot {
chreq := makeExtChangeRequest(newReq)
newInst := store.AddInstructionValue(slot, chreq)
if newInst == nil {
return false
}
newInst.Accept(node, msgProposal, msgLeader)
return true
}
// If we have already chosen an instruction in this slot,
// we can ignore subsequent Accepted messages for it.
if len(slots[relativeSlot]) == 1 && slots[relativeSlot][0].IsChosen() {
return true
}
// Returns if it successfully finds an existing matching value.
for _, value := range slots[relativeSlot] {
chreq := value.ChangeRequest()
if chreq.RequestNode != uint16(*newReq.RequestNode) {
continue
}
if chreq.RequestId != *newReq.RequestId {
continue
}
value.Accept(node, msgProposal, msgLeader)
return true
}
chreq := makeExtChangeRequest(newReq)
newInst := store.AddInstructionValue(slot, chreq)
if newInst == nil {
return false
}
newInst.Accept(node, msgProposal, msgLeader)
return true
}
func handleChosenRequests(slot uint64) {
relativeSlot := int(slot - store.InstructionStart())
instruction := store.InstructionSlots()[relativeSlot][0]
req := instruction.ChangeRequest()
requestTimeoutLock.Lock()
_, exists := getRequestTimeout(req.RequestNode, req.RequestId)
if exists {
log.Print("shared/chrequest: successful request ",
req.RequestId)
cancelRequestTimeout(req.RequestNode, req.RequestId)
}
requestTimeoutLock.Unlock()
}
func handleInstructionChosen(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
var msg fproto.InstructionChosen
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if f.closed {
return
}
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) {
slot := instructions[relativeSlot]
if len(slot) == 1 && slot[0].IsChosen() {
return
}
}
if !config.IsCore() {
// TODO: Should only do this on one follow connection.
var intReq fproto.InstructionRequest
intReq.Slot = msg.Slot
f.conn.SendProto(8, &intReq)
}
timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
f.offerTimers[*msg.Slot] = time.AfterFunc(
timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
func handleApplied(slot uint64, idemChanges []store.Change) {
relativeSlot := int(slot - store.InstructionStart())
slots := store.InstructionSlots()
requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode
requestId := slots[relativeSlot][0].ChangeRequest().RequestId
sessionsLock.Lock()
defer sessionsLock.Unlock()
waitingLock.Lock()
defer waitingLock.Unlock()
// If there is a connection which has this as its waiting request,
// have it complete authentication.
if requestNode == config.Id() {
waitingConn := waiting[requestId]
if waitingConn != nil {
handleAuthComplete(waitingConn, idemChanges)
}
}
ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
for i := range idemChanges {
key := idemChanges[i].Key
value := idemChanges[i].Value
if strings.HasPrefix(key, "attach ") {
// If we were just detached from a session,
// drop any connections associated with it,
// and if it was in our attach timeouts, remove it.
if key == ourAttachStr && value == "" {
sessionId := idemChanges[i].TargetEntity
if conn := sessions[sessionId]; conn != nil {
log.Print("client/listener: " +
"detached from existing " +
"session")
conn.conn.Close()
}
if orphanAttachTimeouts[sessionId] != nil {
orphanAttachTimeouts[sessionId].Stop()
delete(orphanAttachTimeouts, sessionId)
}
}
// If we were just attached to a session,
// but have no connection associated with that session,
// add an attach timeout.
// and if it was in our attach timeouts, remove it.
if key == ourAttachStr && value != "" {
sessionId := idemChanges[i].TargetEntity
if sessions[sessionId] == nil {
startOrphanTimeout(sessionId)
}
}
// Handle a session attaching/detaching from a user.
entityId := idemChanges[i].TargetEntity
entity := store.GetEntity(entityId)
if entity != nil && entity.Value("kind") == "user" {
// Tell following connections when a session
// attaches or detaches from a followed user.
// TODO: Expensive loop,
// should maintain index to avoid this.
for conn, _ := range connections {
following := false
for _, fw := range conn.following {
if fw == entityId {
following = true
break
}
}
if !following {
continue
}
sendUserChange(conn, entityId,
key, value)
}
}
}
}
// If we've created a nameless user, start a timer to delete it.
for i := range idemChanges {
if idemChanges[i].Key != "kind" ||
idemChanges[i].Value != "user" {
continue
}
newUserId := idemChanges[i].TargetEntity
newUser := store.GetEntity(newUserId)
if newUser != nil && newUser.Value("name username") == "" {
startNamelessTimeout(newUserId)
}
}
}
func handleChosen(slot uint64) {
relativeSlot := int(slot - store.InstructionStart())
slots := store.InstructionSlots()
cancelProposalTimeout(slot, slots[relativeSlot][0].ChangeRequest())
}
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
receivedPromises[node] = msg
// If we have promises from a majority of core nodes,
// become leader.
if len(receivedPromises) > len(config.CoreNodes())/2 {
log.Print("core/consensus: became leader")
stopLeaderTimeout()
amLeader = true
proposal, leader := store.Proposal()
// Find a slot number above all those in promise messages,
// and above our first unapplied.
firstUnapplied := store.InstructionFirstUnapplied()
limit := firstUnapplied
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot >= limit {
limit = *accepted.Slot + 1
}
}
}
// Start our next slot after the limit.
nextProposalSlot = limit
// For all slots between this and our first unapplied,
// submit a previously accepted instruction unless we
// know an instruction was already chosen.
// Fills these slots with proposals.
// This is O(n^2) in the number of instructions between our
// first unapplied and limit.
// TODO: Improve worst-case complexity.
start := store.InstructionStart()
slots := store.InstructionSlots()
for i := firstUnapplied; i < limit; i++ {
// If we already have a chosen instruction, skip.
rel := int(i - start)
if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
continue
}
// Find the previously accepted instruction
// accepted with the highest proposal number.
var bestInst *coproto.Instruction
var bp uint64 // Best proposal
var bl uint16 // Best leader
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot != i {
continue
}
if bestInst == nil {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
continue
}
// TODO: This indent is just absurd.
p := *accepted.Proposal
l := uint16(*accepted.Leader)
if store.CompareProposals(p, l,
bp, bl) {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
}
}
}
// If we didn't find an instruction, make an empty one.
if bestInst == nil {
empty := new(coproto.ChangeRequest)
empty.RequestEntity = new(uint64)
empty.RequestNode = new(uint32)
*empty.RequestEntity = uint64(config.Id())
*empty.RequestNode = uint32(config.Id())
bestInst := new(coproto.Instruction)
bestInst.Slot = new(uint64)
*bestInst.Slot = i
bestInst.Request = empty
}
// Add proposal timeout.
req := makeExtChangeRequest(bestInst.Request)
addProposalTimeout(i, req)
// Send proposal.
bestInst.Proposal = new(uint64)
bestInst.Leader = new(uint32)
*bestInst.Proposal = proposal
*bestInst.Leader = uint32(leader)
sendProposal(bestInst)
}
// Discard received promise messages.
receivedPromises = nil
// Make an instruction proposal for each waiting change.
for _, req := range waitingRequests {
slot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(slot, req)
inst := makeInst(slot, proposal, leader, req)
sendProposal(inst)
}
// Clear waiting changes.
waitingRequests = nil
}
}
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Prepare
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
newProposal, newLeader := *msg.Proposal, node
proposal, leader := store.Proposal()
if store.CompareProposals(newProposal, newLeader, proposal, leader) {
log.Print("core/consensus: sending promise to ", newLeader)
// Create a promise message to send back.
var promise coproto.Promise
promise.Proposal = new(uint64)
promise.Leader = new(uint32)
promise.PrevProposal = new(uint64)
promise.PrevLeader = new(uint32)
*promise.Proposal = newProposal
*promise.Leader = uint32(newLeader)
*promise.PrevProposal = proposal
*promise.PrevLeader = uint32(leader)
// Add all the instructions we've previously accepted or chosen.
slots := store.InstructionSlots()
theirFirstUnapplied := *msg.FirstUnapplied
ourStart := store.InstructionStart()
relativeSlot := int(theirFirstUnapplied - ourStart)
if relativeSlot < 0 {
relativeSlot = 0
}
var accepted []*coproto.Instruction
for ; relativeSlot < len(slots); relativeSlot++ {
slot := slots[relativeSlot]
slotNum := ourStart + uint64(relativeSlot)
for i, _ := range slot {
if slot[i].IsChosen() {
appendInst(&accepted, slotNum, slot[i])
break
}
weAccepted := false
for _, node := range slot[i].Accepted() {
if node == config.Id() {
weAccepted = true
break
}
}
if weAccepted {
appendInst(&accepted, slotNum, slot[i])
break
}
}
}
// Send promise message.
conn.SendProto(3, &promise)
// Accept the other node as our new leader.
store.SetProposal(newProposal, newLeader)
} else {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
}
}
func handlePosition(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.Position
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if config.IsCore() && f.node <= 0x2000 {
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
if store.Degraded() && *msg.Degraded {
f.Close()
return
}
start := store.InstructionStart()
if *msg.FirstUnapplied < start {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
burstMsg.Content = []byte{}
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
*msg.Degraded {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.Degraded {
f.Close()
return
}
// Send all chosen instructions above the first unapplied point.
instructions := store.InstructionSlots()
relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
sendInstructionData(f, slot, slotValues[0].ChangeRequest())
}
}
// Sends a burst to the given connection.
// The Bursting message should already have been sent,
// and f.sendingBurst set to true, before calling this asynchronously.
func sendBurst(f *followConn) {
log.Print("shared/follow: sending burst to ", f.node)
globalProp := new(fproto.GlobalProperty)
globalProp.Key = new(string)
globalProp.Value = new(string)
entityProp := new(fproto.EntityProperty)
entityProp.Entity = new(uint64)
entityProp.Key = new(string)
entityProp.Value = new(string)
aborted := false
store.Burst(func(key, value string) bool {
*globalProp.Key = key
*globalProp.Value = value
if !f.conn.SendProto(4, globalProp) {
aborted = true
}
return !aborted
}, func(entity uint64, key, value string) bool {
*entityProp.Entity = entity
*entityProp.Key = key
*entityProp.Value = value
if !f.conn.SendProto(5, entityProp) {
aborted = true
}
return !aborted
})
if aborted {
return
}
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
f.sendingBurst = false
burstDone := new(fproto.BurstDone)
burstDone.FirstUnapplied = new(uint64)
*burstDone.FirstUnapplied = store.InstructionFirstUnapplied()
f.conn.SendProto(6, burstDone)
// Send an instruction chosen message for all chosen instructions above
// our first unapplied point.
instructions := store.InstructionSlots()
firstUnapplied := store.InstructionFirstUnapplied()
relativeSlot := int(firstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
chosen := new(fproto.InstructionChosen)
chosen.Slot = new(uint64)
*chosen.Slot = slot
f.conn.SendProto(7, chosen)
}
}
func handleInstructionData(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
if f.closed {
f.lock.Unlock()
return
}
var msg fproto.InstructionData
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
f.lock.Unlock()
return
}
if f.receivingBurst {
f.waiting = append(f.waiting, &msg)
f.lock.Unlock()
return
}
// We need to unlock before we start mutating the store,
// due to callbacks from the store package to elsewhere.
f.lock.Unlock()
// If we have a chosen instruction in this slot already,
// or it is prior to our instruction start number,
// discard this message.
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) &&
len(instructions[relativeSlot]) == 1 &&
instructions[relativeSlot][0].IsChosen() {
return
}
// Construct a store.ChangeRequest from our
// internal ChangeRequest.
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *msg.Request.RequestEntity
chrequest.RequestNode = uint16(*msg.Request.RequestNode)
chrequest.RequestId = *msg.Request.RequestId
chrequest.Changeset = make([]store.Change,
len(msg.Request.Changeset))
for i, ch := range msg.Request.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
// Add instruction value and immediately choose it.
store.AddInstructionValue(*msg.Slot, chrequest).Choose()
}