func handleApplied(slot uint64, idempotentChanges []store.Change) {
relativeSlot := int(slot - store.InstructionStart())
slots := store.InstructionSlots()
origReq := slots[relativeSlot][0].ChangeRequest()
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = origReq.RequestEntity
chrequest.RequestNode = origReq.RequestNode
chrequest.RequestId = origReq.RequestId
chrequest.Changeset = idempotentChanges
connectionsLock.Lock()
for _, conn := range connections {
conn.lock.Lock()
if conn.sendingBurst {
sendInstructionData(conn, slot, chrequest)
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
// Must be called from inside a transaction.
func makeRequest(changes []store.Change) *store.ChangeRequest {
req := new(store.ChangeRequest)
req.RequestEntity = uint64(config.Id())
req.RequestNode = config.Id()
req.RequestId = store.AllocateRequestId()
req.Changeset = changes
return req
}
func makeExtChangeRequest(intReq *coproto.ChangeRequest) *store.ChangeRequest {
req := new(store.ChangeRequest)
req.RequestEntity = *intReq.RequestEntity
req.RequestNode = uint16(*intReq.RequestNode)
req.RequestId = *intReq.RequestId
req.Changeset = make([]store.Change, len(intReq.Changeset))
for i, intCh := range intReq.Changeset {
req.Changeset[i].TargetEntity = *intCh.TargetEntity
req.Changeset[i].Key = *intCh.Key
req.Changeset[i].Value = *intCh.Value
}
return req
}
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {
// If we are already trying to forward a change forward message with
// the same requesting node and request ID, discard this message.
if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
*forward.Request.RequestId); exists {
return
}
// Everything else in this function runs in a transaction.
// We are read-only.
store.StartTransaction()
defer store.EndTransaction()
// If this is a core node and this node stopped being leader less than
// a Change Timeout Period ago, always add us to the ignore list.
if config.IsCore() && !isIgnored(forward, config.Id()) {
diff := time.Now().Sub(store.StoppedLeading())
if diff < config.CHANGE_TIMEOUT_PERIOD {
forward.Ignores = append(forward.Ignores,
uint32(config.Id()))
}
}
// If all core node IDs are in the forward's ignore list, discard it.
if len(forward.Ignores) == len(config.CoreNodes()) {
log.Print("shared/chrequest: dropped msg due to full ignores")
return
}
// Otherwise, choose a potential leader node.
// This is O(n^2) in the number of core nodes,
// but we don't expect to have many.
chosenNode := uint16(0)
_, leader := store.Proposal()
if leader != 0 && !isIgnored(forward, leader) {
chosenNode = leader
} else {
for _, node := range config.CoreNodes() {
if !isIgnored(forward, node) {
chosenNode = node
break
}
}
}
if chosenNode == 0 {
// Shouldn't happen.
log.Print("shared/chrequest: bug, " +
"couldn't find candidate leader node")
return
}
// If we are the selected leader, construct an external change request,
// and send it on our change request channel.
if chosenNode == config.Id() {
intRequest := forward.Request
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *intRequest.RequestEntity
chrequest.RequestNode = uint16(*intRequest.RequestNode)
chrequest.RequestId = *intRequest.RequestId
chrequest.Changeset = make([]store.Change,
len(intRequest.Changeset))
for i, ch := range intRequest.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
for _, cb := range changeCallbacks {
cb(chrequest)
}
return
}
// Otherwise, we send it on to the selected leader,
// add the selected leader to the ignore list,
// and set a timeout to retry.
sendForward(chosenNode, forward)
forward.Ignores = append(forward.Ignores, uint32(chosenNode))
addForwardTimeout(forward)
}
func handleInstructionData(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
if f.closed {
f.lock.Unlock()
return
}
var msg fproto.InstructionData
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
f.lock.Unlock()
return
}
if f.receivingBurst {
f.waiting = append(f.waiting, &msg)
f.lock.Unlock()
return
}
// We need to unlock before we start mutating the store,
// due to callbacks from the store package to elsewhere.
f.lock.Unlock()
// If we have a chosen instruction in this slot already,
// or it is prior to our instruction start number,
// discard this message.
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) &&
len(instructions[relativeSlot]) == 1 &&
instructions[relativeSlot][0].IsChosen() {
return
}
// Construct a store.ChangeRequest from our
// internal ChangeRequest.
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *msg.Request.RequestEntity
chrequest.RequestNode = uint16(*msg.Request.RequestNode)
chrequest.RequestId = *msg.Request.RequestId
chrequest.Changeset = make([]store.Change,
len(msg.Request.Changeset))
for i, ch := range msg.Request.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
// Add instruction value and immediately choose it.
store.AddInstructionValue(*msg.Slot, chrequest).Choose()
}