func handleFirstUnapplied(f *followConn, content []byte) {
// Client nodes ignore this message, and shouldn't send it.
if !config.IsCore() || f.node > 0x2000 {
return
}
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.FirstUnapplied
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
// Can't trigger callbacks from the store package to elsewhere,
// safe to call while holding f's lock.
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
// Sets the first unapplied slot on the given node.
func SetNodeFirstUnapplied(node uint16, slot uint64) {
if !config.IsCore() {
panic("non-core node tracking first unapplied of other nodes")
}
nodeFirstUnapplied[node] = slot
}
func handleMsg(node uint16, conn *connect.BaseConn, msg *baseproto.Message) {
if *msg.MsgType == 2 {
// Change Forward message.
forward := new(chproto.ChangeForward)
err := proto.Unmarshal(msg.Content, forward)
if err != nil {
conn.Close()
}
// If this is not a core node,
// and the requesting node is not this node,
// discard the message.
if uint16(*forward.Request.RequestNode) != config.Id() &&
!config.IsCore() {
return
}
// Send a Change Forward Ack message to the sender.
ack := new(chproto.ChangeForwardAck)
ack.RequestNode = forward.Request.RequestNode
ack.RequestId = forward.Request.RequestId
conn.SendProto(3, ack)
// Send the forward message to our processing goroutine.
receivedForwardCh <- receivedForward{forward: forward,
node: node}
} else if *msg.MsgType == 3 {
// Change Forward Ack message.
ack := new(chproto.ChangeForwardAck)
err := proto.Unmarshal(msg.Content, ack)
if err != nil {
conn.Close()
}
// Send the ack message to our processing goroutine.
receivedAckCh <- receivedAck{ack: ack, node: node}
} else {
// Unknown message.
conn.Close()
}
}
func handleInstructionChosen(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
var msg fproto.InstructionChosen
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if f.closed {
return
}
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) {
slot := instructions[relativeSlot]
if len(slot) == 1 && slot[0].IsChosen() {
return
}
}
if !config.IsCore() {
// TODO: Should only do this on one follow connection.
var intReq fproto.InstructionRequest
intReq.Slot = msg.Slot
f.conn.SendProto(8, &intReq)
}
timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
f.offerTimers[*msg.Slot] = time.AfterFunc(
timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
func process() {
for {
select {
// Try to make any needed outgoing connections.
case <-tryOutgoingCh:
store.StartTransaction()
connectionsLock.Lock()
coreNodes := config.CoreNodes()
if !store.Degraded() {
if config.IsCore() {
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if connections[node] == nil {
go tryOutgoing(node)
}
}
} else {
// Settle for less than 2 core nodes,
// if there *aren't* 2 core nodes.
targetConns := 2
if targetConns > len(coreNodes) {
targetConns = len(coreNodes)
}
newOutgoing := targetConns -
len(connections)
used := -1
for newOutgoing > 0 {
r := processRand.Intn(
len(coreNodes))
// Don't do the same node
// twice.
if r == used {
continue
}
node := coreNodes[r]
if connections[node] == nil {
used = r
go tryOutgoing(node)
newOutgoing--
}
}
}
} else {
if len(connections) == 0 {
r := processRand.Intn(len(coreNodes))
node := coreNodes[r]
if connections[node] == nil {
go tryOutgoing(node)
}
}
}
connectionsLock.Unlock()
store.EndTransaction()
// After a random time, check again.
randDur := time.Duration(processRand.Intn(19)+1) * time.Second
tryOutgoingCh = time.NewTimer(randDur).C
// New received connection.
case conn := <-receivedConnCh:
conn.offerTimers = make(map[uint64]*time.Timer)
store.StartTransaction()
connectionsLock.Lock()
// If we are degraded, reject the connection,
// unless we have no other connection,
// and it is outbound.
if store.Degraded() &&
!(len(connections) == 0 && conn.outgoing) {
conn.lock.Lock()
conn.Close()
conn.lock.Unlock()
connectionsLock.Unlock()
store.EndTransaction()
break
}
// If we have an existing connection to this node,
// close it.
if connections[conn.node] != nil {
other := connections[conn.node]
other.lock.Lock()
other.Close()
if other.firstUnappliedTimer != nil {
other.firstUnappliedTimer.Stop()
}
other.lock.Unlock()
}
// Add to our connections.
connections[conn.node] = conn
// Send initial position and leader messages.
posMsg := new(fproto.Position)
posMsg.FirstUnapplied = new(uint64)
posMsg.Degraded = new(bool)
*posMsg.FirstUnapplied =
store.InstructionFirstUnapplied()
*posMsg.Degraded = store.Degraded()
conn.conn.SendProto(2, posMsg)
proposal, leader := store.Proposal()
leaderMsg := new(fproto.Leader)
leaderMsg.Proposal = new(uint64)
leaderMsg.Leader = new(uint32)
*leaderMsg.Proposal = proposal
*leaderMsg.Leader = uint32(leader)
conn.conn.SendProto(11, leaderMsg)
connectionsLock.Unlock()
store.EndTransaction()
// Start timer for sending first unapplied
// instruction updates.
if config.IsCore() && conn.node <= 0x2000 {
conn.lock.Lock()
conn.firstUnappliedTimer = time.AfterFunc(
firstUnappliedTimerDuration,
func() { conn.firstUnappliedTimeout() })
conn.lock.Unlock()
}
// Start handling received messages from the connection.
go handleConn(conn)
// Terminated connection.
case conn := <-terminatedConnCh:
connectionsLock.Lock()
if connections[conn.node] == conn {
delete(connections, conn.node)
conn.lock.Lock()
conn.closed = true
if conn.firstUnappliedTimer != nil {
conn.firstUnappliedTimer.Stop()
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
}
}
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {
// If we are already trying to forward a change forward message with
// the same requesting node and request ID, discard this message.
if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
*forward.Request.RequestId); exists {
return
}
// Everything else in this function runs in a transaction.
// We are read-only.
store.StartTransaction()
defer store.EndTransaction()
// If this is a core node and this node stopped being leader less than
// a Change Timeout Period ago, always add us to the ignore list.
if config.IsCore() && !isIgnored(forward, config.Id()) {
diff := time.Now().Sub(store.StoppedLeading())
if diff < config.CHANGE_TIMEOUT_PERIOD {
forward.Ignores = append(forward.Ignores,
uint32(config.Id()))
}
}
// If all core node IDs are in the forward's ignore list, discard it.
if len(forward.Ignores) == len(config.CoreNodes()) {
log.Print("shared/chrequest: dropped msg due to full ignores")
return
}
// Otherwise, choose a potential leader node.
// This is O(n^2) in the number of core nodes,
// but we don't expect to have many.
chosenNode := uint16(0)
_, leader := store.Proposal()
if leader != 0 && !isIgnored(forward, leader) {
chosenNode = leader
} else {
for _, node := range config.CoreNodes() {
if !isIgnored(forward, node) {
chosenNode = node
break
}
}
}
if chosenNode == 0 {
// Shouldn't happen.
log.Print("shared/chrequest: bug, " +
"couldn't find candidate leader node")
return
}
// If we are the selected leader, construct an external change request,
// and send it on our change request channel.
if chosenNode == config.Id() {
intRequest := forward.Request
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *intRequest.RequestEntity
chrequest.RequestNode = uint16(*intRequest.RequestNode)
chrequest.RequestId = *intRequest.RequestId
chrequest.Changeset = make([]store.Change,
len(intRequest.Changeset))
for i, ch := range intRequest.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
for _, cb := range changeCallbacks {
cb(chrequest)
}
return
}
// Otherwise, we send it on to the selected leader,
// add the selected leader to the ignore list,
// and set a timeout to retry.
sendForward(chosenNode, forward)
forward.Ignores = append(forward.Ignores, uint32(chosenNode))
addForwardTimeout(forward)
}
// Sets this instruction value as chosen, clearing all others in the same slot.
// Causes it to be automatically applied to state once all previous instruction
// slots have a chosen value.
func (i *InstructionValue) Choose() {
log.Print("shared/store: choosing instruction in slot ", i.slot)
relativeSlot := i.slot - start
// Drop all other instruction values in the same slot.
if len(slots[relativeSlot]) > 1 {
slots[relativeSlot] = []*InstructionValue{i}
}
// Set the value as chosen.
i.chosen = true
i.chosenTime = time.Now()
i.acceptedBy = nil
// Call instruction chosen callbacks.
for _, cb := range chosenCallbacks {
cb(i.slot)
}
// Apply instructions as far as possible.
tryApply()
// Discard no longer needed instructions, from the start.
newStart := start
for {
relative := newStart - start
// If we've not applied the instruction yet,
// we still need it.
if firstUnapplied == newStart {
break
}
// If the instruction wasn't chosen long enough ago,
// we can't discard it.
minDelay := time.Duration(4) * config.CHANGE_TIMEOUT_PERIOD
timeChosen := slots[relative][0].TimeChosen()
if time.Now().Sub(timeChosen) < minDelay {
break
}
// If this is a core node and the majority of core nodes aren't
// past this point, we have to keep it.
if config.IsCore() {
past := 1 // We are past it.
coreNodes := config.CoreNodes()
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if NodeFirstUnapplied(node) > newStart {
past++
}
}
if past <= len(coreNodes)/2 {
break
}
}
newStart++
}
// If we can discard, do it by reslicing the slots.
// This doesn't result in an immediate free,
// but will when we next expand the slots.
// This avoids extra performance costs for reallocating them now.
if newStart != start {
log.Print("Discarding instructions between ", newStart-1,
" and ", start)
slots = slots[newStart-start:]
start = newStart
}
}
func main() {
haltChan = make(chan struct{})
// Define and parse flags.
id := flag.Uint("id", 0, "Set the node ID of this unanimity instance.")
memprof = flag.Uint("memprof", 0, "Generate a memory profile and "+
"halt after the given instruction slot is applied.")
cpuprof = flag.Uint("cpuprof", 0, "Generate a CPU profile and halt "+
"after the given instruction slot is applied.")
flag.Parse()
// Validate flags.
if *id == 0 || *id > 0xFFFF {
log.Fatal("Invalid node ID specified.")
}
// If both memprof and cpuprof are set, they need to be the same.
if *memprof != 0 && *cpuprof != 0 && *memprof != *cpuprof {
log.Fatal("If both memprof and cpuprof are set they must " +
"match.")
}
// If either memprof or cpuprof are set, hook up the callback
// responsible for terminating the node at the right time.
if *memprof != 0 || *cpuprof != 0 {
store.AddAppliedCallback(handleProfileTime)
// If generating a CPU profile, start it now.
if *cpuprof != 0 {
f, err := os.Create("cpuprofile.prof")
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
}
// Load configuration from config file.
loadConfig()
config.SetId(uint16(*id))
// Load our TLS certificate.
idStr := strconv.FormatUint(uint64(config.Id()), 10)
cert, err := tls.LoadX509KeyPair(idStr+".crt", idStr+".key")
if err != nil {
log.Fatalf("error loading our TLS certificate: %s", err)
}
config.SetCertificate(&cert)
fmt.Printf("Loaded configuration, our ID is %d.\n", config.Id())
if config.IsCore() {
core.Startup()
} else {
client.Startup()
}
<-haltChan
// If we're to generate a memory profile before terminating, do so.
if *memprof != 0 {
f, err := os.Create("memprofile.mprof")
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.WriteHeapProfile(f)
}
}
func handlePosition(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.Position
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if config.IsCore() && f.node <= 0x2000 {
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
if store.Degraded() && *msg.Degraded {
f.Close()
return
}
start := store.InstructionStart()
if *msg.FirstUnapplied < start {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
burstMsg.Content = []byte{}
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
*msg.Degraded {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.Degraded {
f.Close()
return
}
// Send all chosen instructions above the first unapplied point.
instructions := store.InstructionSlots()
relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
sendInstructionData(f, slot, slotValues[0].ChangeRequest())
}
}