Beispiel #1
0
func handleFirstUnapplied(f *followConn, content []byte) {

	// Client nodes ignore this message, and shouldn't send it.
	if !config.IsCore() || f.node > 0x2000 {
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.FirstUnapplied
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	// Can't trigger callbacks from the store package to elsewhere,
	// safe to call while holding f's lock.
	store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
Beispiel #2
0
// Sets the first unapplied slot on the given node.
func SetNodeFirstUnapplied(node uint16, slot uint64) {
	if !config.IsCore() {
		panic("non-core node tracking first unapplied of other nodes")
	}

	nodeFirstUnapplied[node] = slot
}
Beispiel #3
0
func handleMsg(node uint16, conn *connect.BaseConn, msg *baseproto.Message) {
	if *msg.MsgType == 2 {
		// Change Forward message.
		forward := new(chproto.ChangeForward)
		err := proto.Unmarshal(msg.Content, forward)
		if err != nil {
			conn.Close()
		}

		// If this is not a core node,
		// and the requesting node is not this node,
		// discard the message.
		if uint16(*forward.Request.RequestNode) != config.Id() &&
			!config.IsCore() {
			return
		}

		// Send a Change Forward Ack message to the sender.
		ack := new(chproto.ChangeForwardAck)
		ack.RequestNode = forward.Request.RequestNode
		ack.RequestId = forward.Request.RequestId
		conn.SendProto(3, ack)

		// Send the forward message to our processing goroutine.
		receivedForwardCh <- receivedForward{forward: forward,
			node: node}

	} else if *msg.MsgType == 3 {
		// Change Forward Ack message.
		ack := new(chproto.ChangeForwardAck)
		err := proto.Unmarshal(msg.Content, ack)
		if err != nil {
			conn.Close()
		}

		// Send the ack message to our processing goroutine.
		receivedAckCh <- receivedAck{ack: ack, node: node}

	} else {
		// Unknown message.
		conn.Close()
	}
}
Beispiel #4
0
func handleInstructionChosen(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	var msg fproto.InstructionChosen
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if f.closed {
		return
	}

	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		return
	}
	instructions := store.InstructionSlots()
	if relativeSlot < len(instructions) {
		slot := instructions[relativeSlot]
		if len(slot) == 1 && slot[0].IsChosen() {
			return
		}
	}

	if !config.IsCore() {
		// TODO: Should only do this on one follow connection.
		var intReq fproto.InstructionRequest
		intReq.Slot = msg.Slot
		f.conn.SendProto(8, &intReq)
	}

	timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
	f.offerTimers[*msg.Slot] = time.AfterFunc(
		timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
Beispiel #5
0
func process() {

	for {
		select {

		// Try to make any needed outgoing connections.
		case <-tryOutgoingCh:
			store.StartTransaction()
			connectionsLock.Lock()

			coreNodes := config.CoreNodes()
			if !store.Degraded() {
				if config.IsCore() {
					for _, node := range coreNodes {
						if node == config.Id() {
							continue
						}

						if connections[node] == nil {
							go tryOutgoing(node)
						}
					}
				} else {

					// Settle for less than 2 core nodes,
					// if there *aren't* 2 core nodes.
					targetConns := 2
					if targetConns > len(coreNodes) {
						targetConns = len(coreNodes)
					}

					newOutgoing := targetConns -
						len(connections)

					used := -1
					for newOutgoing > 0 {
						r := processRand.Intn(
							len(coreNodes))

						// Don't do the same node
						// twice.
						if r == used {
							continue
						}

						node := coreNodes[r]
						if connections[node] == nil {
							used = r
							go tryOutgoing(node)
							newOutgoing--
						}
					}
				}
			} else {
				if len(connections) == 0 {
					r := processRand.Intn(len(coreNodes))
					node := coreNodes[r]
					if connections[node] == nil {
						go tryOutgoing(node)
					}
				}
			}

			connectionsLock.Unlock()
			store.EndTransaction()

			// After a random time, check again.
			randDur := time.Duration(processRand.Intn(19)+1) * time.Second
			tryOutgoingCh = time.NewTimer(randDur).C

		// New received connection.
		case conn := <-receivedConnCh:

			conn.offerTimers = make(map[uint64]*time.Timer)

			store.StartTransaction()
			connectionsLock.Lock()

			// If we are degraded, reject the connection,
			// unless we have no other connection,
			// and it is outbound.
			if store.Degraded() &&
				!(len(connections) == 0 && conn.outgoing) {

				conn.lock.Lock()
				conn.Close()
				conn.lock.Unlock()
				connectionsLock.Unlock()
				store.EndTransaction()
				break
			}

			// If we have an existing connection to this node,
			// close it.
			if connections[conn.node] != nil {
				other := connections[conn.node]

				other.lock.Lock()

				other.Close()
				if other.firstUnappliedTimer != nil {
					other.firstUnappliedTimer.Stop()
				}

				other.lock.Unlock()
			}

			// Add to our connections.
			connections[conn.node] = conn

			// Send initial position and leader messages.
			posMsg := new(fproto.Position)
			posMsg.FirstUnapplied = new(uint64)
			posMsg.Degraded = new(bool)
			*posMsg.FirstUnapplied =
				store.InstructionFirstUnapplied()
			*posMsg.Degraded = store.Degraded()
			conn.conn.SendProto(2, posMsg)

			proposal, leader := store.Proposal()
			leaderMsg := new(fproto.Leader)
			leaderMsg.Proposal = new(uint64)
			leaderMsg.Leader = new(uint32)
			*leaderMsg.Proposal = proposal
			*leaderMsg.Leader = uint32(leader)
			conn.conn.SendProto(11, leaderMsg)

			connectionsLock.Unlock()
			store.EndTransaction()

			// Start timer for sending first unapplied
			// instruction updates.
			if config.IsCore() && conn.node <= 0x2000 {
				conn.lock.Lock()

				conn.firstUnappliedTimer = time.AfterFunc(
					firstUnappliedTimerDuration,
					func() { conn.firstUnappliedTimeout() })

				conn.lock.Unlock()
			}

			// Start handling received messages from the connection.
			go handleConn(conn)

		// Terminated connection.
		case conn := <-terminatedConnCh:

			connectionsLock.Lock()

			if connections[conn.node] == conn {
				delete(connections, conn.node)

				conn.lock.Lock()

				conn.closed = true
				if conn.firstUnappliedTimer != nil {
					conn.firstUnappliedTimer.Stop()
				}

				conn.lock.Unlock()
			}

			connectionsLock.Unlock()
		}
	}
}
Beispiel #6
0
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {

	// If we are already trying to forward a change forward message with
	// the same requesting node and request ID, discard this message.
	if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
		*forward.Request.RequestId); exists {
		return
	}

	// Everything else in this function runs in a transaction.
	// We are read-only.
	store.StartTransaction()
	defer store.EndTransaction()

	// If this is a core node and this node stopped being leader less than
	// a Change Timeout Period ago, always add us to the ignore list.
	if config.IsCore() && !isIgnored(forward, config.Id()) {
		diff := time.Now().Sub(store.StoppedLeading())
		if diff < config.CHANGE_TIMEOUT_PERIOD {
			forward.Ignores = append(forward.Ignores,
				uint32(config.Id()))
		}
	}

	// If all core node IDs are in the forward's ignore list, discard it.
	if len(forward.Ignores) == len(config.CoreNodes()) {
		log.Print("shared/chrequest: dropped msg due to full ignores")
		return
	}

	// Otherwise, choose a potential leader node.
	// This is O(n^2) in the number of core nodes,
	// but we don't expect to have many.
	chosenNode := uint16(0)
	_, leader := store.Proposal()
	if leader != 0 && !isIgnored(forward, leader) {
		chosenNode = leader
	} else {
		for _, node := range config.CoreNodes() {
			if !isIgnored(forward, node) {
				chosenNode = node
				break
			}
		}
	}
	if chosenNode == 0 {
		// Shouldn't happen.
		log.Print("shared/chrequest: bug, " +
			"couldn't find candidate leader node")
		return
	}

	// If we are the selected leader, construct an external change request,
	// and send it on our change request channel.
	if chosenNode == config.Id() {
		intRequest := forward.Request
		chrequest := new(store.ChangeRequest)
		chrequest.RequestEntity = *intRequest.RequestEntity
		chrequest.RequestNode = uint16(*intRequest.RequestNode)
		chrequest.RequestId = *intRequest.RequestId
		chrequest.Changeset = make([]store.Change,
			len(intRequest.Changeset))

		for i, ch := range intRequest.Changeset {
			chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
			chrequest.Changeset[i].Key = *ch.Key
			chrequest.Changeset[i].Value = *ch.Value
		}

		for _, cb := range changeCallbacks {
			cb(chrequest)
		}

		return
	}

	// Otherwise, we send it on to the selected leader,
	// add the selected leader to the ignore list,
	// and set a timeout to retry.
	sendForward(chosenNode, forward)
	forward.Ignores = append(forward.Ignores, uint32(chosenNode))
	addForwardTimeout(forward)
}
Beispiel #7
0
// Sets this instruction value as chosen, clearing all others in the same slot.
// Causes it to be automatically applied to state once all previous instruction
// slots have a chosen value.
func (i *InstructionValue) Choose() {

	log.Print("shared/store: choosing instruction in slot ", i.slot)

	relativeSlot := i.slot - start

	// Drop all other instruction values in the same slot.
	if len(slots[relativeSlot]) > 1 {
		slots[relativeSlot] = []*InstructionValue{i}
	}

	// Set the value as chosen.
	i.chosen = true
	i.chosenTime = time.Now()
	i.acceptedBy = nil

	// Call instruction chosen callbacks.
	for _, cb := range chosenCallbacks {
		cb(i.slot)
	}

	// Apply instructions as far as possible.
	tryApply()

	// Discard no longer needed instructions, from the start.
	newStart := start
	for {
		relative := newStart - start

		// If we've not applied the instruction yet,
		// we still need it.
		if firstUnapplied == newStart {
			break
		}

		// If the instruction wasn't chosen long enough ago,
		// we can't discard it.
		minDelay := time.Duration(4) * config.CHANGE_TIMEOUT_PERIOD
		timeChosen := slots[relative][0].TimeChosen()
		if time.Now().Sub(timeChosen) < minDelay {
			break
		}

		// If this is a core node and the majority of core nodes aren't
		// past this point, we have to keep it.
		if config.IsCore() {
			past := 1 // We are past it.
			coreNodes := config.CoreNodes()
			for _, node := range coreNodes {
				if node == config.Id() {
					continue
				}

				if NodeFirstUnapplied(node) > newStart {
					past++
				}
			}
			if past <= len(coreNodes)/2 {
				break
			}
		}

		newStart++
	}

	// If we can discard, do it by reslicing the slots.
	// This doesn't result in an immediate free,
	// but will when we next expand the slots.
	// This avoids extra performance costs for reallocating them now.
	if newStart != start {
		log.Print("Discarding instructions between ", newStart-1,
			" and ", start)

		slots = slots[newStart-start:]
		start = newStart
	}

}
Beispiel #8
0
func main() {
	haltChan = make(chan struct{})

	// Define and parse flags.
	id := flag.Uint("id", 0, "Set the node ID of this unanimity instance.")
	memprof = flag.Uint("memprof", 0, "Generate a memory profile and "+
		"halt after the given instruction slot is applied.")
	cpuprof = flag.Uint("cpuprof", 0, "Generate a CPU profile and halt "+
		"after the given instruction slot is applied.")
	flag.Parse()

	// Validate flags.
	if *id == 0 || *id > 0xFFFF {
		log.Fatal("Invalid node ID specified.")
	}

	// If both memprof and cpuprof are set, they need to be the same.
	if *memprof != 0 && *cpuprof != 0 && *memprof != *cpuprof {
		log.Fatal("If both memprof and cpuprof are set they must " +
			"match.")
	}

	// If either memprof or cpuprof are set, hook up the callback
	// responsible for terminating the node at the right time.
	if *memprof != 0 || *cpuprof != 0 {
		store.AddAppliedCallback(handleProfileTime)

		// If generating a CPU profile, start it now.
		if *cpuprof != 0 {
			f, err := os.Create("cpuprofile.prof")
			if err != nil {
				log.Fatal(err)
			}

			pprof.StartCPUProfile(f)
			defer pprof.StopCPUProfile()
		}
	}

	// Load configuration from config file.
	loadConfig()
	config.SetId(uint16(*id))

	// Load our TLS certificate.
	idStr := strconv.FormatUint(uint64(config.Id()), 10)
	cert, err := tls.LoadX509KeyPair(idStr+".crt", idStr+".key")
	if err != nil {
		log.Fatalf("error loading our TLS certificate: %s", err)
	}
	config.SetCertificate(&cert)

	fmt.Printf("Loaded configuration, our ID is %d.\n", config.Id())
	if config.IsCore() {
		core.Startup()
	} else {
		client.Startup()
	}

	<-haltChan

	// If we're to generate a memory profile before terminating, do so.
	if *memprof != 0 {
		f, err := os.Create("memprofile.mprof")
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()

		pprof.WriteHeapProfile(f)
	}
}
Beispiel #9
0
func handlePosition(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.Position
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if config.IsCore() && f.node <= 0x2000 {
		store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
	}

	if store.Degraded() && *msg.Degraded {
		f.Close()
		return
	}

	start := store.InstructionStart()
	if *msg.FirstUnapplied < start {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		burstMsg.Content = []byte{}
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
		*msg.Degraded {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.Degraded {
		f.Close()
		return
	}

	// Send all chosen instructions above the first unapplied point.
	instructions := store.InstructionSlots()
	relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		sendInstructionData(f, slot, slotValues[0].ChangeRequest())
	}
}