Beispiel #1
0
func handleApplied(slot uint64, idempotentChanges []store.Change) {

	relativeSlot := int(slot - store.InstructionStart())
	slots := store.InstructionSlots()

	origReq := slots[relativeSlot][0].ChangeRequest()
	chrequest := new(store.ChangeRequest)
	chrequest.RequestEntity = origReq.RequestEntity
	chrequest.RequestNode = origReq.RequestNode
	chrequest.RequestId = origReq.RequestId
	chrequest.Changeset = idempotentChanges

	connectionsLock.Lock()

	for _, conn := range connections {

		conn.lock.Lock()

		if conn.sendingBurst {
			sendInstructionData(conn, slot, chrequest)
		}

		conn.lock.Unlock()
	}

	connectionsLock.Unlock()

}
Beispiel #2
0
func handleInstructionRequest(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.InstructionRequest
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		f.Close()
		return
	}
	instructions := store.InstructionSlots()
	if relativeSlot >= len(instructions) {
		f.Close()
		return
	}
	slot := instructions[relativeSlot]
	if len(slot) != 1 || !slot[0].IsChosen() {
		f.Close()
		return
	}

	// Convert the change request to our internal format.
	sendInstructionData(f, *msg.Slot, slot[0].ChangeRequest())
}
Beispiel #3
0
// Must be called from the processing goroutine, inside a transaction.
// Returns false if it discarded the accepted message because of it being
// too far in our future. Any other reason for discarding the message,
// as well as the accept succeeding, will return true.
func addAccepted(node uint16, accepted *coproto.Accepted) bool {

	slots := store.InstructionSlots()

	msgProposal, msgLeader := *accepted.Proposal, uint16(*accepted.Leader)
	newReq := accepted.Instruction.Request
	slot := *accepted.Instruction.Slot
	ourStart := store.InstructionStart()
	relativeSlot := int(slot - ourStart)

	// If our instruction slot slice is not this long,
	// we know we do not have any value in this slot yet,
	// and so need to add a new one.
	if len(slots) <= relativeSlot {
		chreq := makeExtChangeRequest(newReq)
		newInst := store.AddInstructionValue(slot, chreq)
		if newInst == nil {
			return false
		}
		newInst.Accept(node, msgProposal, msgLeader)

		return true
	}

	// If we have already chosen an instruction in this slot,
	// we can ignore subsequent Accepted messages for it.
	if len(slots[relativeSlot]) == 1 && slots[relativeSlot][0].IsChosen() {
		return true
	}

	// Returns if it successfully finds an existing matching value.
	for _, value := range slots[relativeSlot] {
		chreq := value.ChangeRequest()
		if chreq.RequestNode != uint16(*newReq.RequestNode) {
			continue
		}
		if chreq.RequestId != *newReq.RequestId {
			continue
		}

		value.Accept(node, msgProposal, msgLeader)
		return true
	}

	chreq := makeExtChangeRequest(newReq)
	newInst := store.AddInstructionValue(slot, chreq)
	if newInst == nil {
		return false
	}
	newInst.Accept(node, msgProposal, msgLeader)

	return true
}
Beispiel #4
0
func handleChosenRequests(slot uint64) {

	relativeSlot := int(slot - store.InstructionStart())
	instruction := store.InstructionSlots()[relativeSlot][0]
	req := instruction.ChangeRequest()

	requestTimeoutLock.Lock()

	_, exists := getRequestTimeout(req.RequestNode, req.RequestId)
	if exists {
		log.Print("shared/chrequest: successful request ",
			req.RequestId)

		cancelRequestTimeout(req.RequestNode, req.RequestId)
	}

	requestTimeoutLock.Unlock()
}
Beispiel #5
0
func handleInstructionChosen(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	var msg fproto.InstructionChosen
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if f.closed {
		return
	}

	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		return
	}
	instructions := store.InstructionSlots()
	if relativeSlot < len(instructions) {
		slot := instructions[relativeSlot]
		if len(slot) == 1 && slot[0].IsChosen() {
			return
		}
	}

	if !config.IsCore() {
		// TODO: Should only do this on one follow connection.
		var intReq fproto.InstructionRequest
		intReq.Slot = msg.Slot
		f.conn.SendProto(8, &intReq)
	}

	timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
	f.offerTimers[*msg.Slot] = time.AfterFunc(
		timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
Beispiel #6
0
func handleApplied(slot uint64, idemChanges []store.Change) {

	relativeSlot := int(slot - store.InstructionStart())
	slots := store.InstructionSlots()
	requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode
	requestId := slots[relativeSlot][0].ChangeRequest().RequestId

	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	waitingLock.Lock()
	defer waitingLock.Unlock()

	// If there is a connection which has this as its waiting request,
	// have it complete authentication.
	if requestNode == config.Id() {
		waitingConn := waiting[requestId]
		if waitingConn != nil {
			handleAuthComplete(waitingConn, idemChanges)
		}
	}

	ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
	for i := range idemChanges {

		key := idemChanges[i].Key
		value := idemChanges[i].Value

		if strings.HasPrefix(key, "attach ") {

			// If we were just detached from a session,
			// drop any connections associated with it,
			// and if it was in our attach timeouts, remove it.
			if key == ourAttachStr && value == "" {

				sessionId := idemChanges[i].TargetEntity
				if conn := sessions[sessionId]; conn != nil {
					log.Print("client/listener: " +
						"detached from existing " +
						"session")

					conn.conn.Close()
				}

				if orphanAttachTimeouts[sessionId] != nil {
					orphanAttachTimeouts[sessionId].Stop()
					delete(orphanAttachTimeouts, sessionId)
				}
			}

			// If we were just attached to a session,
			// but have no connection associated with that session,
			// add an attach timeout.
			// and if it was in our attach timeouts, remove it.
			if key == ourAttachStr && value != "" {

				sessionId := idemChanges[i].TargetEntity
				if sessions[sessionId] == nil {
					startOrphanTimeout(sessionId)
				}
			}

			// Handle a session attaching/detaching from a user.
			entityId := idemChanges[i].TargetEntity
			entity := store.GetEntity(entityId)
			if entity != nil && entity.Value("kind") == "user" {

				// Tell following connections when a session
				// attaches or detaches from a followed user.
				// TODO: Expensive loop,
				// should maintain index to avoid this.
				for conn, _ := range connections {
					following := false
					for _, fw := range conn.following {
						if fw == entityId {
							following = true
							break
						}
					}

					if !following {
						continue
					}

					sendUserChange(conn, entityId,
						key, value)
				}
			}
		}
	}

	// If we've created a nameless user, start a timer to delete it.
	for i := range idemChanges {
		if idemChanges[i].Key != "kind" ||
			idemChanges[i].Value != "user" {

			continue
		}

		newUserId := idemChanges[i].TargetEntity
		newUser := store.GetEntity(newUserId)
		if newUser != nil && newUser.Value("name username") == "" {
			startNamelessTimeout(newUserId)
		}
	}
}
Beispiel #7
0
func handleChosen(slot uint64) {
	relativeSlot := int(slot - store.InstructionStart())
	slots := store.InstructionSlots()

	cancelProposalTimeout(slot, slots[relativeSlot][0].ChangeRequest())
}
Beispiel #8
0
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
	receivedPromises[node] = msg

	// If we have promises from a majority of core nodes,
	// become leader.
	if len(receivedPromises) > len(config.CoreNodes())/2 {

		log.Print("core/consensus: became leader")

		stopLeaderTimeout()
		amLeader = true
		proposal, leader := store.Proposal()

		// Find a slot number above all those in promise messages,
		// and above our first unapplied.
		firstUnapplied := store.InstructionFirstUnapplied()
		limit := firstUnapplied
		for _, msg := range receivedPromises {
			for _, accepted := range msg.Accepted {
				if *accepted.Slot >= limit {
					limit = *accepted.Slot + 1
				}
			}
		}

		// Start our next slot after the limit.
		nextProposalSlot = limit

		// For all slots between this and our first unapplied,
		// submit a previously accepted instruction unless we
		// know an instruction was already chosen.
		// Fills these slots with proposals.
		// This is O(n^2) in the number of instructions between our
		// first unapplied and limit.
		// TODO: Improve worst-case complexity.
		start := store.InstructionStart()
		slots := store.InstructionSlots()
		for i := firstUnapplied; i < limit; i++ {

			// If we already have a chosen instruction, skip.
			rel := int(i - start)
			if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
				continue
			}

			// Find the previously accepted instruction
			// accepted with the highest proposal number.
			var bestInst *coproto.Instruction
			var bp uint64 // Best proposal
			var bl uint16 // Best leader

			for _, msg := range receivedPromises {
				for _, accepted := range msg.Accepted {
					if *accepted.Slot != i {
						continue
					}
					if bestInst == nil {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
						continue
					}

					// TODO: This indent is just absurd.
					p := *accepted.Proposal
					l := uint16(*accepted.Leader)
					if store.CompareProposals(p, l,
						bp, bl) {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
					}
				}
			}

			// If we didn't find an instruction, make an empty one.
			if bestInst == nil {
				empty := new(coproto.ChangeRequest)
				empty.RequestEntity = new(uint64)
				empty.RequestNode = new(uint32)
				*empty.RequestEntity = uint64(config.Id())
				*empty.RequestNode = uint32(config.Id())

				bestInst := new(coproto.Instruction)
				bestInst.Slot = new(uint64)
				*bestInst.Slot = i
				bestInst.Request = empty
			}

			// Add proposal timeout.
			req := makeExtChangeRequest(bestInst.Request)
			addProposalTimeout(i, req)

			// Send proposal.
			bestInst.Proposal = new(uint64)
			bestInst.Leader = new(uint32)
			*bestInst.Proposal = proposal
			*bestInst.Leader = uint32(leader)
			sendProposal(bestInst)
		}

		// Discard received promise messages.
		receivedPromises = nil

		// Make an instruction proposal for each waiting change.
		for _, req := range waitingRequests {
			slot := nextProposalSlot
			nextProposalSlot++

			addProposalTimeout(slot, req)

			inst := makeInst(slot, proposal, leader, req)

			sendProposal(inst)
		}

		// Clear waiting changes.
		waitingRequests = nil
	}
}
Beispiel #9
0
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Prepare
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	newProposal, newLeader := *msg.Proposal, node
	proposal, leader := store.Proposal()
	if store.CompareProposals(newProposal, newLeader, proposal, leader) {

		log.Print("core/consensus: sending promise to ", newLeader)

		// Create a promise message to send back.
		var promise coproto.Promise
		promise.Proposal = new(uint64)
		promise.Leader = new(uint32)
		promise.PrevProposal = new(uint64)
		promise.PrevLeader = new(uint32)
		*promise.Proposal = newProposal
		*promise.Leader = uint32(newLeader)
		*promise.PrevProposal = proposal
		*promise.PrevLeader = uint32(leader)

		// Add all the instructions we've previously accepted or chosen.
		slots := store.InstructionSlots()
		theirFirstUnapplied := *msg.FirstUnapplied
		ourStart := store.InstructionStart()
		relativeSlot := int(theirFirstUnapplied - ourStart)
		if relativeSlot < 0 {
			relativeSlot = 0
		}
		var accepted []*coproto.Instruction
		for ; relativeSlot < len(slots); relativeSlot++ {
			slot := slots[relativeSlot]
			slotNum := ourStart + uint64(relativeSlot)
			for i, _ := range slot {
				if slot[i].IsChosen() {
					appendInst(&accepted, slotNum, slot[i])
					break
				}

				weAccepted := false
				for _, node := range slot[i].Accepted() {
					if node == config.Id() {
						weAccepted = true
						break
					}
				}

				if weAccepted {
					appendInst(&accepted, slotNum, slot[i])
					break
				}
			}
		}

		// Send promise message.
		conn.SendProto(3, &promise)

		// Accept the other node as our new leader.
		store.SetProposal(newProposal, newLeader)
	} else {
		var nack coproto.Nack
		nack.Proposal = new(uint64)
		nack.Leader = new(uint32)
		*nack.Proposal = proposal
		*nack.Leader = uint32(leader)
		conn.SendProto(6, &nack)
	}
}
Beispiel #10
0
func handlePosition(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.Position
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if config.IsCore() && f.node <= 0x2000 {
		store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
	}

	if store.Degraded() && *msg.Degraded {
		f.Close()
		return
	}

	start := store.InstructionStart()
	if *msg.FirstUnapplied < start {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		burstMsg.Content = []byte{}
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
		*msg.Degraded {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.Degraded {
		f.Close()
		return
	}

	// Send all chosen instructions above the first unapplied point.
	instructions := store.InstructionSlots()
	relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		sendInstructionData(f, slot, slotValues[0].ChangeRequest())
	}
}
Beispiel #11
0
// Sends a burst to the given connection.
// The Bursting message should already have been sent,
// and f.sendingBurst set to true, before calling this asynchronously.
func sendBurst(f *followConn) {

	log.Print("shared/follow: sending burst to ", f.node)

	globalProp := new(fproto.GlobalProperty)
	globalProp.Key = new(string)
	globalProp.Value = new(string)

	entityProp := new(fproto.EntityProperty)
	entityProp.Entity = new(uint64)
	entityProp.Key = new(string)
	entityProp.Value = new(string)

	aborted := false
	store.Burst(func(key, value string) bool {
		*globalProp.Key = key
		*globalProp.Value = value

		if !f.conn.SendProto(4, globalProp) {
			aborted = true
		}
		return !aborted
	}, func(entity uint64, key, value string) bool {
		*entityProp.Entity = entity
		*entityProp.Key = key
		*entityProp.Value = value

		if !f.conn.SendProto(5, entityProp) {
			aborted = true
		}
		return !aborted
	})

	if aborted {
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	f.sendingBurst = false

	burstDone := new(fproto.BurstDone)
	burstDone.FirstUnapplied = new(uint64)
	*burstDone.FirstUnapplied = store.InstructionFirstUnapplied()
	f.conn.SendProto(6, burstDone)

	// Send an instruction chosen message for all chosen instructions above
	// our first unapplied point.
	instructions := store.InstructionSlots()
	firstUnapplied := store.InstructionFirstUnapplied()
	relativeSlot := int(firstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		chosen := new(fproto.InstructionChosen)
		chosen.Slot = new(uint64)
		*chosen.Slot = slot
		f.conn.SendProto(7, chosen)
	}
}
Beispiel #12
0
func handleInstructionData(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()

	if f.closed {
		f.lock.Unlock()
		return
	}

	var msg fproto.InstructionData
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		f.lock.Unlock()
		return
	}

	if f.receivingBurst {
		f.waiting = append(f.waiting, &msg)
		f.lock.Unlock()
		return
	}

	// We need to unlock before we start mutating the store,
	// due to callbacks from the store package to elsewhere.
	f.lock.Unlock()

	// If we have a chosen instruction in this slot already,
	// or it is prior to our instruction start number,
	// discard this message.
	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		return
	}

	instructions := store.InstructionSlots()
	if relativeSlot < len(instructions) &&
		len(instructions[relativeSlot]) == 1 &&
		instructions[relativeSlot][0].IsChosen() {

		return
	}

	// Construct a store.ChangeRequest from our
	// internal ChangeRequest.
	chrequest := new(store.ChangeRequest)
	chrequest.RequestEntity = *msg.Request.RequestEntity
	chrequest.RequestNode = uint16(*msg.Request.RequestNode)
	chrequest.RequestId = *msg.Request.RequestId
	chrequest.Changeset = make([]store.Change,
		len(msg.Request.Changeset))

	for i, ch := range msg.Request.Changeset {
		chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
		chrequest.Changeset[i].Key = *ch.Key
		chrequest.Changeset[i].Value = *ch.Value
	}

	// Add instruction value and immediately choose it.
	store.AddInstructionValue(*msg.Slot, chrequest).Choose()
}