示例#1
0
// Can only be called from the handling goroutine for conn,
// inside a transaction.
func followUser(conn *userConn, followId uint64) {
	conn.following = append(conn.following, followId)

	user := store.GetEntity(followId)

	// Send the user's username.
	var dataMsg cliproto_down.UserData
	dataMsg.UserId = new(uint64)
	dataMsg.Key = new(string)
	dataMsg.Value = new(string)
	dataMsg.FirstUnapplied = new(uint64)
	*dataMsg.UserId = followId
	*dataMsg.Key = "name username"
	*dataMsg.Value = user.Value(*dataMsg.Key)
	*dataMsg.FirstUnapplied = store.InstructionFirstUnapplied()
	conn.conn.SendProto(6, &dataMsg)

	// Send information on all the user's sessions.
	user.Attached(func(key string) {
		*dataMsg.Key = key
		*dataMsg.Value = "true"
		conn.conn.SendProto(6, &dataMsg)
	})

	// Send a done message to indicate that we are finished.
	var doneMsg cliproto_down.UserDataDone
	doneMsg.UserId = dataMsg.UserId
	doneMsg.FirstUnapplied = dataMsg.FirstUnapplied
	conn.conn.SendProto(7, &doneMsg)
}
示例#2
0
// Must be called inside transaction.
func sendFollowUserIdFail(conn *userConn, userId uint64, reason string) {
	var msg cliproto_down.FollowUserIdFailed
	msg.UserId = new(uint64)
	msg.Reason = new(string)
	msg.FirstUnapplied = new(uint64)
	*msg.UserId = userId
	*msg.Reason = reason
	*msg.FirstUnapplied = store.InstructionFirstUnapplied()

	conn.conn.SendProto(5, &msg)
}
示例#3
0
// Must be called inside transaction.
func sendUserChange(conn *userConn, entityId uint64, key, value string) {
	var msg cliproto_down.UserDataChange
	msg.UserId = new(uint64)
	msg.Key = new(string)
	msg.Value = new(string)
	msg.FirstUnapplied = new(uint64)
	*msg.UserId = entityId
	*msg.Key = key
	*msg.Value = value
	*msg.FirstUnapplied = store.InstructionFirstUnapplied()
	conn.conn.SendProto(8, &msg)
}
示例#4
0
func (f *followConn) firstUnappliedTimeout() {

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	msg := new(fproto.FirstUnapplied)
	msg.FirstUnapplied = new(uint64)
	*msg.FirstUnapplied = store.InstructionFirstUnapplied()
	f.conn.SendProto(10, msg)

	f.firstUnappliedTimer = time.AfterFunc(firstUnappliedTimerDuration,
		func() { f.firstUnappliedTimeout() })

}
示例#5
0
func process() {

	for {
		select {

		// Try to make any needed outgoing connections.
		case <-tryOutgoingCh:
			store.StartTransaction()
			connectionsLock.Lock()

			coreNodes := config.CoreNodes()
			if !store.Degraded() {
				if config.IsCore() {
					for _, node := range coreNodes {
						if node == config.Id() {
							continue
						}

						if connections[node] == nil {
							go tryOutgoing(node)
						}
					}
				} else {

					// Settle for less than 2 core nodes,
					// if there *aren't* 2 core nodes.
					targetConns := 2
					if targetConns > len(coreNodes) {
						targetConns = len(coreNodes)
					}

					newOutgoing := targetConns -
						len(connections)

					used := -1
					for newOutgoing > 0 {
						r := processRand.Intn(
							len(coreNodes))

						// Don't do the same node
						// twice.
						if r == used {
							continue
						}

						node := coreNodes[r]
						if connections[node] == nil {
							used = r
							go tryOutgoing(node)
							newOutgoing--
						}
					}
				}
			} else {
				if len(connections) == 0 {
					r := processRand.Intn(len(coreNodes))
					node := coreNodes[r]
					if connections[node] == nil {
						go tryOutgoing(node)
					}
				}
			}

			connectionsLock.Unlock()
			store.EndTransaction()

			// After a random time, check again.
			randDur := time.Duration(processRand.Intn(19)+1) * time.Second
			tryOutgoingCh = time.NewTimer(randDur).C

		// New received connection.
		case conn := <-receivedConnCh:

			conn.offerTimers = make(map[uint64]*time.Timer)

			store.StartTransaction()
			connectionsLock.Lock()

			// If we are degraded, reject the connection,
			// unless we have no other connection,
			// and it is outbound.
			if store.Degraded() &&
				!(len(connections) == 0 && conn.outgoing) {

				conn.lock.Lock()
				conn.Close()
				conn.lock.Unlock()
				connectionsLock.Unlock()
				store.EndTransaction()
				break
			}

			// If we have an existing connection to this node,
			// close it.
			if connections[conn.node] != nil {
				other := connections[conn.node]

				other.lock.Lock()

				other.Close()
				if other.firstUnappliedTimer != nil {
					other.firstUnappliedTimer.Stop()
				}

				other.lock.Unlock()
			}

			// Add to our connections.
			connections[conn.node] = conn

			// Send initial position and leader messages.
			posMsg := new(fproto.Position)
			posMsg.FirstUnapplied = new(uint64)
			posMsg.Degraded = new(bool)
			*posMsg.FirstUnapplied =
				store.InstructionFirstUnapplied()
			*posMsg.Degraded = store.Degraded()
			conn.conn.SendProto(2, posMsg)

			proposal, leader := store.Proposal()
			leaderMsg := new(fproto.Leader)
			leaderMsg.Proposal = new(uint64)
			leaderMsg.Leader = new(uint32)
			*leaderMsg.Proposal = proposal
			*leaderMsg.Leader = uint32(leader)
			conn.conn.SendProto(11, leaderMsg)

			connectionsLock.Unlock()
			store.EndTransaction()

			// Start timer for sending first unapplied
			// instruction updates.
			if config.IsCore() && conn.node <= 0x2000 {
				conn.lock.Lock()

				conn.firstUnappliedTimer = time.AfterFunc(
					firstUnappliedTimerDuration,
					func() { conn.firstUnappliedTimeout() })

				conn.lock.Unlock()
			}

			// Start handling received messages from the connection.
			go handleConn(conn)

		// Terminated connection.
		case conn := <-terminatedConnCh:

			connectionsLock.Lock()

			if connections[conn.node] == conn {
				delete(connections, conn.node)

				conn.lock.Lock()

				conn.closed = true
				if conn.firstUnappliedTimer != nil {
					conn.firstUnappliedTimer.Stop()
				}

				conn.lock.Unlock()
			}

			connectionsLock.Unlock()
		}
	}
}
示例#6
0
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
	receivedPromises[node] = msg

	// If we have promises from a majority of core nodes,
	// become leader.
	if len(receivedPromises) > len(config.CoreNodes())/2 {

		log.Print("core/consensus: became leader")

		stopLeaderTimeout()
		amLeader = true
		proposal, leader := store.Proposal()

		// Find a slot number above all those in promise messages,
		// and above our first unapplied.
		firstUnapplied := store.InstructionFirstUnapplied()
		limit := firstUnapplied
		for _, msg := range receivedPromises {
			for _, accepted := range msg.Accepted {
				if *accepted.Slot >= limit {
					limit = *accepted.Slot + 1
				}
			}
		}

		// Start our next slot after the limit.
		nextProposalSlot = limit

		// For all slots between this and our first unapplied,
		// submit a previously accepted instruction unless we
		// know an instruction was already chosen.
		// Fills these slots with proposals.
		// This is O(n^2) in the number of instructions between our
		// first unapplied and limit.
		// TODO: Improve worst-case complexity.
		start := store.InstructionStart()
		slots := store.InstructionSlots()
		for i := firstUnapplied; i < limit; i++ {

			// If we already have a chosen instruction, skip.
			rel := int(i - start)
			if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
				continue
			}

			// Find the previously accepted instruction
			// accepted with the highest proposal number.
			var bestInst *coproto.Instruction
			var bp uint64 // Best proposal
			var bl uint16 // Best leader

			for _, msg := range receivedPromises {
				for _, accepted := range msg.Accepted {
					if *accepted.Slot != i {
						continue
					}
					if bestInst == nil {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
						continue
					}

					// TODO: This indent is just absurd.
					p := *accepted.Proposal
					l := uint16(*accepted.Leader)
					if store.CompareProposals(p, l,
						bp, bl) {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
					}
				}
			}

			// If we didn't find an instruction, make an empty one.
			if bestInst == nil {
				empty := new(coproto.ChangeRequest)
				empty.RequestEntity = new(uint64)
				empty.RequestNode = new(uint32)
				*empty.RequestEntity = uint64(config.Id())
				*empty.RequestNode = uint32(config.Id())

				bestInst := new(coproto.Instruction)
				bestInst.Slot = new(uint64)
				*bestInst.Slot = i
				bestInst.Request = empty
			}

			// Add proposal timeout.
			req := makeExtChangeRequest(bestInst.Request)
			addProposalTimeout(i, req)

			// Send proposal.
			bestInst.Proposal = new(uint64)
			bestInst.Leader = new(uint32)
			*bestInst.Proposal = proposal
			*bestInst.Leader = uint32(leader)
			sendProposal(bestInst)
		}

		// Discard received promise messages.
		receivedPromises = nil

		// Make an instruction proposal for each waiting change.
		for _, req := range waitingRequests {
			slot := nextProposalSlot
			nextProposalSlot++

			addProposalTimeout(slot, req)

			inst := makeInst(slot, proposal, leader, req)

			sendProposal(inst)
		}

		// Clear waiting changes.
		waitingRequests = nil
	}
}
示例#7
0
// Top-level function of the processing goroutine.
func process() {

	connections = make(map[uint16][]*connect.BaseConn)

	// On startup, make an outgoing connection attempt to all other
	// core nodes, before continuing.
	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		conn, err := connect.Dial(
			connect.CONSENSUS_PROTOCOL, node)
		if err != nil {
			// Can't reach the other node.
			continue
		}

		log.Print("core/consensus: made outgoing connection to ", node)
		connections[node] = append(connections[node], conn)
		go handleConn(node, conn)
	}

	// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
	// Largely arbitrary.
	reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)

	for {
		select {

		// Connection retry tick.
		// We should try to make an outgoing connection to any node
		// that we do not have at least one connection to.
		// We need to make these asynchronously, because connections
		// are slow.
		case <-reconnectTicker:
			for _, node := range config.CoreNodes() {
				if node == config.Id() {
					continue
				}
				if len(connections[node]) > 0 {
					continue
				}

				go outgoingConn(node)
			}

		// New change request, for us to propose as leader.
		case req := <-newChangeCh:

			store.StartTransaction()

			if !amLeader && receivedPromises != nil {
				waitingRequests = append(waitingRequests, req)
			} else if !amLeader {
				waitingRequests = append(waitingRequests, req)

				// Start attempting to be leader.
				m := make(map[uint16]*coproto.Promise)
				receivedPromises = m

				proposal, _ := store.Proposal()
				proposal++
				store.SetProposal(proposal, config.Id())
				firstUn := store.InstructionFirstUnapplied()

				// Send prepare messages to all other nodes.
				var prepare coproto.Prepare
				prepare.Proposal = new(uint64)
				prepare.FirstUnapplied = new(uint64)
				*prepare.Proposal = proposal
				*prepare.FirstUnapplied = firstUn

				for _, node := range config.CoreNodes() {
					if node == config.Id() {
						continue
					}

					if len(connections[node]) != 0 {
						c := connections[node][0]
						c.SendProto(2, &prepare)
					}
				}

				// Behave as if we got a promise message
				// from ourselves.
				var promise coproto.Promise
				promise.Proposal = prepare.Proposal
				promise.PrevProposal = promise.Proposal
				promise.Leader = new(uint32)
				*promise.Leader = uint32(config.Id())
				promise.PrevLeader = promise.Leader
				addPromise(config.Id(), &promise)
			} else {
				newSlot := nextProposalSlot
				nextProposalSlot++

				addProposalTimeout(newSlot, req)

				proposal, leader := store.Proposal()
				inst := makeInst(newSlot, proposal, leader,
					req)

				sendProposal(inst)
			}

			store.EndTransaction()

		// Leadership attempt timed out.
		case timedOutProposal := <-leaderTimeoutCh:

			store.StartTransaction()

			proposal, leader := store.Proposal()

			// If the proposal has changed since that
			// leadership attempt, ignore it.
			if leader != config.Id() {
				return
			}
			if proposal != timedOutProposal {
				return
			}

			// If we successfully became leader, ignore it.
			if amLeader {
				return
			}

			// Otherwise, stop our attempt to become leader.
			stopBeingLeader()

			store.EndTransaction()

		// Proposal timed out.
		case timeout := <-proposalTimeoutCh:

			store.StartTransaction()

			// If this timeout was not canceled, a proposal failed.
			// We stop being leader.
			if proposalTimeouts[timeout.slot] == timeout {
				stopBeingLeader()
			}

			store.EndTransaction()

		// New received connection.
		case receivedConn := <-receivedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			connections[node] = append(connections[node], conn)

		// Received message.
		case recvMsg := <-receivedMsgCh:
			node := recvMsg.node
			conn := recvMsg.conn
			msg := recvMsg.msg

			switch *msg.MsgType {
			case 2:
				processPrepare(node, conn, msg.Content)
			case 3:
				processPromise(node, conn, msg.Content)
			case 4:
				processAccept(node, conn, msg.Content)
			case 5:
				processAccepted(node, conn, msg.Content)
			case 6:
				processNack(node, conn, msg.Content)
			default:
				// Unknown message.
				conn.Close()
			}

		// Terminate received connection.
		case terminatedConn := <-terminatedConnCh:
			node := terminatedConn.node
			conn := terminatedConn.conn

			for i, other := range connections[node] {
				if other != conn {
					continue
				}

				conns := connections[node]
				conns = append(conns[:i], conns[i+1:]...)
				connections[node] = conns
				break
			}
		}
	}
}
示例#8
0
func handlePosition(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.Position
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if config.IsCore() && f.node <= 0x2000 {
		store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
	}

	if store.Degraded() && *msg.Degraded {
		f.Close()
		return
	}

	start := store.InstructionStart()
	if *msg.FirstUnapplied < start {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		burstMsg.Content = []byte{}
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
		*msg.Degraded {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.Degraded {
		f.Close()
		return
	}

	// Send all chosen instructions above the first unapplied point.
	instructions := store.InstructionSlots()
	relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		sendInstructionData(f, slot, slotValues[0].ChangeRequest())
	}
}
示例#9
0
// Sends a burst to the given connection.
// The Bursting message should already have been sent,
// and f.sendingBurst set to true, before calling this asynchronously.
func sendBurst(f *followConn) {

	log.Print("shared/follow: sending burst to ", f.node)

	globalProp := new(fproto.GlobalProperty)
	globalProp.Key = new(string)
	globalProp.Value = new(string)

	entityProp := new(fproto.EntityProperty)
	entityProp.Entity = new(uint64)
	entityProp.Key = new(string)
	entityProp.Value = new(string)

	aborted := false
	store.Burst(func(key, value string) bool {
		*globalProp.Key = key
		*globalProp.Value = value

		if !f.conn.SendProto(4, globalProp) {
			aborted = true
		}
		return !aborted
	}, func(entity uint64, key, value string) bool {
		*entityProp.Entity = entity
		*entityProp.Key = key
		*entityProp.Value = value

		if !f.conn.SendProto(5, entityProp) {
			aborted = true
		}
		return !aborted
	})

	if aborted {
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	f.sendingBurst = false

	burstDone := new(fproto.BurstDone)
	burstDone.FirstUnapplied = new(uint64)
	*burstDone.FirstUnapplied = store.InstructionFirstUnapplied()
	f.conn.SendProto(6, burstDone)

	// Send an instruction chosen message for all chosen instructions above
	// our first unapplied point.
	instructions := store.InstructionSlots()
	firstUnapplied := store.InstructionFirstUnapplied()
	relativeSlot := int(firstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		chosen := new(fproto.InstructionChosen)
		chosen.Slot = new(uint64)
		*chosen.Slot = slot
		f.conn.SendProto(7, chosen)
	}
}