Example #1
0
// May only be called by the processing goroutine, in a transaction.
func sendProposal(inst *coproto.Instruction) {

	// We must be leader.
	proposal, leader := store.Proposal()
	if leader != config.Id() || !amLeader {
		panic("tried to send accept messages while not leader")
	}

	// Send accept messages to all other core nodes.
	var accept coproto.Accept
	accept.Proposal = new(uint64)
	*accept.Proposal = proposal
	accept.Instruction = inst

	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		if len(connections[node]) != 0 {
			c := connections[node][0]
			c.SendProto(4, &accept)
		}
	}

	// Behave as if we received our own accept message.
	addAccept(config.Id(), &accept)
}
Example #2
0
// Must be called from the processing goroutine, inside a transaction.
// Handle an accept message that we've decided to respond to.
// Called directly when we send accepted messages to other nodes.
func addAccept(node uint16, msg *coproto.Accept) {

	// Send an accepted message to all other core nodes reachable.
	var accepted coproto.Accepted
	accepted.Proposal = msg.Proposal
	accepted.Leader = new(uint32)
	*accepted.Leader = uint32(node)
	accepted.Instruction = msg.Instruction

	// We must do this before sending accepted messages.
	// We behave as if we received one from ourselves.
	if !addAccepted(config.Id(), &accepted) {
		// Required too large an instruction slots slice.
		// We have NOT accepted this accept message,
		// and must NOT send accepted messages.
		return
	}

	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		if len(connections[node]) > 0 {
			connections[node][0].SendProto(5, &accepted)
		}
	}
}
Example #3
0
// Must be called from inside a transaction.
func makeRequest(changes []store.Change) *store.ChangeRequest {
	req := new(store.ChangeRequest)
	req.RequestEntity = uint64(config.Id())
	req.RequestNode = config.Id()
	req.RequestId = store.AllocateRequestId()
	req.Changeset = changes

	return req
}
Example #4
0
// Create change request creating new session entity,
// attached to user entity, this node ID attached to it.
// And the session then set as transient.
func makeNewSessionRequest(userId uint64) *store.ChangeRequest {
	chset := make([]store.Change, 5)

	// Create new entity.
	// Entity ID 1 now refers to this within the changeset.
	chset[0].TargetEntity = 1
	chset[0].Key = "id"
	chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10)

	// Make new entity a session entity.
	chset[1].TargetEntity = 1
	chset[1].Key = "kind"
	chset[1].Value = "session"

	// Attach session entity to user.
	chset[2].TargetEntity = userId
	chset[2].Key = "attach 1"
	chset[2].Value = "true"

	// Attach node ID to session entity.
	idStr := strconv.FormatUint(uint64(config.Id()), 10)
	chset[3].TargetEntity = 1
	chset[3].Key = "attach " + idStr
	chset[3].Value = "true"

	// Set session entity as transient.
	chset[4].TargetEntity = 1
	chset[4].Key = "transient"
	chset[4].Value = "true"

	return makeRequest(chset)
}
Example #5
0
// Checks for "attach <id>" entries attaching this node ID to sessions we lack
// a connection to. Starts a timer to delete them if present.
// Must be called during a transaction, holding the session lock.
func checkOrphanAttaches() {
	attachedTo := store.AllAttachedTo(uint64(config.Id()))

	for _, session := range attachedTo {
		if sessions[session] == nil {
			startOrphanTimeout(session)
		}
	}
}
Example #6
0
func SetProposal(newProposal uint64, newLeader uint16) {
	if intLeader == config.Id() && newLeader != intLeader {
		lastLeader = time.Now()
	}

	if newProposal != intProposal || newLeader != intLeader {
		intProposal = newProposal
		intLeader = newLeader

		log.Print("shared/store: new proposal and leader ",
			intProposal, " ", intLeader)

		for _, cb := range proposalCallbacks {
			cb()
		}
	}
}
Example #7
0
func handleMsg(node uint16, conn *connect.BaseConn, msg *baseproto.Message) {
	if *msg.MsgType == 2 {
		// Change Forward message.
		forward := new(chproto.ChangeForward)
		err := proto.Unmarshal(msg.Content, forward)
		if err != nil {
			conn.Close()
		}

		// If this is not a core node,
		// and the requesting node is not this node,
		// discard the message.
		if uint16(*forward.Request.RequestNode) != config.Id() &&
			!config.IsCore() {
			return
		}

		// Send a Change Forward Ack message to the sender.
		ack := new(chproto.ChangeForwardAck)
		ack.RequestNode = forward.Request.RequestNode
		ack.RequestId = forward.Request.RequestId
		conn.SendProto(3, ack)

		// Send the forward message to our processing goroutine.
		receivedForwardCh <- receivedForward{forward: forward,
			node: node}

	} else if *msg.MsgType == 3 {
		// Change Forward Ack message.
		ack := new(chproto.ChangeForwardAck)
		err := proto.Unmarshal(msg.Content, ack)
		if err != nil {
			conn.Close()
		}

		// Send the ack message to our processing goroutine.
		receivedAckCh <- receivedAck{ack: ack, node: node}

	} else {
		// Unknown message.
		conn.Close()
	}
}
Example #8
0
func orphanTimeout(id uint64) {
	store.StartTransaction()
	defer store.EndTransaction()

	// If the timeout has been removed, do nothing.
	if namelessRemoveTimeouts[id] == nil {
		return
	}

	// Detach ourselves from the session.
	// TODO: Should make sure we only do this once.
	ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
	session := store.GetEntity(id)
	if session != nil {
		chset := make([]store.Change, 1)
		chset[0].TargetEntity = id
		chset[0].Key = ourAttachStr
		chset[0].Value = ""

		req := makeRequest(chset)
		go chrequest.Request(req)
	}
}
Example #9
0
// Create change request creating new session entity,
// attached to user entity, this node ID attached to it.
// And the session then set as transient.
func makeNewUserRequest(username, pass, salt string,
	transient bool) *store.ChangeRequest {
	count := 10
	if transient {
		// Add an extra change for setting the user as transient.
		count++
	}
	chset := make([]store.Change, count)

	// Create new entity.
	// Entity ID 1 now refers to this within the changeset.
	chset[0].TargetEntity = 1
	chset[0].Key = "id"
	chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10)

	// Make new entity a user entity.
	chset[1].TargetEntity = 1
	chset[1].Key = "kind"
	chset[1].Value = "user"

	// Set username.
	chset[2].TargetEntity = 1
	chset[2].Key = "name username"
	chset[2].Value = username

	// Set password.
	chset[3].TargetEntity = 1
	chset[3].Key = "auth password"
	chset[3].Value = pass

	// Set salt.
	chset[4].TargetEntity = 1
	chset[4].Key = "auth salt"
	chset[4].Value = salt

	// Create new entity.
	// Entity ID 2 now refers to this within the changeset.
	chset[5].TargetEntity = 2
	chset[5].Key = "id"
	chset[5].Value = strconv.FormatUint(chset[0].TargetEntity, 10)

	// Make new entity a session entity.
	chset[6].TargetEntity = 2
	chset[6].Key = "kind"
	chset[6].Value = "session"

	// Attach session entity to user.
	chset[7].TargetEntity = 1
	chset[7].Key = "attach 2"
	chset[7].Value = "true"

	// Attach node ID to session entity.
	idStr := strconv.FormatUint(uint64(config.Id()), 10)
	chset[8].TargetEntity = 2
	chset[8].Key = "attach " + idStr
	chset[8].Value = "true"

	// Set session entity as transient.
	chset[9].TargetEntity = 2
	chset[9].Key = "transient"
	chset[9].Value = "true"

	// Set user entity as transient, if the user is to be so.
	if transient {
		chset[10].TargetEntity = 1
		chset[10].Key = "transient"
		chset[10].Value = "true"
	}

	return makeRequest(chset)
}
Example #10
0
// Main function for the handling goroutine for conn.
func handleConn(conn *userConn) {

	// Do cleanup in a defer, so if they crash us we still tidy up here.
	// A small nod towards tolerating bad input, with much more needed.
	defer func() {
		// Remove the connection from our connection set if present.
		// It will not be present if we are degraded or similar.
		connectionsLock.Lock()

		if _, exists := connections[conn]; exists {
			delete(connections, conn)
		}

		// Remove the connection from our waiting map if present.
		// This must happen before session removal, as otherwise
		// auth could complete between the session check and this one.
		waitingLock.Lock()
		if conn.waitingAuth != nil {
			waitingConn := waiting[conn.waitingAuth.requestId]
			if waitingConn == conn {
				delete(waiting, conn.waitingAuth.requestId)
			}
		}
		waitingLock.Unlock()

		// Remove the connection from our sessions map if present.
		// It will not be present if we are degraded,
		// or have replaced this connection.
		sessionsLock.Lock()
		if conn.session != 0 {
			sessionConn := sessions[conn.session]
			if sessionConn == conn {
				delete(sessions, conn.session)

				// Send a change detaching this node
				// from that session.
				idStr := strconv.FormatUint(uint64(config.Id()),
					10)
				ourAttachStr := "attach " + idStr

				chset := make([]store.Change, 1)
				chset[0].TargetEntity = conn.session
				chset[0].Key = ourAttachStr
				chset[0].Value = ""

				req := makeRequest(chset)
				go chrequest.Request(req)
			}

		}
		sessionsLock.Unlock()

		connectionsLock.Unlock()
	}()

	for {
		select {
		case msg, ok := <-conn.conn.Received:
			if !ok {
				return
			}

			switch *msg.MsgType {
			case 2:
				handleAuth(conn, msg.Content)
			case 3:
				handleFollowUsername(conn, msg.Content)
			case 4:
				handleFollowUser(conn, msg.Content)
			case 5:
				handleStopFollowingUser(conn, msg.Content)
			case 6:
				handleSend(conn, msg.Content)
			default:
				conn.conn.Close()
			}

		case userMsg := <-conn.deliver:

			// Get the sender's user ID.
			store.StartTransaction()
			attachedTo := store.AllAttachedTo(userMsg.Sender)
			store.EndTransaction()

			// We don't know who the sender is. Drop message.
			if len(attachedTo) == 0 {
				break
			}

			// Deliver any user messages given to us.
			var deliverMsg cliproto_down.Received
			deliverMsg.SenderUserId = new(uint64)
			deliverMsg.SenderSessionId = new(uint64)
			deliverMsg.Tag = new(string)
			deliverMsg.Content = new(string)
			*deliverMsg.SenderUserId = attachedTo[0]
			*deliverMsg.SenderSessionId = userMsg.Sender
			*deliverMsg.Tag = userMsg.Tag
			*deliverMsg.Content = userMsg.Content

			conn.conn.SendProto(10, &deliverMsg)
		}
	}

}
Example #11
0
func process() {

	for {
		select {

		// Try to make any needed outgoing connections.
		case <-tryOutgoingCh:
			store.StartTransaction()
			connectionsLock.Lock()

			coreNodes := config.CoreNodes()
			if !store.Degraded() {
				if config.IsCore() {
					for _, node := range coreNodes {
						if node == config.Id() {
							continue
						}

						if connections[node] == nil {
							go tryOutgoing(node)
						}
					}
				} else {

					// Settle for less than 2 core nodes,
					// if there *aren't* 2 core nodes.
					targetConns := 2
					if targetConns > len(coreNodes) {
						targetConns = len(coreNodes)
					}

					newOutgoing := targetConns -
						len(connections)

					used := -1
					for newOutgoing > 0 {
						r := processRand.Intn(
							len(coreNodes))

						// Don't do the same node
						// twice.
						if r == used {
							continue
						}

						node := coreNodes[r]
						if connections[node] == nil {
							used = r
							go tryOutgoing(node)
							newOutgoing--
						}
					}
				}
			} else {
				if len(connections) == 0 {
					r := processRand.Intn(len(coreNodes))
					node := coreNodes[r]
					if connections[node] == nil {
						go tryOutgoing(node)
					}
				}
			}

			connectionsLock.Unlock()
			store.EndTransaction()

			// After a random time, check again.
			randDur := time.Duration(processRand.Intn(19)+1) * time.Second
			tryOutgoingCh = time.NewTimer(randDur).C

		// New received connection.
		case conn := <-receivedConnCh:

			conn.offerTimers = make(map[uint64]*time.Timer)

			store.StartTransaction()
			connectionsLock.Lock()

			// If we are degraded, reject the connection,
			// unless we have no other connection,
			// and it is outbound.
			if store.Degraded() &&
				!(len(connections) == 0 && conn.outgoing) {

				conn.lock.Lock()
				conn.Close()
				conn.lock.Unlock()
				connectionsLock.Unlock()
				store.EndTransaction()
				break
			}

			// If we have an existing connection to this node,
			// close it.
			if connections[conn.node] != nil {
				other := connections[conn.node]

				other.lock.Lock()

				other.Close()
				if other.firstUnappliedTimer != nil {
					other.firstUnappliedTimer.Stop()
				}

				other.lock.Unlock()
			}

			// Add to our connections.
			connections[conn.node] = conn

			// Send initial position and leader messages.
			posMsg := new(fproto.Position)
			posMsg.FirstUnapplied = new(uint64)
			posMsg.Degraded = new(bool)
			*posMsg.FirstUnapplied =
				store.InstructionFirstUnapplied()
			*posMsg.Degraded = store.Degraded()
			conn.conn.SendProto(2, posMsg)

			proposal, leader := store.Proposal()
			leaderMsg := new(fproto.Leader)
			leaderMsg.Proposal = new(uint64)
			leaderMsg.Leader = new(uint32)
			*leaderMsg.Proposal = proposal
			*leaderMsg.Leader = uint32(leader)
			conn.conn.SendProto(11, leaderMsg)

			connectionsLock.Unlock()
			store.EndTransaction()

			// Start timer for sending first unapplied
			// instruction updates.
			if config.IsCore() && conn.node <= 0x2000 {
				conn.lock.Lock()

				conn.firstUnappliedTimer = time.AfterFunc(
					firstUnappliedTimerDuration,
					func() { conn.firstUnappliedTimeout() })

				conn.lock.Unlock()
			}

			// Start handling received messages from the connection.
			go handleConn(conn)

		// Terminated connection.
		case conn := <-terminatedConnCh:

			connectionsLock.Lock()

			if connections[conn.node] == conn {
				delete(connections, conn.node)

				conn.lock.Lock()

				conn.closed = true
				if conn.firstUnappliedTimer != nil {
					conn.firstUnappliedTimer.Stop()
				}

				conn.lock.Unlock()
			}

			connectionsLock.Unlock()
		}
	}
}
Example #12
0
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
	receivedPromises[node] = msg

	// If we have promises from a majority of core nodes,
	// become leader.
	if len(receivedPromises) > len(config.CoreNodes())/2 {

		log.Print("core/consensus: became leader")

		stopLeaderTimeout()
		amLeader = true
		proposal, leader := store.Proposal()

		// Find a slot number above all those in promise messages,
		// and above our first unapplied.
		firstUnapplied := store.InstructionFirstUnapplied()
		limit := firstUnapplied
		for _, msg := range receivedPromises {
			for _, accepted := range msg.Accepted {
				if *accepted.Slot >= limit {
					limit = *accepted.Slot + 1
				}
			}
		}

		// Start our next slot after the limit.
		nextProposalSlot = limit

		// For all slots between this and our first unapplied,
		// submit a previously accepted instruction unless we
		// know an instruction was already chosen.
		// Fills these slots with proposals.
		// This is O(n^2) in the number of instructions between our
		// first unapplied and limit.
		// TODO: Improve worst-case complexity.
		start := store.InstructionStart()
		slots := store.InstructionSlots()
		for i := firstUnapplied; i < limit; i++ {

			// If we already have a chosen instruction, skip.
			rel := int(i - start)
			if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
				continue
			}

			// Find the previously accepted instruction
			// accepted with the highest proposal number.
			var bestInst *coproto.Instruction
			var bp uint64 // Best proposal
			var bl uint16 // Best leader

			for _, msg := range receivedPromises {
				for _, accepted := range msg.Accepted {
					if *accepted.Slot != i {
						continue
					}
					if bestInst == nil {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
						continue
					}

					// TODO: This indent is just absurd.
					p := *accepted.Proposal
					l := uint16(*accepted.Leader)
					if store.CompareProposals(p, l,
						bp, bl) {
						bestInst = accepted
						bp = *accepted.Proposal
						bl = uint16(*accepted.Leader)
					}
				}
			}

			// If we didn't find an instruction, make an empty one.
			if bestInst == nil {
				empty := new(coproto.ChangeRequest)
				empty.RequestEntity = new(uint64)
				empty.RequestNode = new(uint32)
				*empty.RequestEntity = uint64(config.Id())
				*empty.RequestNode = uint32(config.Id())

				bestInst := new(coproto.Instruction)
				bestInst.Slot = new(uint64)
				*bestInst.Slot = i
				bestInst.Request = empty
			}

			// Add proposal timeout.
			req := makeExtChangeRequest(bestInst.Request)
			addProposalTimeout(i, req)

			// Send proposal.
			bestInst.Proposal = new(uint64)
			bestInst.Leader = new(uint32)
			*bestInst.Proposal = proposal
			*bestInst.Leader = uint32(leader)
			sendProposal(bestInst)
		}

		// Discard received promise messages.
		receivedPromises = nil

		// Make an instruction proposal for each waiting change.
		for _, req := range waitingRequests {
			slot := nextProposalSlot
			nextProposalSlot++

			addProposalTimeout(slot, req)

			inst := makeInst(slot, proposal, leader, req)

			sendProposal(inst)
		}

		// Clear waiting changes.
		waitingRequests = nil
	}
}
Example #13
0
// Top-level function of the processing goroutine.
func process() {

	connections = make(map[uint16][]*connect.BaseConn)

	// On startup, make an outgoing connection attempt to all other
	// core nodes, before continuing.
	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		conn, err := connect.Dial(
			connect.CONSENSUS_PROTOCOL, node)
		if err != nil {
			// Can't reach the other node.
			continue
		}

		log.Print("core/consensus: made outgoing connection to ", node)
		connections[node] = append(connections[node], conn)
		go handleConn(node, conn)
	}

	// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
	// Largely arbitrary.
	reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)

	for {
		select {

		// Connection retry tick.
		// We should try to make an outgoing connection to any node
		// that we do not have at least one connection to.
		// We need to make these asynchronously, because connections
		// are slow.
		case <-reconnectTicker:
			for _, node := range config.CoreNodes() {
				if node == config.Id() {
					continue
				}
				if len(connections[node]) > 0 {
					continue
				}

				go outgoingConn(node)
			}

		// New change request, for us to propose as leader.
		case req := <-newChangeCh:

			store.StartTransaction()

			if !amLeader && receivedPromises != nil {
				waitingRequests = append(waitingRequests, req)
			} else if !amLeader {
				waitingRequests = append(waitingRequests, req)

				// Start attempting to be leader.
				m := make(map[uint16]*coproto.Promise)
				receivedPromises = m

				proposal, _ := store.Proposal()
				proposal++
				store.SetProposal(proposal, config.Id())
				firstUn := store.InstructionFirstUnapplied()

				// Send prepare messages to all other nodes.
				var prepare coproto.Prepare
				prepare.Proposal = new(uint64)
				prepare.FirstUnapplied = new(uint64)
				*prepare.Proposal = proposal
				*prepare.FirstUnapplied = firstUn

				for _, node := range config.CoreNodes() {
					if node == config.Id() {
						continue
					}

					if len(connections[node]) != 0 {
						c := connections[node][0]
						c.SendProto(2, &prepare)
					}
				}

				// Behave as if we got a promise message
				// from ourselves.
				var promise coproto.Promise
				promise.Proposal = prepare.Proposal
				promise.PrevProposal = promise.Proposal
				promise.Leader = new(uint32)
				*promise.Leader = uint32(config.Id())
				promise.PrevLeader = promise.Leader
				addPromise(config.Id(), &promise)
			} else {
				newSlot := nextProposalSlot
				nextProposalSlot++

				addProposalTimeout(newSlot, req)

				proposal, leader := store.Proposal()
				inst := makeInst(newSlot, proposal, leader,
					req)

				sendProposal(inst)
			}

			store.EndTransaction()

		// Leadership attempt timed out.
		case timedOutProposal := <-leaderTimeoutCh:

			store.StartTransaction()

			proposal, leader := store.Proposal()

			// If the proposal has changed since that
			// leadership attempt, ignore it.
			if leader != config.Id() {
				return
			}
			if proposal != timedOutProposal {
				return
			}

			// If we successfully became leader, ignore it.
			if amLeader {
				return
			}

			// Otherwise, stop our attempt to become leader.
			stopBeingLeader()

			store.EndTransaction()

		// Proposal timed out.
		case timeout := <-proposalTimeoutCh:

			store.StartTransaction()

			// If this timeout was not canceled, a proposal failed.
			// We stop being leader.
			if proposalTimeouts[timeout.slot] == timeout {
				stopBeingLeader()
			}

			store.EndTransaction()

		// New received connection.
		case receivedConn := <-receivedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			connections[node] = append(connections[node], conn)

		// Received message.
		case recvMsg := <-receivedMsgCh:
			node := recvMsg.node
			conn := recvMsg.conn
			msg := recvMsg.msg

			switch *msg.MsgType {
			case 2:
				processPrepare(node, conn, msg.Content)
			case 3:
				processPromise(node, conn, msg.Content)
			case 4:
				processAccept(node, conn, msg.Content)
			case 5:
				processAccepted(node, conn, msg.Content)
			case 6:
				processNack(node, conn, msg.Content)
			default:
				// Unknown message.
				conn.Close()
			}

		// Terminate received connection.
		case terminatedConn := <-terminatedConnCh:
			node := terminatedConn.node
			conn := terminatedConn.conn

			for i, other := range connections[node] {
				if other != conn {
					continue
				}

				conns := connections[node]
				conns = append(conns[:i], conns[i+1:]...)
				connections[node] = conns
				break
			}
		}
	}
}
Example #14
0
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Prepare
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	newProposal, newLeader := *msg.Proposal, node
	proposal, leader := store.Proposal()
	if store.CompareProposals(newProposal, newLeader, proposal, leader) {

		log.Print("core/consensus: sending promise to ", newLeader)

		// Create a promise message to send back.
		var promise coproto.Promise
		promise.Proposal = new(uint64)
		promise.Leader = new(uint32)
		promise.PrevProposal = new(uint64)
		promise.PrevLeader = new(uint32)
		*promise.Proposal = newProposal
		*promise.Leader = uint32(newLeader)
		*promise.PrevProposal = proposal
		*promise.PrevLeader = uint32(leader)

		// Add all the instructions we've previously accepted or chosen.
		slots := store.InstructionSlots()
		theirFirstUnapplied := *msg.FirstUnapplied
		ourStart := store.InstructionStart()
		relativeSlot := int(theirFirstUnapplied - ourStart)
		if relativeSlot < 0 {
			relativeSlot = 0
		}
		var accepted []*coproto.Instruction
		for ; relativeSlot < len(slots); relativeSlot++ {
			slot := slots[relativeSlot]
			slotNum := ourStart + uint64(relativeSlot)
			for i, _ := range slot {
				if slot[i].IsChosen() {
					appendInst(&accepted, slotNum, slot[i])
					break
				}

				weAccepted := false
				for _, node := range slot[i].Accepted() {
					if node == config.Id() {
						weAccepted = true
						break
					}
				}

				if weAccepted {
					appendInst(&accepted, slotNum, slot[i])
					break
				}
			}
		}

		// Send promise message.
		conn.SendProto(3, &promise)

		// Accept the other node as our new leader.
		store.SetProposal(newProposal, newLeader)
	} else {
		var nack coproto.Nack
		nack.Proposal = new(uint64)
		nack.Leader = new(uint32)
		*nack.Proposal = proposal
		*nack.Leader = uint32(leader)
		conn.SendProto(6, &nack)
	}
}
Example #15
0
func handleProposalChange() {
	_, leader := store.Proposal()
	if amLeader && leader != config.Id() {
		stopBeingLeader()
	}
}
Example #16
0
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {

	// If we are already trying to forward a change forward message with
	// the same requesting node and request ID, discard this message.
	if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
		*forward.Request.RequestId); exists {
		return
	}

	// Everything else in this function runs in a transaction.
	// We are read-only.
	store.StartTransaction()
	defer store.EndTransaction()

	// If this is a core node and this node stopped being leader less than
	// a Change Timeout Period ago, always add us to the ignore list.
	if config.IsCore() && !isIgnored(forward, config.Id()) {
		diff := time.Now().Sub(store.StoppedLeading())
		if diff < config.CHANGE_TIMEOUT_PERIOD {
			forward.Ignores = append(forward.Ignores,
				uint32(config.Id()))
		}
	}

	// If all core node IDs are in the forward's ignore list, discard it.
	if len(forward.Ignores) == len(config.CoreNodes()) {
		log.Print("shared/chrequest: dropped msg due to full ignores")
		return
	}

	// Otherwise, choose a potential leader node.
	// This is O(n^2) in the number of core nodes,
	// but we don't expect to have many.
	chosenNode := uint16(0)
	_, leader := store.Proposal()
	if leader != 0 && !isIgnored(forward, leader) {
		chosenNode = leader
	} else {
		for _, node := range config.CoreNodes() {
			if !isIgnored(forward, node) {
				chosenNode = node
				break
			}
		}
	}
	if chosenNode == 0 {
		// Shouldn't happen.
		log.Print("shared/chrequest: bug, " +
			"couldn't find candidate leader node")
		return
	}

	// If we are the selected leader, construct an external change request,
	// and send it on our change request channel.
	if chosenNode == config.Id() {
		intRequest := forward.Request
		chrequest := new(store.ChangeRequest)
		chrequest.RequestEntity = *intRequest.RequestEntity
		chrequest.RequestNode = uint16(*intRequest.RequestNode)
		chrequest.RequestId = *intRequest.RequestId
		chrequest.Changeset = make([]store.Change,
			len(intRequest.Changeset))

		for i, ch := range intRequest.Changeset {
			chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
			chrequest.Changeset[i].Key = *ch.Key
			chrequest.Changeset[i].Value = *ch.Value
		}

		for _, cb := range changeCallbacks {
			cb(chrequest)
		}

		return
	}

	// Otherwise, we send it on to the selected leader,
	// add the selected leader to the ignore list,
	// and set a timeout to retry.
	sendForward(chosenNode, forward)
	forward.Ignores = append(forward.Ignores, uint32(chosenNode))
	addForwardTimeout(forward)
}
Example #17
0
func main() {
	haltChan = make(chan struct{})

	// Define and parse flags.
	id := flag.Uint("id", 0, "Set the node ID of this unanimity instance.")
	memprof = flag.Uint("memprof", 0, "Generate a memory profile and "+
		"halt after the given instruction slot is applied.")
	cpuprof = flag.Uint("cpuprof", 0, "Generate a CPU profile and halt "+
		"after the given instruction slot is applied.")
	flag.Parse()

	// Validate flags.
	if *id == 0 || *id > 0xFFFF {
		log.Fatal("Invalid node ID specified.")
	}

	// If both memprof and cpuprof are set, they need to be the same.
	if *memprof != 0 && *cpuprof != 0 && *memprof != *cpuprof {
		log.Fatal("If both memprof and cpuprof are set they must " +
			"match.")
	}

	// If either memprof or cpuprof are set, hook up the callback
	// responsible for terminating the node at the right time.
	if *memprof != 0 || *cpuprof != 0 {
		store.AddAppliedCallback(handleProfileTime)

		// If generating a CPU profile, start it now.
		if *cpuprof != 0 {
			f, err := os.Create("cpuprofile.prof")
			if err != nil {
				log.Fatal(err)
			}

			pprof.StartCPUProfile(f)
			defer pprof.StopCPUProfile()
		}
	}

	// Load configuration from config file.
	loadConfig()
	config.SetId(uint16(*id))

	// Load our TLS certificate.
	idStr := strconv.FormatUint(uint64(config.Id()), 10)
	cert, err := tls.LoadX509KeyPair(idStr+".crt", idStr+".key")
	if err != nil {
		log.Fatalf("error loading our TLS certificate: %s", err)
	}
	config.SetCertificate(&cert)

	fmt.Printf("Loaded configuration, our ID is %d.\n", config.Id())
	if config.IsCore() {
		core.Startup()
	} else {
		client.Startup()
	}

	<-haltChan

	// If we're to generate a memory profile before terminating, do so.
	if *memprof != 0 {
		f, err := os.Create("memprofile.mprof")
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()

		pprof.WriteHeapProfile(f)
	}
}
Example #18
0
// Blocks listening for connections of that given protocol type,
// and calls the specified handler when one is received,
// passing the node ID of the connecting node, or 0 for the Client Protocol.
func Listen(protocol int, handler func(id uint16, b *BaseConn)) {

	ip := config.NodeIP(config.Id())
	ipStr := ip.String()
	port := getProtocolPort(protocol)
	portStr := strconv.FormatInt(int64(port), 10)

	tlsConfig := new(tls.Config)
	tlsConfig.Certificates = []tls.Certificate{*config.Certificate()}
	if protocol != CLIENT_PROTOCOL {
		tlsConfig.ClientAuth = tls.RequireAnyClientCert
	}

	listener, err := tls.Listen("tcp", ipStr+":"+portStr, tlsConfig)
	if err != nil {
		log.Fatal(err)
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal(err)
		}
		tlsConn := conn.(*tls.Conn)

		err = tlsConn.Handshake()
		if err != nil {
			tlsConn.Close()
			continue
		}

		if protocol != CLIENT_PROTOCOL {
			// Check this connecting node has authenticated.
			state := tlsConn.ConnectionState()
			if len(state.PeerCertificates) == 0 {
				tlsConn.Close()
				continue
			}
			cert := state.PeerCertificates[0]

			// Identify which node they authenticated as.
			matched := false
			for _, node := range config.Nodes() {
				var verifyOpts x509.VerifyOptions
				verifyOpts.Intermediates = new(x509.CertPool)
				verifyOpts.Roots = config.NodeCertPool(node)
				chains, err := cert.Verify(verifyOpts)
				if err != nil {
					continue
				}

				// Matched the node. Start the handler.
				if len(chains) > 0 {
					matched = true
					go handler(node, newBaseConn(tlsConn))
					break
				}
			}

			// No matching node found. Close the connection.
			if !matched {
				tlsConn.Close()
			}
		} else {
			// We don't authenticate clients.
			// Just run the handler.
			handler(0, newBaseConn(tlsConn))
		}
	}
}
Example #19
0
// Sets this instruction value as chosen, clearing all others in the same slot.
// Causes it to be automatically applied to state once all previous instruction
// slots have a chosen value.
func (i *InstructionValue) Choose() {

	log.Print("shared/store: choosing instruction in slot ", i.slot)

	relativeSlot := i.slot - start

	// Drop all other instruction values in the same slot.
	if len(slots[relativeSlot]) > 1 {
		slots[relativeSlot] = []*InstructionValue{i}
	}

	// Set the value as chosen.
	i.chosen = true
	i.chosenTime = time.Now()
	i.acceptedBy = nil

	// Call instruction chosen callbacks.
	for _, cb := range chosenCallbacks {
		cb(i.slot)
	}

	// Apply instructions as far as possible.
	tryApply()

	// Discard no longer needed instructions, from the start.
	newStart := start
	for {
		relative := newStart - start

		// If we've not applied the instruction yet,
		// we still need it.
		if firstUnapplied == newStart {
			break
		}

		// If the instruction wasn't chosen long enough ago,
		// we can't discard it.
		minDelay := time.Duration(4) * config.CHANGE_TIMEOUT_PERIOD
		timeChosen := slots[relative][0].TimeChosen()
		if time.Now().Sub(timeChosen) < minDelay {
			break
		}

		// If this is a core node and the majority of core nodes aren't
		// past this point, we have to keep it.
		if config.IsCore() {
			past := 1 // We are past it.
			coreNodes := config.CoreNodes()
			for _, node := range coreNodes {
				if node == config.Id() {
					continue
				}

				if NodeFirstUnapplied(node) > newStart {
					past++
				}
			}
			if past <= len(coreNodes)/2 {
				break
			}
		}

		newStart++
	}

	// If we can discard, do it by reslicing the slots.
	// This doesn't result in an immediate free,
	// but will when we next expand the slots.
	// This avoids extra performance costs for reallocating them now.
	if newStart != start {
		log.Print("Discarding instructions between ", newStart-1,
			" and ", start)

		slots = slots[newStart-start:]
		start = newStart
	}

}
Example #20
0
func handleApplied(slot uint64, idemChanges []store.Change) {

	relativeSlot := int(slot - store.InstructionStart())
	slots := store.InstructionSlots()
	requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode
	requestId := slots[relativeSlot][0].ChangeRequest().RequestId

	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	waitingLock.Lock()
	defer waitingLock.Unlock()

	// If there is a connection which has this as its waiting request,
	// have it complete authentication.
	if requestNode == config.Id() {
		waitingConn := waiting[requestId]
		if waitingConn != nil {
			handleAuthComplete(waitingConn, idemChanges)
		}
	}

	ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
	for i := range idemChanges {

		key := idemChanges[i].Key
		value := idemChanges[i].Value

		if strings.HasPrefix(key, "attach ") {

			// If we were just detached from a session,
			// drop any connections associated with it,
			// and if it was in our attach timeouts, remove it.
			if key == ourAttachStr && value == "" {

				sessionId := idemChanges[i].TargetEntity
				if conn := sessions[sessionId]; conn != nil {
					log.Print("client/listener: " +
						"detached from existing " +
						"session")

					conn.conn.Close()
				}

				if orphanAttachTimeouts[sessionId] != nil {
					orphanAttachTimeouts[sessionId].Stop()
					delete(orphanAttachTimeouts, sessionId)
				}
			}

			// If we were just attached to a session,
			// but have no connection associated with that session,
			// add an attach timeout.
			// and if it was in our attach timeouts, remove it.
			if key == ourAttachStr && value != "" {

				sessionId := idemChanges[i].TargetEntity
				if sessions[sessionId] == nil {
					startOrphanTimeout(sessionId)
				}
			}

			// Handle a session attaching/detaching from a user.
			entityId := idemChanges[i].TargetEntity
			entity := store.GetEntity(entityId)
			if entity != nil && entity.Value("kind") == "user" {

				// Tell following connections when a session
				// attaches or detaches from a followed user.
				// TODO: Expensive loop,
				// should maintain index to avoid this.
				for conn, _ := range connections {
					following := false
					for _, fw := range conn.following {
						if fw == entityId {
							following = true
							break
						}
					}

					if !following {
						continue
					}

					sendUserChange(conn, entityId,
						key, value)
				}
			}
		}
	}

	// If we've created a nameless user, start a timer to delete it.
	for i := range idemChanges {
		if idemChanges[i].Key != "kind" ||
			idemChanges[i].Value != "user" {

			continue
		}

		newUserId := idemChanges[i].TargetEntity
		newUser := store.GetEntity(newUserId)
		if newUser != nil && newUser.Value("name username") == "" {
			startNamelessTimeout(newUserId)
		}
	}
}
Example #21
0
// Can only be called from the handling goroutine for conn.
func handleAuth(conn *userConn, content []byte) {
	var msg cliproto_up.Authenticate
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.conn.Close()
		return
	}

	// Try to get information, then release locks and hash the password.
	// If the situation changes we may have to hash it again anyway,
	// but scrypt hashing is extremely expensive and we want to try to
	// do this without holding our locks in the vast majority of cases.
	var tryPassGenerated bool
	var trySaltGenerated bool
	var tryPass, trySalt, tryKey string
	if *msg.Password != "" {
		tryPass = *msg.Password

		store.StartTransaction()

		userId := store.NameLookup("user", "name username",
			*msg.Username)
		if userId != 0 {
			user := store.GetEntity(userId)
			if user != nil {
				trySalt = user.Value("auth salt")
			}
		}

		store.EndTransaction()

		if trySalt == "" {
			trySaltGenerated = true

			var err error
			trySalt, tryKey, err = genRandomSalt(conn,
				[]byte(tryPass))

			if err != nil {
				return
			}
		} else {
			var err error
			tryPass, err = genKey(conn, []byte(tryPass),
				[]byte(trySalt))

			if err != nil {
				return
			}
		}

	} else {
		tryPassGenerated = true
		trySaltGenerated = true

		var err error
		tryPass, trySalt, tryKey, err = genRandomPass(conn)
		if err != nil {
			return
		}
	}

	// TODO: Validate username and password against constraints.

	// We hold this through quite a lot of logic.
	// Would be good to be locked less.
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()
	waitingLock.Lock()
	defer waitingLock.Unlock()

	if conn.session != 0 || conn.waitingAuth != nil {
		conn.conn.Close()
		return
	}

	userId := store.NameLookup("user", "name username", *msg.Username)
	if userId != 0 {
		if *msg.Password == "" {
			sendAuthFail(conn, "Invalid Password")
			return
		}

		// The user already exists.
		user := store.GetEntity(userId)

		// Try to authenticate them to it.
		var key string

		// If their salt and password matches our attempt above,
		// we can just take that key.
		salt := user.Value("auth salt")
		if trySalt == salt && tryPass == *msg.Password {
			key = tryKey
		} else {
			saltBytes := []byte(user.Value("auth salt"))
			passBytes := []byte(*msg.Password)
			var err error
			key, err = genKey(conn, passBytes, saltBytes)
			if err != nil {
				return
			}
		}

		if user.Value("auth password") != string(key) {
			sendAuthFail(conn, "Invalid Password")
			return
		}

		// It's the real user.
		if *msg.SessionId != 0 {

			// They are attaching to an existing session.
			// Check it exists and is attached to this user.
			strSessionId := strconv.FormatUint(*msg.SessionId, 10)
			if user.Value("attach "+strSessionId) == "" {
				sendAuthFail(conn, "Invalid Session")
				return
			}

			// The session does exist.
			conn.session = *msg.SessionId

			// If this node is already attached to
			// the session, drop the other connection.
			if sessions[conn.session] != nil {
				sessions[conn.session].conn.Close()
			} else {
				// Create change attaching this node ID
				// to the session.
				id := config.Id()
				idStr := strconv.FormatUint(uint64(id), 10)

				chset := make([]store.Change, 1)
				chset[0].TargetEntity = conn.session
				chset[0].Key = "attach " + idStr
				chset[0].Value = "true"
				req := makeRequest(chset)
				go chrequest.Request(req)
			}

			// Put us in the sessions map.
			sessions[conn.session] = conn

			// Tell the client they authenticated successfully.
			sendAuthSuccess(conn, "")
		} else {
			// They are creating a new session.
			req := makeNewSessionRequest(userId)
			go chrequest.Request(req)

			// Stuff details in waiting auth.
			conn.waitingAuth = new(authData)
			conn.waitingAuth.msg = msg
			conn.waitingAuth.requestId = req.RequestId
			waiting[conn.waitingAuth.requestId] = conn
		}
	} else {
		// The user does not already exist.
		// Check they weren't trying to attach to a session.
		if *msg.SessionId != 0 {
			sendAuthFail(conn, "User Does Not Exist")
			return
		}

		// We're creating a new user.
		newUser := *msg.Username
		newPass := *msg.Password

		if !strings.HasPrefix(newUser, "Guest-") {

			// We're creating a new non-guest user.
			// Make sure they have a password.
			if newPass == "" {
				sendAuthFail(conn, "No Password")
				return
			}

			var salt string
			var hash string
			if tryPass == newPass && trySaltGenerated {
				salt = trySalt
				hash = tryKey
			} else {
				passBytes := []byte(newPass)

				var err error
				salt, hash, err = genRandomSalt(conn, passBytes)
				if err != nil {
					return
				}
			}

			// Create the new user.
			req := makeNewUserRequest(newUser, hash, salt, false)
			go chrequest.Request(req)

			// Stuff details in waiting auth.
			conn.waitingAuth = new(authData)
			conn.waitingAuth.msg = msg
			conn.waitingAuth.requestId = req.RequestId
			waiting[conn.waitingAuth.requestId] = conn

			return
		}

		// We're creating a new guest user.
		// Guests get automatic passwords, and can't set them.
		if newPass != "" {
			sendAuthFail(conn, "Cannot Set Password For Guest User")
			return
		}

		var hash string
		var salt string
		if tryPassGenerated && trySaltGenerated {
			newPass = tryPass
			salt = trySalt
			hash = tryKey
		} else {
			var err error
			newPass, salt, hash, err = genRandomPass(conn)
			if err != nil {
				return
			}
		}

		waitingLock.Lock()

		// Create the new user.
		req := makeNewUserRequest(newUser, hash, salt, true)
		go chrequest.Request(req)

		// Stuff details in waiting auth.
		conn.waitingAuth = new(authData)
		conn.waitingAuth.msg = msg
		conn.waitingAuth.requestId = req.RequestId
		waiting[conn.waitingAuth.requestId] = conn

		waitingLock.Unlock()

		return
	}
}
Example #22
0
// If the leader ID matches our ID,
// updates our "last stopped leading" time to now.
// Call when, whether or not we've a new leader,
// we consider ourselves to have stopped being the leader.
func StopLeading() {
	if intLeader == config.Id() {
		lastLeader = time.Now()
	}
}
Example #23
0
func process() {

	// Try to make an outgoing connection to all other client nodes,
	// if we're not in a degraded state.
	store.StartTransaction()
	if !store.Degraded() {

		for _, node := range config.ClientNodes() {
			if node == config.Id() {
				continue
			}

			conn, err := connect.Dial(connect.RELAY_PROTOCOL, node)
			if err != nil {
				// No luck connecting.
				continue
			}

			connections[node] = append(connections[node], conn)
			go handleConn(node, conn)
		}
	}
	store.EndTransaction()

	// Retry connections once per config.CHANGE_TIMEOUT_PERIOD
	// Largely arbitrary.
	reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)

	for {
		select {

		// Connection retry tick.
		// If not degraded, try to make an outgoing connection to any
		// client node that we do not have at least one connection to.
		case <-reconnectTicker:

			store.StartTransaction()

			// Do not attempt to make connections while degraded.
			if store.Degraded() {
				store.EndTransaction()
				break
			}

			for _, node := range config.ClientNodes() {
				if node == config.Id() {
					continue
				}
				if len(connections[node]) > 0 {
					continue
				}

				conn, err := connect.Dial(
					connect.RELAY_PROTOCOL, node)

				if err != nil {
					// No luck connecting.
					continue
				}

				connections[node] =
					append(connections[node], conn)
				go handleConn(node, conn)
			}

			store.EndTransaction()

		// New received connection.
		case receivedConn := <-receivedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			store.StartTransaction()

			// If we are degraded, reject the connection.
			if store.Degraded() {
				conn.Close()
				store.EndTransaction()
				break
			}

			// Add to our connections.
			connections[node] = append(connections[node], conn)

			store.EndTransaction()

		// Terminated connection.
		case receivedConn := <-terminatedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			store.StartTransaction()

			// Remove this connection from our connection list.
			index := -1
			for i, _ := range connections[node] {
				if conn == connections[node][i] {
					index = i
					break
				}
			}
			if index != -1 {
				connections[node] =
					append(connections[node][:index],
						connections[node][index+1:]...)
			}

			store.EndTransaction()
		}
	}
}