Пример #1
0
// Must be called while holding no locks and not in a transaction.
func deliver(msg *relay.UserMessage) {

	// If we have a connection to the recipient session,
	// deliver the message.
	sessionsLock.Lock()

	userConn, exists := sessions[msg.Recipient]
	if exists {
		// Try to deliver the message to this connection,
		// without blocking.
		// If its delivery channel is full, just drop it.
		select {
		case userConn.deliver <- msg:
			break
		default:
			break
		}
	}

	sessionsLock.Unlock()

	if exists {
		return
	}

	// Otherwise, decrement TTL. If it's 0, discard the message.
	msg.Ttl--
	if msg.Ttl == 0 {
		return
	}

	// Otherwise, look for another node which is connected to this session.
	store.StartTransaction()

	session := store.GetEntity(msg.Recipient)
	if session == nil || session.Value("kind") != "session" {
		// Unknown session; discard the message.
		store.EndTransaction()
		return
	}

	nodes := session.AllAttached()
	store.EndTransaction()

	if len(nodes) == 0 {
		// Shouldn't happen, sessions are transient,
		// should be deleted before reaching here.
		// Just discard the message.
		return
	}

	// Forward the message to a random one of those nodes.
	r := rand.Intn(len(nodes))
	relay.Forward(uint16(nodes[r]), msg)
}
Пример #2
0
func Forward(node uint16, userMsg *UserMessage) {
	store.StartTransaction()
	defer store.EndTransaction()

	// While degraded, we drop all messages.
	if store.Degraded() {
		return
	}

	// If we have no connection to that node, we drop the message.
	if len(connections[node]) == 0 {
		return
	}

	// Otherwise, send to the given node.
	var forward rproto.Forward
	forward.Sender = new(uint64)
	forward.Recipient = new(uint64)
	forward.Tag = new(string)
	forward.Content = new(string)
	forward.Ttl = new(uint32)
	*forward.Sender = userMsg.Sender
	*forward.Recipient = userMsg.Recipient
	*forward.Tag = userMsg.Tag
	*forward.Content = userMsg.Content
	*forward.Ttl = uint32(userMsg.Ttl)

	connections[node][0].SendProto(2, &forward)
}
Пример #3
0
func handleInstructionRequest(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.InstructionRequest
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		f.Close()
		return
	}
	instructions := store.InstructionSlots()
	if relativeSlot >= len(instructions) {
		f.Close()
		return
	}
	slot := instructions[relativeSlot]
	if len(slot) != 1 || !slot[0].IsChosen() {
		f.Close()
		return
	}

	// Convert the change request to our internal format.
	sendInstructionData(f, *msg.Slot, slot[0].ChangeRequest())
}
Пример #4
0
// Must be called from the processing goroutine.
func processAccept(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Accept
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	proposal, leader := store.Proposal()
	msgProposal, msgLeader := *msg.Proposal, node
	if proposal != msgProposal || leader != msgLeader {
		// Send a nack message and return,
		// if this accept relates to an earlier proposal.
		if store.CompareProposals(proposal, leader, msgProposal,
			msgLeader) {

			var nack coproto.Nack
			nack.Proposal = new(uint64)
			nack.Leader = new(uint32)
			*nack.Proposal = proposal
			*nack.Leader = uint32(leader)
			conn.SendProto(6, &nack)

			return
		}

		store.SetProposal(msgProposal, msgLeader)
	}

	addAccept(node, &msg)
}
Пример #5
0
// Must be called from the processing goroutine.
func processNack(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Nack
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	// If we don't consider ourselves the leader, discard.
	if !amLeader {
		return
	}

	msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
	proposal, leader := store.Proposal()
	if msgProposal == proposal && msgLeader == leader {
		return
	}
	if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
		stopBeingLeader()
		store.SetProposal(msgProposal, msgLeader)
	}
}
Пример #6
0
// Can only be called from the handling goroutine for conn.
func handleStopFollowingUser(conn *userConn, content []byte) {
	var msg cliproto_up.StopFollowingUser
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.conn.Close()
		return
	}

	// Start transaction.
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	// Authentication check.
	if conn.session == 0 {
		conn.conn.Close()
		return
	}

	// If the ID exists in our following list, remove it.
	for i, existing := range conn.following {
		if existing == *msg.UserId {
			conn.following = append(conn.following[:i],
				conn.following[i+1:]...)
		}
	}

	// Send "stopped following" message.
	sendStoppedFollowing(conn, *msg.UserId, "By Request")
}
Пример #7
0
func handleFirstUnapplied(f *followConn, content []byte) {

	// Client nodes ignore this message, and shouldn't send it.
	if !config.IsCore() || f.node > 0x2000 {
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.FirstUnapplied
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	// Can't trigger callbacks from the store package to elsewhere,
	// safe to call while holding f's lock.
	store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
Пример #8
0
// Must be called from the processing goroutine.
func processPromise(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Promise
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	if receivedPromises == nil {
		// Not attempting to become leader.
		log.Print("core/consensus: discarded promise, not becoming "+
			"leader, from ", node)
		return
	}

	proposal, leader := store.Proposal()
	if proposal != *msg.Proposal || leader != uint16(*msg.Leader) {
		log.Print("core/consensus: rejected promise for wrong "+
			"proposal number from ", node)
		return
	}
	if receivedPromises[node] != nil {
		// Protocol violation; shouldn't get duplicate promises.
		log.Print("core/consensus: PROTOCOL VIOLATION: received "+
			"duplicate promise from node ", node)
		return
	}

	log.Print("core/consensus: received promise from node ", node)
	addPromise(node, &msg)
}
Пример #9
0
// Must be called from the processing goroutine.
func processAccepted(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Accepted
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	addAccepted(node, &msg)
}
Пример #10
0
func incomingConn(node uint16, conn *connect.BaseConn) {

	store.StartTransaction()
	if store.Degraded() {
		conn.Close()
		store.EndTransaction()
		return
	}

	userConn := new(userConn)
	userConn.conn = conn
	userConn.deliver = make(chan *relay.UserMessage, 100)

	// Add to connections set.
	connectionsLock.Lock()
	connections[userConn] = true
	connectionsLock.Unlock()

	store.EndTransaction()

	go handleConn(userConn)
}
Пример #11
0
func handleBurstDone(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()

	if f.closed {
		f.lock.Unlock()
		return
	}

	var msg fproto.BurstDone
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		f.lock.Unlock()
		return
	}

	if !f.receivingBurst {
		f.Close()
		f.lock.Unlock()
		return
	}
	f.receivingBurst = false

	wasWaiting := f.waiting
	f.waiting = nil

	// We need to unlock before we start mutating the store,
	// due to callbacks from the store package to elsewhere.
	f.lock.Unlock()

	chrequests := make([]store.ChangeRequest, len(wasWaiting))
	for i, data := range wasWaiting {
		req := data.Request
		chrequests[i].RequestEntity = *req.RequestEntity
		chrequests[i].RequestNode = uint16(*req.RequestNode)
		chrequests[i].RequestId = *req.RequestId
		chrequests[i].Changeset =
			make([]store.Change, len(req.Changeset))

		chset := chrequests[i].Changeset
		for j, ch := range req.Changeset {
			chset[j].TargetEntity = *ch.TargetEntity
			chset[j].Key = *ch.Key
			chset[j].Value = *ch.Value
		}
	}
	store.EndBurst(*msg.FirstUnapplied, chrequests)
}
Пример #12
0
func handleBursting(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()

	if f.closed {
		f.lock.Unlock()
		return
	}

	f.receivingBurst = true

	f.lock.Unlock()

	store.Degrade()
}
Пример #13
0
func Startup() {
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	// If undegraded, check for attached users with no session,
	// or nameless users in the store.
	if !store.Degraded() {
		checkOrphanAttaches()
		checkNameless()
	}

	// Start accepting client protocol connections.
	go connect.Listen(connect.CLIENT_PROTOCOL, incomingConn)
}
Пример #14
0
func handleLeader(f *followConn, content []byte) {
	var msg fproto.Leader
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.lock.Lock()
		f.Close()
		f.lock.Unlock()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	proposal, leader := store.Proposal()
	msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
	if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
		store.SetProposal(msgProposal, msgLeader)
	}
}
Пример #15
0
func (f *followConn) firstUnappliedTimeout() {

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	msg := new(fproto.FirstUnapplied)
	msg.FirstUnapplied = new(uint64)
	*msg.FirstUnapplied = store.InstructionFirstUnapplied()
	f.conn.SendProto(10, msg)

	f.firstUnappliedTimer = time.AfterFunc(firstUnappliedTimerDuration,
		func() { f.firstUnappliedTimeout() })

}
Пример #16
0
func handleInstructionChosen(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	var msg fproto.InstructionChosen
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if f.closed {
		return
	}

	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		return
	}
	instructions := store.InstructionSlots()
	if relativeSlot < len(instructions) {
		slot := instructions[relativeSlot]
		if len(slot) == 1 && slot[0].IsChosen() {
			return
		}
	}

	if !config.IsCore() {
		// TODO: Should only do this on one follow connection.
		var intReq fproto.InstructionRequest
		intReq.Slot = msg.Slot
		f.conn.SendProto(8, &intReq)
	}

	timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
	f.offerTimers[*msg.Slot] = time.AfterFunc(
		timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
Пример #17
0
func namelessTimeout(id uint64) {
	store.StartTransaction()
	defer store.EndTransaction()

	// If the timeout has been removed, do nothing.
	if namelessRemoveTimeouts[id] == nil {
		return
	}

	// Remove nameless user.
	// TODO: Should make sure we only do this once.
	user := store.GetEntity(id)
	if user != nil {
		chset := make([]store.Change, 1)
		chset[0].TargetEntity = id
		chset[0].Key = "id"
		chset[0].Value = ""

		req := makeRequest(chset)
		go chrequest.Request(req)
	}
}
Пример #18
0
func handleEntityProperty(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.EntityProperty
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if !f.receivingBurst {
		f.Close()
		return
	}

	store.BurstEntity(*msg.Entity, *msg.Key, *msg.Value)
}
Пример #19
0
func orphanTimeout(id uint64) {
	store.StartTransaction()
	defer store.EndTransaction()

	// If the timeout has been removed, do nothing.
	if namelessRemoveTimeouts[id] == nil {
		return
	}

	// Detach ourselves from the session.
	// TODO: Should make sure we only do this once.
	ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
	session := store.GetEntity(id)
	if session != nil {
		chset := make([]store.Change, 1)
		chset[0].TargetEntity = id
		chset[0].Key = ourAttachStr
		chset[0].Value = ""

		req := makeRequest(chset)
		go chrequest.Request(req)
	}
}
Пример #20
0
// Can only be called from the handling goroutine for conn.
func handleFollowUser(conn *userConn, content []byte) {
	var msg cliproto_up.FollowUser
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.conn.Close()
		return
	}

	// Start transaction.
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	// Authentication check.
	if conn.session == 0 {
		conn.conn.Close()
		return
	}

	// Check we're not already following this user.
	// If we are, discard the message.
	for _, existing := range conn.following {
		if existing == *msg.UserId {
			return
		}
	}

	// Check this ID is actually a user entity.
	otherUser := store.GetEntity(*msg.UserId)
	if otherUser == nil || otherUser.Value("kind") != "user" {
		sendFollowUserIdFail(conn, *msg.UserId, "No Such User")
		return
	}

	// Start following this user.
	followUser(conn, *msg.UserId)
}
Пример #21
0
// Can only be called from the handling goroutine for conn.
func handleFollowUsername(conn *userConn, content []byte) {
	var msg cliproto_up.FollowUsername
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.conn.Close()
		return
	}
	// Start transaction.
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()

	// Authentication check.
	if conn.session == 0 {
		conn.conn.Close()
		return
	}

	// Lookup this user.
	followId := store.NameLookup("user", "name username", *msg.Username)
	if followId == 0 {
		sendFollowUsernameFail(conn, *msg.Username, "No Such User")
		return
	}

	// Check we're not already following this user.
	// If we are, discard the message.
	for _, existing := range conn.following {
		if existing == followId {
			return
		}
	}

	// Start following this user.
	followUser(conn, followId)
}
Пример #22
0
// Can only be called from the handling goroutine for conn.
func handleAuth(conn *userConn, content []byte) {
	var msg cliproto_up.Authenticate
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.conn.Close()
		return
	}

	// Try to get information, then release locks and hash the password.
	// If the situation changes we may have to hash it again anyway,
	// but scrypt hashing is extremely expensive and we want to try to
	// do this without holding our locks in the vast majority of cases.
	var tryPassGenerated bool
	var trySaltGenerated bool
	var tryPass, trySalt, tryKey string
	if *msg.Password != "" {
		tryPass = *msg.Password

		store.StartTransaction()

		userId := store.NameLookup("user", "name username",
			*msg.Username)
		if userId != 0 {
			user := store.GetEntity(userId)
			if user != nil {
				trySalt = user.Value("auth salt")
			}
		}

		store.EndTransaction()

		if trySalt == "" {
			trySaltGenerated = true

			var err error
			trySalt, tryKey, err = genRandomSalt(conn,
				[]byte(tryPass))

			if err != nil {
				return
			}
		} else {
			var err error
			tryPass, err = genKey(conn, []byte(tryPass),
				[]byte(trySalt))

			if err != nil {
				return
			}
		}

	} else {
		tryPassGenerated = true
		trySaltGenerated = true

		var err error
		tryPass, trySalt, tryKey, err = genRandomPass(conn)
		if err != nil {
			return
		}
	}

	// TODO: Validate username and password against constraints.

	// We hold this through quite a lot of logic.
	// Would be good to be locked less.
	store.StartTransaction()
	defer store.EndTransaction()
	sessionsLock.Lock()
	defer sessionsLock.Unlock()
	waitingLock.Lock()
	defer waitingLock.Unlock()

	if conn.session != 0 || conn.waitingAuth != nil {
		conn.conn.Close()
		return
	}

	userId := store.NameLookup("user", "name username", *msg.Username)
	if userId != 0 {
		if *msg.Password == "" {
			sendAuthFail(conn, "Invalid Password")
			return
		}

		// The user already exists.
		user := store.GetEntity(userId)

		// Try to authenticate them to it.
		var key string

		// If their salt and password matches our attempt above,
		// we can just take that key.
		salt := user.Value("auth salt")
		if trySalt == salt && tryPass == *msg.Password {
			key = tryKey
		} else {
			saltBytes := []byte(user.Value("auth salt"))
			passBytes := []byte(*msg.Password)
			var err error
			key, err = genKey(conn, passBytes, saltBytes)
			if err != nil {
				return
			}
		}

		if user.Value("auth password") != string(key) {
			sendAuthFail(conn, "Invalid Password")
			return
		}

		// It's the real user.
		if *msg.SessionId != 0 {

			// They are attaching to an existing session.
			// Check it exists and is attached to this user.
			strSessionId := strconv.FormatUint(*msg.SessionId, 10)
			if user.Value("attach "+strSessionId) == "" {
				sendAuthFail(conn, "Invalid Session")
				return
			}

			// The session does exist.
			conn.session = *msg.SessionId

			// If this node is already attached to
			// the session, drop the other connection.
			if sessions[conn.session] != nil {
				sessions[conn.session].conn.Close()
			} else {
				// Create change attaching this node ID
				// to the session.
				id := config.Id()
				idStr := strconv.FormatUint(uint64(id), 10)

				chset := make([]store.Change, 1)
				chset[0].TargetEntity = conn.session
				chset[0].Key = "attach " + idStr
				chset[0].Value = "true"
				req := makeRequest(chset)
				go chrequest.Request(req)
			}

			// Put us in the sessions map.
			sessions[conn.session] = conn

			// Tell the client they authenticated successfully.
			sendAuthSuccess(conn, "")
		} else {
			// They are creating a new session.
			req := makeNewSessionRequest(userId)
			go chrequest.Request(req)

			// Stuff details in waiting auth.
			conn.waitingAuth = new(authData)
			conn.waitingAuth.msg = msg
			conn.waitingAuth.requestId = req.RequestId
			waiting[conn.waitingAuth.requestId] = conn
		}
	} else {
		// The user does not already exist.
		// Check they weren't trying to attach to a session.
		if *msg.SessionId != 0 {
			sendAuthFail(conn, "User Does Not Exist")
			return
		}

		// We're creating a new user.
		newUser := *msg.Username
		newPass := *msg.Password

		if !strings.HasPrefix(newUser, "Guest-") {

			// We're creating a new non-guest user.
			// Make sure they have a password.
			if newPass == "" {
				sendAuthFail(conn, "No Password")
				return
			}

			var salt string
			var hash string
			if tryPass == newPass && trySaltGenerated {
				salt = trySalt
				hash = tryKey
			} else {
				passBytes := []byte(newPass)

				var err error
				salt, hash, err = genRandomSalt(conn, passBytes)
				if err != nil {
					return
				}
			}

			// Create the new user.
			req := makeNewUserRequest(newUser, hash, salt, false)
			go chrequest.Request(req)

			// Stuff details in waiting auth.
			conn.waitingAuth = new(authData)
			conn.waitingAuth.msg = msg
			conn.waitingAuth.requestId = req.RequestId
			waiting[conn.waitingAuth.requestId] = conn

			return
		}

		// We're creating a new guest user.
		// Guests get automatic passwords, and can't set them.
		if newPass != "" {
			sendAuthFail(conn, "Cannot Set Password For Guest User")
			return
		}

		var hash string
		var salt string
		if tryPassGenerated && trySaltGenerated {
			newPass = tryPass
			salt = trySalt
			hash = tryKey
		} else {
			var err error
			newPass, salt, hash, err = genRandomPass(conn)
			if err != nil {
				return
			}
		}

		waitingLock.Lock()

		// Create the new user.
		req := makeNewUserRequest(newUser, hash, salt, true)
		go chrequest.Request(req)

		// Stuff details in waiting auth.
		conn.waitingAuth = new(authData)
		conn.waitingAuth.msg = msg
		conn.waitingAuth.requestId = req.RequestId
		waiting[conn.waitingAuth.requestId] = conn

		waitingLock.Unlock()

		return
	}
}
Пример #23
0
func handlePosition(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.closed {
		return
	}

	var msg fproto.Position
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		return
	}

	if config.IsCore() && f.node <= 0x2000 {
		store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
	}

	if store.Degraded() && *msg.Degraded {
		f.Close()
		return
	}

	start := store.InstructionStart()
	if *msg.FirstUnapplied < start {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		burstMsg.Content = []byte{}
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
		*msg.Degraded {

		f.sendingBurst = true
		burstMsg := new(baseproto.Message)
		burstMsg.MsgType = new(uint32)
		*burstMsg.MsgType = 3
		f.conn.Send(burstMsg)

		go sendBurst(f)

		return

	} else if *msg.Degraded {
		f.Close()
		return
	}

	// Send all chosen instructions above the first unapplied point.
	instructions := store.InstructionSlots()
	relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		sendInstructionData(f, slot, slotValues[0].ChangeRequest())
	}
}
Пример #24
0
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {

	// If we are already trying to forward a change forward message with
	// the same requesting node and request ID, discard this message.
	if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
		*forward.Request.RequestId); exists {
		return
	}

	// Everything else in this function runs in a transaction.
	// We are read-only.
	store.StartTransaction()
	defer store.EndTransaction()

	// If this is a core node and this node stopped being leader less than
	// a Change Timeout Period ago, always add us to the ignore list.
	if config.IsCore() && !isIgnored(forward, config.Id()) {
		diff := time.Now().Sub(store.StoppedLeading())
		if diff < config.CHANGE_TIMEOUT_PERIOD {
			forward.Ignores = append(forward.Ignores,
				uint32(config.Id()))
		}
	}

	// If all core node IDs are in the forward's ignore list, discard it.
	if len(forward.Ignores) == len(config.CoreNodes()) {
		log.Print("shared/chrequest: dropped msg due to full ignores")
		return
	}

	// Otherwise, choose a potential leader node.
	// This is O(n^2) in the number of core nodes,
	// but we don't expect to have many.
	chosenNode := uint16(0)
	_, leader := store.Proposal()
	if leader != 0 && !isIgnored(forward, leader) {
		chosenNode = leader
	} else {
		for _, node := range config.CoreNodes() {
			if !isIgnored(forward, node) {
				chosenNode = node
				break
			}
		}
	}
	if chosenNode == 0 {
		// Shouldn't happen.
		log.Print("shared/chrequest: bug, " +
			"couldn't find candidate leader node")
		return
	}

	// If we are the selected leader, construct an external change request,
	// and send it on our change request channel.
	if chosenNode == config.Id() {
		intRequest := forward.Request
		chrequest := new(store.ChangeRequest)
		chrequest.RequestEntity = *intRequest.RequestEntity
		chrequest.RequestNode = uint16(*intRequest.RequestNode)
		chrequest.RequestId = *intRequest.RequestId
		chrequest.Changeset = make([]store.Change,
			len(intRequest.Changeset))

		for i, ch := range intRequest.Changeset {
			chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
			chrequest.Changeset[i].Key = *ch.Key
			chrequest.Changeset[i].Value = *ch.Value
		}

		for _, cb := range changeCallbacks {
			cb(chrequest)
		}

		return
	}

	// Otherwise, we send it on to the selected leader,
	// add the selected leader to the ignore list,
	// and set a timeout to retry.
	sendForward(chosenNode, forward)
	forward.Ignores = append(forward.Ignores, uint32(chosenNode))
	addForwardTimeout(forward)
}
Пример #25
0
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Prepare
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	newProposal, newLeader := *msg.Proposal, node
	proposal, leader := store.Proposal()
	if store.CompareProposals(newProposal, newLeader, proposal, leader) {

		log.Print("core/consensus: sending promise to ", newLeader)

		// Create a promise message to send back.
		var promise coproto.Promise
		promise.Proposal = new(uint64)
		promise.Leader = new(uint32)
		promise.PrevProposal = new(uint64)
		promise.PrevLeader = new(uint32)
		*promise.Proposal = newProposal
		*promise.Leader = uint32(newLeader)
		*promise.PrevProposal = proposal
		*promise.PrevLeader = uint32(leader)

		// Add all the instructions we've previously accepted or chosen.
		slots := store.InstructionSlots()
		theirFirstUnapplied := *msg.FirstUnapplied
		ourStart := store.InstructionStart()
		relativeSlot := int(theirFirstUnapplied - ourStart)
		if relativeSlot < 0 {
			relativeSlot = 0
		}
		var accepted []*coproto.Instruction
		for ; relativeSlot < len(slots); relativeSlot++ {
			slot := slots[relativeSlot]
			slotNum := ourStart + uint64(relativeSlot)
			for i, _ := range slot {
				if slot[i].IsChosen() {
					appendInst(&accepted, slotNum, slot[i])
					break
				}

				weAccepted := false
				for _, node := range slot[i].Accepted() {
					if node == config.Id() {
						weAccepted = true
						break
					}
				}

				if weAccepted {
					appendInst(&accepted, slotNum, slot[i])
					break
				}
			}
		}

		// Send promise message.
		conn.SendProto(3, &promise)

		// Accept the other node as our new leader.
		store.SetProposal(newProposal, newLeader)
	} else {
		var nack coproto.Nack
		nack.Proposal = new(uint64)
		nack.Leader = new(uint32)
		*nack.Proposal = proposal
		*nack.Leader = uint32(leader)
		conn.SendProto(6, &nack)
	}
}
Пример #26
0
// Main function for the handling goroutine for conn.
func handleConn(conn *userConn) {

	// Do cleanup in a defer, so if they crash us we still tidy up here.
	// A small nod towards tolerating bad input, with much more needed.
	defer func() {
		// Remove the connection from our connection set if present.
		// It will not be present if we are degraded or similar.
		connectionsLock.Lock()

		if _, exists := connections[conn]; exists {
			delete(connections, conn)
		}

		// Remove the connection from our waiting map if present.
		// This must happen before session removal, as otherwise
		// auth could complete between the session check and this one.
		waitingLock.Lock()
		if conn.waitingAuth != nil {
			waitingConn := waiting[conn.waitingAuth.requestId]
			if waitingConn == conn {
				delete(waiting, conn.waitingAuth.requestId)
			}
		}
		waitingLock.Unlock()

		// Remove the connection from our sessions map if present.
		// It will not be present if we are degraded,
		// or have replaced this connection.
		sessionsLock.Lock()
		if conn.session != 0 {
			sessionConn := sessions[conn.session]
			if sessionConn == conn {
				delete(sessions, conn.session)

				// Send a change detaching this node
				// from that session.
				idStr := strconv.FormatUint(uint64(config.Id()),
					10)
				ourAttachStr := "attach " + idStr

				chset := make([]store.Change, 1)
				chset[0].TargetEntity = conn.session
				chset[0].Key = ourAttachStr
				chset[0].Value = ""

				req := makeRequest(chset)
				go chrequest.Request(req)
			}

		}
		sessionsLock.Unlock()

		connectionsLock.Unlock()
	}()

	for {
		select {
		case msg, ok := <-conn.conn.Received:
			if !ok {
				return
			}

			switch *msg.MsgType {
			case 2:
				handleAuth(conn, msg.Content)
			case 3:
				handleFollowUsername(conn, msg.Content)
			case 4:
				handleFollowUser(conn, msg.Content)
			case 5:
				handleStopFollowingUser(conn, msg.Content)
			case 6:
				handleSend(conn, msg.Content)
			default:
				conn.conn.Close()
			}

		case userMsg := <-conn.deliver:

			// Get the sender's user ID.
			store.StartTransaction()
			attachedTo := store.AllAttachedTo(userMsg.Sender)
			store.EndTransaction()

			// We don't know who the sender is. Drop message.
			if len(attachedTo) == 0 {
				break
			}

			// Deliver any user messages given to us.
			var deliverMsg cliproto_down.Received
			deliverMsg.SenderUserId = new(uint64)
			deliverMsg.SenderSessionId = new(uint64)
			deliverMsg.Tag = new(string)
			deliverMsg.Content = new(string)
			*deliverMsg.SenderUserId = attachedTo[0]
			*deliverMsg.SenderSessionId = userMsg.Sender
			*deliverMsg.Tag = userMsg.Tag
			*deliverMsg.Content = userMsg.Content

			conn.conn.SendProto(10, &deliverMsg)
		}
	}

}
Пример #27
0
// Sends a burst to the given connection.
// The Bursting message should already have been sent,
// and f.sendingBurst set to true, before calling this asynchronously.
func sendBurst(f *followConn) {

	log.Print("shared/follow: sending burst to ", f.node)

	globalProp := new(fproto.GlobalProperty)
	globalProp.Key = new(string)
	globalProp.Value = new(string)

	entityProp := new(fproto.EntityProperty)
	entityProp.Entity = new(uint64)
	entityProp.Key = new(string)
	entityProp.Value = new(string)

	aborted := false
	store.Burst(func(key, value string) bool {
		*globalProp.Key = key
		*globalProp.Value = value

		if !f.conn.SendProto(4, globalProp) {
			aborted = true
		}
		return !aborted
	}, func(entity uint64, key, value string) bool {
		*entityProp.Entity = entity
		*entityProp.Key = key
		*entityProp.Value = value

		if !f.conn.SendProto(5, entityProp) {
			aborted = true
		}
		return !aborted
	})

	if aborted {
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()
	defer f.lock.Unlock()

	f.sendingBurst = false

	burstDone := new(fproto.BurstDone)
	burstDone.FirstUnapplied = new(uint64)
	*burstDone.FirstUnapplied = store.InstructionFirstUnapplied()
	f.conn.SendProto(6, burstDone)

	// Send an instruction chosen message for all chosen instructions above
	// our first unapplied point.
	instructions := store.InstructionSlots()
	firstUnapplied := store.InstructionFirstUnapplied()
	relativeSlot := int(firstUnapplied - store.InstructionStart())
	for ; relativeSlot < len(instructions); relativeSlot++ {
		slot := store.InstructionStart() + uint64(relativeSlot)
		slotValues := instructions[relativeSlot]
		if len(slotValues) != 1 || !slotValues[0].IsChosen() {
			continue
		}

		// Convert the change request to our internal format.
		chosen := new(fproto.InstructionChosen)
		chosen.Slot = new(uint64)
		*chosen.Slot = slot
		f.conn.SendProto(7, chosen)
	}
}
Пример #28
0
func handleInstructionData(f *followConn, content []byte) {
	store.StartTransaction()
	defer store.EndTransaction()
	f.lock.Lock()

	if f.closed {
		f.lock.Unlock()
		return
	}

	var msg fproto.InstructionData
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.Close()
		f.lock.Unlock()
		return
	}

	if f.receivingBurst {
		f.waiting = append(f.waiting, &msg)
		f.lock.Unlock()
		return
	}

	// We need to unlock before we start mutating the store,
	// due to callbacks from the store package to elsewhere.
	f.lock.Unlock()

	// If we have a chosen instruction in this slot already,
	// or it is prior to our instruction start number,
	// discard this message.
	relativeSlot := int(*msg.Slot - store.InstructionStart())
	if relativeSlot < 0 {
		return
	}

	instructions := store.InstructionSlots()
	if relativeSlot < len(instructions) &&
		len(instructions[relativeSlot]) == 1 &&
		instructions[relativeSlot][0].IsChosen() {

		return
	}

	// Construct a store.ChangeRequest from our
	// internal ChangeRequest.
	chrequest := new(store.ChangeRequest)
	chrequest.RequestEntity = *msg.Request.RequestEntity
	chrequest.RequestNode = uint16(*msg.Request.RequestNode)
	chrequest.RequestId = *msg.Request.RequestId
	chrequest.Changeset = make([]store.Change,
		len(msg.Request.Changeset))

	for i, ch := range msg.Request.Changeset {
		chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
		chrequest.Changeset[i].Key = *ch.Key
		chrequest.Changeset[i].Value = *ch.Value
	}

	// Add instruction value and immediately choose it.
	store.AddInstructionValue(*msg.Slot, chrequest).Choose()
}
Пример #29
0
func process() {

	for {
		select {

		// Try to make any needed outgoing connections.
		case <-tryOutgoingCh:
			store.StartTransaction()
			connectionsLock.Lock()

			coreNodes := config.CoreNodes()
			if !store.Degraded() {
				if config.IsCore() {
					for _, node := range coreNodes {
						if node == config.Id() {
							continue
						}

						if connections[node] == nil {
							go tryOutgoing(node)
						}
					}
				} else {

					// Settle for less than 2 core nodes,
					// if there *aren't* 2 core nodes.
					targetConns := 2
					if targetConns > len(coreNodes) {
						targetConns = len(coreNodes)
					}

					newOutgoing := targetConns -
						len(connections)

					used := -1
					for newOutgoing > 0 {
						r := processRand.Intn(
							len(coreNodes))

						// Don't do the same node
						// twice.
						if r == used {
							continue
						}

						node := coreNodes[r]
						if connections[node] == nil {
							used = r
							go tryOutgoing(node)
							newOutgoing--
						}
					}
				}
			} else {
				if len(connections) == 0 {
					r := processRand.Intn(len(coreNodes))
					node := coreNodes[r]
					if connections[node] == nil {
						go tryOutgoing(node)
					}
				}
			}

			connectionsLock.Unlock()
			store.EndTransaction()

			// After a random time, check again.
			randDur := time.Duration(processRand.Intn(19)+1) * time.Second
			tryOutgoingCh = time.NewTimer(randDur).C

		// New received connection.
		case conn := <-receivedConnCh:

			conn.offerTimers = make(map[uint64]*time.Timer)

			store.StartTransaction()
			connectionsLock.Lock()

			// If we are degraded, reject the connection,
			// unless we have no other connection,
			// and it is outbound.
			if store.Degraded() &&
				!(len(connections) == 0 && conn.outgoing) {

				conn.lock.Lock()
				conn.Close()
				conn.lock.Unlock()
				connectionsLock.Unlock()
				store.EndTransaction()
				break
			}

			// If we have an existing connection to this node,
			// close it.
			if connections[conn.node] != nil {
				other := connections[conn.node]

				other.lock.Lock()

				other.Close()
				if other.firstUnappliedTimer != nil {
					other.firstUnappliedTimer.Stop()
				}

				other.lock.Unlock()
			}

			// Add to our connections.
			connections[conn.node] = conn

			// Send initial position and leader messages.
			posMsg := new(fproto.Position)
			posMsg.FirstUnapplied = new(uint64)
			posMsg.Degraded = new(bool)
			*posMsg.FirstUnapplied =
				store.InstructionFirstUnapplied()
			*posMsg.Degraded = store.Degraded()
			conn.conn.SendProto(2, posMsg)

			proposal, leader := store.Proposal()
			leaderMsg := new(fproto.Leader)
			leaderMsg.Proposal = new(uint64)
			leaderMsg.Leader = new(uint32)
			*leaderMsg.Proposal = proposal
			*leaderMsg.Leader = uint32(leader)
			conn.conn.SendProto(11, leaderMsg)

			connectionsLock.Unlock()
			store.EndTransaction()

			// Start timer for sending first unapplied
			// instruction updates.
			if config.IsCore() && conn.node <= 0x2000 {
				conn.lock.Lock()

				conn.firstUnappliedTimer = time.AfterFunc(
					firstUnappliedTimerDuration,
					func() { conn.firstUnappliedTimeout() })

				conn.lock.Unlock()
			}

			// Start handling received messages from the connection.
			go handleConn(conn)

		// Terminated connection.
		case conn := <-terminatedConnCh:

			connectionsLock.Lock()

			if connections[conn.node] == conn {
				delete(connections, conn.node)

				conn.lock.Lock()

				conn.closed = true
				if conn.firstUnappliedTimer != nil {
					conn.firstUnappliedTimer.Stop()
				}

				conn.lock.Unlock()
			}

			connectionsLock.Unlock()
		}
	}
}
Пример #30
0
// Top-level function of the processing goroutine.
func process() {

	connections = make(map[uint16][]*connect.BaseConn)

	// On startup, make an outgoing connection attempt to all other
	// core nodes, before continuing.
	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		conn, err := connect.Dial(
			connect.CONSENSUS_PROTOCOL, node)
		if err != nil {
			// Can't reach the other node.
			continue
		}

		log.Print("core/consensus: made outgoing connection to ", node)
		connections[node] = append(connections[node], conn)
		go handleConn(node, conn)
	}

	// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
	// Largely arbitrary.
	reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)

	for {
		select {

		// Connection retry tick.
		// We should try to make an outgoing connection to any node
		// that we do not have at least one connection to.
		// We need to make these asynchronously, because connections
		// are slow.
		case <-reconnectTicker:
			for _, node := range config.CoreNodes() {
				if node == config.Id() {
					continue
				}
				if len(connections[node]) > 0 {
					continue
				}

				go outgoingConn(node)
			}

		// New change request, for us to propose as leader.
		case req := <-newChangeCh:

			store.StartTransaction()

			if !amLeader && receivedPromises != nil {
				waitingRequests = append(waitingRequests, req)
			} else if !amLeader {
				waitingRequests = append(waitingRequests, req)

				// Start attempting to be leader.
				m := make(map[uint16]*coproto.Promise)
				receivedPromises = m

				proposal, _ := store.Proposal()
				proposal++
				store.SetProposal(proposal, config.Id())
				firstUn := store.InstructionFirstUnapplied()

				// Send prepare messages to all other nodes.
				var prepare coproto.Prepare
				prepare.Proposal = new(uint64)
				prepare.FirstUnapplied = new(uint64)
				*prepare.Proposal = proposal
				*prepare.FirstUnapplied = firstUn

				for _, node := range config.CoreNodes() {
					if node == config.Id() {
						continue
					}

					if len(connections[node]) != 0 {
						c := connections[node][0]
						c.SendProto(2, &prepare)
					}
				}

				// Behave as if we got a promise message
				// from ourselves.
				var promise coproto.Promise
				promise.Proposal = prepare.Proposal
				promise.PrevProposal = promise.Proposal
				promise.Leader = new(uint32)
				*promise.Leader = uint32(config.Id())
				promise.PrevLeader = promise.Leader
				addPromise(config.Id(), &promise)
			} else {
				newSlot := nextProposalSlot
				nextProposalSlot++

				addProposalTimeout(newSlot, req)

				proposal, leader := store.Proposal()
				inst := makeInst(newSlot, proposal, leader,
					req)

				sendProposal(inst)
			}

			store.EndTransaction()

		// Leadership attempt timed out.
		case timedOutProposal := <-leaderTimeoutCh:

			store.StartTransaction()

			proposal, leader := store.Proposal()

			// If the proposal has changed since that
			// leadership attempt, ignore it.
			if leader != config.Id() {
				return
			}
			if proposal != timedOutProposal {
				return
			}

			// If we successfully became leader, ignore it.
			if amLeader {
				return
			}

			// Otherwise, stop our attempt to become leader.
			stopBeingLeader()

			store.EndTransaction()

		// Proposal timed out.
		case timeout := <-proposalTimeoutCh:

			store.StartTransaction()

			// If this timeout was not canceled, a proposal failed.
			// We stop being leader.
			if proposalTimeouts[timeout.slot] == timeout {
				stopBeingLeader()
			}

			store.EndTransaction()

		// New received connection.
		case receivedConn := <-receivedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			connections[node] = append(connections[node], conn)

		// Received message.
		case recvMsg := <-receivedMsgCh:
			node := recvMsg.node
			conn := recvMsg.conn
			msg := recvMsg.msg

			switch *msg.MsgType {
			case 2:
				processPrepare(node, conn, msg.Content)
			case 3:
				processPromise(node, conn, msg.Content)
			case 4:
				processAccept(node, conn, msg.Content)
			case 5:
				processAccepted(node, conn, msg.Content)
			case 6:
				processNack(node, conn, msg.Content)
			default:
				// Unknown message.
				conn.Close()
			}

		// Terminate received connection.
		case terminatedConn := <-terminatedConnCh:
			node := terminatedConn.node
			conn := terminatedConn.conn

			for i, other := range connections[node] {
				if other != conn {
					continue
				}

				conns := connections[node]
				conns = append(conns[:i], conns[i+1:]...)
				connections[node] = conns
				break
			}
		}
	}
}