// Must be called while holding no locks and not in a transaction.
func deliver(msg *relay.UserMessage) {
// If we have a connection to the recipient session,
// deliver the message.
sessionsLock.Lock()
userConn, exists := sessions[msg.Recipient]
if exists {
// Try to deliver the message to this connection,
// without blocking.
// If its delivery channel is full, just drop it.
select {
case userConn.deliver <- msg:
break
default:
break
}
}
sessionsLock.Unlock()
if exists {
return
}
// Otherwise, decrement TTL. If it's 0, discard the message.
msg.Ttl--
if msg.Ttl == 0 {
return
}
// Otherwise, look for another node which is connected to this session.
store.StartTransaction()
session := store.GetEntity(msg.Recipient)
if session == nil || session.Value("kind") != "session" {
// Unknown session; discard the message.
store.EndTransaction()
return
}
nodes := session.AllAttached()
store.EndTransaction()
if len(nodes) == 0 {
// Shouldn't happen, sessions are transient,
// should be deleted before reaching here.
// Just discard the message.
return
}
// Forward the message to a random one of those nodes.
r := rand.Intn(len(nodes))
relay.Forward(uint16(nodes[r]), msg)
}
func Forward(node uint16, userMsg *UserMessage) {
store.StartTransaction()
defer store.EndTransaction()
// While degraded, we drop all messages.
if store.Degraded() {
return
}
// If we have no connection to that node, we drop the message.
if len(connections[node]) == 0 {
return
}
// Otherwise, send to the given node.
var forward rproto.Forward
forward.Sender = new(uint64)
forward.Recipient = new(uint64)
forward.Tag = new(string)
forward.Content = new(string)
forward.Ttl = new(uint32)
*forward.Sender = userMsg.Sender
*forward.Recipient = userMsg.Recipient
*forward.Tag = userMsg.Tag
*forward.Content = userMsg.Content
*forward.Ttl = uint32(userMsg.Ttl)
connections[node][0].SendProto(2, &forward)
}
func handleInstructionRequest(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.InstructionRequest
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
f.Close()
return
}
instructions := store.InstructionSlots()
if relativeSlot >= len(instructions) {
f.Close()
return
}
slot := instructions[relativeSlot]
if len(slot) != 1 || !slot[0].IsChosen() {
f.Close()
return
}
// Convert the change request to our internal format.
sendInstructionData(f, *msg.Slot, slot[0].ChangeRequest())
}
// Must be called from the processing goroutine.
func processAccept(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Accept
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
proposal, leader := store.Proposal()
msgProposal, msgLeader := *msg.Proposal, node
if proposal != msgProposal || leader != msgLeader {
// Send a nack message and return,
// if this accept relates to an earlier proposal.
if store.CompareProposals(proposal, leader, msgProposal,
msgLeader) {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
return
}
store.SetProposal(msgProposal, msgLeader)
}
addAccept(node, &msg)
}
// Must be called from the processing goroutine.
func processNack(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Nack
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
// If we don't consider ourselves the leader, discard.
if !amLeader {
return
}
msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
proposal, leader := store.Proposal()
if msgProposal == proposal && msgLeader == leader {
return
}
if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
stopBeingLeader()
store.SetProposal(msgProposal, msgLeader)
}
}
// Can only be called from the handling goroutine for conn.
func handleStopFollowingUser(conn *userConn, content []byte) {
var msg cliproto_up.StopFollowingUser
if err := proto.Unmarshal(content, &msg); err != nil {
conn.conn.Close()
return
}
// Start transaction.
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
// Authentication check.
if conn.session == 0 {
conn.conn.Close()
return
}
// If the ID exists in our following list, remove it.
for i, existing := range conn.following {
if existing == *msg.UserId {
conn.following = append(conn.following[:i],
conn.following[i+1:]...)
}
}
// Send "stopped following" message.
sendStoppedFollowing(conn, *msg.UserId, "By Request")
}
func handleFirstUnapplied(f *followConn, content []byte) {
// Client nodes ignore this message, and shouldn't send it.
if !config.IsCore() || f.node > 0x2000 {
return
}
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.FirstUnapplied
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
// Can't trigger callbacks from the store package to elsewhere,
// safe to call while holding f's lock.
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
// Must be called from the processing goroutine.
func processPromise(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Promise
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
if receivedPromises == nil {
// Not attempting to become leader.
log.Print("core/consensus: discarded promise, not becoming "+
"leader, from ", node)
return
}
proposal, leader := store.Proposal()
if proposal != *msg.Proposal || leader != uint16(*msg.Leader) {
log.Print("core/consensus: rejected promise for wrong "+
"proposal number from ", node)
return
}
if receivedPromises[node] != nil {
// Protocol violation; shouldn't get duplicate promises.
log.Print("core/consensus: PROTOCOL VIOLATION: received "+
"duplicate promise from node ", node)
return
}
log.Print("core/consensus: received promise from node ", node)
addPromise(node, &msg)
}
// Must be called from the processing goroutine.
func processAccepted(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Accepted
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
addAccepted(node, &msg)
}
func incomingConn(node uint16, conn *connect.BaseConn) {
store.StartTransaction()
if store.Degraded() {
conn.Close()
store.EndTransaction()
return
}
userConn := new(userConn)
userConn.conn = conn
userConn.deliver = make(chan *relay.UserMessage, 100)
// Add to connections set.
connectionsLock.Lock()
connections[userConn] = true
connectionsLock.Unlock()
store.EndTransaction()
go handleConn(userConn)
}
func handleBurstDone(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
if f.closed {
f.lock.Unlock()
return
}
var msg fproto.BurstDone
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
f.lock.Unlock()
return
}
if !f.receivingBurst {
f.Close()
f.lock.Unlock()
return
}
f.receivingBurst = false
wasWaiting := f.waiting
f.waiting = nil
// We need to unlock before we start mutating the store,
// due to callbacks from the store package to elsewhere.
f.lock.Unlock()
chrequests := make([]store.ChangeRequest, len(wasWaiting))
for i, data := range wasWaiting {
req := data.Request
chrequests[i].RequestEntity = *req.RequestEntity
chrequests[i].RequestNode = uint16(*req.RequestNode)
chrequests[i].RequestId = *req.RequestId
chrequests[i].Changeset =
make([]store.Change, len(req.Changeset))
chset := chrequests[i].Changeset
for j, ch := range req.Changeset {
chset[j].TargetEntity = *ch.TargetEntity
chset[j].Key = *ch.Key
chset[j].Value = *ch.Value
}
}
store.EndBurst(*msg.FirstUnapplied, chrequests)
}
func handleBursting(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
if f.closed {
f.lock.Unlock()
return
}
f.receivingBurst = true
f.lock.Unlock()
store.Degrade()
}
func Startup() {
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
// If undegraded, check for attached users with no session,
// or nameless users in the store.
if !store.Degraded() {
checkOrphanAttaches()
checkNameless()
}
// Start accepting client protocol connections.
go connect.Listen(connect.CLIENT_PROTOCOL, incomingConn)
}
func handleLeader(f *followConn, content []byte) {
var msg fproto.Leader
if err := proto.Unmarshal(content, &msg); err != nil {
f.lock.Lock()
f.Close()
f.lock.Unlock()
return
}
store.StartTransaction()
defer store.EndTransaction()
proposal, leader := store.Proposal()
msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
store.SetProposal(msgProposal, msgLeader)
}
}
func (f *followConn) firstUnappliedTimeout() {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
msg := new(fproto.FirstUnapplied)
msg.FirstUnapplied = new(uint64)
*msg.FirstUnapplied = store.InstructionFirstUnapplied()
f.conn.SendProto(10, msg)
f.firstUnappliedTimer = time.AfterFunc(firstUnappliedTimerDuration,
func() { f.firstUnappliedTimeout() })
}
func handleInstructionChosen(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
var msg fproto.InstructionChosen
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if f.closed {
return
}
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) {
slot := instructions[relativeSlot]
if len(slot) == 1 && slot[0].IsChosen() {
return
}
}
if !config.IsCore() {
// TODO: Should only do this on one follow connection.
var intReq fproto.InstructionRequest
intReq.Slot = msg.Slot
f.conn.SendProto(8, &intReq)
}
timeout := config.ROUND_TRIP_TIMEOUT_PERIOD
f.offerTimers[*msg.Slot] = time.AfterFunc(
timeout, func() { f.offerTimeout(*msg.Slot, timeout) })
}
func namelessTimeout(id uint64) {
store.StartTransaction()
defer store.EndTransaction()
// If the timeout has been removed, do nothing.
if namelessRemoveTimeouts[id] == nil {
return
}
// Remove nameless user.
// TODO: Should make sure we only do this once.
user := store.GetEntity(id)
if user != nil {
chset := make([]store.Change, 1)
chset[0].TargetEntity = id
chset[0].Key = "id"
chset[0].Value = ""
req := makeRequest(chset)
go chrequest.Request(req)
}
}
func handleEntityProperty(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.EntityProperty
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if !f.receivingBurst {
f.Close()
return
}
store.BurstEntity(*msg.Entity, *msg.Key, *msg.Value)
}
func orphanTimeout(id uint64) {
store.StartTransaction()
defer store.EndTransaction()
// If the timeout has been removed, do nothing.
if namelessRemoveTimeouts[id] == nil {
return
}
// Detach ourselves from the session.
// TODO: Should make sure we only do this once.
ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
session := store.GetEntity(id)
if session != nil {
chset := make([]store.Change, 1)
chset[0].TargetEntity = id
chset[0].Key = ourAttachStr
chset[0].Value = ""
req := makeRequest(chset)
go chrequest.Request(req)
}
}
// Can only be called from the handling goroutine for conn.
func handleFollowUser(conn *userConn, content []byte) {
var msg cliproto_up.FollowUser
if err := proto.Unmarshal(content, &msg); err != nil {
conn.conn.Close()
return
}
// Start transaction.
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
// Authentication check.
if conn.session == 0 {
conn.conn.Close()
return
}
// Check we're not already following this user.
// If we are, discard the message.
for _, existing := range conn.following {
if existing == *msg.UserId {
return
}
}
// Check this ID is actually a user entity.
otherUser := store.GetEntity(*msg.UserId)
if otherUser == nil || otherUser.Value("kind") != "user" {
sendFollowUserIdFail(conn, *msg.UserId, "No Such User")
return
}
// Start following this user.
followUser(conn, *msg.UserId)
}
// Can only be called from the handling goroutine for conn.
func handleFollowUsername(conn *userConn, content []byte) {
var msg cliproto_up.FollowUsername
if err := proto.Unmarshal(content, &msg); err != nil {
conn.conn.Close()
return
}
// Start transaction.
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
// Authentication check.
if conn.session == 0 {
conn.conn.Close()
return
}
// Lookup this user.
followId := store.NameLookup("user", "name username", *msg.Username)
if followId == 0 {
sendFollowUsernameFail(conn, *msg.Username, "No Such User")
return
}
// Check we're not already following this user.
// If we are, discard the message.
for _, existing := range conn.following {
if existing == followId {
return
}
}
// Start following this user.
followUser(conn, followId)
}
// Can only be called from the handling goroutine for conn.
func handleAuth(conn *userConn, content []byte) {
var msg cliproto_up.Authenticate
if err := proto.Unmarshal(content, &msg); err != nil {
conn.conn.Close()
return
}
// Try to get information, then release locks and hash the password.
// If the situation changes we may have to hash it again anyway,
// but scrypt hashing is extremely expensive and we want to try to
// do this without holding our locks in the vast majority of cases.
var tryPassGenerated bool
var trySaltGenerated bool
var tryPass, trySalt, tryKey string
if *msg.Password != "" {
tryPass = *msg.Password
store.StartTransaction()
userId := store.NameLookup("user", "name username",
*msg.Username)
if userId != 0 {
user := store.GetEntity(userId)
if user != nil {
trySalt = user.Value("auth salt")
}
}
store.EndTransaction()
if trySalt == "" {
trySaltGenerated = true
var err error
trySalt, tryKey, err = genRandomSalt(conn,
[]byte(tryPass))
if err != nil {
return
}
} else {
var err error
tryPass, err = genKey(conn, []byte(tryPass),
[]byte(trySalt))
if err != nil {
return
}
}
} else {
tryPassGenerated = true
trySaltGenerated = true
var err error
tryPass, trySalt, tryKey, err = genRandomPass(conn)
if err != nil {
return
}
}
// TODO: Validate username and password against constraints.
// We hold this through quite a lot of logic.
// Would be good to be locked less.
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
waitingLock.Lock()
defer waitingLock.Unlock()
if conn.session != 0 || conn.waitingAuth != nil {
conn.conn.Close()
return
}
userId := store.NameLookup("user", "name username", *msg.Username)
if userId != 0 {
if *msg.Password == "" {
sendAuthFail(conn, "Invalid Password")
return
}
// The user already exists.
user := store.GetEntity(userId)
// Try to authenticate them to it.
var key string
// If their salt and password matches our attempt above,
// we can just take that key.
salt := user.Value("auth salt")
if trySalt == salt && tryPass == *msg.Password {
key = tryKey
} else {
saltBytes := []byte(user.Value("auth salt"))
passBytes := []byte(*msg.Password)
var err error
key, err = genKey(conn, passBytes, saltBytes)
if err != nil {
return
}
}
if user.Value("auth password") != string(key) {
sendAuthFail(conn, "Invalid Password")
return
}
// It's the real user.
if *msg.SessionId != 0 {
// They are attaching to an existing session.
// Check it exists and is attached to this user.
strSessionId := strconv.FormatUint(*msg.SessionId, 10)
if user.Value("attach "+strSessionId) == "" {
sendAuthFail(conn, "Invalid Session")
return
}
// The session does exist.
conn.session = *msg.SessionId
// If this node is already attached to
// the session, drop the other connection.
if sessions[conn.session] != nil {
sessions[conn.session].conn.Close()
} else {
// Create change attaching this node ID
// to the session.
id := config.Id()
idStr := strconv.FormatUint(uint64(id), 10)
chset := make([]store.Change, 1)
chset[0].TargetEntity = conn.session
chset[0].Key = "attach " + idStr
chset[0].Value = "true"
req := makeRequest(chset)
go chrequest.Request(req)
}
// Put us in the sessions map.
sessions[conn.session] = conn
// Tell the client they authenticated successfully.
sendAuthSuccess(conn, "")
} else {
// They are creating a new session.
req := makeNewSessionRequest(userId)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
}
} else {
// The user does not already exist.
// Check they weren't trying to attach to a session.
if *msg.SessionId != 0 {
sendAuthFail(conn, "User Does Not Exist")
return
}
// We're creating a new user.
newUser := *msg.Username
newPass := *msg.Password
if !strings.HasPrefix(newUser, "Guest-") {
// We're creating a new non-guest user.
// Make sure they have a password.
if newPass == "" {
sendAuthFail(conn, "No Password")
return
}
var salt string
var hash string
if tryPass == newPass && trySaltGenerated {
salt = trySalt
hash = tryKey
} else {
passBytes := []byte(newPass)
var err error
salt, hash, err = genRandomSalt(conn, passBytes)
if err != nil {
return
}
}
// Create the new user.
req := makeNewUserRequest(newUser, hash, salt, false)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
return
}
// We're creating a new guest user.
// Guests get automatic passwords, and can't set them.
if newPass != "" {
sendAuthFail(conn, "Cannot Set Password For Guest User")
return
}
var hash string
var salt string
if tryPassGenerated && trySaltGenerated {
newPass = tryPass
salt = trySalt
hash = tryKey
} else {
var err error
newPass, salt, hash, err = genRandomPass(conn)
if err != nil {
return
}
}
waitingLock.Lock()
// Create the new user.
req := makeNewUserRequest(newUser, hash, salt, true)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
waitingLock.Unlock()
return
}
}
func handlePosition(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
if f.closed {
return
}
var msg fproto.Position
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
return
}
if config.IsCore() && f.node <= 0x2000 {
store.SetNodeFirstUnapplied(f.node, *msg.FirstUnapplied)
}
if store.Degraded() && *msg.Degraded {
f.Close()
return
}
start := store.InstructionStart()
if *msg.FirstUnapplied < start {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
burstMsg.Content = []byte{}
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.FirstUnapplied <= store.InstructionFirstUnapplied() &&
*msg.Degraded {
f.sendingBurst = true
burstMsg := new(baseproto.Message)
burstMsg.MsgType = new(uint32)
*burstMsg.MsgType = 3
f.conn.Send(burstMsg)
go sendBurst(f)
return
} else if *msg.Degraded {
f.Close()
return
}
// Send all chosen instructions above the first unapplied point.
instructions := store.InstructionSlots()
relativeSlot := int(*msg.FirstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
sendInstructionData(f, slot, slotValues[0].ChangeRequest())
}
}
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {
// If we are already trying to forward a change forward message with
// the same requesting node and request ID, discard this message.
if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
*forward.Request.RequestId); exists {
return
}
// Everything else in this function runs in a transaction.
// We are read-only.
store.StartTransaction()
defer store.EndTransaction()
// If this is a core node and this node stopped being leader less than
// a Change Timeout Period ago, always add us to the ignore list.
if config.IsCore() && !isIgnored(forward, config.Id()) {
diff := time.Now().Sub(store.StoppedLeading())
if diff < config.CHANGE_TIMEOUT_PERIOD {
forward.Ignores = append(forward.Ignores,
uint32(config.Id()))
}
}
// If all core node IDs are in the forward's ignore list, discard it.
if len(forward.Ignores) == len(config.CoreNodes()) {
log.Print("shared/chrequest: dropped msg due to full ignores")
return
}
// Otherwise, choose a potential leader node.
// This is O(n^2) in the number of core nodes,
// but we don't expect to have many.
chosenNode := uint16(0)
_, leader := store.Proposal()
if leader != 0 && !isIgnored(forward, leader) {
chosenNode = leader
} else {
for _, node := range config.CoreNodes() {
if !isIgnored(forward, node) {
chosenNode = node
break
}
}
}
if chosenNode == 0 {
// Shouldn't happen.
log.Print("shared/chrequest: bug, " +
"couldn't find candidate leader node")
return
}
// If we are the selected leader, construct an external change request,
// and send it on our change request channel.
if chosenNode == config.Id() {
intRequest := forward.Request
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *intRequest.RequestEntity
chrequest.RequestNode = uint16(*intRequest.RequestNode)
chrequest.RequestId = *intRequest.RequestId
chrequest.Changeset = make([]store.Change,
len(intRequest.Changeset))
for i, ch := range intRequest.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
for _, cb := range changeCallbacks {
cb(chrequest)
}
return
}
// Otherwise, we send it on to the selected leader,
// add the selected leader to the ignore list,
// and set a timeout to retry.
sendForward(chosenNode, forward)
forward.Ignores = append(forward.Ignores, uint32(chosenNode))
addForwardTimeout(forward)
}
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Prepare
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
newProposal, newLeader := *msg.Proposal, node
proposal, leader := store.Proposal()
if store.CompareProposals(newProposal, newLeader, proposal, leader) {
log.Print("core/consensus: sending promise to ", newLeader)
// Create a promise message to send back.
var promise coproto.Promise
promise.Proposal = new(uint64)
promise.Leader = new(uint32)
promise.PrevProposal = new(uint64)
promise.PrevLeader = new(uint32)
*promise.Proposal = newProposal
*promise.Leader = uint32(newLeader)
*promise.PrevProposal = proposal
*promise.PrevLeader = uint32(leader)
// Add all the instructions we've previously accepted or chosen.
slots := store.InstructionSlots()
theirFirstUnapplied := *msg.FirstUnapplied
ourStart := store.InstructionStart()
relativeSlot := int(theirFirstUnapplied - ourStart)
if relativeSlot < 0 {
relativeSlot = 0
}
var accepted []*coproto.Instruction
for ; relativeSlot < len(slots); relativeSlot++ {
slot := slots[relativeSlot]
slotNum := ourStart + uint64(relativeSlot)
for i, _ := range slot {
if slot[i].IsChosen() {
appendInst(&accepted, slotNum, slot[i])
break
}
weAccepted := false
for _, node := range slot[i].Accepted() {
if node == config.Id() {
weAccepted = true
break
}
}
if weAccepted {
appendInst(&accepted, slotNum, slot[i])
break
}
}
}
// Send promise message.
conn.SendProto(3, &promise)
// Accept the other node as our new leader.
store.SetProposal(newProposal, newLeader)
} else {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
}
}
// Main function for the handling goroutine for conn.
func handleConn(conn *userConn) {
// Do cleanup in a defer, so if they crash us we still tidy up here.
// A small nod towards tolerating bad input, with much more needed.
defer func() {
// Remove the connection from our connection set if present.
// It will not be present if we are degraded or similar.
connectionsLock.Lock()
if _, exists := connections[conn]; exists {
delete(connections, conn)
}
// Remove the connection from our waiting map if present.
// This must happen before session removal, as otherwise
// auth could complete between the session check and this one.
waitingLock.Lock()
if conn.waitingAuth != nil {
waitingConn := waiting[conn.waitingAuth.requestId]
if waitingConn == conn {
delete(waiting, conn.waitingAuth.requestId)
}
}
waitingLock.Unlock()
// Remove the connection from our sessions map if present.
// It will not be present if we are degraded,
// or have replaced this connection.
sessionsLock.Lock()
if conn.session != 0 {
sessionConn := sessions[conn.session]
if sessionConn == conn {
delete(sessions, conn.session)
// Send a change detaching this node
// from that session.
idStr := strconv.FormatUint(uint64(config.Id()),
10)
ourAttachStr := "attach " + idStr
chset := make([]store.Change, 1)
chset[0].TargetEntity = conn.session
chset[0].Key = ourAttachStr
chset[0].Value = ""
req := makeRequest(chset)
go chrequest.Request(req)
}
}
sessionsLock.Unlock()
connectionsLock.Unlock()
}()
for {
select {
case msg, ok := <-conn.conn.Received:
if !ok {
return
}
switch *msg.MsgType {
case 2:
handleAuth(conn, msg.Content)
case 3:
handleFollowUsername(conn, msg.Content)
case 4:
handleFollowUser(conn, msg.Content)
case 5:
handleStopFollowingUser(conn, msg.Content)
case 6:
handleSend(conn, msg.Content)
default:
conn.conn.Close()
}
case userMsg := <-conn.deliver:
// Get the sender's user ID.
store.StartTransaction()
attachedTo := store.AllAttachedTo(userMsg.Sender)
store.EndTransaction()
// We don't know who the sender is. Drop message.
if len(attachedTo) == 0 {
break
}
// Deliver any user messages given to us.
var deliverMsg cliproto_down.Received
deliverMsg.SenderUserId = new(uint64)
deliverMsg.SenderSessionId = new(uint64)
deliverMsg.Tag = new(string)
deliverMsg.Content = new(string)
*deliverMsg.SenderUserId = attachedTo[0]
*deliverMsg.SenderSessionId = userMsg.Sender
*deliverMsg.Tag = userMsg.Tag
*deliverMsg.Content = userMsg.Content
conn.conn.SendProto(10, &deliverMsg)
}
}
}
// Sends a burst to the given connection.
// The Bursting message should already have been sent,
// and f.sendingBurst set to true, before calling this asynchronously.
func sendBurst(f *followConn) {
log.Print("shared/follow: sending burst to ", f.node)
globalProp := new(fproto.GlobalProperty)
globalProp.Key = new(string)
globalProp.Value = new(string)
entityProp := new(fproto.EntityProperty)
entityProp.Entity = new(uint64)
entityProp.Key = new(string)
entityProp.Value = new(string)
aborted := false
store.Burst(func(key, value string) bool {
*globalProp.Key = key
*globalProp.Value = value
if !f.conn.SendProto(4, globalProp) {
aborted = true
}
return !aborted
}, func(entity uint64, key, value string) bool {
*entityProp.Entity = entity
*entityProp.Key = key
*entityProp.Value = value
if !f.conn.SendProto(5, entityProp) {
aborted = true
}
return !aborted
})
if aborted {
return
}
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
defer f.lock.Unlock()
f.sendingBurst = false
burstDone := new(fproto.BurstDone)
burstDone.FirstUnapplied = new(uint64)
*burstDone.FirstUnapplied = store.InstructionFirstUnapplied()
f.conn.SendProto(6, burstDone)
// Send an instruction chosen message for all chosen instructions above
// our first unapplied point.
instructions := store.InstructionSlots()
firstUnapplied := store.InstructionFirstUnapplied()
relativeSlot := int(firstUnapplied - store.InstructionStart())
for ; relativeSlot < len(instructions); relativeSlot++ {
slot := store.InstructionStart() + uint64(relativeSlot)
slotValues := instructions[relativeSlot]
if len(slotValues) != 1 || !slotValues[0].IsChosen() {
continue
}
// Convert the change request to our internal format.
chosen := new(fproto.InstructionChosen)
chosen.Slot = new(uint64)
*chosen.Slot = slot
f.conn.SendProto(7, chosen)
}
}
func handleInstructionData(f *followConn, content []byte) {
store.StartTransaction()
defer store.EndTransaction()
f.lock.Lock()
if f.closed {
f.lock.Unlock()
return
}
var msg fproto.InstructionData
if err := proto.Unmarshal(content, &msg); err != nil {
f.Close()
f.lock.Unlock()
return
}
if f.receivingBurst {
f.waiting = append(f.waiting, &msg)
f.lock.Unlock()
return
}
// We need to unlock before we start mutating the store,
// due to callbacks from the store package to elsewhere.
f.lock.Unlock()
// If we have a chosen instruction in this slot already,
// or it is prior to our instruction start number,
// discard this message.
relativeSlot := int(*msg.Slot - store.InstructionStart())
if relativeSlot < 0 {
return
}
instructions := store.InstructionSlots()
if relativeSlot < len(instructions) &&
len(instructions[relativeSlot]) == 1 &&
instructions[relativeSlot][0].IsChosen() {
return
}
// Construct a store.ChangeRequest from our
// internal ChangeRequest.
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *msg.Request.RequestEntity
chrequest.RequestNode = uint16(*msg.Request.RequestNode)
chrequest.RequestId = *msg.Request.RequestId
chrequest.Changeset = make([]store.Change,
len(msg.Request.Changeset))
for i, ch := range msg.Request.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
// Add instruction value and immediately choose it.
store.AddInstructionValue(*msg.Slot, chrequest).Choose()
}
func process() {
for {
select {
// Try to make any needed outgoing connections.
case <-tryOutgoingCh:
store.StartTransaction()
connectionsLock.Lock()
coreNodes := config.CoreNodes()
if !store.Degraded() {
if config.IsCore() {
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if connections[node] == nil {
go tryOutgoing(node)
}
}
} else {
// Settle for less than 2 core nodes,
// if there *aren't* 2 core nodes.
targetConns := 2
if targetConns > len(coreNodes) {
targetConns = len(coreNodes)
}
newOutgoing := targetConns -
len(connections)
used := -1
for newOutgoing > 0 {
r := processRand.Intn(
len(coreNodes))
// Don't do the same node
// twice.
if r == used {
continue
}
node := coreNodes[r]
if connections[node] == nil {
used = r
go tryOutgoing(node)
newOutgoing--
}
}
}
} else {
if len(connections) == 0 {
r := processRand.Intn(len(coreNodes))
node := coreNodes[r]
if connections[node] == nil {
go tryOutgoing(node)
}
}
}
connectionsLock.Unlock()
store.EndTransaction()
// After a random time, check again.
randDur := time.Duration(processRand.Intn(19)+1) * time.Second
tryOutgoingCh = time.NewTimer(randDur).C
// New received connection.
case conn := <-receivedConnCh:
conn.offerTimers = make(map[uint64]*time.Timer)
store.StartTransaction()
connectionsLock.Lock()
// If we are degraded, reject the connection,
// unless we have no other connection,
// and it is outbound.
if store.Degraded() &&
!(len(connections) == 0 && conn.outgoing) {
conn.lock.Lock()
conn.Close()
conn.lock.Unlock()
connectionsLock.Unlock()
store.EndTransaction()
break
}
// If we have an existing connection to this node,
// close it.
if connections[conn.node] != nil {
other := connections[conn.node]
other.lock.Lock()
other.Close()
if other.firstUnappliedTimer != nil {
other.firstUnappliedTimer.Stop()
}
other.lock.Unlock()
}
// Add to our connections.
connections[conn.node] = conn
// Send initial position and leader messages.
posMsg := new(fproto.Position)
posMsg.FirstUnapplied = new(uint64)
posMsg.Degraded = new(bool)
*posMsg.FirstUnapplied =
store.InstructionFirstUnapplied()
*posMsg.Degraded = store.Degraded()
conn.conn.SendProto(2, posMsg)
proposal, leader := store.Proposal()
leaderMsg := new(fproto.Leader)
leaderMsg.Proposal = new(uint64)
leaderMsg.Leader = new(uint32)
*leaderMsg.Proposal = proposal
*leaderMsg.Leader = uint32(leader)
conn.conn.SendProto(11, leaderMsg)
connectionsLock.Unlock()
store.EndTransaction()
// Start timer for sending first unapplied
// instruction updates.
if config.IsCore() && conn.node <= 0x2000 {
conn.lock.Lock()
conn.firstUnappliedTimer = time.AfterFunc(
firstUnappliedTimerDuration,
func() { conn.firstUnappliedTimeout() })
conn.lock.Unlock()
}
// Start handling received messages from the connection.
go handleConn(conn)
// Terminated connection.
case conn := <-terminatedConnCh:
connectionsLock.Lock()
if connections[conn.node] == conn {
delete(connections, conn.node)
conn.lock.Lock()
conn.closed = true
if conn.firstUnappliedTimer != nil {
conn.firstUnappliedTimer.Stop()
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
}
}
// Top-level function of the processing goroutine.
func process() {
connections = make(map[uint16][]*connect.BaseConn)
// On startup, make an outgoing connection attempt to all other
// core nodes, before continuing.
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(
connect.CONSENSUS_PROTOCOL, node)
if err != nil {
// Can't reach the other node.
continue
}
log.Print("core/consensus: made outgoing connection to ", node)
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// We should try to make an outgoing connection to any node
// that we do not have at least one connection to.
// We need to make these asynchronously, because connections
// are slow.
case <-reconnectTicker:
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
go outgoingConn(node)
}
// New change request, for us to propose as leader.
case req := <-newChangeCh:
store.StartTransaction()
if !amLeader && receivedPromises != nil {
waitingRequests = append(waitingRequests, req)
} else if !amLeader {
waitingRequests = append(waitingRequests, req)
// Start attempting to be leader.
m := make(map[uint16]*coproto.Promise)
receivedPromises = m
proposal, _ := store.Proposal()
proposal++
store.SetProposal(proposal, config.Id())
firstUn := store.InstructionFirstUnapplied()
// Send prepare messages to all other nodes.
var prepare coproto.Prepare
prepare.Proposal = new(uint64)
prepare.FirstUnapplied = new(uint64)
*prepare.Proposal = proposal
*prepare.FirstUnapplied = firstUn
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) != 0 {
c := connections[node][0]
c.SendProto(2, &prepare)
}
}
// Behave as if we got a promise message
// from ourselves.
var promise coproto.Promise
promise.Proposal = prepare.Proposal
promise.PrevProposal = promise.Proposal
promise.Leader = new(uint32)
*promise.Leader = uint32(config.Id())
promise.PrevLeader = promise.Leader
addPromise(config.Id(), &promise)
} else {
newSlot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(newSlot, req)
proposal, leader := store.Proposal()
inst := makeInst(newSlot, proposal, leader,
req)
sendProposal(inst)
}
store.EndTransaction()
// Leadership attempt timed out.
case timedOutProposal := <-leaderTimeoutCh:
store.StartTransaction()
proposal, leader := store.Proposal()
// If the proposal has changed since that
// leadership attempt, ignore it.
if leader != config.Id() {
return
}
if proposal != timedOutProposal {
return
}
// If we successfully became leader, ignore it.
if amLeader {
return
}
// Otherwise, stop our attempt to become leader.
stopBeingLeader()
store.EndTransaction()
// Proposal timed out.
case timeout := <-proposalTimeoutCh:
store.StartTransaction()
// If this timeout was not canceled, a proposal failed.
// We stop being leader.
if proposalTimeouts[timeout.slot] == timeout {
stopBeingLeader()
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
connections[node] = append(connections[node], conn)
// Received message.
case recvMsg := <-receivedMsgCh:
node := recvMsg.node
conn := recvMsg.conn
msg := recvMsg.msg
switch *msg.MsgType {
case 2:
processPrepare(node, conn, msg.Content)
case 3:
processPromise(node, conn, msg.Content)
case 4:
processAccept(node, conn, msg.Content)
case 5:
processAccepted(node, conn, msg.Content)
case 6:
processNack(node, conn, msg.Content)
default:
// Unknown message.
conn.Close()
}
// Terminate received connection.
case terminatedConn := <-terminatedConnCh:
node := terminatedConn.node
conn := terminatedConn.conn
for i, other := range connections[node] {
if other != conn {
continue
}
conns := connections[node]
conns = append(conns[:i], conns[i+1:]...)
connections[node] = conns
break
}
}
}
}