// May only be called by the processing goroutine, in a transaction.
func sendProposal(inst *coproto.Instruction) {
// We must be leader.
proposal, leader := store.Proposal()
if leader != config.Id() || !amLeader {
panic("tried to send accept messages while not leader")
}
// Send accept messages to all other core nodes.
var accept coproto.Accept
accept.Proposal = new(uint64)
*accept.Proposal = proposal
accept.Instruction = inst
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) != 0 {
c := connections[node][0]
c.SendProto(4, &accept)
}
}
// Behave as if we received our own accept message.
addAccept(config.Id(), &accept)
}
// Must be called from the processing goroutine, inside a transaction.
// Handle an accept message that we've decided to respond to.
// Called directly when we send accepted messages to other nodes.
func addAccept(node uint16, msg *coproto.Accept) {
// Send an accepted message to all other core nodes reachable.
var accepted coproto.Accepted
accepted.Proposal = msg.Proposal
accepted.Leader = new(uint32)
*accepted.Leader = uint32(node)
accepted.Instruction = msg.Instruction
// We must do this before sending accepted messages.
// We behave as if we received one from ourselves.
if !addAccepted(config.Id(), &accepted) {
// Required too large an instruction slots slice.
// We have NOT accepted this accept message,
// and must NOT send accepted messages.
return
}
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
connections[node][0].SendProto(5, &accepted)
}
}
}
// Must be called from inside a transaction.
func makeRequest(changes []store.Change) *store.ChangeRequest {
req := new(store.ChangeRequest)
req.RequestEntity = uint64(config.Id())
req.RequestNode = config.Id()
req.RequestId = store.AllocateRequestId()
req.Changeset = changes
return req
}
// Create change request creating new session entity,
// attached to user entity, this node ID attached to it.
// And the session then set as transient.
func makeNewSessionRequest(userId uint64) *store.ChangeRequest {
chset := make([]store.Change, 5)
// Create new entity.
// Entity ID 1 now refers to this within the changeset.
chset[0].TargetEntity = 1
chset[0].Key = "id"
chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10)
// Make new entity a session entity.
chset[1].TargetEntity = 1
chset[1].Key = "kind"
chset[1].Value = "session"
// Attach session entity to user.
chset[2].TargetEntity = userId
chset[2].Key = "attach 1"
chset[2].Value = "true"
// Attach node ID to session entity.
idStr := strconv.FormatUint(uint64(config.Id()), 10)
chset[3].TargetEntity = 1
chset[3].Key = "attach " + idStr
chset[3].Value = "true"
// Set session entity as transient.
chset[4].TargetEntity = 1
chset[4].Key = "transient"
chset[4].Value = "true"
return makeRequest(chset)
}
// Checks for "attach <id>" entries attaching this node ID to sessions we lack
// a connection to. Starts a timer to delete them if present.
// Must be called during a transaction, holding the session lock.
func checkOrphanAttaches() {
attachedTo := store.AllAttachedTo(uint64(config.Id()))
for _, session := range attachedTo {
if sessions[session] == nil {
startOrphanTimeout(session)
}
}
}
func SetProposal(newProposal uint64, newLeader uint16) {
if intLeader == config.Id() && newLeader != intLeader {
lastLeader = time.Now()
}
if newProposal != intProposal || newLeader != intLeader {
intProposal = newProposal
intLeader = newLeader
log.Print("shared/store: new proposal and leader ",
intProposal, " ", intLeader)
for _, cb := range proposalCallbacks {
cb()
}
}
}
func handleMsg(node uint16, conn *connect.BaseConn, msg *baseproto.Message) {
if *msg.MsgType == 2 {
// Change Forward message.
forward := new(chproto.ChangeForward)
err := proto.Unmarshal(msg.Content, forward)
if err != nil {
conn.Close()
}
// If this is not a core node,
// and the requesting node is not this node,
// discard the message.
if uint16(*forward.Request.RequestNode) != config.Id() &&
!config.IsCore() {
return
}
// Send a Change Forward Ack message to the sender.
ack := new(chproto.ChangeForwardAck)
ack.RequestNode = forward.Request.RequestNode
ack.RequestId = forward.Request.RequestId
conn.SendProto(3, ack)
// Send the forward message to our processing goroutine.
receivedForwardCh <- receivedForward{forward: forward,
node: node}
} else if *msg.MsgType == 3 {
// Change Forward Ack message.
ack := new(chproto.ChangeForwardAck)
err := proto.Unmarshal(msg.Content, ack)
if err != nil {
conn.Close()
}
// Send the ack message to our processing goroutine.
receivedAckCh <- receivedAck{ack: ack, node: node}
} else {
// Unknown message.
conn.Close()
}
}
func orphanTimeout(id uint64) {
store.StartTransaction()
defer store.EndTransaction()
// If the timeout has been removed, do nothing.
if namelessRemoveTimeouts[id] == nil {
return
}
// Detach ourselves from the session.
// TODO: Should make sure we only do this once.
ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
session := store.GetEntity(id)
if session != nil {
chset := make([]store.Change, 1)
chset[0].TargetEntity = id
chset[0].Key = ourAttachStr
chset[0].Value = ""
req := makeRequest(chset)
go chrequest.Request(req)
}
}
// Create change request creating new session entity,
// attached to user entity, this node ID attached to it.
// And the session then set as transient.
func makeNewUserRequest(username, pass, salt string,
transient bool) *store.ChangeRequest {
count := 10
if transient {
// Add an extra change for setting the user as transient.
count++
}
chset := make([]store.Change, count)
// Create new entity.
// Entity ID 1 now refers to this within the changeset.
chset[0].TargetEntity = 1
chset[0].Key = "id"
chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10)
// Make new entity a user entity.
chset[1].TargetEntity = 1
chset[1].Key = "kind"
chset[1].Value = "user"
// Set username.
chset[2].TargetEntity = 1
chset[2].Key = "name username"
chset[2].Value = username
// Set password.
chset[3].TargetEntity = 1
chset[3].Key = "auth password"
chset[3].Value = pass
// Set salt.
chset[4].TargetEntity = 1
chset[4].Key = "auth salt"
chset[4].Value = salt
// Create new entity.
// Entity ID 2 now refers to this within the changeset.
chset[5].TargetEntity = 2
chset[5].Key = "id"
chset[5].Value = strconv.FormatUint(chset[0].TargetEntity, 10)
// Make new entity a session entity.
chset[6].TargetEntity = 2
chset[6].Key = "kind"
chset[6].Value = "session"
// Attach session entity to user.
chset[7].TargetEntity = 1
chset[7].Key = "attach 2"
chset[7].Value = "true"
// Attach node ID to session entity.
idStr := strconv.FormatUint(uint64(config.Id()), 10)
chset[8].TargetEntity = 2
chset[8].Key = "attach " + idStr
chset[8].Value = "true"
// Set session entity as transient.
chset[9].TargetEntity = 2
chset[9].Key = "transient"
chset[9].Value = "true"
// Set user entity as transient, if the user is to be so.
if transient {
chset[10].TargetEntity = 1
chset[10].Key = "transient"
chset[10].Value = "true"
}
return makeRequest(chset)
}
// Main function for the handling goroutine for conn.
func handleConn(conn *userConn) {
// Do cleanup in a defer, so if they crash us we still tidy up here.
// A small nod towards tolerating bad input, with much more needed.
defer func() {
// Remove the connection from our connection set if present.
// It will not be present if we are degraded or similar.
connectionsLock.Lock()
if _, exists := connections[conn]; exists {
delete(connections, conn)
}
// Remove the connection from our waiting map if present.
// This must happen before session removal, as otherwise
// auth could complete between the session check and this one.
waitingLock.Lock()
if conn.waitingAuth != nil {
waitingConn := waiting[conn.waitingAuth.requestId]
if waitingConn == conn {
delete(waiting, conn.waitingAuth.requestId)
}
}
waitingLock.Unlock()
// Remove the connection from our sessions map if present.
// It will not be present if we are degraded,
// or have replaced this connection.
sessionsLock.Lock()
if conn.session != 0 {
sessionConn := sessions[conn.session]
if sessionConn == conn {
delete(sessions, conn.session)
// Send a change detaching this node
// from that session.
idStr := strconv.FormatUint(uint64(config.Id()),
10)
ourAttachStr := "attach " + idStr
chset := make([]store.Change, 1)
chset[0].TargetEntity = conn.session
chset[0].Key = ourAttachStr
chset[0].Value = ""
req := makeRequest(chset)
go chrequest.Request(req)
}
}
sessionsLock.Unlock()
connectionsLock.Unlock()
}()
for {
select {
case msg, ok := <-conn.conn.Received:
if !ok {
return
}
switch *msg.MsgType {
case 2:
handleAuth(conn, msg.Content)
case 3:
handleFollowUsername(conn, msg.Content)
case 4:
handleFollowUser(conn, msg.Content)
case 5:
handleStopFollowingUser(conn, msg.Content)
case 6:
handleSend(conn, msg.Content)
default:
conn.conn.Close()
}
case userMsg := <-conn.deliver:
// Get the sender's user ID.
store.StartTransaction()
attachedTo := store.AllAttachedTo(userMsg.Sender)
store.EndTransaction()
// We don't know who the sender is. Drop message.
if len(attachedTo) == 0 {
break
}
// Deliver any user messages given to us.
var deliverMsg cliproto_down.Received
deliverMsg.SenderUserId = new(uint64)
deliverMsg.SenderSessionId = new(uint64)
deliverMsg.Tag = new(string)
deliverMsg.Content = new(string)
*deliverMsg.SenderUserId = attachedTo[0]
*deliverMsg.SenderSessionId = userMsg.Sender
*deliverMsg.Tag = userMsg.Tag
*deliverMsg.Content = userMsg.Content
conn.conn.SendProto(10, &deliverMsg)
}
}
}
func process() {
for {
select {
// Try to make any needed outgoing connections.
case <-tryOutgoingCh:
store.StartTransaction()
connectionsLock.Lock()
coreNodes := config.CoreNodes()
if !store.Degraded() {
if config.IsCore() {
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if connections[node] == nil {
go tryOutgoing(node)
}
}
} else {
// Settle for less than 2 core nodes,
// if there *aren't* 2 core nodes.
targetConns := 2
if targetConns > len(coreNodes) {
targetConns = len(coreNodes)
}
newOutgoing := targetConns -
len(connections)
used := -1
for newOutgoing > 0 {
r := processRand.Intn(
len(coreNodes))
// Don't do the same node
// twice.
if r == used {
continue
}
node := coreNodes[r]
if connections[node] == nil {
used = r
go tryOutgoing(node)
newOutgoing--
}
}
}
} else {
if len(connections) == 0 {
r := processRand.Intn(len(coreNodes))
node := coreNodes[r]
if connections[node] == nil {
go tryOutgoing(node)
}
}
}
connectionsLock.Unlock()
store.EndTransaction()
// After a random time, check again.
randDur := time.Duration(processRand.Intn(19)+1) * time.Second
tryOutgoingCh = time.NewTimer(randDur).C
// New received connection.
case conn := <-receivedConnCh:
conn.offerTimers = make(map[uint64]*time.Timer)
store.StartTransaction()
connectionsLock.Lock()
// If we are degraded, reject the connection,
// unless we have no other connection,
// and it is outbound.
if store.Degraded() &&
!(len(connections) == 0 && conn.outgoing) {
conn.lock.Lock()
conn.Close()
conn.lock.Unlock()
connectionsLock.Unlock()
store.EndTransaction()
break
}
// If we have an existing connection to this node,
// close it.
if connections[conn.node] != nil {
other := connections[conn.node]
other.lock.Lock()
other.Close()
if other.firstUnappliedTimer != nil {
other.firstUnappliedTimer.Stop()
}
other.lock.Unlock()
}
// Add to our connections.
connections[conn.node] = conn
// Send initial position and leader messages.
posMsg := new(fproto.Position)
posMsg.FirstUnapplied = new(uint64)
posMsg.Degraded = new(bool)
*posMsg.FirstUnapplied =
store.InstructionFirstUnapplied()
*posMsg.Degraded = store.Degraded()
conn.conn.SendProto(2, posMsg)
proposal, leader := store.Proposal()
leaderMsg := new(fproto.Leader)
leaderMsg.Proposal = new(uint64)
leaderMsg.Leader = new(uint32)
*leaderMsg.Proposal = proposal
*leaderMsg.Leader = uint32(leader)
conn.conn.SendProto(11, leaderMsg)
connectionsLock.Unlock()
store.EndTransaction()
// Start timer for sending first unapplied
// instruction updates.
if config.IsCore() && conn.node <= 0x2000 {
conn.lock.Lock()
conn.firstUnappliedTimer = time.AfterFunc(
firstUnappliedTimerDuration,
func() { conn.firstUnappliedTimeout() })
conn.lock.Unlock()
}
// Start handling received messages from the connection.
go handleConn(conn)
// Terminated connection.
case conn := <-terminatedConnCh:
connectionsLock.Lock()
if connections[conn.node] == conn {
delete(connections, conn.node)
conn.lock.Lock()
conn.closed = true
if conn.firstUnappliedTimer != nil {
conn.firstUnappliedTimer.Stop()
}
conn.lock.Unlock()
}
connectionsLock.Unlock()
}
}
}
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
receivedPromises[node] = msg
// If we have promises from a majority of core nodes,
// become leader.
if len(receivedPromises) > len(config.CoreNodes())/2 {
log.Print("core/consensus: became leader")
stopLeaderTimeout()
amLeader = true
proposal, leader := store.Proposal()
// Find a slot number above all those in promise messages,
// and above our first unapplied.
firstUnapplied := store.InstructionFirstUnapplied()
limit := firstUnapplied
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot >= limit {
limit = *accepted.Slot + 1
}
}
}
// Start our next slot after the limit.
nextProposalSlot = limit
// For all slots between this and our first unapplied,
// submit a previously accepted instruction unless we
// know an instruction was already chosen.
// Fills these slots with proposals.
// This is O(n^2) in the number of instructions between our
// first unapplied and limit.
// TODO: Improve worst-case complexity.
start := store.InstructionStart()
slots := store.InstructionSlots()
for i := firstUnapplied; i < limit; i++ {
// If we already have a chosen instruction, skip.
rel := int(i - start)
if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
continue
}
// Find the previously accepted instruction
// accepted with the highest proposal number.
var bestInst *coproto.Instruction
var bp uint64 // Best proposal
var bl uint16 // Best leader
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot != i {
continue
}
if bestInst == nil {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
continue
}
// TODO: This indent is just absurd.
p := *accepted.Proposal
l := uint16(*accepted.Leader)
if store.CompareProposals(p, l,
bp, bl) {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
}
}
}
// If we didn't find an instruction, make an empty one.
if bestInst == nil {
empty := new(coproto.ChangeRequest)
empty.RequestEntity = new(uint64)
empty.RequestNode = new(uint32)
*empty.RequestEntity = uint64(config.Id())
*empty.RequestNode = uint32(config.Id())
bestInst := new(coproto.Instruction)
bestInst.Slot = new(uint64)
*bestInst.Slot = i
bestInst.Request = empty
}
// Add proposal timeout.
req := makeExtChangeRequest(bestInst.Request)
addProposalTimeout(i, req)
// Send proposal.
bestInst.Proposal = new(uint64)
bestInst.Leader = new(uint32)
*bestInst.Proposal = proposal
*bestInst.Leader = uint32(leader)
sendProposal(bestInst)
}
// Discard received promise messages.
receivedPromises = nil
// Make an instruction proposal for each waiting change.
for _, req := range waitingRequests {
slot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(slot, req)
inst := makeInst(slot, proposal, leader, req)
sendProposal(inst)
}
// Clear waiting changes.
waitingRequests = nil
}
}
// Top-level function of the processing goroutine.
func process() {
connections = make(map[uint16][]*connect.BaseConn)
// On startup, make an outgoing connection attempt to all other
// core nodes, before continuing.
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(
connect.CONSENSUS_PROTOCOL, node)
if err != nil {
// Can't reach the other node.
continue
}
log.Print("core/consensus: made outgoing connection to ", node)
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// We should try to make an outgoing connection to any node
// that we do not have at least one connection to.
// We need to make these asynchronously, because connections
// are slow.
case <-reconnectTicker:
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
go outgoingConn(node)
}
// New change request, for us to propose as leader.
case req := <-newChangeCh:
store.StartTransaction()
if !amLeader && receivedPromises != nil {
waitingRequests = append(waitingRequests, req)
} else if !amLeader {
waitingRequests = append(waitingRequests, req)
// Start attempting to be leader.
m := make(map[uint16]*coproto.Promise)
receivedPromises = m
proposal, _ := store.Proposal()
proposal++
store.SetProposal(proposal, config.Id())
firstUn := store.InstructionFirstUnapplied()
// Send prepare messages to all other nodes.
var prepare coproto.Prepare
prepare.Proposal = new(uint64)
prepare.FirstUnapplied = new(uint64)
*prepare.Proposal = proposal
*prepare.FirstUnapplied = firstUn
for _, node := range config.CoreNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) != 0 {
c := connections[node][0]
c.SendProto(2, &prepare)
}
}
// Behave as if we got a promise message
// from ourselves.
var promise coproto.Promise
promise.Proposal = prepare.Proposal
promise.PrevProposal = promise.Proposal
promise.Leader = new(uint32)
*promise.Leader = uint32(config.Id())
promise.PrevLeader = promise.Leader
addPromise(config.Id(), &promise)
} else {
newSlot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(newSlot, req)
proposal, leader := store.Proposal()
inst := makeInst(newSlot, proposal, leader,
req)
sendProposal(inst)
}
store.EndTransaction()
// Leadership attempt timed out.
case timedOutProposal := <-leaderTimeoutCh:
store.StartTransaction()
proposal, leader := store.Proposal()
// If the proposal has changed since that
// leadership attempt, ignore it.
if leader != config.Id() {
return
}
if proposal != timedOutProposal {
return
}
// If we successfully became leader, ignore it.
if amLeader {
return
}
// Otherwise, stop our attempt to become leader.
stopBeingLeader()
store.EndTransaction()
// Proposal timed out.
case timeout := <-proposalTimeoutCh:
store.StartTransaction()
// If this timeout was not canceled, a proposal failed.
// We stop being leader.
if proposalTimeouts[timeout.slot] == timeout {
stopBeingLeader()
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
connections[node] = append(connections[node], conn)
// Received message.
case recvMsg := <-receivedMsgCh:
node := recvMsg.node
conn := recvMsg.conn
msg := recvMsg.msg
switch *msg.MsgType {
case 2:
processPrepare(node, conn, msg.Content)
case 3:
processPromise(node, conn, msg.Content)
case 4:
processAccept(node, conn, msg.Content)
case 5:
processAccepted(node, conn, msg.Content)
case 6:
processNack(node, conn, msg.Content)
default:
// Unknown message.
conn.Close()
}
// Terminate received connection.
case terminatedConn := <-terminatedConnCh:
node := terminatedConn.node
conn := terminatedConn.conn
for i, other := range connections[node] {
if other != conn {
continue
}
conns := connections[node]
conns = append(conns[:i], conns[i+1:]...)
connections[node] = conns
break
}
}
}
}
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Prepare
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
newProposal, newLeader := *msg.Proposal, node
proposal, leader := store.Proposal()
if store.CompareProposals(newProposal, newLeader, proposal, leader) {
log.Print("core/consensus: sending promise to ", newLeader)
// Create a promise message to send back.
var promise coproto.Promise
promise.Proposal = new(uint64)
promise.Leader = new(uint32)
promise.PrevProposal = new(uint64)
promise.PrevLeader = new(uint32)
*promise.Proposal = newProposal
*promise.Leader = uint32(newLeader)
*promise.PrevProposal = proposal
*promise.PrevLeader = uint32(leader)
// Add all the instructions we've previously accepted or chosen.
slots := store.InstructionSlots()
theirFirstUnapplied := *msg.FirstUnapplied
ourStart := store.InstructionStart()
relativeSlot := int(theirFirstUnapplied - ourStart)
if relativeSlot < 0 {
relativeSlot = 0
}
var accepted []*coproto.Instruction
for ; relativeSlot < len(slots); relativeSlot++ {
slot := slots[relativeSlot]
slotNum := ourStart + uint64(relativeSlot)
for i, _ := range slot {
if slot[i].IsChosen() {
appendInst(&accepted, slotNum, slot[i])
break
}
weAccepted := false
for _, node := range slot[i].Accepted() {
if node == config.Id() {
weAccepted = true
break
}
}
if weAccepted {
appendInst(&accepted, slotNum, slot[i])
break
}
}
}
// Send promise message.
conn.SendProto(3, &promise)
// Accept the other node as our new leader.
store.SetProposal(newProposal, newLeader)
} else {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
}
}
func handleProposalChange() {
_, leader := store.Proposal()
if amLeader && leader != config.Id() {
stopBeingLeader()
}
}
// Handle a received change forward. Decides which node is responsible for it.
// Must only be called from the processing goroutine.
func processForward(forward *chproto.ChangeForward) {
// If we are already trying to forward a change forward message with
// the same requesting node and request ID, discard this message.
if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode),
*forward.Request.RequestId); exists {
return
}
// Everything else in this function runs in a transaction.
// We are read-only.
store.StartTransaction()
defer store.EndTransaction()
// If this is a core node and this node stopped being leader less than
// a Change Timeout Period ago, always add us to the ignore list.
if config.IsCore() && !isIgnored(forward, config.Id()) {
diff := time.Now().Sub(store.StoppedLeading())
if diff < config.CHANGE_TIMEOUT_PERIOD {
forward.Ignores = append(forward.Ignores,
uint32(config.Id()))
}
}
// If all core node IDs are in the forward's ignore list, discard it.
if len(forward.Ignores) == len(config.CoreNodes()) {
log.Print("shared/chrequest: dropped msg due to full ignores")
return
}
// Otherwise, choose a potential leader node.
// This is O(n^2) in the number of core nodes,
// but we don't expect to have many.
chosenNode := uint16(0)
_, leader := store.Proposal()
if leader != 0 && !isIgnored(forward, leader) {
chosenNode = leader
} else {
for _, node := range config.CoreNodes() {
if !isIgnored(forward, node) {
chosenNode = node
break
}
}
}
if chosenNode == 0 {
// Shouldn't happen.
log.Print("shared/chrequest: bug, " +
"couldn't find candidate leader node")
return
}
// If we are the selected leader, construct an external change request,
// and send it on our change request channel.
if chosenNode == config.Id() {
intRequest := forward.Request
chrequest := new(store.ChangeRequest)
chrequest.RequestEntity = *intRequest.RequestEntity
chrequest.RequestNode = uint16(*intRequest.RequestNode)
chrequest.RequestId = *intRequest.RequestId
chrequest.Changeset = make([]store.Change,
len(intRequest.Changeset))
for i, ch := range intRequest.Changeset {
chrequest.Changeset[i].TargetEntity = *ch.TargetEntity
chrequest.Changeset[i].Key = *ch.Key
chrequest.Changeset[i].Value = *ch.Value
}
for _, cb := range changeCallbacks {
cb(chrequest)
}
return
}
// Otherwise, we send it on to the selected leader,
// add the selected leader to the ignore list,
// and set a timeout to retry.
sendForward(chosenNode, forward)
forward.Ignores = append(forward.Ignores, uint32(chosenNode))
addForwardTimeout(forward)
}
func main() {
haltChan = make(chan struct{})
// Define and parse flags.
id := flag.Uint("id", 0, "Set the node ID of this unanimity instance.")
memprof = flag.Uint("memprof", 0, "Generate a memory profile and "+
"halt after the given instruction slot is applied.")
cpuprof = flag.Uint("cpuprof", 0, "Generate a CPU profile and halt "+
"after the given instruction slot is applied.")
flag.Parse()
// Validate flags.
if *id == 0 || *id > 0xFFFF {
log.Fatal("Invalid node ID specified.")
}
// If both memprof and cpuprof are set, they need to be the same.
if *memprof != 0 && *cpuprof != 0 && *memprof != *cpuprof {
log.Fatal("If both memprof and cpuprof are set they must " +
"match.")
}
// If either memprof or cpuprof are set, hook up the callback
// responsible for terminating the node at the right time.
if *memprof != 0 || *cpuprof != 0 {
store.AddAppliedCallback(handleProfileTime)
// If generating a CPU profile, start it now.
if *cpuprof != 0 {
f, err := os.Create("cpuprofile.prof")
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
}
// Load configuration from config file.
loadConfig()
config.SetId(uint16(*id))
// Load our TLS certificate.
idStr := strconv.FormatUint(uint64(config.Id()), 10)
cert, err := tls.LoadX509KeyPair(idStr+".crt", idStr+".key")
if err != nil {
log.Fatalf("error loading our TLS certificate: %s", err)
}
config.SetCertificate(&cert)
fmt.Printf("Loaded configuration, our ID is %d.\n", config.Id())
if config.IsCore() {
core.Startup()
} else {
client.Startup()
}
<-haltChan
// If we're to generate a memory profile before terminating, do so.
if *memprof != 0 {
f, err := os.Create("memprofile.mprof")
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.WriteHeapProfile(f)
}
}
// Blocks listening for connections of that given protocol type,
// and calls the specified handler when one is received,
// passing the node ID of the connecting node, or 0 for the Client Protocol.
func Listen(protocol int, handler func(id uint16, b *BaseConn)) {
ip := config.NodeIP(config.Id())
ipStr := ip.String()
port := getProtocolPort(protocol)
portStr := strconv.FormatInt(int64(port), 10)
tlsConfig := new(tls.Config)
tlsConfig.Certificates = []tls.Certificate{*config.Certificate()}
if protocol != CLIENT_PROTOCOL {
tlsConfig.ClientAuth = tls.RequireAnyClientCert
}
listener, err := tls.Listen("tcp", ipStr+":"+portStr, tlsConfig)
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept()
if err != nil {
log.Fatal(err)
}
tlsConn := conn.(*tls.Conn)
err = tlsConn.Handshake()
if err != nil {
tlsConn.Close()
continue
}
if protocol != CLIENT_PROTOCOL {
// Check this connecting node has authenticated.
state := tlsConn.ConnectionState()
if len(state.PeerCertificates) == 0 {
tlsConn.Close()
continue
}
cert := state.PeerCertificates[0]
// Identify which node they authenticated as.
matched := false
for _, node := range config.Nodes() {
var verifyOpts x509.VerifyOptions
verifyOpts.Intermediates = new(x509.CertPool)
verifyOpts.Roots = config.NodeCertPool(node)
chains, err := cert.Verify(verifyOpts)
if err != nil {
continue
}
// Matched the node. Start the handler.
if len(chains) > 0 {
matched = true
go handler(node, newBaseConn(tlsConn))
break
}
}
// No matching node found. Close the connection.
if !matched {
tlsConn.Close()
}
} else {
// We don't authenticate clients.
// Just run the handler.
handler(0, newBaseConn(tlsConn))
}
}
}
// Sets this instruction value as chosen, clearing all others in the same slot.
// Causes it to be automatically applied to state once all previous instruction
// slots have a chosen value.
func (i *InstructionValue) Choose() {
log.Print("shared/store: choosing instruction in slot ", i.slot)
relativeSlot := i.slot - start
// Drop all other instruction values in the same slot.
if len(slots[relativeSlot]) > 1 {
slots[relativeSlot] = []*InstructionValue{i}
}
// Set the value as chosen.
i.chosen = true
i.chosenTime = time.Now()
i.acceptedBy = nil
// Call instruction chosen callbacks.
for _, cb := range chosenCallbacks {
cb(i.slot)
}
// Apply instructions as far as possible.
tryApply()
// Discard no longer needed instructions, from the start.
newStart := start
for {
relative := newStart - start
// If we've not applied the instruction yet,
// we still need it.
if firstUnapplied == newStart {
break
}
// If the instruction wasn't chosen long enough ago,
// we can't discard it.
minDelay := time.Duration(4) * config.CHANGE_TIMEOUT_PERIOD
timeChosen := slots[relative][0].TimeChosen()
if time.Now().Sub(timeChosen) < minDelay {
break
}
// If this is a core node and the majority of core nodes aren't
// past this point, we have to keep it.
if config.IsCore() {
past := 1 // We are past it.
coreNodes := config.CoreNodes()
for _, node := range coreNodes {
if node == config.Id() {
continue
}
if NodeFirstUnapplied(node) > newStart {
past++
}
}
if past <= len(coreNodes)/2 {
break
}
}
newStart++
}
// If we can discard, do it by reslicing the slots.
// This doesn't result in an immediate free,
// but will when we next expand the slots.
// This avoids extra performance costs for reallocating them now.
if newStart != start {
log.Print("Discarding instructions between ", newStart-1,
" and ", start)
slots = slots[newStart-start:]
start = newStart
}
}
func handleApplied(slot uint64, idemChanges []store.Change) {
relativeSlot := int(slot - store.InstructionStart())
slots := store.InstructionSlots()
requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode
requestId := slots[relativeSlot][0].ChangeRequest().RequestId
sessionsLock.Lock()
defer sessionsLock.Unlock()
waitingLock.Lock()
defer waitingLock.Unlock()
// If there is a connection which has this as its waiting request,
// have it complete authentication.
if requestNode == config.Id() {
waitingConn := waiting[requestId]
if waitingConn != nil {
handleAuthComplete(waitingConn, idemChanges)
}
}
ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10)
for i := range idemChanges {
key := idemChanges[i].Key
value := idemChanges[i].Value
if strings.HasPrefix(key, "attach ") {
// If we were just detached from a session,
// drop any connections associated with it,
// and if it was in our attach timeouts, remove it.
if key == ourAttachStr && value == "" {
sessionId := idemChanges[i].TargetEntity
if conn := sessions[sessionId]; conn != nil {
log.Print("client/listener: " +
"detached from existing " +
"session")
conn.conn.Close()
}
if orphanAttachTimeouts[sessionId] != nil {
orphanAttachTimeouts[sessionId].Stop()
delete(orphanAttachTimeouts, sessionId)
}
}
// If we were just attached to a session,
// but have no connection associated with that session,
// add an attach timeout.
// and if it was in our attach timeouts, remove it.
if key == ourAttachStr && value != "" {
sessionId := idemChanges[i].TargetEntity
if sessions[sessionId] == nil {
startOrphanTimeout(sessionId)
}
}
// Handle a session attaching/detaching from a user.
entityId := idemChanges[i].TargetEntity
entity := store.GetEntity(entityId)
if entity != nil && entity.Value("kind") == "user" {
// Tell following connections when a session
// attaches or detaches from a followed user.
// TODO: Expensive loop,
// should maintain index to avoid this.
for conn, _ := range connections {
following := false
for _, fw := range conn.following {
if fw == entityId {
following = true
break
}
}
if !following {
continue
}
sendUserChange(conn, entityId,
key, value)
}
}
}
}
// If we've created a nameless user, start a timer to delete it.
for i := range idemChanges {
if idemChanges[i].Key != "kind" ||
idemChanges[i].Value != "user" {
continue
}
newUserId := idemChanges[i].TargetEntity
newUser := store.GetEntity(newUserId)
if newUser != nil && newUser.Value("name username") == "" {
startNamelessTimeout(newUserId)
}
}
}
// Can only be called from the handling goroutine for conn.
func handleAuth(conn *userConn, content []byte) {
var msg cliproto_up.Authenticate
if err := proto.Unmarshal(content, &msg); err != nil {
conn.conn.Close()
return
}
// Try to get information, then release locks and hash the password.
// If the situation changes we may have to hash it again anyway,
// but scrypt hashing is extremely expensive and we want to try to
// do this without holding our locks in the vast majority of cases.
var tryPassGenerated bool
var trySaltGenerated bool
var tryPass, trySalt, tryKey string
if *msg.Password != "" {
tryPass = *msg.Password
store.StartTransaction()
userId := store.NameLookup("user", "name username",
*msg.Username)
if userId != 0 {
user := store.GetEntity(userId)
if user != nil {
trySalt = user.Value("auth salt")
}
}
store.EndTransaction()
if trySalt == "" {
trySaltGenerated = true
var err error
trySalt, tryKey, err = genRandomSalt(conn,
[]byte(tryPass))
if err != nil {
return
}
} else {
var err error
tryPass, err = genKey(conn, []byte(tryPass),
[]byte(trySalt))
if err != nil {
return
}
}
} else {
tryPassGenerated = true
trySaltGenerated = true
var err error
tryPass, trySalt, tryKey, err = genRandomPass(conn)
if err != nil {
return
}
}
// TODO: Validate username and password against constraints.
// We hold this through quite a lot of logic.
// Would be good to be locked less.
store.StartTransaction()
defer store.EndTransaction()
sessionsLock.Lock()
defer sessionsLock.Unlock()
waitingLock.Lock()
defer waitingLock.Unlock()
if conn.session != 0 || conn.waitingAuth != nil {
conn.conn.Close()
return
}
userId := store.NameLookup("user", "name username", *msg.Username)
if userId != 0 {
if *msg.Password == "" {
sendAuthFail(conn, "Invalid Password")
return
}
// The user already exists.
user := store.GetEntity(userId)
// Try to authenticate them to it.
var key string
// If their salt and password matches our attempt above,
// we can just take that key.
salt := user.Value("auth salt")
if trySalt == salt && tryPass == *msg.Password {
key = tryKey
} else {
saltBytes := []byte(user.Value("auth salt"))
passBytes := []byte(*msg.Password)
var err error
key, err = genKey(conn, passBytes, saltBytes)
if err != nil {
return
}
}
if user.Value("auth password") != string(key) {
sendAuthFail(conn, "Invalid Password")
return
}
// It's the real user.
if *msg.SessionId != 0 {
// They are attaching to an existing session.
// Check it exists and is attached to this user.
strSessionId := strconv.FormatUint(*msg.SessionId, 10)
if user.Value("attach "+strSessionId) == "" {
sendAuthFail(conn, "Invalid Session")
return
}
// The session does exist.
conn.session = *msg.SessionId
// If this node is already attached to
// the session, drop the other connection.
if sessions[conn.session] != nil {
sessions[conn.session].conn.Close()
} else {
// Create change attaching this node ID
// to the session.
id := config.Id()
idStr := strconv.FormatUint(uint64(id), 10)
chset := make([]store.Change, 1)
chset[0].TargetEntity = conn.session
chset[0].Key = "attach " + idStr
chset[0].Value = "true"
req := makeRequest(chset)
go chrequest.Request(req)
}
// Put us in the sessions map.
sessions[conn.session] = conn
// Tell the client they authenticated successfully.
sendAuthSuccess(conn, "")
} else {
// They are creating a new session.
req := makeNewSessionRequest(userId)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
}
} else {
// The user does not already exist.
// Check they weren't trying to attach to a session.
if *msg.SessionId != 0 {
sendAuthFail(conn, "User Does Not Exist")
return
}
// We're creating a new user.
newUser := *msg.Username
newPass := *msg.Password
if !strings.HasPrefix(newUser, "Guest-") {
// We're creating a new non-guest user.
// Make sure they have a password.
if newPass == "" {
sendAuthFail(conn, "No Password")
return
}
var salt string
var hash string
if tryPass == newPass && trySaltGenerated {
salt = trySalt
hash = tryKey
} else {
passBytes := []byte(newPass)
var err error
salt, hash, err = genRandomSalt(conn, passBytes)
if err != nil {
return
}
}
// Create the new user.
req := makeNewUserRequest(newUser, hash, salt, false)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
return
}
// We're creating a new guest user.
// Guests get automatic passwords, and can't set them.
if newPass != "" {
sendAuthFail(conn, "Cannot Set Password For Guest User")
return
}
var hash string
var salt string
if tryPassGenerated && trySaltGenerated {
newPass = tryPass
salt = trySalt
hash = tryKey
} else {
var err error
newPass, salt, hash, err = genRandomPass(conn)
if err != nil {
return
}
}
waitingLock.Lock()
// Create the new user.
req := makeNewUserRequest(newUser, hash, salt, true)
go chrequest.Request(req)
// Stuff details in waiting auth.
conn.waitingAuth = new(authData)
conn.waitingAuth.msg = msg
conn.waitingAuth.requestId = req.RequestId
waiting[conn.waitingAuth.requestId] = conn
waitingLock.Unlock()
return
}
}
// If the leader ID matches our ID,
// updates our "last stopped leading" time to now.
// Call when, whether or not we've a new leader,
// we consider ourselves to have stopped being the leader.
func StopLeading() {
if intLeader == config.Id() {
lastLeader = time.Now()
}
}
func process() {
// Try to make an outgoing connection to all other client nodes,
// if we're not in a degraded state.
store.StartTransaction()
if !store.Degraded() {
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
conn, err := connect.Dial(connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] = append(connections[node], conn)
go handleConn(node, conn)
}
}
store.EndTransaction()
// Retry connections once per config.CHANGE_TIMEOUT_PERIOD
// Largely arbitrary.
reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)
for {
select {
// Connection retry tick.
// If not degraded, try to make an outgoing connection to any
// client node that we do not have at least one connection to.
case <-reconnectTicker:
store.StartTransaction()
// Do not attempt to make connections while degraded.
if store.Degraded() {
store.EndTransaction()
break
}
for _, node := range config.ClientNodes() {
if node == config.Id() {
continue
}
if len(connections[node]) > 0 {
continue
}
conn, err := connect.Dial(
connect.RELAY_PROTOCOL, node)
if err != nil {
// No luck connecting.
continue
}
connections[node] =
append(connections[node], conn)
go handleConn(node, conn)
}
store.EndTransaction()
// New received connection.
case receivedConn := <-receivedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// If we are degraded, reject the connection.
if store.Degraded() {
conn.Close()
store.EndTransaction()
break
}
// Add to our connections.
connections[node] = append(connections[node], conn)
store.EndTransaction()
// Terminated connection.
case receivedConn := <-terminatedConnCh:
node := receivedConn.node
conn := receivedConn.conn
store.StartTransaction()
// Remove this connection from our connection list.
index := -1
for i, _ := range connections[node] {
if conn == connections[node][i] {
index = i
break
}
}
if index != -1 {
connections[node] =
append(connections[node][:index],
connections[node][index+1:]...)
}
store.EndTransaction()
}
}
}
